Makefile

SHELL := /bin/bash
.SHELLFLAGS := -o pipefail -c

# `_comma` lets us pass a literal comma into $(subst) where the function-call
# syntax would otherwise interpret it as an argument separator. Used to convert
# autocollie's csv-joined env values (e.g. `pe=0.5,zip=2.0`) back into the
# space-separated form make's $(foreach) expects.
_comma := ,
.PHONY: azoth-backtest azoth-build-allowlist azoth-allowlist-monitor azoth-allowlist-tune azoth-full-train azoth-fast-train azoth-publish-train _azoth-train azoth-general azoth-general-fold-a azoth-general-fold-b azoth-oof-merge-general evaluate explain inspect errors scan traits thresholds thresholds-refresh filetype-matrix elf-model-benchmark elf-route-optimization azoth-specialists azoth-specialists-fold-a azoth-specialists-fold-b azoth-prefill-specialist-features azoth-oof-route-scores azoth-calibrate azoth-diagnostics azoth-policies azoth-deploy azoth-deploy-final azoth-check-bundle-layout azoth-shap false-positives false-negatives near-false-positives near-false-negatives false-positives-archive false-negatives-archive near-false-positives-archive near-false-negatives-archive false-positives-triage false-negatives-triage near-false-positives-triage mislabeled-triage benchmark build-splits experiment ablate ablation demo-db test lint clean deploy verify-litmus venv help repin azoth-clean-bundle

VENV_DIR ?= .venv
PYTHON ?= $(VENV_DIR)/bin/python
DB ?= postgres://hopper@localhost:5432/hopper
PROFILE_PERCENT ?= 1
PROFILE_WORKERS ?= 48
PROFILE_PROFILER ?= cprofile
PROFILE_STOP_STAGE ?= 9
PROFILE_TARGET_MINUTES ?= 15
PROFILE_FIT_MEMORY_GB ?=
PROFILE_EXTRACT_WORKERS_PER_FIT ?=
PROFILE_SCORING_WORKERS ?= 6 12 6
MODEL ?= azoth
LEARNER ?= $(if $(filter azoth%,$(MODEL)),azoth,$(MODEL))
OUT_ROOT ?= out/models
OUT_DIR ?= $(if $(filter azoth,$(MODEL)),$(OUT_ROOT)/azoth/general,$(OUT_ROOT)/$(MODEL))
# Resolve the bundle's model file. Single-seed bundles ship `model.txt`
# directly under $(OUT_DIR); multi-seed bundles ship `models/seed_NN.txt`
# (the deployed Azoth general is 3-seed). Auto-pick whichever is on disk:
# `model.txt` if present, else the lowest-numbered seed, else fall back
# to the legacy `model.txt` name (callers get a clear FileNotFoundError
# rather than a misleading "missing arg" failure).
MODEL_FILE_EXT := $(if $(filter azoth,$(LEARNER)),txt,json)
MODEL_FILE ?= $(patsubst $(OUT_DIR)/%,%,$(or \
	$(wildcard $(OUT_DIR)/model.$(MODEL_FILE_EXT)), \
	$(firstword $(sort $(wildcard $(OUT_DIR)/models/seed_*.$(MODEL_FILE_EXT)))), \
	$(OUT_DIR)/model.$(MODEL_FILE_EXT)))
LOG_DIR ?= $(OUT_DIR)/logs
EXP_OUT_DIR ?= out/experiments/$(MODEL)
AZOTH_FOLD_A_EXP_OUT_DIR ?= out/experiments/azoth.oof-fold-a
AZOTH_FOLD_B_EXP_OUT_DIR ?= out/experiments/azoth.oof-fold-b
EXP_LOG_DIR ?= $(EXP_OUT_DIR)/logs
THRESHOLD_SCORES ?= $(OUT_DIR)/threshold_scores.npz
THRESHOLD_MAX_ID ?=
THRESHOLD_MAX_ID_ARG := $(if $(THRESHOLD_MAX_ID),--max-id $(THRESHOLD_MAX_ID),)
TOP_ERRORS ?= 250
SKIP ?= 0
THRESHOLD_TOP_ERRORS ?= 0
FILETYPE_MATRIX_OUTPUT ?= $(OUT_DIR)/filetype_metrics.json
FILETYPE_MATRIX_CSV ?= $(OUT_DIR)/filetype_metrics.csv
FILETYPE_MATRIX_MIN_COUNT ?= 25
ELF_BENCHMARK_OUTPUT ?= $(OUT_ROOT)/elf_model_benchmark.json
ELF_BENCHMARK_GENERAL_DIR ?= $(OUT_ROOT)/azoth/general
ELF_BENCHMARK_BINARY_DIR ?= $(OUT_ROOT)/azoth-binary-cpu
ELF_BENCHMARK_ELF_DIR ?= $(OUT_ROOT)/azoth-elf-cpu
ELF_BENCHMARK_FOLDS ?= 2
ELF_BENCHMARK_ESTIMATORS ?= 400
ELF_BENCHMARK_MAX_DEPTH ?= 12
ELF_BENCHMARK_LEARNING_RATE ?= 0.05
ELF_BENCHMARK_EARLY_STOPPING ?= 50
ELF_BENCHMARK_NUM_LEAVES ?= 96
ELF_BENCHMARK_MIN_CHILD_SAMPLES ?= 100
AZOTH_ROOT ?= $(OUT_ROOT)/azoth
# Run-isolation layout. Each fresh train allocates a directory under
# $(AZOTH_RUNS_ROOT)/ and writes the entire bundle there; on success
# azoth-publish atomically updates $(OUT_ROOT)/azoth to point at it.
# See scripts/azoth_publish_run.py and the azoth-run-new / azoth-publish
# targets below.
AZOTH_RUNS_ROOT ?= $(OUT_ROOT)/azoth-runs
# Successful run dirs to retain under $(AZOTH_RUNS_ROOT) after a publish;
# older ones (and legacy-* archives) are pruned. The live symlink target is
# always kept regardless of age.
AZOTH_KEEP_RUNS ?= 5
AZOTH_SPECIALISTS_SUMMARY ?= $(AZOTH_ROOT)/specialists.json
AZOTH_GENERAL_DIR ?= $(AZOTH_ROOT)/general
AZOTH_GENERAL_SCORES ?= $(AZOTH_GENERAL_DIR)/threshold_scores.npz
AZOTH_CONFIG ?= $(AZOTH_ROOT)/config.json
AZOTH_SCORE_TABLE ?= $(AZOTH_ROOT)/score_table.npz
AZOTH_DIAGNOSTICS ?= $(AZOTH_ROOT)/route_diagnostics.md
AZOTH_DIAGNOSTICS_CSV ?= $(AZOTH_ROOT)/route_diagnostics.csv
AZOTH_SLICE_METRICS ?= $(AZOTH_ROOT)/slice_metrics.md
AZOTH_SLICE_METRICS_CSV ?= $(AZOTH_ROOT)/slice_metrics.csv
AZOTH_ROUTE_POLICIES ?= $(AZOTH_ROOT)/route_policies.json
AZOTH_ROUTE_POLICIES_CSV ?= $(AZOTH_ROOT)/route_policies.csv

# Discover features from each route's training rows. Frequency cutoffs and
# per-family caps bound the vocabulary; a deployed model is not required.
# A fixed allowlist is an optional optimization, explicitly selected for both
# experiments and full training. Empty means no additional name filter.
AZOTH_ALLOWED_FEATURES_FILE ?=
export COLLIMATOR_ALLOWED_FEATURES_FILE := $(AZOTH_ALLOWED_FEATURES_FILE)

# Optional size optimization: discover candidates from current training rows,
# then retain frequent columns plus names used by existing tree splits.
# AZOTH_ALLOWLIST_SIZE is the aggressiveness knob (memory vs tail) that autocollie
# should sweep + validate against FP/100M; it is NOT pinned here.
AZOTH_ALLOWLIST_SIZE ?= 15000
AZOTH_ALLOWLIST_SAMPLE_PER_TYPE ?= 6000
AZOTH_ALLOWLIST_SAMPLING_WORKERS ?= 4
AZOTH_ALLOWLIST_MAX_DB_WORKERS ?= 24
AZOTH_ALLOWLIST_CACHE_DIR ?= out/cache/azoth-allowlist
AZOTH_ALLOWLIST_CACHE_MAX_AGE ?= 3600
AZOTH_ALLOWLIST_REFRESH ?= 0
azoth-build-allowlist: venv check-db
	$(EXPERIMENT_FEATURE_ENV) $(PYTHON) scripts/azoth_build_feature_allowlist.py \
		--bundle $(AZOTH_DEPLOY_DIR) \
		--db $(DB) \
		--workers $(if $(WORKERS),$(WORKERS),16) \
		--target-size $(AZOTH_ALLOWLIST_SIZE) \
		--sample-per-type $(AZOTH_ALLOWLIST_SAMPLE_PER_TYPE) \
		--sampling-workers $(AZOTH_ALLOWLIST_SAMPLING_WORKERS) \
		--max-db-workers $(AZOTH_ALLOWLIST_MAX_DB_WORKERS) \
		--cache-dir $(AZOTH_ALLOWLIST_CACHE_DIR) \
		--cache-max-age $(AZOTH_ALLOWLIST_CACHE_MAX_AGE) \
		$(if $(filter 1 true yes,$(AZOTH_ALLOWLIST_REFRESH)),--refresh,) \
		--output out/models/azoth/allowed_features.json
	@echo "to use it for the next retrain:"
	@echo "  make azoth-full-train AZOTH_ALLOWED_FEATURES_FILE=out/models/azoth/allowed_features.json"
	@echo "then gate before deploy: scripts/check_azoth_regression.py (already in the deploy flow)"

# Non-blocking drift monitor: re-extracts the proxy routes from the deployed
# bundle + current corpus, then scores the CURRENT pin ($(AZOTH_ALLOWED_FEATURES_FILE))
# full-vs-pruned. Read-only — it never changes the pin or deploys; it just
# reports worst_dR@1e-3 and an OK / RE-TUNE verdict so you (or autocollie) know
# when the committed level has drifted. Run it after a deploy cycle. The proxy
# screens coarse quality only (R@1e-3); the L50 tail stays owned by the gate.
AZOTH_MONITOR_ROUTES ?= general,elf,source
azoth-allowlist-monitor: venv check-db
	@test -n "$(AZOTH_ALLOWED_FEATURES_FILE)" || { echo "error: set AZOTH_ALLOWED_FEATURES_FILE to the allowlist to monitor"; exit 1; }
	-$(PYTHON) scripts/azoth_allowlist_experiment.py \
		--prepare --routes $(AZOTH_MONITOR_ROUTES) --db $(DB) \
		--workers $(if $(WORKERS),$(WORKERS),16) \
		--committed $(AZOTH_ALLOWED_FEATURES_FILE) \
		--out out/allowlist_exp/monitor.json
	@echo "monitor report: out/allowlist_exp/monitor.json (non-blocking; never changes the pin)"

# The allowlist "virtual route": sweep candidate levels on the proxy routes and
# write the SMALLEST allowlist whose worst proxy R@1e-3 drop stays within
# AZOTH_TUNE_TOLERANCE. Produces a CANDIDATE pin only — promotion is gated
# (validate via a check_azoth_regression'd full-train, never auto-adopted).
# This is the collimator-side entry point autocollie drives for the level knob;
# the proxy screens coarse quality, the gate owns the L50 tail.
AZOTH_TUNE_LEVELS ?= 8000,11000,14000,17000,20000
AZOTH_TUNE_TOLERANCE ?= -0.002
azoth-allowlist-tune: venv check-db
	-$(PYTHON) scripts/azoth_allowlist_experiment.py \
		--prepare --routes $(AZOTH_MONITOR_ROUTES) --db $(DB) \
		--workers $(if $(WORKERS),$(WORKERS),16) \
		--tune $(AZOTH_TUNE_LEVELS) --tolerance $(AZOTH_TUNE_TOLERANCE) \
		--out out/models/azoth/allowed_features_candidate.json
	@echo "candidate pin: out/models/azoth/allowed_features_candidate.json (NOT adopted — gated)"
	@echo "validate via gated retrain:"
	@echo "  make azoth-full-train AZOTH_ALLOWED_FEATURES_FILE=out/models/azoth/allowed_features_candidate.json EXP_RERUN=1"

AZOTH_ROUTE_POLICIES_MD ?= $(AZOTH_ROOT)/route_policies.md
AZOTH_GLOBAL_POLICY_METRICS ?= $(AZOTH_ROOT)/global_policy_metrics.json
AZOTH_GLOBAL_POLICY_METRICS_MD ?= $(AZOTH_ROOT)/global_policy_metrics.md
AZOTH_ROUTED_METRICS_ARGS ?=
# When set, compute_routed_metrics and azoth_route_policy_search carry
# forward unchanged-filetype entries from this bundle's previous outputs
# instead of recomputing them. Honest invalidation: any model/feature
# spec hash diff triggers recomputation for the affected filetypes.
# Major wall-time savings on single-route promotes (~3-5 min off each
# of policy_search and compute_routed_metrics).
#
# Defaults to $(OUT_ROOT)/azoth — the SOURCE training bundle is what
# the candidate was cloned from (autocollie promote does
# cloneTreeFast(baselineRoot, candidateRoot) → only the changed route's
# files differ from this baseline). Using the live deployed bundle as
# the comparison point fails: it may not have been refreshed in days,
# so its model files genuinely differ on many routes from the training
# baseline, defeating the impact detection.
#
# Empty disables the optimization.
AZOTH_PREVIOUS_BUNDLE ?= $(OUT_ROOT)/azoth
AZOTH_PREVIOUS_BUNDLE_ARG = $(if $(AZOTH_PREVIOUS_BUNDLE),--previous-bundle $(AZOTH_PREVIOUS_BUNDLE),)
# Carry exact auxiliary artifacts from the preceding training bundle into a
# fresh run. Their scores and policies are recalibrated; existing gates remain.
AZOTH_RETAIN_AUXILIARIES_ARG = $(if $(and $(AZOTH_PREVIOUS_BUNDLE),$(wildcard $(AZOTH_PREVIOUS_BUNDLE)/config.json)),--retain-auxiliaries-from "$(AZOTH_PREVIOUS_BUNDLE)",)
# Impact filtering is valid for a candidate cloned from another bundle. A full
# build compared with itself must still check every filetype against deployed.
AZOTH_REGRESSION_SOURCE_ARG = $(if $(and $(AZOTH_PREVIOUS_BUNDLE),$(filter-out $(realpath $(AZOTH_ROOT)),$(realpath $(AZOTH_PREVIOUS_BUNDLE)))),--source-bundle $(AZOTH_PREVIOUS_BUNDLE),)
AZOTH_VALIDATE_ROUTED_METRICS_ARGS ?= --no-ci --no-stacked
AZOTH_VALIDATE_DIAGNOSTICS ?= 0
# Whether azoth-deploy emits route_diagnostics.{md,csv} and slice_metrics.{md,csv}.
# Diagnostics are human-debugging artifacts that nothing downstream consumes —
# regression check, policy search, and routed metrics all read directly from
# config.json + score_table.npz. Generating them on every deploy adds many
# minutes to the critical path for output most deploys never read. Off by
# default; run `make azoth-diagnostics` standalone when you actually need
# the report (the deployed bundle has everything it needs to regenerate them
# on demand). Set to 1/true/yes to opt back in.
AZOTH_DEPLOY_DIAGNOSTICS ?= 0
# Low-water-mark for the regression gate: a pinned reference bundle's
# route_policy_eval_oof.json that no future deploy is allowed to fall more
# than --lwm-tolerance below (per filetype). Lives outside $(AZOTH_ROOT)
# so azoth-clean-bundle doesn't wipe it. Captured via
# `make azoth-set-low-water-mark`. If the file doesn't exist the
# regression gate silently skips the LWM check, so this is fully
# opt-in — set the LWM once you have a deploy you want to lock in.
AZOTH_LOW_WATER_MARK_DIR ?= $(OUT_ROOT)/azoth_low_water_mark
# Check all saved levels; only L0–L25 can block. Higher levels are report only.
# Extra args appended to the check_azoth_regression.py deploy gate. Default is
# --net-improvement-fallback: a net-positive deploy (malware-weighted TP delta
# > 0) ships even with per-filetype regressions, PROVIDED no high-volume route
# (>= --catastrophe-min-mal malware) craters past --max-net-route-regression.
# Small-route craters are dataset-level noise (sampling variance on a few
# hundred samples) and don't block; a real regression on a route big enough to
# move the average still does. Override to e.g.
#   AZOTH_REGRESSION_ARGS="--recall-tolerance 0 --lwm-tolerance 0"
# for a stricter per-filetype gate, or set AZOTH_ALLOW_REGRESSION=1 to bypass
# entirely. litmus validate + severity-sync stay on regardless.
AZOTH_REGRESSION_ARGS ?= --net-improvement-fallback
# Report held-out FP counts and per-filetype allowance overruns at L25.
# Installation is gated by detection regressions and runtime validation;
# the development FP allowance is not a mandatory held-out install budget.
# Operators can opt into the stricter count gate explicitly:
#   AZOTH_BUDGET_GATE_ARGS=--fail-on-budget
AZOTH_BUDGET_GATE_ARGS ?=
AZOTH_POLICY_OVERRIDE_ROUTE ?=
# scan v2 reads its default bundle from <data>/atomdrift/scan/models/<bundle>,
# where <bundle> is the last segment of SCAN_MODELS_REPO (default repo
# github.com/atomdrift-project/azoth.git -> "azoth"). Deploy there so `ascan scan`
# picks up the bundle at runtime without SCAN_MODELS_DIR.
AZOTH_DEPLOY_DIR ?= $(XDG_DATA_HOME)/atomdrift/scan/models/azoth
ELF_ROUTE_OUTPUT_DIR ?= $(AZOTH_ROOT)/elf_route_optimization
ELF_ROUTE_OUTPUT ?= $(AZOTH_ROOT)/elf_route_optimization.json
ELF_ROUTE_TEACHER_DIR ?= $(AZOTH_ROOT)/filetypes/elf
AZOTH_REFRESH_SCORES ?= 0
AZOTH_REFRESH_SCORES_ARG := $(if $(filter 1 true yes,$(AZOTH_REFRESH_SCORES)),--refresh,)
AZOTH_REFRESH_ROUTE ?=
AZOTH_SKIP_LEVEL_CALIBRATION ?= 0
AZOTH_SKIP_LEVEL_CALIBRATION_ARG := $(if $(filter 1 true yes,$(AZOTH_SKIP_LEVEL_CALIBRATION)),--skip-level-calibration,)
AZOTH_SKIP_LITMUS_VALIDATE ?= 0
AZOTH_FEATURE_CACHE_DIR ?= out/cache/azoth-route-features
# Days of route-feature matrices to retain under $(AZOTH_FEATURE_CACHE_DIR).
# Entries are keyed route+max_id+spec_hash+rows_hash, and max_id is the live
# max(id) at run time, so every run mints a generation that can never be hit
# again unless THRESHOLD_MAX_ID pins that exact value. Only same-day reuse
# (general -> specialists -> calibrate) matters; older generations are dead
# weight at ~20G apparent each. Pruned by azoth-prune-feature-cache.
AZOTH_KEEP_CACHE_DAYS ?= 2
AZOTH_SPECIALIST_FOLDS ?= 0
AZOTH_SPECIALIST_ESTIMATORS ?= 400
AZOTH_SPECIALIST_MAX_DEPTH ?= 12
AZOTH_SPECIALIST_LEARNING_RATE ?= 0.05
AZOTH_SPECIALIST_EARLY_STOPPING ?= 50
AZOTH_SPECIALIST_NUM_LEAVES ?= 96
AZOTH_SPECIALIST_MIN_CHILD_SAMPLES ?= 100
AZOTH_SPECIALIST_MIN_BAD ?= 50
AZOTH_SPECIALIST_MIN_GOOD ?= 50
# Multi-seed averaging (item A): K extra seeds trained against the same matrix.
# Default 2 → 3 trained seeds, averaged at predict time. Reduces seed-driven
# prediction variance by ~3× at 3× training cost; matrix extraction (the
# slow part) is shared across seeds. K=2 is the sweet spot — most independent
# variance is captured in the first 3 samples; K>2 hits diminishing returns
# because tree boosters share enough hyperparam-driven structure that the
# noise terms don't actually decorrelate to zero. Setting K=0 reverts to the
# legacy single-model layout (model.txt) — useful for debugging, since
# averaged bundles smear blame across members when something regresses.
#
# 2026-05 update: the default is now 0 (single seed). Observed seed-to-seed
# variance on holdout is ~3pp F1 (verified on go/python/pe specialists from a
# k=3 fold run); averaging trims that to ~0.1-0.5pp at the headline recall,
# which is smaller than the per-pipeline-run noise our eval harness can
# resolve. Setting K=2 burns ~18 GPU hours per publish-train for an unverified
# fraction-of-a-percent of recall — not worth it now that we have honest
# OOF + recall-monotone floor + learned_blend in the pipeline. Bump back to
# K=2 if you measure a real regression on test partition; the infrastructure
# is in place to A/B this cleanly.
AZOTH_SPECIALIST_N_SEED_EXTRAS ?= 2
# Reject pathological seed fits before they enter the averaged specialist
# bundle. A rejected member is replaced with the next deterministic seed, up
# to this per-route retry budget; exhaustion omits the route so the runtime
# falls back to its filegroup/general ensemble instead of shipping bad scores.
AZOTH_SPECIALIST_SEED_HEALTH_MIN_ROC_AUC ?= 0.5
AZOTH_SPECIALIST_SEED_RETRY_BUDGET ?= 3
# OOF specialist training (fold-A / fold-B / route-score merge) doesn't ship
# the resulting bundles to deploy — only their OOF predictions get merged.
# Variance reduction from multi-seed averaging doesn't compound through the
# merge (the OOF prediction for each row already comes from a single fold
# model), so the extra seeds add training cost for no calibration benefit.
# Default to 0 extras (single seed) for OOF runs; override to 2 if you want
# parity with production training for an apples-to-apples comparison.
AZOTH_OOF_SEED_EXTRAS ?= 2
# Companion knob for the fold GENERAL trainings: the standard
# azoth_train_best.py invocation sets EXP_SEED_SEARCH_K=3 (train 3 seeds,
# ship the best). Fold bundles aren't shipped — only their OOF predictions
# merge into general/threshold_scores.npz — so the K-1 discarded seeds
# are pure waste during OOF. Production training keeps K=3.
AZOTH_OOF_SEED_SEARCH_K ?= 1
# Skip the per-specialist benchmark extract+score during fold training.
# benchmark.json's diagnostic metrics are only consumed by
# write_azoth_readmes (which only sees the production bundle), so the
# fold pair's benchmark passes are pure overhead. Saves ~30-60min per
# fold pair. Production specialists.json keeps its full metrics.
AZOTH_OOF_SKIP_BENCHMARK ?= 1
AZOTH_OOF_SKIP_BENCHMARK_ARG := $(if $(filter 1 true yes,$(AZOTH_OOF_SKIP_BENCHMARK)),--skip-benchmark,)
# Feature cache for cross-fold sharing. azoth-prefill-specialist-features
# pre-populates this directory; both fold trainings then read from it via
# --feature-cache-dir, skipping per-fold extraction. Set empty to disable.
AZOTH_SPECIALIST_FEATURE_CACHE_DIR ?= out/cache/azoth-route-features
AZOTH_SPECIALIST_FEATURE_CACHE_ARG := $(if $(AZOTH_SPECIALIST_FEATURE_CACHE_DIR),--feature-cache-dir $(AZOTH_SPECIALIST_FEATURE_CACHE_DIR),)
AZOTH_SPECIALIST_ONLY ?=
AZOTH_SPECIALIST_MASK_SPEC ?=
# Explicit operator overrides win over discovered configurations. Keep these
# empty by default so historical experiments actually determine each route.
AZOTH_SPECIALIST_TRAIN_OVERRIDE ?=
AZOTH_SPECIALIST_FEATURE_ENV ?=
# Where autocollie-driven experiments write run JSONs. Pointing the suite at
# this dir lets it auto-pick each route's best supported historical experiment
# and replay its train_config + feature_env, so `make azoth-specialist-suite`
# uses autocollie's wins by default. Unset to revert to pure CLI/Makefile
# defaults (legacy behavior, autocollie discoveries are dropped on the floor).
AZOTH_AUTOCOLLIE_RUNS_DIR ?= out/experiments/azoth/runs
# The promoted baselines that pick each route's config. Passed explicitly to
# every suite: the suite's fallback looks under its own --output-root, which
# for the fold suites is azoth.oof-fold-{a,b} — so folds silently trained ~15
# routes with a different config than prod, and those OOF scores set prod's
# thresholds.
AZOTH_PROMOTED_BASELINES ?= $(AZOTH_ROOT)/.autocollie/promoted_baselines.json
# Both fold suites fit the same routes on half the rows each, so they share
# one peak-memory file: fold-B is packed by what fold-A measured that night.
AZOTH_FOLD_FIT_MEM_FILE ?= out/cache/fit-mem/fold.json
AZOTH_SPECIALIST_FIT_MEM_FILE ?=
AZOTH_SPECIALIST_SKIP_EXISTING ?= 1
AZOTH_SPECIALIST_SKIP_EXISTING_ARG := $(if $(filter 1 true yes,$(AZOTH_SPECIALIST_SKIP_EXISTING)),--skip-existing,)
# Concurrent specialist trainings. Default derived with the other auto-
# saturation knobs further down (NPROC / EXP_LGBM_THREADS). Set to 1 when
# training on GPU (``--device cuda``) so concurrent jobs don't fight for
# memory.
AZOTH_FILEGROUP_SCORE_FILTER ?= 0
AZOTH_FILEGROUP_SCORE_FILTER_ARG := $(if $(filter 1 true yes,$(AZOTH_FILEGROUP_SCORE_FILTER)),--filegroup-score-filter,)
SAMPLES_DIR ?= /data/samples
FALSE_POSITIVES_ARCHIVE ?= /tmp/false-positives.tgz
FALSE_NEGATIVES_ARCHIVE ?= /tmp/false-negatives.tgz
NEAR_FALSE_POSITIVES_ARCHIVE ?= /tmp/near-false-positives.tgz
NEAR_FALSE_NEGATIVES_ARCHIVE ?= /tmp/near-false-negatives.tgz
NEAR_FALSE_POSITIVES_TRIAGE_DIR ?= /var/tmp/near-false-positives
# {FALSE_POSITIVES,FALSE_NEGATIVES,MISLABELED}_TRIAGE_DIR are now derived
# inside the mislabeled-triage target block from SCOPE/LEVEL/SEVERITY so
# parallel triages under different scopes don't collide on /tmp paths.
SAMPLE ?=
FILE ?=
CLEAVE ?= cleave
DEMO_DB ?= out/demo.db
# Auto-saturate cores. One principle: each LightGBM fit uses 8 threads
# (the empirical sweet spot — past that returns diminish), and concurrent
# fits fill the rest. So total active threads ≈ NPROC. Override any of
# these on the command line if a specific run needs different shape.
NPROC := $(shell nproc 2>/dev/null || echo 4)
EXP_LGBM_THREADS ?= 8
# Scoring is limited by corpus/matrix memory and DB throughput, not fit cores.
# Keep training concurrency independent of the conservative calibration default.
AZOTH_CALIBRATE_PARALLELISM ?= 2
AZOTH_POLICY_PARALLELISM ?= 0
AZOTH_SPECIALIST_PARALLELISM ?= $(shell expr $(NPROC) / $(EXP_LGBM_THREADS))
WORKERS ?= $(NPROC)
EXP_WORKERS ?= $(WORKERS)
AZOTH_THRESHOLD_WORKERS ?= $(EXP_WORKERS)
WORKERS_ARG := $(if $(WORKERS),--workers $(WORKERS),)
EXP_WORKERS_ARG := $(if $(EXP_WORKERS),--workers $(EXP_WORKERS),)
AZOTH_THRESHOLD_WORKERS_ARG := $(if $(AZOTH_THRESHOLD_WORKERS),--workers $(AZOTH_THRESHOLD_WORKERS),)
SEED ?= 42
# Training device. `auto` runs the pick_device() heuristic in
# src/collimator/model.py: it picks CUDA only when the workload is dense
# and large enough to benefit (LightGBM's CUDA path produces garbage on
# sparse high-dim inputs — constant-prediction models, SIGFPE in fit).
# Today every collimator route is sparse + narrow-N, so this resolves to
# CPU in practice. Override with DEVICE=cuda to force-test the GPU path
# or DEVICE=cpu to skip the heuristic.
DEVICE ?= auto
DROP_FEATURE_PREFIXES ?=
# Default azoth screening profile: a probe-sized run for bulk iteration.
# Confirm winners with a different seed, an explicit larger sample, or make azoth-full-train.
EXP_TRAIN_SAMPLES ?= 150000
EXP_MAX_TEST_SAMPLES ?= 40000
EXP_TOTAL_LIMIT ?= 0
EXP_MAX_ID ?=
EXP_REFRESH_CACHE_SNAPSHOT ?= 0
EXP_REFRESH_CACHE_SNAPSHOT_ARG := $(if $(filter 1 true yes,$(EXP_REFRESH_CACHE_SNAPSHOT)),--refresh-cache-snapshot,)
EXP_RERUN ?= 0
EXP_RERUN_ARG := $(if $(filter 1 true yes,$(EXP_RERUN)),--rerun-existing,)
EXP_ROUTE ?= general
EXP_IDEA ?= $(if $(EXP_TAG),$(patsubst _%,%,$(EXP_TAG)),adhoc)
# Match full training's explicit filter, if any. Otherwise each experiment
# discovers its vocabulary from its training rows and feature knobs.
EXP_ALLOWED_FEATURES_FILE ?= $(abspath $(AZOTH_ALLOWED_FEATURES_FILE))
EXP_FOLDS ?= 0
EXP_HOLDOUT_FRACTION ?= 0.12
EXP_ESTIMATORS ?= 180
EXP_MAX_DEPTH ?= 12
EXP_LEARNING_RATE ?= 0.05
EXP_EARLY_STOPPING ?= 25
EXP_NUM_LEAVES ?= $(if $(filter azoth,$(LEARNER)),96,)
EXP_MIN_CHILD_SAMPLES ?= $(if $(filter azoth,$(LEARNER)),100,)
EXP_MIN_CHILD_WEIGHT ?=
EXP_COLSAMPLE_BYTREE ?= 0.8
EXP_SUBSAMPLE ?= 0.8
EXP_GAMMA ?= 0.0
EXP_REG_ALPHA ?= 0.0
EXP_REG_LAMBDA ?= 1.0
EXP_THRESHOLD_MODE ?= fbeta
EXP_THRESHOLD_FPR_TARGET ?=
EXP_HARD_NEGATIVE_FRACTION ?= 0.0
EXP_HARD_NEGATIVE_WEIGHT ?= 1.0
EXP_BENIGN_FILETYPE_WEIGHT ?=
EXP_MONOTONE_JSON ?=
EXP_SCALE_POS_WEIGHT_MULT ?= 1.0
EXP_MAX_DELTA_STEP ?= 0
EXP_BOOSTING_TYPE ?= gbdt
EXP_EXTRA_TREES ?= 0
EXP_SEED_SEARCH_K ?= 1
EXP_ENSEMBLE_SEEDS ?=
# When EXP_SEED_SEARCH_K>1, set EXP_SAVE_ALL_SEEDS=1 to deploy the averaged
# ensemble (item A) instead of picking-best. Bundle layout becomes
# models/seed_<S>.txt; litmus averages at predict time.
EXP_SAVE_ALL_SEEDS ?= 0
EXP_TEST_NATURAL_PREVALENCE ?= 0
EXP_BETA ?= 2.0
# Asymmetric label-quality gate: keep ALL benigns (EXP_MIN_SAMPLE_SCORE=0) but
# require malware to clear a higher bar. A low-score benign is still benign, and
# score=0 is where the clean curated corpus lives (foraged/harvest/distro/pypi,
# ~2.1M benigns — files cleave found nothing notable in). Excluding them starves
# benign-poor routes into collapse: at floor 1, filetypes/rtf kept only 76
# benigns vs ~5000 malware and went to AUC 0.50 / 100% benign-FP; at floor 0
# (449 benigns) the same route is perfect. Malicious git checkouts / multi-file
# samples label every file `bad`, but most are benign filler that scores low —
# they don't separate from real benigns and poison weak routes (e.g. filetypes/c
# had 2,367 of 6,599 "malware" at score=1). Dropping malware below this score
# from TRAINING (test set unchanged) focuses the model on confident malware.
# autocollie tunes this (knobs.json: min_malware_score, 0-9). Conceptually the
# non-archive analog of skip-benign-archive-item.
# 2026-06-05: this is now the PRIMARY label-quality gate (benigns dropped to
# the >= 1 floor via COLLIMATOR_MIN_SAMPLE_SCORE — filtering them starves
# routes into FPs). A score-sweep (scripts/sweep_malware_score.py) over go/c/
# ruby showed the score=1 malware is genuine noise: flooring go's hostiles at
# 4 lifts recall@0.1%FP from 0.645 -> 0.94 (+30pp at ship operating point),
# helps c slightly, and leaves ruby's tail recall flat. 4 is the global
# default; autocollie tunes per-route (go wants ~5, ruby ~2).
EXP_MIN_MALWARE_SCORE ?= 4
# A Make default must not overwrite autocollie's selected per-route cutoff.
# An explicit command-line/environment value still overrides every route.
AZOTH_MALWARE_SCORE_OVERRIDE = $(if $(filter file default undefined,$(origin EXP_MIN_MALWARE_SCORE)),,$(EXP_MIN_MALWARE_SCORE))
# Ablation 2026-04-10: silent_packer (Exp 43) and mtime_kurtosis (Exp 44) were
# net-negative at 75k experiment scale. air_gap_signal (Exp 46) and the
# extreme-features bundle (Exps 48/49/54/55/56) are kept ON.
EXP_SILENT_PACKER_SIGNAL ?= 0
EXP_MTIME_KURTOSIS ?= 0
EXP_AIR_GAP_SIGNAL ?= 1
EXP_EXTREME_FEATURES ?= 1
# Per-feature toggles within EXTREME_FEATURES bundle (Exps 48/49/51/54/55/56).
# Each defaults to inheriting from EXTREME_FEATURES; override individually for ablations.
EXP_ANACHRONISTIC_INJECTION ?=
EXP_CODE_ENTROPY_SPIKE ?=
EXP_FOREIGN_BINARY_SIGNAL ?=
EXP_EXTENSION_MISMATCH_SIGNAL ?=
EXP_HOSTILE_FINDING_DENSITY ?=
EXP_HOSTILE_DEPTH_WEIGHT ?=
# v16 default OFF: drops 163k inter:{ft}*{element/skeleton} features that contribute essentially nothing.
EXP_FILETYPE_INTERACTIONS ?= 0
EXP_FORMAT_HINTS ?= 0
# Default-on toggles in features.py — overridable for ablations.
EXP_BLINDFOLD ?= 1
EXP_SCORE_WEIGHTED_TRAITS ?= 1
EXP_SOFT_PRESENCE ?= 1
EXP_REPETITION_PENALTY_FEATURES ?= 1
EXP_FILE_SEVERITY_DISTRIBUTION ?= 1
# Size-invariant crit-tier severity fractions (opt-in; autocollie A/B candidate).
EXP_SEVERITY_FRACTION_FEATURES ?= 0
EXP_HOSTILE_WEIGHTED_DENSITY ?= 1
EXP_HOSTILE_ESCALATION_FEATURES ?= 1
EXP_SUSPICIOUS_BREADTH_DENSITY ?= 1
EXP_STRUCT_FILE_RISK_COVERAGE ?= 1
EXP_TOP_K_RISK_FILES ?= 1
EXP_MIN_SAMPLE_SCORE ?= 0
EXP_TAXONOMY_FEATURES ?= 0
EXP_EXTENDED_METRICS ?= 1
EXP_METRIC_MIN_FREQ_PCT ?= 5
EXP_EMBER_LITE_FEATURES ?= 0
EXP_BIGRAM_MAX ?= 5000
EXP_BIGRAM_MIN_FREQ ?= 5
EXP_TRIGRAM_MAX ?= 500
EXP_TRIGRAM_MAX_BENIGN_FRAC ?= 0.01
EXP_CONFIDENCE_WEIGHTED_NGRAMS ?= 0
EXP_OBJECTIVE_TRIGRAMS ?= 0
EXP_SUSPICIOUS_TRIGRAMS ?= 0
EXP_ATTACK_NGRAMS ?= 0
# Previously hardcoded to 1 in the experiment recipe; now toggleable so
# autocollie can ablate them. Defaults preserve historical behavior.
EXP_ATTACK_FEATURES ?= 1
EXP_ATTACK_CODE_NGRAMS ?= 1
EXP_CRIT_CATEGORY_NGRAMS ?= 1
EXP_TRAIT_PATH_DEPTH ?= 3
EXP_TIERED_CRIT_BIGRAMS ?= 1
EXP_TIERED_BIGRAM_MIN_CRIT ?= 3
EXP_TIERED_BIGRAM_MAX ?= 5000
EXP_TIERED_BIGRAM_MIN_FREQ ?= 5
EXP_TIERED_CRIT_TRIGRAMS ?= 0
EXP_TIERED_TRIGRAM_MIN_CRIT ?= 3
EXP_TIERED_TRIGRAM_MAX ?= 5000
EXP_TIERED_TRIGRAM_MIN_FREQ ?= 5
EXP_SYMBOL_VOCAB ?= 0
EXP_SYMBOL_VOCAB_MAX ?= 5000
EXP_SYMBOL_MIN_FREQ ?= 5
EXP_KV_VOCAB ?= 0
EXP_KV_VOCAB_MAX ?= 5000
EXP_KV_MIN_FREQ ?= 5
EXP_KV_SHAPE_FEATURES ?= 0
EXP_RULE_KEYWORDS ?= 0
EXP_DESCRIPTION_KEYWORDS ?= 0
EXP_FULL_RULE_KEYWORDS ?= 0
EXP_RULE_DESCRIPTION_KEYWORDS ?= 0
EXP_CONTEXT_KEYWORDS ?= 0
EXP_ARCHIVE_MEMBER_CONTEXT ?= 0
EXP_KEYWORD_VOCAB_MAX ?= 10000
EXP_KEYWORD_MIN_FREQ ?= 5
EXP_KEYWORD_SELECTION ?= frequency
EXP_KEYWORD_CANDIDATE_MAX ?= 100000
EXP_RULE_KEYWORD_VOCAB_MAX ?= 0
EXP_RULE_KEYWORD_MIN_FREQ ?= 0
EXP_RULE_KEYWORD_SELECTION ?= inherit
EXP_DESCRIPTION_KEYWORD_VOCAB_MAX ?= 0
EXP_DESCRIPTION_KEYWORD_MIN_FREQ ?= 0
EXP_DESCRIPTION_KEYWORD_SELECTION ?= inherit
EXP_FULL_RULE_KEYWORD_VOCAB_MAX ?= 0
EXP_FULL_RULE_KEYWORD_MIN_FREQ ?= 0
EXP_FULL_RULE_KEYWORD_SELECTION ?= inherit
EXP_RULE_DESCRIPTION_KEYWORD_VOCAB_MAX ?= 0
EXP_RULE_DESCRIPTION_KEYWORD_MIN_FREQ ?= 0
EXP_RULE_DESCRIPTION_KEYWORD_SELECTION ?= inherit
EXP_CONTEXT_KEYWORD_VOCAB_MAX ?= 0
EXP_CONTEXT_KEYWORD_MIN_FREQ ?= 0
EXP_CONTEXT_KEYWORD_SELECTION ?= inherit
# Batch 1 — cheap metric extracts (default off; autocollie can toggle each).
EXP_PE_FORMAT_FLAGS ?= 0
EXP_PE_TEMPORAL_ANOMALY ?= 0
EXP_TEXT_METRICS_FULL ?= 0
EXP_OVERLAY_SIGNAL ?= 0
EXP_METRIC_RATIO_FEATURES ?= 0
EXP_SIZE_NORMALIZED_METRICS ?= 0
EXP_NONSTANDARD_SECTION_SIGNAL ?= 0
EXP_LINE_LENGTH_BUCKETS ?= 0
# Batch 2 — allowlist + filter knobs (default off / no filter).
EXP_EXTENDED_METRICS_INCLUDE ?=
EXP_TOP_K_RISK_FILES_MIN_CRIT ?= 0
EXP_METRIC_CORRELATION_PAIRS ?=
EXP_KV_VALUE_SPLIT ?= 0
# Batch 3 — symbol & string n-grams (default off; trigram_min_freq is the
# symmetry knob and defaults to the previously-hardcoded value of 5).
EXP_SYMBOL_BIGRAMS ?= 0
EXP_SYMBOL_BIGRAM_MAX ?= 5000
EXP_SYMBOL_MIN_FREQ_BIGRAM ?= 10
EXP_SYMBOL_TRIGRAMS ?= 0
EXP_SYMBOL_TRIGRAM_MAX ?= 2000
EXP_SYMBOL_MIN_FREQ_TRIGRAM ?= 10
EXP_TRIGRAM_MIN_FREQ ?= 5
EXP_TIERED_CRIT_QUADGRAMS ?= 0
EXP_TIERED_QUADGRAM_MIN_CRIT ?= 3
EXP_TIERED_QUADGRAM_MAX ?= 5000
EXP_TIERED_QUADGRAM_MIN_FREQ ?= 5
# Batch 4 — trait & taxonomy extensions (default off / no overrides).
EXP_MBC_ID_VOCAB ?= 0
EXP_TRAIT_CONFIDENCE_MOMENTS ?= 0
EXP_TRAIT_ID_LEXICAL_DISTANCE ?= 0
EXP_DOCUMENT_OBFUSCATION_FEATURES ?= 0
EXP_TIERED_BIGRAM_BRANCH_MIN_CRIT ?=
EXP_DISABLE_FEATURE_GROUPS ?= clusters
# packaged_capability compute mode: zero | chars | tokens | paths | findings
EXP_PACKAGED_CAPABILITY_MODE ?= paths
# Experiment data cache directory. When set, corpus selections and extracted
# matrices are cached to disk so repeated experiments skip expensive DB scans.
EXP_CACHE_DIR ?= out/cache/experiment/$(MODEL)
ABLATE_CACHE_DIR ?= $(EXP_CACHE_DIR)
ABLATE_MAX_ID ?=
ALLOWED_FEATURES ?= src/collimator/allowed_features.json

# Validate DB is set for targets that need it
check-db:
ifndef DB
	$(error DB is required. Usage: make azoth-full-train DB=postgres://hopper@localhost/hopper)
endif

# Fail if the newest sample is older than 24 hours (replication may be broken)
check-db-fresh: check-db
	@age_hours=$$(psql "$(DB)" -tAc \
		"SELECT EXTRACT(EPOCH FROM now() - MAX(updated_at)) / 3600 FROM samples;") ; \
	age_hours=$${age_hours%.*} ; \
	if [ "$$age_hours" -gt 2 ] 2>/dev/null; then \
		echo "ERROR: newest sample is $${age_hours}h old (>2h). Check logical replication status:" >&2 ; \
		echo "  sudo -u postgres psql -d hopper -c \"SELECT * FROM pg_stat_subscription;\"" >&2 ; \
		echo "  tail -20 /var/log/postgresql/postgresql-17-main.log" >&2 ; \
		exit 1 ; \
	fi

venv: $(VENV_DIR)/.deps.stamp

$(VENV_DIR)/bin/python:
	# A venv can be left with a dangling interpreter symlink after its base
	# Python is removed (or the venv is moved). Clear it before recreating so
	# `venv` does not trip over the stale bin/python3 entry.
	rm -f $(VENV_DIR)/.deps.stamp
	python3 -m venv --clear $(VENV_DIR)

$(VENV_DIR)/.deps.stamp: requirements.txt pyproject.toml | $(VENV_DIR)/bin/python
	# Bootstrap pip if missing — handles the case where the venv was created
	# by `uv venv` (which omits pip) instead of `python -m venv`. Using
	# `python -m pip` rather than `pip` keeps us agnostic to the unversioned
	# `pip` symlink, which `ensurepip` doesn't always install.
	$(VENV_DIR)/bin/python -m ensurepip --upgrade
	$(VENV_DIR)/bin/python -m pip install --upgrade pip
	$(VENV_DIR)/bin/python -m pip install -r requirements.txt
	$(VENV_DIR)/bin/python -m pip install -e .
	touch $(VENV_DIR)/.deps.stamp

# azoth-{full,fast}-train: the two top-level "give me the latest best deployed
# model" entry points.  Both run the same chain — replay autocollie's
# highest-F1 historical run for general → retrain every specialist with
# autocollie's per-route best → calibrate + deploy — but at different
# fidelities.  Pick one explicitly; there is no `make azoth-train` shortcut
# because the speed/quality trade-off is real and worth being conscious of.
#
#   make azoth-full-train  — train on the full labeled corpus (~2M rows in
#     natural distribution).  ~7-8 hours end-to-end at K=3 seeds.  Use this
#     for deploy-bound retrains: the natural distribution preserves the
#     benign tail that 50/50 sampling discards, which is what determines
#     L25 (one calibration FP plus 24 extrapolated FP/100M) threshold quality.
#
#   make azoth-fast-train  — train on a 600k 50/50-balanced sample.  ~5 hours
#     end-to-end at K=3 seeds.  Same fidelity autocollie's promote step uses
#     internally, so this matches what an autocollie auto-promote would have
#     produced.  Use for fast iteration; the candidate is still deployable.
#
# Both:
#   * Always retrain from scratch on current DB state (snapshot-pinned, no
#     model cache).
#   * K=3 multi-seed averaging.
#   * Run azoth-specialists + azoth-deploy on completion.
#   * Override individual knobs via DEPLOY_ESTIMATORS, DEPLOY_TRAIN_SAMPLES,
#     DEPLOY_MAX_TEST_SAMPLES if you need a custom profile.

DEPLOY_ESTIMATORS               ?= 400
DEPLOY_TRAIN_SAMPLES_FULL       ?= 0
DEPLOY_MAX_TEST_SAMPLES_FULL    ?= 0
DEPLOY_TRAIN_SAMPLES_FAST       ?= 600000
DEPLOY_MAX_TEST_SAMPLES_FAST    ?= 80000
# Default the fidelity for any standalone azoth-general / azoth-general-fold-*
# invocation to the FULL settings. azoth-full-train passes the _FULL values
# explicitly, and azoth-publish-train hands them to the pipeline script as
# PUBLISH_TRAIN_SAMPLES / PUBLISH_MAX_TEST_SAMPLES (which re-applies them on
# each general's command line), so this default just covers a sub-target called
# directly or scripts/azoth_oof_pipeline.sh run standalone.
# Setting them empty (the prior behavior) would propagate "--train-samples ''"
# through to the experiment CLI and abort it with an empty-arg error.
DEPLOY_TRAIN_SAMPLES            ?= $(DEPLOY_TRAIN_SAMPLES_FULL)
DEPLOY_MAX_TEST_SAMPLES         ?= $(DEPLOY_MAX_TEST_SAMPLES_FULL)

# Both build the ENTIRE bundle into an isolated run dir
# ($(AZOTH_RUNS_ROOT)/<id>/) and only flip the canonical $(OUT_ROOT)/azoth
# symlink on success via azoth-publish. An interrupted or failed train never
# touches the live bundle, so the previous model stays published and
# serveable (this is what would have saved the in-place wipe that stranded a
# crashed train). Successful runs are retained $(AZOTH_KEEP_RUNS) deep for
# rollback. See the azoth-run-new / azoth-publish / azoth-prune-runs block.
azoth-full-train: venv check-db
	@echo "azoth-full-train: building into $(AZOTH_RUN_DIR) (canonical bundle untouched until success)"
	@mkdir -p $(AZOTH_RUN_DIR)
	$(MAKE) _azoth-train \
		AZOTH_ROOT=$(AZOTH_RUN_DIR) \
		AZOTH_PROMOTED_BASELINES="$(AZOTH_PROMOTED_BASELINES)" \
		DEPLOY_TRAIN_SAMPLES=$(DEPLOY_TRAIN_SAMPLES_FULL) \
		DEPLOY_MAX_TEST_SAMPLES=$(DEPLOY_MAX_TEST_SAMPLES_FULL)
	$(MAKE) azoth-publish AZOTH_ROOT=$(AZOTH_RUN_DIR)
	$(MAKE) azoth-prune-runs
	$(MAKE) azoth-prune-feature-cache
	@echo "azoth-full-train: published $(OUT_ROOT)/azoth -> $(AZOTH_RUN_DIR)"

azoth-fast-train: venv check-db
	@echo "azoth-fast-train: building into $(AZOTH_RUN_DIR) (canonical bundle untouched until success)"
	@mkdir -p $(AZOTH_RUN_DIR)
	$(MAKE) _azoth-train \
		AZOTH_ROOT=$(AZOTH_RUN_DIR) \
		AZOTH_PROMOTED_BASELINES="$(AZOTH_PROMOTED_BASELINES)" \
		DEPLOY_TRAIN_SAMPLES=$(DEPLOY_TRAIN_SAMPLES_FAST) \
		DEPLOY_MAX_TEST_SAMPLES=$(DEPLOY_MAX_TEST_SAMPLES_FAST)
	$(MAKE) azoth-publish AZOTH_ROOT=$(AZOTH_RUN_DIR)
	$(MAKE) azoth-prune-runs
	$(MAKE) azoth-prune-feature-cache
	@echo "azoth-fast-train: published $(OUT_ROOT)/azoth -> $(AZOTH_RUN_DIR)"

.PHONY: azoth-profile azoth-profile-scoring
azoth-profile-scoring: venv check-db
	@test -n "$(PROFILE_DIR)" || { echo "Set PROFILE_DIR to a new output directory"; exit 1; }
	$(VENV_DIR)/bin/python scripts/benchmark_threshold_scoring.py \
		--db "$(DB)" --model "$(AZOTH_GENERAL_DIR)" \
		--percent "$(PROFILE_PERCENT)" --workers $(PROFILE_SCORING_WORKERS) \
		--output "$(PROFILE_DIR)"

azoth-profile: venv check-db
	$(VENV_DIR)/bin/python scripts/azoth_profile.py --db "$(DB)" \
		--percent "$(PROFILE_PERCENT)" --workers "$(PROFILE_WORKERS)" \
		--profiler "$(PROFILE_PROFILER)" --stop-stage "$(PROFILE_STOP_STAGE)" \
		--target-minutes "$(PROFILE_TARGET_MINUTES)" \
		$(if $(PROFILE_FIT_MEMORY_GB),--fit-memory-gb "$(PROFILE_FIT_MEMORY_GB)",) \
		$(if $(PROFILE_EXTRACT_WORKERS_PER_FIT),--extract-workers-per-fit "$(PROFILE_EXTRACT_WORKERS_PER_FIT)",) \
		$(if $(PROFILE_DIR),--output "$(PROFILE_DIR)",) \
		$(if $(PROFILE_PIN),--pin "$(PROFILE_PIN)",) \
		$(if $(filter 1,$(PROFILE_DRY_RUN)),--dry-run,)

# azoth-publish-train additionally computes k=2 OOF diagnostics.
# Fold-model scores cannot calibrate the separately trained production models:
# their score distributions differ. Publication always fits production ONNX
# ensemble scores on dev, then applies the saved policies to locked test.
# Use azoth-full-train when fold diagnostics are unnecessary.
#
# The recipe used to inline all nine steps in sequence, which idled 126 of 128
# cores between LightGBM phases for 7h29m of trainings. It now delegates to
# scripts/azoth_oof_pipeline.sh, which runs the same steps as two concurrent
# dependency chains (prod general->specialists || fold-A gen->fold-B gen->
# fold-A spec->fold-B spec) and pairs the two independent scoring stages. Same
# work, same models, ~11h -> ~7h. The scheduling and the RAM-splitting that
# makes it safe are documented in that script's header. PARALLEL_CHAINS=0
# restores the old strictly-sequential order.
#
# MAKEFLAGS is deliberately NOT cleared: it is how a command-line override
# (`make azoth-publish-train THRESHOLD_MAX_ID=... AZOTH_SPECIALIST_PARALLELISM=...`)
# reaches the script's sub-makes. The per-chain resource split still wins over
# it, because the script passes those on each sub-make's own command line and an
# explicit assignment outranks an inherited one.
#
# WORKERS is additionally handed over through the environment: the script halves
# it per chain, and $(WORKERS) here is the resolved value whether it came from
# the environment, the command line, or the nproc default — none of which the
# script could see on its own when the setting arrived via MAKEFLAGS.
azoth-publish-train: venv check-db
	@echo "azoth-publish-train: starting full k=2 OOF run (general + specialists)."
	@echo "  3x general + 3x specialists (folds skip benchmark) + OOF merge/route-scores + calibrate + deploy."
	@# Read once: edits during hours of training must not shift Bash's file offset.
	@pipeline=$$(cat scripts/azoth_oof_pipeline.sh) && \
	bash -n <<< "$$pipeline" && \
	WORKERS=$(WORKERS) \
	PUBLISH_TRAIN_SAMPLES=$(DEPLOY_TRAIN_SAMPLES_FULL) \
	PUBLISH_MAX_TEST_SAMPLES=$(DEPLOY_MAX_TEST_SAMPLES_FULL) \
	bash -c "$$pipeline" scripts/azoth_oof_pipeline.sh
	@echo "azoth-publish-train: complete (validation-only=$(or $(AZOTH_VALIDATE_ONLY),0))."
	@echo "azoth-publish-train: archived fold bundles at out/models/azoth.oof-fold-{a,b}/"

# azoth-general: train the general model at deploy fidelity, promote it
# into the source bundle slot, and rebuild threshold_scores against the
# fresh model.
#
# This is the GENERAL-ONLY half of what _azoth-train used to do, split
# out so the OOF-publish flow can train fold-A/-B GENERAL bundles
# without burning compute on specialists that get thrown away. See the
# refactored azoth-publish-train below.
#
# Three steps:
#   1. Train general at deploy fidelity (writes to out/experiments/azoth/).
#   2. Promote the freshly-trained general into the source bundle slot
#      (out/models/azoth/general/) so azoth-deploy actually picks it up —
#      `make experiment` writes to a different location than azoth-deploy
#      reads from, and without this step the deploy ships a stale general.
#   3. Rescore threshold_scores against the freshly-promoted model.
#
# Set AZOTH_GENERAL_SKIP_RESCORE=1 to skip step 3 — used by the fold
# variants below since azoth_oof_score.py does its own corpus-wide
# scoring after merging the two fold bundles, making this rescore a
# multi-hour wasted pass for fold builds.
azoth-general: venv check-db azoth-clean-bundle
	$(PYTHON) scripts/azoth_train_best.py \
		--runs-dir "$(AZOTH_AUTOCOLLIE_RUNS_DIR)" \
		--promoted-baselines "$(AZOTH_PROMOTED_BASELINES)" \
		--route general \
		--set DB=$(DB) \
		--set EXP_ALLOWED_FEATURES_FILE=$(abspath $(AZOTH_ALLOWED_FEATURES_FILE)) \
		--set COLLIMATOR_MIN_SAMPLE_SCORE=$(EXP_MIN_SAMPLE_SCORE) \
		--default-min-malware-score $(EXP_MIN_MALWARE_SCORE) \
		$(if $(AZOTH_MALWARE_SCORE_OVERRIDE),--set EXP_MIN_MALWARE_SCORE=$(AZOTH_MALWARE_SCORE_OVERRIDE),) \
		--set EXP_TRAIN_SAMPLES=$(DEPLOY_TRAIN_SAMPLES) \
		--set EXP_MAX_TEST_SAMPLES=$(DEPLOY_MAX_TEST_SAMPLES) \
		--set EXP_ESTIMATORS=$(DEPLOY_ESTIMATORS) \
		$(if $(AZOTH_GENERAL_SEED_SEARCH_K),--set EXP_SEED_SEARCH_K=$(AZOTH_GENERAL_SEED_SEARCH_K),) \
		$(if $(AZOTH_GENERAL_SEED_SEARCH_K),--set EXP_SAVE_ALL_SEEDS=0,) \
		$(if $(WORKERS),--set EXP_WORKERS=$(WORKERS),)
	@# Promote the freshly-trained general into the source bundle slot.
	@# `make experiment` wrote to out/experiments/azoth/{models/,model.txt,
	@# feature_spec.json}; copy whichever layout it used into the deployed
	@# location, replacing whatever was there. Atomic-ish: the per-file
	@# `cp` and `rm -rf` are sequenced so an interrupted run leaves the
	@# slot in a clean (older) state rather than half-overwritten.
	@# Promote from $(EXP_OUT_DIR) — NOT a hardcoded out/experiments/azoth.
	@# Hardcoding here used to silently break fold parallelism: both
	@# fold-A and fold-B wrote to the same path and clobbered each
	@# other's freshly-trained models. Honoring $(EXP_OUT_DIR) lets
	@# the fold targets point each invocation at its own workspace.
	@mkdir -p $(AZOTH_GENERAL_DIR)
	@if [ -d $(EXP_OUT_DIR)/models ]; then \
		rm -rf $(AZOTH_GENERAL_DIR)/models $(AZOTH_GENERAL_DIR)/model.txt $(AZOTH_GENERAL_DIR)/model.json $(AZOTH_GENERAL_DIR)/model.onnx; \
		cp -a $(EXP_OUT_DIR)/models $(AZOTH_GENERAL_DIR)/models; \
	elif [ -f $(EXP_OUT_DIR)/model.txt ]; then \
		rm -rf $(AZOTH_GENERAL_DIR)/models $(AZOTH_GENERAL_DIR)/model.txt $(AZOTH_GENERAL_DIR)/model.json $(AZOTH_GENERAL_DIR)/model.onnx; \
		cp -a $(EXP_OUT_DIR)/model.txt $(AZOTH_GENERAL_DIR)/model.txt; \
	else \
		echo "error: azoth_train_best did not produce a general model in $(EXP_OUT_DIR)/"; \
		exit 1; \
	fi
	cp -a $(EXP_OUT_DIR)/feature_spec.json $(AZOTH_GENERAL_DIR)/feature_spec.json
	@if [ -f $(EXP_OUT_DIR)/model.onnx ]; then cp -a $(EXP_OUT_DIR)/model.onnx $(AZOTH_GENERAL_DIR)/model.onnx; fi
	@rm -f $(AZOTH_GENERAL_DIR)/training_provenance.json
	@if [ -f $(EXP_OUT_DIR)/training_provenance.json ]; then cp -a $(EXP_OUT_DIR)/training_provenance.json $(AZOTH_GENERAL_DIR)/training_provenance.json; fi
	@# Rebuild the general's threshold_scores cache against the freshly-
	@# promoted model. Without this, azoth-calibrate would consume a stale
	@# score table whose probabilities map to a previous model — calibrators
	@# and L0..L100 thresholds would be fit on the wrong score distribution.
	@if [ "$(AZOTH_GENERAL_SKIP_RESCORE)" = "1" ]; then \
		echo "AZOTH_GENERAL_SKIP_RESCORE=1: skipping threshold_scores rescore"; \
	else \
		set -e; \
		echo "rescoring general against full labeled corpus -> $(AZOTH_GENERAL_SCORES)"; \
		$(PYTHON) -u -m collimator tune-thresholds --db $(DB) \
		    --model "$(AZOTH_GENERAL_DIR)" --scores-only \
		    --spec $(AZOTH_GENERAL_DIR)/feature_spec.json \
		    $(WORKERS_ARG) \
		    --scores-cache $(AZOTH_GENERAL_SCORES) \
		    --refresh-cache \
		    --output $(AZOTH_GENERAL_DIR)/threshold_tuning.json; \
	fi

# Fold-aware general training. Each variant trains general with one OOF
# fold held out, then archives the bundle to azoth.oof-fold-{a,b}/ for
# later consumption by azoth-oof-merge-general. Skips the threshold_scores
# rescore — those scores get replaced by the OOF merge anyway, so doing it
# here is a multi-hour wasted pass.
#
# Why a separate target rather than a shell variable: the fold-A and
# fold-B bundles need DISTINCT on-disk locations (different stash dirs).
# A single parameterized target couldn't run both back-to-back without
# the second one overwriting the first's bundle slot.
azoth-general-fold-a: venv check-db
	@# Run training in a fold-specific workspace and deploy slot so that
	@# concurrent chains (PARALLEL_CHAINS=1) don't race over EXP_OUT_DIR /
	@# AZOTH_ROOT. The EXP_CACHE_DIR is intentionally NOT fold-suffixed —
	@# corpus + matrix caches are content-hashed (oof_fold_exclude is in
	@# the key), so both folds can share that directory and benefit from
	@# cross-run cache hits.
	EXP_OOF_FOLD_EXCLUDE=0 $(MAKE) azoth-general \
		AZOTH_ROOT=$(OUT_ROOT)/azoth.oof-fold-a \
		EXP_OUT_DIR=$(AZOTH_FOLD_A_EXP_OUT_DIR) \
		AZOTH_GENERAL_SKIP_RESCORE=1 \
		AZOTH_GENERAL_SEED_SEARCH_K=$(AZOTH_OOF_SEED_SEARCH_K)
	@echo "fold-A bundle ready at $(OUT_ROOT)/azoth.oof-fold-a/"

azoth-general-fold-b: venv check-db
	EXP_OOF_FOLD_EXCLUDE=1 $(MAKE) azoth-general \
		AZOTH_ROOT=$(OUT_ROOT)/azoth.oof-fold-b \
		EXP_OUT_DIR=$(AZOTH_FOLD_B_EXP_OUT_DIR) \
		AZOTH_GENERAL_SKIP_RESCORE=1 \
		AZOTH_GENERAL_SEED_SEARCH_K=$(AZOTH_OOF_SEED_SEARCH_K)
	@echo "fold-B bundle ready at $(OUT_ROOT)/azoth.oof-fold-b/"

# Merge the two fold-trained general bundles into honest OOF probabilities.
# Reads from the stash dirs created by azoth-general-fold-{a,b}, writes
# diagnostic scores under oof_route_scores/general, separate from calibration. Pulled
# out of azoth-publish-train so it's individually rerunnable when only
# the merge needs refreshing.
azoth-oof-merge-general: venv check-db
	$(PYTHON) scripts/azoth_oof_score.py \
		--db $(DB) \
		--fold-a-bundle $(OUT_ROOT)/azoth.oof-fold-a \
		--fold-b-bundle $(OUT_ROOT)/azoth.oof-fold-b \
		--prod-bundle $(AZOTH_ROOT) \
		--output $(AZOTH_OOF_ROUTE_SCORES_DIR)/general/threshold_scores.npz \
		$(WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG)

# _azoth-train: shared body for the two named targets above.  Not a public
# entry point — pick azoth-full-train or azoth-fast-train explicitly.
#
# Calls azoth-general (which does train + promote + rescore), then trains
# specialists and deploys. The split lets fold-aware OOF training reuse
# the general half without the specialist+deploy tail.
_azoth-train: azoth-general
	$(MAKE) azoth-specialists AZOTH_SPECIALIST_SKIP_EXISTING=0
	$(MAKE) azoth-deploy
	$(MAKE) azoth-shap

# Per-route SHAP feature importance for the whole bundle. Pure inference on the
# trained LightGBM boosters (~seconds/route), so it runs on EVERY train — right
# after azoth-deploy, so it covers exactly the routes that survived the
# deploy-time weakness prune. Written into each route dir under $(AZOTH_ROOT)
# (i.e. out/) and INTENTIONALLY NOT staged into the deployed bundle:
# stage_azoth_runtime_bundle.py omits shap_importance.json, since SHAP is an
# internal debugging aid (ascan --extra), not a published artifact. Fatal on
# error so a systemic break (e.g. the XGBoost->LightGBM regression) surfaces
# loudly instead of silently leaving stale SHAP behind. Uses the first seed per
# route — global importance is stable across seeds.
AZOTH_SHAP_SCAN_LIMIT ?= 10000
azoth-shap: venv check-db
	@for d in $(AZOTH_ROOT)/general $(AZOTH_ROOT)/filegroups/* $(AZOTH_ROOT)/filetypes/*; do \
		[ -d "$$d" ] || continue; \
		[ -f "$$d/feature_spec.json" ] || continue; \
		m="$$d/models/seed_42.txt"; [ -f "$$m" ] || m="$$d/model.txt"; \
		if [ ! -f "$$m" ]; then echo "azoth-shap: skip $$d (no .txt booster)"; continue; fi; \
		echo "azoth-shap: $$d"; \
		$(PYTHON) -m collimator explain --db $(DB) --model "$$m" \
			--spec "$$d/feature_spec.json" --output "$$d" \
			--scan-limit $(AZOTH_SHAP_SCAN_LIMIT) || exit 1; \
	done
	@echo "azoth-shap: wrote per-route shap_importance.json under $(AZOTH_ROOT) (kept in out/, not staged into deploy)"

evaluate: venv check-db
	$(PYTHON) -m collimator evaluate --db $(DB) --model $(OUT_DIR)/model.onnx --spec $(OUT_DIR)/feature_spec.json

explain: venv check-db
	$(PYTHON) -m collimator explain --db $(DB) --model $(OUT_DIR)/$(MODEL_FILE) --spec $(OUT_DIR)/feature_spec.json --output $(OUT_DIR)

inspect: venv check-db
ifndef SAMPLE
	$(error SAMPLE is required. Usage: make inspect DB=... SAMPLE=<sha256>)
endif
	$(PYTHON) -m collimator inspect --db $(DB) --sample $(SAMPLE) --model $(OUT_DIR)/$(MODEL_FILE) --spec $(OUT_DIR)/feature_spec.json

errors: venv check-db
	$(PYTHON) -m collimator errors --db $(DB) --model $(OUT_DIR)/$(MODEL_FILE) --spec $(OUT_DIR)/feature_spec.json

traits: venv check-db
	$(PYTHON) -m collimator traits --db $(DB)

thresholds: venv check-db
	$(PYTHON) -u -m collimator tune-thresholds --db $(DB) \
		--model $(OUT_DIR)/$(MODEL_FILE) \
		--spec $(OUT_DIR)/feature_spec.json \
		$(WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		--scores-cache $(THRESHOLD_SCORES) \
		--top-errors $(THRESHOLD_TOP_ERRORS) \
		--output $(OUT_DIR)/threshold_tuning.json

thresholds-refresh: venv check-db
	$(PYTHON) -u -m collimator tune-thresholds --db $(DB) \
		--model $(OUT_DIR)/$(MODEL_FILE) \
		--spec $(OUT_DIR)/feature_spec.json \
		$(WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		--scores-cache $(THRESHOLD_SCORES) \
		--refresh-cache \
		--top-errors $(THRESHOLD_TOP_ERRORS) \
		--output $(OUT_DIR)/threshold_tuning.json

# repin: drop the pinned snapshot_max_id so the next experiment-style
# invocation re-queries the live max(id). Chain it before the target you
# actually want to run, e.g.:
#
#   make repin azoth-full-train DB=...
#
# Make builds command-line goals left-to-right by default, so `repin` runs
# first, then the next target rebuilds the pin from the current DB.
# Prints the current pin (so you have a revert path) before clearing or
# re-setting. Pass PIN_TO=<id> to atomically replace the pin with a
# specific id (useful when reverting to the value an earlier `make repin`
# printed). Default behavior with no PIN_TO clears the pin so the next
# experiment re-queries max(id).
repin:
	@if [ -f $(EXP_CACHE_DIR)/snapshot_max_id.txt ]; then \
		echo "previous pin: $$(cat $(EXP_CACHE_DIR)/snapshot_max_id.txt) (revert with: make repin PIN_TO=$$(cat $(EXP_CACHE_DIR)/snapshot_max_id.txt))"; \
	else \
		echo "previous pin: (none)"; \
	fi
	@if [ -n "$(PIN_TO)" ]; then \
		mkdir -p $(EXP_CACHE_DIR); \
		echo "$(PIN_TO)" > $(EXP_CACHE_DIR)/snapshot_max_id.txt; \
		echo "set snapshot pin: $(EXP_CACHE_DIR)/snapshot_max_id.txt -> $(PIN_TO)"; \
	else \
		rm -f $(EXP_CACHE_DIR)/snapshot_max_id.txt; \
		echo "cleared snapshot pin: $(EXP_CACHE_DIR)/snapshot_max_id.txt (next invocation will re-query max(id))"; \
	fi

# azoth-clean-bundle: wipe regen-able deployed artifacts from the source
# bundle slot at $(AZOTH_ROOT). Caches under */cache/ are preserved (rebuilding
# feature matrices is expensive and they're keyed by snapshot+model anyway).
# Run before any retrain to guarantee no leftover seed_*.txt, calibrator.json,
# benchmark.json, etc. from a prior generation pollutes the new bundle.
#
# Categories cleaned:
#   - top-level deployed artifacts (config.json, score_table.npz, *.md, *.csv, *.json)
#   - per-route deployed artifacts (model.{txt,json,onnx}, feature_spec.json, calibrator.json,
#     benchmark.json, README.md, threshold_scores.npz, calibration_scores.npz,
#     threshold_tuning.json)
#   - per-route models/ subdirectories (catches multi-seed leftovers like
#     seed_42.txt + a stale seed_45.txt from an earlier K=4 run)
#
# Categories preserved:
#   - $(AZOTH_GENERAL_DIR)/cache/  (route feature matrices, snapshot-keyed)
#   - configured auxiliary artifacts and .retained_components/ identity manifest
#   - any sibling directories not matching the above patterns
azoth-clean-bundle: venv
	$(PYTHON) scripts/clean_azoth_bundle.py --azoth-root "$(AZOTH_ROOT)"

# azoth-check-bundle-layout: fail fast on an ambiguous bundle layout — a stale
# top-level model.{onnx,txt,json} (a pre-multi-seed fossil) left sitting next to
# a freshly-trained models/seed_* dir. bundle.model_files() rejects this, but not
# until write_azoth_readmes runs ~40 stages into a deploy. azoth-clean-bundle now
# sweeps the fossil on retrain; this guards the deploy/validate paths against a
# bundle that reached them without a clean (e.g. a hand-assembled AZOTH_ROOT).
# Same route set as azoth-clean-bundle so the two stay symmetric.
azoth-check-bundle-layout:
	@for d in $(AZOTH_ROOT)/general $(AZOTH_ROOT)/filegroups/* $(AZOTH_ROOT)/filetypes/*; do \
	    [ -d "$$d" ] || continue; \
	    _legacy=$$(find "$$d" -maxdepth 1 -type f \( -name model.onnx -o -name model.txt -o -name model.json \) 2>/dev/null); \
	    _seeds=$$(find "$$d/models" -maxdepth 1 -type f -name 'seed_*' 2>/dev/null); \
	    if [ -n "$$_legacy" ] && [ -n "$$_seeds" ]; then \
	        echo "error: $$d has both legacy (model.*) and multi-seed (models/seed_*) artifacts — ambiguous bundle layout."; \
	        echo "       stale: $$_legacy"; \
	        echo "       fix: remove the stale legacy file, or run 'make azoth-clean-bundle AZOTH_ROOT=$(AZOTH_ROOT)' first."; \
	        exit 1; \
	    fi; \
	done

filetype-matrix: venv check-db
	$(PYTHON) scripts/filetype_metric_matrix.py \
		--db $(DB) \
		--scores-cache $(THRESHOLD_SCORES) \
		--thresholds $(OUT_DIR)/threshold_tuning.json \
		--output $(FILETYPE_MATRIX_OUTPUT) \
		--csv-output $(FILETYPE_MATRIX_CSV) \
		--min-count $(FILETYPE_MATRIX_MIN_COUNT)

# Same env-resolution pattern as `fixture` — use the deployed model's
# feature env (autocollie's best general spec) so this benchmark sees the
# same features the production pipeline does.
elf-model-benchmark: venv check-db
	$(PYTHON) scripts/azoth_train_best.py \
		--runs-dir "$(AZOTH_AUTOCOLLIE_RUNS_DIR)" \
		--route general \
		--exec $(PYTHON) scripts/elf_model_benchmark.py \
		--db $(DB) \
		$(EXP_WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		--general-model $(ELF_BENCHMARK_GENERAL_DIR)/model.txt \
		--general-spec $(ELF_BENCHMARK_GENERAL_DIR)/feature_spec.json \
		--binary-output $(ELF_BENCHMARK_BINARY_DIR) \
		--elf-output $(ELF_BENCHMARK_ELF_DIR) \
		--output $(ELF_BENCHMARK_OUTPUT) \
		--seed $(SEED) \
		--n-folds $(ELF_BENCHMARK_FOLDS) \
		--n-estimators $(ELF_BENCHMARK_ESTIMATORS) \
		--max-depth $(ELF_BENCHMARK_MAX_DEPTH) \
		--learning-rate $(ELF_BENCHMARK_LEARNING_RATE) \
		--early-stopping-rounds $(ELF_BENCHMARK_EARLY_STOPPING) \
		--num-leaves $(ELF_BENCHMARK_NUM_LEAVES) \
		--min-child-samples $(ELF_BENCHMARK_MIN_CHILD_SAMPLES) \
		$(if $(DEVICE),--device $(DEVICE),)

elf-route-optimization: venv check-db
	$(PYTHON) scripts/elf_ensemble_experiments.py \
		--db $(DB) \
		--general-scores $(AZOTH_GENERAL_SCORES) \
		--general-spec $(AZOTH_GENERAL_DIR)/feature_spec.json \
		--teacher-model $(ELF_ROUTE_TEACHER_DIR)/model.txt \
		--teacher-spec $(ELF_ROUTE_TEACHER_DIR)/feature_spec.json \
		--output-dir $(ELF_ROUTE_OUTPUT_DIR) \
		--output $(ELF_ROUTE_OUTPUT) \
		$(EXP_WORKERS_ARG) \
		--seed $(SEED)

# Same env-resolution pattern as `fixture` and `elf-model-benchmark` —
# the suite trains every specialist with the deployed model's feature env.
# Per-route train_config and feature_env overrides for individual specialists
# come in via --autocollie-best-runs-dir (read from the suite, see the flag
# in scripts/azoth_specialist_suite.py).
azoth-specialists: venv check-db
	$(EXPERIMENT_FEATURE_ENV) $(PYTHON) scripts/azoth_specialist_suite.py \
		--db $(DB) \
		$(EXP_WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		--output-root $(AZOTH_ROOT) \
		--summary $(AZOTH_SPECIALISTS_SUMMARY) \
		--general-dir $(AZOTH_GENERAL_DIR) \
		--seed $(SEED) \
		--n-folds $(AZOTH_SPECIALIST_FOLDS) \
		--n-estimators $(AZOTH_SPECIALIST_ESTIMATORS) \
		--max-depth $(AZOTH_SPECIALIST_MAX_DEPTH) \
		--learning-rate $(AZOTH_SPECIALIST_LEARNING_RATE) \
		--early-stopping-rounds $(AZOTH_SPECIALIST_EARLY_STOPPING) \
		--num-leaves $(AZOTH_SPECIALIST_NUM_LEAVES) \
		--min-child-samples $(AZOTH_SPECIALIST_MIN_CHILD_SAMPLES) \
		--n-seed-extras $(AZOTH_SPECIALIST_N_SEED_EXTRAS) \
		$(if $(AZOTH_SPECIALIST_FIT_MEM_FILE),--fit-mem-file $(AZOTH_SPECIALIST_FIT_MEM_FILE),) \
		--seed-health-min-roc-auc $(AZOTH_SPECIALIST_SEED_HEALTH_MIN_ROC_AUC) \
		--seed-retry-budget $(AZOTH_SPECIALIST_SEED_RETRY_BUDGET) \
		--parallelism $(AZOTH_SPECIALIST_PARALLELISM) \
		--lgbm-threads $(EXP_LGBM_THREADS) \
		--min-bad $(AZOTH_SPECIALIST_MIN_BAD) \
		--min-good $(AZOTH_SPECIALIST_MIN_GOOD) \
		--default-min-malware-score $(EXP_MIN_MALWARE_SCORE) \
		$(if $(AZOTH_MALWARE_SCORE_OVERRIDE),--min-malware-score $(AZOTH_MALWARE_SCORE_OVERRIDE),) \
		$(foreach target,$(AZOTH_SPECIALIST_ONLY),--only $(target)) \
		$(foreach mask,$(AZOTH_SPECIALIST_MASK_SPEC),--mask-spec $(mask)) \
		$(foreach override,$(AZOTH_SPECIALIST_TRAIN_OVERRIDE),--train-override $(override)) \
		$(foreach env,$(AZOTH_SPECIALIST_FEATURE_ENV),--feature-env $(env)) \
		$(if $(AZOTH_AUTOCOLLIE_RUNS_DIR),--autocollie-best-runs-dir $(AZOTH_AUTOCOLLIE_RUNS_DIR),) \
		--promoted-baselines $(AZOTH_PROMOTED_BASELINES) \
		$(AZOTH_SPECIALIST_SKIP_EXISTING_ARG) \
		$(AZOTH_FILEGROUP_SCORE_FILTER_ARG) \
		$(AZOTH_SPECIALIST_FEATURE_CACHE_ARG) \
		$(if $(DEVICE),--device $(DEVICE),)

# Rewrite specialists.json so the route list matches the CURRENT
# normalize_archive_filetype rule, without retraining. Use after
# changing the rule (e.g. extending or shrinking the
# PURE_COMPRESSION_SUFFIXES set) to consolidate the bundle's routes
# in-place. Existing on-disk specialist models are left untouched —
# variant entries that fold into a container (tar.gz → tar) are dropped
# from the summary but their model dirs stay until the next full
# `make azoth-specialists` pass overwrites or removes them.
azoth-rediscover: venv check-db
	$(PYTHON) scripts/azoth_rediscover_routes.py \
		--db $(DB) \
		--azoth-root $(AZOTH_ROOT) \
		--summary $(AZOTH_SPECIALISTS_SUMMARY) \
		$(THRESHOLD_MAX_ID_ARG) \
		--min-bad $(AZOTH_SPECIALIST_MIN_BAD) \
		--min-good $(AZOTH_SPECIALIST_MIN_GOOD)

# Fold-aware specialist training for OOF score generation. Mirror of
# azoth-specialists with EXP_OOF_FOLD_EXCLUDE wired through (honored in
# azoth_specialist_suite._fetch_rows via _read_oof_exclude). Each fold
# bundle excludes one half of train+dev so the OTHER half can be scored
# OOF afterwards via azoth-oof-route-scores. Outputs go to azoth-fold-a/
# and azoth-fold-b/ so the production bundle under $(AZOTH_ROOT) is left
# alone — the deployed model stays single-fold; the fold bundles exist
# only to feed honest score merging downstream.
azoth-specialists-fold-a: venv check-db
	$(EXPERIMENT_FEATURE_ENV) EXP_OOF_FOLD_EXCLUDE=0 $(PYTHON) scripts/azoth_specialist_suite.py \
		--db $(DB) \
		$(EXP_WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		--output-root $(OUT_ROOT)/azoth.oof-fold-a \
		--summary $(OUT_ROOT)/azoth.oof-fold-a/specialists.json \
		--general-dir $(OUT_ROOT)/azoth.oof-fold-a/general \
		--seed $(SEED) \
		--n-folds $(AZOTH_SPECIALIST_FOLDS) \
		--n-estimators $(AZOTH_SPECIALIST_ESTIMATORS) \
		--max-depth $(AZOTH_SPECIALIST_MAX_DEPTH) \
		--learning-rate $(AZOTH_SPECIALIST_LEARNING_RATE) \
		--early-stopping-rounds $(AZOTH_SPECIALIST_EARLY_STOPPING) \
		--num-leaves $(AZOTH_SPECIALIST_NUM_LEAVES) \
		--min-child-samples $(AZOTH_SPECIALIST_MIN_CHILD_SAMPLES) \
		--n-seed-extras $(AZOTH_OOF_SEED_EXTRAS) \
		--seed-health-min-roc-auc $(AZOTH_SPECIALIST_SEED_HEALTH_MIN_ROC_AUC) \
		--seed-retry-budget $(AZOTH_SPECIALIST_SEED_RETRY_BUDGET) \
		--parallelism $(AZOTH_SPECIALIST_PARALLELISM) \
		--lgbm-threads $(EXP_LGBM_THREADS) \
		--min-bad $(AZOTH_SPECIALIST_MIN_BAD) \
		--min-good $(AZOTH_SPECIALIST_MIN_GOOD) \
		--default-min-malware-score $(EXP_MIN_MALWARE_SCORE) \
		$(if $(AZOTH_MALWARE_SCORE_OVERRIDE),--min-malware-score $(AZOTH_MALWARE_SCORE_OVERRIDE),) \
		$(foreach target,$(AZOTH_SPECIALIST_ONLY),--only $(target)) \
		$(foreach mask,$(AZOTH_SPECIALIST_MASK_SPEC),--mask-spec $(mask)) \
		$(foreach override,$(AZOTH_SPECIALIST_TRAIN_OVERRIDE),--train-override $(override)) \
		$(foreach env,$(AZOTH_SPECIALIST_FEATURE_ENV),--feature-env $(env)) \
		$(if $(AZOTH_AUTOCOLLIE_RUNS_DIR),--autocollie-best-runs-dir $(AZOTH_AUTOCOLLIE_RUNS_DIR),) \
		--promoted-baselines $(AZOTH_PROMOTED_BASELINES) \
		--fit-mem-file $(AZOTH_FOLD_FIT_MEM_FILE) \
		$(AZOTH_SPECIALIST_SKIP_EXISTING_ARG) \
		$(AZOTH_FILEGROUP_SCORE_FILTER_ARG) \
		$(AZOTH_OOF_SKIP_BENCHMARK_ARG) \
		$(AZOTH_SPECIALIST_FEATURE_CACHE_ARG) \
		$(if $(DEVICE),--device $(DEVICE),)

azoth-specialists-fold-b: venv check-db
	$(EXPERIMENT_FEATURE_ENV) EXP_OOF_FOLD_EXCLUDE=1 $(PYTHON) scripts/azoth_specialist_suite.py \
		--db $(DB) \
		$(EXP_WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		--output-root $(OUT_ROOT)/azoth.oof-fold-b \
		--summary $(OUT_ROOT)/azoth.oof-fold-b/specialists.json \
		--general-dir $(OUT_ROOT)/azoth.oof-fold-b/general \
		--seed $(SEED) \
		--n-folds $(AZOTH_SPECIALIST_FOLDS) \
		--n-estimators $(AZOTH_SPECIALIST_ESTIMATORS) \
		--max-depth $(AZOTH_SPECIALIST_MAX_DEPTH) \
		--learning-rate $(AZOTH_SPECIALIST_LEARNING_RATE) \
		--early-stopping-rounds $(AZOTH_SPECIALIST_EARLY_STOPPING) \
		--num-leaves $(AZOTH_SPECIALIST_NUM_LEAVES) \
		--min-child-samples $(AZOTH_SPECIALIST_MIN_CHILD_SAMPLES) \
		--n-seed-extras $(AZOTH_OOF_SEED_EXTRAS) \
		--seed-health-min-roc-auc $(AZOTH_SPECIALIST_SEED_HEALTH_MIN_ROC_AUC) \
		--seed-retry-budget $(AZOTH_SPECIALIST_SEED_RETRY_BUDGET) \
		--parallelism $(AZOTH_SPECIALIST_PARALLELISM) \
		--lgbm-threads $(EXP_LGBM_THREADS) \
		--min-bad $(AZOTH_SPECIALIST_MIN_BAD) \
		--min-good $(AZOTH_SPECIALIST_MIN_GOOD) \
		--default-min-malware-score $(EXP_MIN_MALWARE_SCORE) \
		$(if $(AZOTH_MALWARE_SCORE_OVERRIDE),--min-malware-score $(AZOTH_MALWARE_SCORE_OVERRIDE),) \
		$(foreach target,$(AZOTH_SPECIALIST_ONLY),--only $(target)) \
		$(foreach mask,$(AZOTH_SPECIALIST_MASK_SPEC),--mask-spec $(mask)) \
		$(foreach override,$(AZOTH_SPECIALIST_TRAIN_OVERRIDE),--train-override $(override)) \
		$(foreach env,$(AZOTH_SPECIALIST_FEATURE_ENV),--feature-env $(env)) \
		$(if $(AZOTH_AUTOCOLLIE_RUNS_DIR),--autocollie-best-runs-dir $(AZOTH_AUTOCOLLIE_RUNS_DIR),) \
		--promoted-baselines $(AZOTH_PROMOTED_BASELINES) \
		--fit-mem-file $(AZOTH_FOLD_FIT_MEM_FILE) \
		$(AZOTH_SPECIALIST_SKIP_EXISTING_ARG) \
		$(AZOTH_FILEGROUP_SCORE_FILTER_ARG) \
		$(AZOTH_OOF_SKIP_BENCHMARK_ARG) \
		$(AZOTH_SPECIALIST_FEATURE_CACHE_ARG) \
		$(if $(DEVICE),--device $(DEVICE),)

# Pre-build the per-route feature cache before fold-A / fold-B specialist
# training. Without this, each fold does its own extract_partitioned_from_db
# pass and the train+dev overlap (~88% of rows) gets extracted twice.
# Pre-fill extracts the union once and saves a fold-specific matrix under
# each fold's cache key, so both subsequent fold trainings hit the cache.
#
# Safe to run anytime — the worst case is a stale cache entry that
# fails its row-count check at load time and triggers re-extraction.
azoth-prefill-specialist-features: venv check-db
	$(PYTHON) scripts/azoth_prefill_specialist_features.py \
		--db $(DB) \
		--general-dir $(AZOTH_GENERAL_DIR) \
		--feature-cache-dir $(AZOTH_SPECIALIST_FEATURE_CACHE_DIR) \
		--min-bad $(AZOTH_SPECIALIST_MIN_BAD) \
		--min-good $(AZOTH_SPECIALIST_MIN_GOOD) \
		$(EXP_WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		$(if $(filter 1 true yes,$(AZOTH_FILEGROUP_SCORE_FILTER)),--filegroup-score-filter,) \
		$(foreach target,$(AZOTH_SPECIALIST_ONLY),--only $(target))

# Merge the two fold-trained specialist bundles into per-route OOF
# threshold_scores.npz files under $(AZOTH_ROOT)/oof_route_scores/. Pass
# --prod-root so test rows get scored with the deployed bundle (which
# never saw test rows at training time, so its predictions on them are
# legitimately OOS). The downstream calibrate-ensemble run then reads
# these files via --oof-route-scores-dir.
AZOTH_OOF_ROUTE_SCORES_DIR ?= $(AZOTH_ROOT)/oof_route_scores
azoth-oof-route-scores: venv check-db
	$(PYTHON) scripts/azoth_oof_score_routes.py \
		--db $(DB) \
		--fold-a-root $(OUT_ROOT)/azoth.oof-fold-a \
		--fold-b-root $(OUT_ROOT)/azoth.oof-fold-b \
		--prod-root $(AZOTH_ROOT) \
		--summary $(AZOTH_SPECIALISTS_SUMMARY) \
		--output-dir $(AZOTH_OOF_ROUTE_SCORES_DIR) \
		$(EXP_WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG)

# Fit thresholds on held-out dev scores from the exact production ensemble.
AZOTH_CALIBRATE_PARTITION ?= dev
# Optional scoring scope; dev+test retain all calibration and reporting rows.
# Empty preserves the full-corpus cache used by other diagnostic commands.
AZOTH_SCORE_PARTITIONS ?=
AZOTH_SCORE_PARTITIONS_ARG = $(if $(AZOTH_SCORE_PARTITIONS),--score-partitions $(AZOTH_SCORE_PARTITIONS),)
# Concurrent route scoring defaults to two jobs, with shared row-index maps
# and disk-backed feature matrices. Override after measuring DB/memory headroom.

# Retain the old option so callers receive an explicit error if they attempt
# to calibrate production models with diagnostic OOF scores.
AZOTH_OOF_ROUTE_SCORES_ARG := $(if $(filter 1,$(AZOTH_USE_OOF_ROUTE_SCORES)),--oof-route-scores-dir $(AZOTH_OOF_ROUTE_SCORES_DIR),)

azoth-calibrate: venv check-db
	$(PYTHON) scripts/azoth_calibrate_ensemble.py \
		--db $(DB) --azoth-root $(AZOTH_ROOT) --summary $(AZOTH_SPECIALISTS_SUMMARY) \
		--general-scores $(AZOTH_GENERAL_SCORES) --partition $(AZOTH_CALIBRATE_PARTITION) \
		$(AZOTH_RETAIN_AUXILIARIES_ARG) \
		$(AZOTH_OOF_ROUTE_SCORES_ARG) --check-inputs-only
	@# Cache reuse verifies ensemble, feature code, corpus, and canonical partitions.
	$(PYTHON) -m collimator tune-thresholds \
		--db $(DB) \
		--model "$(AZOTH_GENERAL_DIR)" --scores-only \
		--spec "$(AZOTH_GENERAL_DIR)/feature_spec.json" \
		$(AZOTH_SCORE_PARTITIONS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		$(AZOTH_THRESHOLD_WORKERS_ARG) \
		--scores-cache "$(AZOTH_GENERAL_SCORES)" \
		--output "$(AZOTH_GENERAL_DIR)/threshold_tuning.json"
	$(PYTHON) scripts/azoth_calibrate_ensemble.py \
		--db $(DB) \
		$(EXP_WORKERS_ARG) \
		--azoth-root $(AZOTH_ROOT) \
		--summary $(AZOTH_SPECIALISTS_SUMMARY) \
		--general-scores $(AZOTH_GENERAL_SCORES) \
		--output $(AZOTH_CONFIG) \
		--score-table $(AZOTH_SCORE_TABLE) \
		--partition $(AZOTH_CALIBRATE_PARTITION) \
		$(AZOTH_RETAIN_AUXILIARIES_ARG) \
		--parallelism $(AZOTH_CALIBRATE_PARALLELISM) \
		$(AZOTH_REFRESH_SCORES_ARG) \
		$(AZOTH_SKIP_LEVEL_CALIBRATION_ARG) \
		$(AZOTH_OOF_ROUTE_SCORES_ARG) \
		$(foreach route,$(AZOTH_REFRESH_ROUTE),--refresh-route $(route)) \
		--feature-cache-dir $(AZOTH_FEATURE_CACHE_DIR)
	@# Honest test-bucket evaluation: same dev-fit thresholds applied to
	@# the locked test partition. Output goes to $(AZOTH_ROOT)/test_metrics.json
	@# alongside config.json, reusing verified production score caches.
	$(PYTHON) scripts/azoth_calibrate_ensemble.py \
		--db $(DB) \
		$(EXP_WORKERS_ARG) \
		--azoth-root $(AZOTH_ROOT) \
		--summary $(AZOTH_SPECIALISTS_SUMMARY) \
		--general-scores $(AZOTH_GENERAL_SCORES) \
		--output $(AZOTH_CONFIG) \
		--score-table $(AZOTH_SCORE_TABLE) \
		--partition test \
		--parallelism $(AZOTH_CALIBRATE_PARALLELISM) \
		--apply-thresholds-from $(AZOTH_CONFIG) \
		$(AZOTH_OOF_ROUTE_SCORES_ARG) \
		--feature-cache-dir $(AZOTH_FEATURE_CACHE_DIR)

# azoth-set-low-water-mark pins the current deployed-bundle's
# route_policy_eval_oof.json as the low-water-mark for the regression
# gate. From this point on every deploy must keep per-filetype recall
# within --lwm-tolerance (default 0.9pp) of this snapshot, regardless
# of how the live deployed bundle drifts. Re-run this target whenever
# you want to advance the floor (e.g., after several deploys land
# stable improvements you want to lock in). Idempotent.
azoth-set-low-water-mark:
	@test -f "$(AZOTH_ROOT)/route_policy_eval_oof.json" || { \
	  echo "error: $(AZOTH_ROOT)/route_policy_eval_oof.json not found — run make azoth-deploy first"; \
	  exit 1; \
	}
	mkdir -p "$(AZOTH_LOW_WATER_MARK_DIR)"
	cp "$(AZOTH_ROOT)/route_policy_eval_oof.json" "$(AZOTH_LOW_WATER_MARK_DIR)/route_policy_eval_oof.json"
	@echo "low-water-mark pinned: $(AZOTH_LOW_WATER_MARK_DIR)/route_policy_eval_oof.json"
	@echo "  source: $(AZOTH_ROOT)/route_policy_eval_oof.json"
	@echo "  future deploys must stay within --lwm-tolerance of this snapshot."

azoth-diagnostics: venv
	$(PYTHON) scripts/azoth_route_diagnostics.py \
		--config $(AZOTH_CONFIG) \
		--score-table $(AZOTH_SCORE_TABLE) \
		--output $(AZOTH_DIAGNOSTICS) \
		--csv $(AZOTH_DIAGNOSTICS_CSV) \
		--slice-output $(AZOTH_SLICE_METRICS) \
		--slice-csv $(AZOTH_SLICE_METRICS_CSV)

azoth-policies: venv
	$(PYTHON) scripts/azoth_route_policy_search.py \
		--parallelism $(AZOTH_POLICY_PARALLELISM) \
		$(if $(AZOTH_POLICY_OVERRIDE_ROUTE),--db $(DB),) \
		--config $(AZOTH_CONFIG) \
		--score-table $(AZOTH_SCORE_TABLE) \
		--output $(AZOTH_ROUTE_POLICIES) \
		--csv $(AZOTH_ROUTE_POLICIES_CSV) \
		--markdown $(AZOTH_ROUTE_POLICIES_MD) \
		$(foreach route,$(AZOTH_POLICY_OVERRIDE_ROUTE),--override-route $(route)) \
		$(AZOTH_PREVIOUS_BUNDLE_ARG) \
		$(EXP_WORKERS_ARG)

# azoth-validate runs every gate that azoth-deploy runs (calibrate, route
# diagnostics, policy search, global FP/M with --fail-on-budget, bundle
# validator, and litmus parity unless AZOTH_SKIP_LITMUS_VALIDATE=1), but
# stops short of copying anything into $(AZOTH_DEPLOY_DIR). Used by
# autocollie's auto-promote path to vet a candidate bundle without touching
# the live deploy.
.PHONY: azoth-validate
azoth-validate: azoth-calibrate
	@test -f $(AZOTH_ROOT)/config.json || { echo "error: $(AZOTH_ROOT)/config.json not found"; exit 1; }
	@test -f $(AZOTH_ROOT)/score_table.npz || { echo "error: $(AZOTH_ROOT)/score_table.npz not found"; exit 1; }
	@test -f $(AZOTH_ROOT)/specialists.json || { echo "error: $(AZOTH_ROOT)/specialists.json not found"; exit 1; }
	@test -f $(AZOTH_ROOT)/general/model.txt || ls $(AZOTH_ROOT)/general/models/seed_*.txt >/dev/null 2>&1 || { echo "error: $(AZOTH_ROOT)/general missing model.txt or models/seed_*.txt"; exit 1; }
	@test -f $(AZOTH_ROOT)/general/feature_spec.json || { echo "error: $(AZOTH_ROOT)/general/feature_spec.json not found"; exit 1; }
	@$(MAKE) azoth-check-bundle-layout
	@if [ "$(AZOTH_VALIDATE_DIAGNOSTICS)" = "1" ] || [ "$(AZOTH_VALIDATE_DIAGNOSTICS)" = "true" ] || [ "$(AZOTH_VALIDATE_DIAGNOSTICS)" = "yes" ]; then \
	  $(PYTHON) scripts/azoth_route_diagnostics.py \
	    --config $(AZOTH_CONFIG) \
	    --score-table $(AZOTH_SCORE_TABLE) \
	    --output $(AZOTH_DIAGNOSTICS) \
	    --csv $(AZOTH_DIAGNOSTICS_CSV) \
	    --slice-output $(AZOTH_SLICE_METRICS) \
	    --slice-csv $(AZOTH_SLICE_METRICS_CSV); \
	else \
	  printf '# Azoth Route Diagnostics\n\nSkipped during fast azoth-validate. Run `make azoth-diagnostics AZOTH_ROOT=%s` for the full report.\n' "$(AZOTH_ROOT)" > "$(AZOTH_DIAGNOSTICS)"; \
	  printf 'status,message\nskipped,fast azoth-validate\n' > "$(AZOTH_DIAGNOSTICS_CSV)"; \
	  printf '# Azoth Slice Metrics\n\nSkipped during fast azoth-validate. Run `make azoth-diagnostics AZOTH_ROOT=%s` for the full report.\n' "$(AZOTH_ROOT)" > "$(AZOTH_SLICE_METRICS)"; \
	  printf 'status,message\nskipped,fast azoth-validate\n' > "$(AZOTH_SLICE_METRICS_CSV)"; \
	fi
	$(PYTHON) scripts/azoth_route_policy_search.py \
		--parallelism $(AZOTH_POLICY_PARALLELISM) \
		$(if $(AZOTH_POLICY_OVERRIDE_ROUTE),--db $(DB),) \
		--config $(AZOTH_CONFIG) \
		--score-table $(AZOTH_SCORE_TABLE) \
		--output $(AZOTH_ROUTE_POLICIES) \
		--csv $(AZOTH_ROUTE_POLICIES_CSV) \
		--markdown $(AZOTH_ROUTE_POLICIES_MD) \
		$(foreach route,$(AZOTH_POLICY_OVERRIDE_ROUTE),--override-route $(route)) \
		$(AZOTH_PREVIOUS_BUNDLE_ARG) \
		$(EXP_WORKERS_ARG)
	$(PYTHON) scripts/azoth_policy_global_metrics.py \
		--config $(AZOTH_CONFIG) \
		--policy $(AZOTH_ROUTE_POLICIES) \
		--score-table $(AZOTH_SCORE_TABLE) \
		--output $(AZOTH_GLOBAL_POLICY_METRICS) \
		--markdown $(AZOTH_GLOBAL_POLICY_METRICS_MD) \
		$(AZOTH_BUDGET_GATE_ARGS)
	$(PYTHON) scripts/compute_routed_metrics.py --azoth-root $(AZOTH_ROOT) --db $(DB) $(AZOTH_VALIDATE_ROUTED_METRICS_ARGS) $(AZOTH_ROUTED_METRICS_ARGS)
	$(PYTHON) scripts/azoth_route_policy_eval.py \
		--score-table $(AZOTH_ROOT)/score_table.npz \
		--general-scores $(AZOTH_ROOT)/general/threshold_scores.npz \
		--route-policies $(AZOTH_ROUTE_POLICIES) \
		--partition test \
		--output-md $(AZOTH_ROOT)/route_policy_eval_oof.md \
		--output-json $(AZOTH_ROOT)/route_policy_eval_oof.json
	$(PYTHON) scripts/write_azoth_readmes.py --azoth-root $(AZOTH_ROOT)
	@# Auto-convert any LightGBM .txt seeds to .onnx siblings before staging
	@# trips its ONNX-only guard. Same fix as in azoth-deploy-final, applied
	@# here too because autocollie's promote-validate path goes through
	@# azoth-validate (not azoth-deploy-final), and the candidate bundle from
	@# promote-full-train ships .txt seeds. Idempotent: no-op when every seed
	@# already has its .onnx sibling.
	$(PYTHON) scripts/convert_bundle_to_onnx.py --azoth-root $(AZOTH_ROOT) --db $(DB)
	@# Conversion must preserve the calibrated ensemble; parity failures stop publication.
	$(PYTHON) scripts/write_azoth_readmes.py --azoth-root $(AZOTH_ROOT)
	@# SCAN_NO_UPDATE=1: see the note in azoth-deploy-final — the litmus checks
	@# below hand $$_STAGE to atomscan via SCAN_MODELS_DIR, which also makes it the
	@# auto-updater's install target. autocollie's promote gate runs THIS target.
	@_STAGE=$$(mktemp -d) && \
	  export SCAN_NO_UPDATE=1 && \
	  $(PYTHON) scripts/stage_azoth_runtime_bundle.py "$(AZOTH_ROOT)" "$$_STAGE" && \
	  _STAGED=$$(md5sum "$$_STAGE/config.json" | cut -d' ' -f1) && \
	  cp "$(AZOTH_DIAGNOSTICS)" "$$_STAGE/route_diagnostics.md" && \
	  cp "$(AZOTH_SLICE_METRICS)" "$$_STAGE/slice_metrics.md" && \
	  cp "$(AZOTH_ROUTE_POLICIES_MD)" "$$_STAGE/route_policies.md" && \
	  cp "$(AZOTH_GLOBAL_POLICY_METRICS_MD)" "$$_STAGE/global_policy_metrics.md" && \
	  $(PYTHON) scripts/validate_azoth_bundle.py "$$_STAGE" && \
	  $(PYTHON) scripts/check_azoth_regression.py --staged "$$_STAGE" --deployed "$(AZOTH_DEPLOY_DIR)" --low-water-mark "$(AZOTH_LOW_WATER_MARK_DIR)" $(AZOTH_REGRESSION_SOURCE_ARG) $(AZOTH_REGRESSION_ARGS) && \
	  if [ "$(AZOTH_SKIP_LITMUS_VALIDATE)" = "1" ] || [ "$(AZOTH_SKIP_LITMUS_VALIDATE)" = "true" ] || [ "$(AZOTH_SKIP_LITMUS_VALIDATE)" = "yes" ]; then \
	    echo "Skipping litmus deployed-ensemble compatibility checks (AZOTH_SKIP_LITMUS_VALIDATE=$(AZOTH_SKIP_LITMUS_VALIDATE))"; \
	  else \
	    echo "Running litmus deployed-ensemble compatibility checks against staged copy..." && \
	    ( cd $(LITMUS_DIR) && SCAN_MODELS_DIR="$$_STAGE" cargo test --release --test scan_no_deadlock -- --ignored ) && \
	    $(PYTHON) scripts/verify_azoth_litmus_runtime.py --litmus-dir $(LITMUS_DIR) --models-dir "$$_STAGE" --required-model az/native --required-model az/elf; \
	  fi && \
	  _NOW=$$(md5sum "$$_STAGE/config.json" | cut -d' ' -f1) && \
	  if [ "$$_NOW" != "$$_STAGED" ]; then \
	    echo "error: staged config.json changed under us ($$_STAGED -> $$_NOW) — the gates"; \
	    echo "       above validated a bundle that is no longer in $$_STAGE."; \
	    exit 1; \
	  fi && \
	  rm -rf "$$_STAGE" && \
	  echo "azoth-validate: all gates passed for $(AZOTH_ROOT)" \
	|| { ec=$$?; rm -rf "$$_STAGE"; exit $$ec; }

# Run isolation. Each fresh train picks a unique RUN_ID and writes its
# entire bundle into $(AZOTH_RUNS_ROOT)/$(RUN_ID)/, then atomically
# updates $(AZOTH_ROOT) (the canonical symlink) on success. Failed or
# in-flight runs leave the canonical bundle untouched.
#
# Typical usage from a shell:
#   RUN_DIR=$$(make -s azoth-run-new)
#   make azoth-specialists AZOTH_ROOT="$$RUN_DIR"   # if retraining
#   make azoth-deploy      AZOTH_ROOT="$$RUN_DIR"
#   make azoth-publish     AZOTH_ROOT="$$RUN_DIR"
#
# Or with the chained convenience target:
#   make azoth-publish-deploy
#     # allocates a run dir, runs azoth-deploy into it, publishes on success
#
# Resume an existing run by exporting AZOTH_RUN_ID=<id> before make,
# which freezes the timestamped path used by these targets.
ifeq ($(origin AZOTH_RUN_ID), undefined)
AZOTH_RUN_ID := $(shell date -u +%Y%m%dT%H%M%SZ)-$(shell openssl rand -hex 4 2>/dev/null || hexdump -n4 -e '4/1 "%02x"' /dev/urandom)
endif
AZOTH_RUN_DIR := $(AZOTH_RUNS_ROOT)/$(AZOTH_RUN_ID)

.PHONY: azoth-run-new azoth-publish azoth-publish-deploy azoth-prune-runs azoth-prune-feature-cache

# Create (if needed) and print the path to the run dir for AZOTH_RUN_ID.
# Capture it with $(shell …) or make -s.
azoth-run-new:
	@mkdir -p $(AZOTH_RUN_DIR)
	@echo $(AZOTH_RUN_DIR)

# Validate that AZOTH_ROOT looks like a complete bundle, then atomically
# point $(OUT_ROOT)/azoth at it. Errors out if the bundle is incomplete.
azoth-publish: venv
	@test -z "$(COLLIMATOR_SAMPLE_PERCENT)$(COLLIMATOR_PROFILE_DIR)" || { echo "Refusing to publish a sampled/profiling run"; exit 1; }
	$(PYTHON) scripts/azoth_publish_run.py $(AZOTH_ROOT) --link $(OUT_ROOT)/azoth

# Retain only the newest $(AZOTH_KEEP_RUNS) run dirs under $(AZOTH_RUNS_ROOT)
# (legacy-* archives included); the dir the canonical symlink points at is
# always kept regardless of age. Safe to run anytime.
azoth-prune-runs:
	@set -e; \
	link=$(OUT_ROOT)/azoth; \
	current=""; \
	[ -L "$$link" ] && current=$$(readlink -f "$$link") || true; \
	echo "azoth-prune-runs: keeping newest $(AZOTH_KEEP_RUNS) under $(AZOTH_RUNS_ROOT) (+ current symlink target)"; \
	ls -1dt $(AZOTH_RUNS_ROOT)/*/ 2>/dev/null | sed 's#/*$$##' | awk 'NR>$(AZOTH_KEEP_RUNS)' | while read d; do \
	  if [ -n "$$current" ] && [ "$$(readlink -f "$$d")" = "$$current" ]; then echo "  keep (current) $$d"; continue; fi; \
	  echo "  rm -rf $$d"; rm -rf "$$d"; \
	done; \
	true

# Drop route-feature matrices older than $(AZOTH_KEEP_CACHE_DAYS) days. Safe to
# run anytime, including mid-train: generations are disjoint by filename, and a
# running stage's own generation is always today's. Set AZOTH_KEEP_CACHE_DAYS=0
# to wipe the cache entirely (next run re-extracts from the DB).
azoth-prune-feature-cache:
	@set -e; \
	dir=$(AZOTH_FEATURE_CACHE_DIR); \
	if [ ! -d "$$dir" ]; then \
	  echo "azoth-prune-feature-cache: $$dir does not exist; nothing to do"; \
	  exit 0; \
	fi; \
	before=$$(du -sh "$$dir" 2>/dev/null | cut -f1); \
	n=$$(find "$$dir" -maxdepth 1 -type f -mtime +$(AZOTH_KEEP_CACHE_DAYS) | wc -l); \
	echo "azoth-prune-feature-cache: $$dir is $$before; dropping $$n file(s) older than $(AZOTH_KEEP_CACHE_DAYS) day(s)"; \
	find "$$dir" -maxdepth 1 -type f -mtime +$(AZOTH_KEEP_CACHE_DAYS) -delete; \
	echo "azoth-prune-feature-cache: now $$(du -sh "$$dir" 2>/dev/null | cut -f1)"

# Out-of-time backtest of a route's operating point. Every number the pipeline
# publishes is fitted and measured on the same benign pool; this fits on benign
# ingested up to a cutoff and measures on what arrived after, which is the split
# deploy actually faces. Reads score_table.npz + hopper ingest metadata — no
# retraining, no feature extraction, seconds per experiment once the metadata
# cache is warm.
#
# Always read the `validate` block first: it checks the harness reproduces the
# thresholds azoth_route_policy_search.py shipped. If that says DRIFT, nothing
# below it is trustworthy.
#
# Usage:
#   make azoth-backtest ROUTE=pe
#   make azoth-backtest ROUTE=pe BACKTEST_ARGS="--experiment pool-hygiene"
#   make azoth-backtest ROUTE=pe BACKTEST_ARGS="--exclude-feed win32"
BACKTEST_LEVEL ?= 25
BACKTEST_HOLDOUT_DAYS ?= 30
BACKTEST_OUTPUT ?= out/backtest/$(ROUTE).json
BACKTEST_ARGS ?=
azoth-backtest: venv
	@test -n "$(ROUTE)" || { echo "error: ROUTE= is required (a file type, e.g. ROUTE=pe)"; exit 2; }
	$(PYTHON) scripts/azoth_backtest.py \
		--file-type $(ROUTE) \
		--db $(DB) \
		--score-table $(AZOTH_ROOT)/score_table.npz \
		--level $(BACKTEST_LEVEL) \
		--holdout-days $(BACKTEST_HOLDOUT_DAYS) \
		--output $(BACKTEST_OUTPUT) \
		$(BACKTEST_ARGS)

# Apples-to-apples baseline scoring for autocollie. Given a route, an optional
# row-ids file, and a deploy-root (defaults to AZOTH_ROOT, i.e. the currently
# trained bundle), score the deployed model on those rows and write metrics to
# OUTPUT. First call per (route, model_hash) populates a cache under
# out/cache/autocollie-baseline; subsequent calls hit the cache.
#
# Usage:
#   make azoth-score-route ROUTE=filetypes/pe \
#       ROW_IDS_FILE=/tmp/rows.txt OUTPUT=/tmp/baseline.json
azoth-score-route: venv
	@test -n "$(ROUTE)"  || { echo "error: ROUTE= is required"; exit 2; }
	@test -n "$(OUTPUT)" || { echo "error: OUTPUT= is required"; exit 2; }
	$(PYTHON) scripts/azoth_score_deployed.py \
		--route $(ROUTE) \
		--db $(DB) \
		--deploy-root $(AZOTH_ROOT) \
		--workers $(EXP_WORKERS) \
		$(if $(ROW_IDS_FILE),--row-ids-file $(ROW_IDS_FILE),) \
		--output $(OUTPUT)

# Convenience: full deploy into a fresh run dir, then publish.
azoth-publish-deploy: venv
	@echo "publish-deploy: allocating $(AZOTH_RUN_DIR)"
	@mkdir -p $(AZOTH_RUN_DIR)
	$(MAKE) azoth-deploy AZOTH_ROOT=$(AZOTH_RUN_DIR)
	$(MAKE) azoth-publish AZOTH_ROOT=$(AZOTH_RUN_DIR)
	@echo "publish-deploy: $(OUT_ROOT)/azoth now points to $(AZOTH_RUN_DIR)"

azoth-deploy: azoth-calibrate
	@test -f $(AZOTH_ROOT)/config.json || { echo "error: $(AZOTH_ROOT)/config.json not found"; exit 1; }
	@test -f $(AZOTH_ROOT)/score_table.npz || { echo "error: $(AZOTH_ROOT)/score_table.npz not found"; exit 1; }
	@test -f $(AZOTH_ROOT)/specialists.json || { echo "error: $(AZOTH_ROOT)/specialists.json not found"; exit 1; }
	@test -f $(AZOTH_ROOT)/general/model.txt || ls $(AZOTH_ROOT)/general/models/seed_*.txt >/dev/null 2>&1 || { echo "error: $(AZOTH_ROOT)/general missing model.txt or models/seed_*.txt"; exit 1; }
	@test -f $(AZOTH_ROOT)/general/feature_spec.json || { echo "error: $(AZOTH_ROOT)/general/feature_spec.json not found"; exit 1; }
	@$(MAKE) azoth-check-bundle-layout
	@if [ "$(AZOTH_DEPLOY_DIAGNOSTICS)" = "1" ] || [ "$(AZOTH_DEPLOY_DIAGNOSTICS)" = "true" ] || [ "$(AZOTH_DEPLOY_DIAGNOSTICS)" = "yes" ]; then \
	  $(PYTHON) scripts/azoth_route_diagnostics.py \
	    --config $(AZOTH_CONFIG) \
	    --score-table $(AZOTH_SCORE_TABLE) \
	    --output $(AZOTH_DIAGNOSTICS) \
	    --csv $(AZOTH_DIAGNOSTICS_CSV) \
	    --slice-output $(AZOTH_SLICE_METRICS) \
	    --slice-csv $(AZOTH_SLICE_METRICS_CSV); \
	else \
	  printf '# Azoth Route Diagnostics\n\nSkipped during deploy (AZOTH_DEPLOY_DIAGNOSTICS=0). Run `make azoth-diagnostics AZOTH_ROOT=%s` for the full report.\n' "$(AZOTH_ROOT)" > "$(AZOTH_DIAGNOSTICS)"; \
	  printf 'status,message\nskipped,AZOTH_DEPLOY_DIAGNOSTICS=0\n' > "$(AZOTH_DIAGNOSTICS_CSV)"; \
	  printf '# Azoth Slice Metrics\n\nSkipped during deploy (AZOTH_DEPLOY_DIAGNOSTICS=0). Run `make azoth-diagnostics AZOTH_ROOT=%s` for the full report.\n' "$(AZOTH_ROOT)" > "$(AZOTH_SLICE_METRICS)"; \
	  printf 'status,message\nskipped,AZOTH_DEPLOY_DIAGNOSTICS=0\n' > "$(AZOTH_SLICE_METRICS_CSV)"; \
	fi
	$(PYTHON) scripts/azoth_route_policy_search.py \
		--parallelism $(AZOTH_POLICY_PARALLELISM) \
		$(if $(AZOTH_POLICY_OVERRIDE_ROUTE),--db $(DB),) \
		--config $(AZOTH_CONFIG) \
		--score-table $(AZOTH_SCORE_TABLE) \
		--output $(AZOTH_ROUTE_POLICIES) \
		--csv $(AZOTH_ROUTE_POLICIES_CSV) \
		--markdown $(AZOTH_ROUTE_POLICIES_MD) \
		$(foreach route,$(AZOTH_POLICY_OVERRIDE_ROUTE),--override-route $(route)) \
		$(AZOTH_PREVIOUS_BUNDLE_ARG) \
		$(EXP_WORKERS_ARG)
	$(PYTHON) scripts/azoth_policy_global_metrics.py \
		--config $(AZOTH_CONFIG) \
		--policy $(AZOTH_ROUTE_POLICIES) \
		--score-table $(AZOTH_SCORE_TABLE) \
		--output $(AZOTH_GLOBAL_POLICY_METRICS) \
		--markdown $(AZOTH_GLOBAL_POLICY_METRICS_MD) \
		$(AZOTH_BUDGET_GATE_ARGS)
	$(PYTHON) scripts/compute_routed_metrics.py --azoth-root $(AZOTH_ROOT) --db $(DB) $(AZOTH_PREVIOUS_BUNDLE_ARG) $(AZOTH_ROUTED_METRICS_ARGS)
	$(PYTHON) scripts/azoth_route_policy_eval.py \
		--score-table $(AZOTH_ROOT)/score_table.npz \
		--general-scores $(AZOTH_ROOT)/general/threshold_scores.npz \
		--route-policies $(AZOTH_ROUTE_POLICIES) \
		--partition test \
		--output-md $(AZOTH_ROOT)/route_policy_eval_oof.md \
		--output-json $(AZOTH_ROOT)/route_policy_eval_oof.json
	$(PYTHON) scripts/write_azoth_readmes.py --azoth-root $(AZOTH_ROOT)
	$(MAKE) azoth-deploy-final

# azoth-deploy-final reruns only the post-generation deploy stages: stage the
# curated runtime/docs bundle, validate it, run litmus compatibility checks,
# mirror it into $(AZOTH_DEPLOY_DIR), and verify litmus's default deployed path.
# It intentionally skips azoth-calibrate, diagnostics, policy search, and routed
# metrics; use after those artifacts already exist. It DOES regenerate READMEs,
# but only after convert (the prune step) so the docs match the final bundle.
#
# The litmus checks include `litmus validate` (fatal) against the STAGED bundle
# before the rsync. `scan` only WARNs when an offset-written feature family
# (present/maxcrit/bigram/trigram/unsigned_bigram) is present-but-non-contiguous
# in vocab order — the failure mode an out-of-sync allowlist prune produces —
# whereas validate's ExtractContext::validate_layout() rejects it, so a bundle
# that would run silently degraded never reaches $(AZOTH_DEPLOY_DIR).
#
# --checksum is a FIX, not a tuning knob (added 2026-08-07). Plain `rsync -a`
# decides what to copy from size and mtime alone, so a file whose content
# changed while its size and mtime happened to match is skipped silently — no
# error, no output, exit 0. Demonstrated directly: two 8-byte files with
# differing content and identical mtimes, `rsync -a` leaves the destination
# stale and reports nothing; add --checksum and it transfers (`>fc........`).
# That is the exact signature of the two silent no-ops below, and while the
# staged tree is deleted by the EXIT trap before it can be examined (so this
# cannot be proven retroactively), the mechanism fits. Cost is negligible and
# the failure mode it removes is not: the bundle is 206 MB and hashes in
# ~0.6 s, against a 9-hour publish. It can only ever copy MORE files, never
# fewer, so it cannot introduce a partial deploy.
#
# Mirror instrumentation (added 2026-08-07). This rsync has twice reported
# success while transferring almost nothing — 2026-07-21 and again 2026-08-07,
# where a full 9-hour publish passed every gate and left the deploy dir on a
# 3-day-old bundle. Both investigations stalled for want of evidence: the recipe
# printed the same "Deployed ..." line whether it moved 210 files or one. Three
# additions make the next occurrence self-diagnosing rather than forensic:
#
#   * the staged-bundle line reports which run dir was staged (AZOTH_ROOT is a
#     symlink into $(AZOTH_RUNS_ROOT), so "which build shipped" is not obvious)
#     plus its file count and size — that alone separates "staging produced an
#     incomplete tree" from "the mirror dropped it";
#   * rsync --itemize-changes names every file transferred, and --stats gives
#     the totals;
#   * the staged config.json md5 is captured the moment the stage is built and
#     asserted twice, because rsync exiting 0 has never been evidence that
#     anything shipped. Before the mirror: the stage must still hold the bundle
#     every gate above validated (it does not, if something rewrote $_STAGE
#     mid-flight). After the mirror: the deploy dir must equal that same md5.
#     Both are fatal. Note the invariant is deployed == STAGED, not
#     deployed != PRE — the old pre/post comparison had to warn rather than
#     fail, because re-running deploy-final against an already-deployed bundle
#     is a legitimate no-op that it could not tell apart from a mirror that
#     shipped nothing. Comparing against the staged fingerprint has no such
#     exception: the legitimate re-run passes, both no-op modes fail.
azoth-deploy-final: venv
	@test -z "$(COLLIMATOR_SAMPLE_PERCENT)$(COLLIMATOR_PROFILE_DIR)" || { echo "Refusing to deploy a sampled/profiling run"; exit 1; }
	@test -f $(AZOTH_ROOT)/config.json || { echo "error: $(AZOTH_ROOT)/config.json not found"; exit 1; }
	@test -f $(AZOTH_ROOT)/score_table.npz || { echo "error: $(AZOTH_ROOT)/score_table.npz not found"; exit 1; }
	@test -f $(AZOTH_ROOT)/specialists.json || { echo "error: $(AZOTH_ROOT)/specialists.json not found"; exit 1; }
	@test -f $(AZOTH_ROOT)/general/model.txt || ls $(AZOTH_ROOT)/general/models/seed_*.txt >/dev/null 2>&1 || { echo "error: $(AZOTH_ROOT)/general missing model.txt or models/seed_*.txt"; exit 1; }
	@test -f $(AZOTH_ROOT)/general/feature_spec.json || { echo "error: $(AZOTH_ROOT)/general/feature_spec.json not found"; exit 1; }
	@$(MAKE) azoth-check-bundle-layout
	@test -f $(AZOTH_DIAGNOSTICS) || { echo "error: $(AZOTH_DIAGNOSTICS) not found; run make azoth-deploy or regenerate diagnostics first"; exit 1; }
	@test -f $(AZOTH_SLICE_METRICS) || { echo "error: $(AZOTH_SLICE_METRICS) not found; run make azoth-deploy or regenerate diagnostics first"; exit 1; }
	@test -f $(AZOTH_ROUTE_POLICIES) || { echo "error: $(AZOTH_ROUTE_POLICIES) not found; run make azoth-deploy or make azoth-policies first"; exit 1; }
	@test -f $(AZOTH_ROUTE_POLICIES_MD) || { echo "error: $(AZOTH_ROUTE_POLICIES_MD) not found; run make azoth-deploy or make azoth-policies first"; exit 1; }
	@test -f $(AZOTH_GLOBAL_POLICY_METRICS_MD) || { echo "error: $(AZOTH_GLOBAL_POLICY_METRICS_MD) not found; run make azoth-deploy first"; exit 1; }
	@# Auto-convert any LightGBM .txt seeds to .onnx siblings. The deployed
	@# bundle is ONNX-only (litmus dropped the LightGBM/XGBoost loaders), so a
	@# training pipeline that emits only .txt would otherwise trip the staging
	@# guard in stage_azoth_runtime_bundle.py. Idempotent: skips files that
	@# already have an .onnx sibling (typical case for re-runs). Parity-
	@# checked: refuses to ship if any prob delta exceeds the converter's
	@# default tolerance.
	$(PYTHON) scripts/convert_bundle_to_onnx.py --azoth-root $(AZOTH_ROOT) --db $(DB)
	@# Conversion must preserve the calibrated ensemble; parity failures stop publication.
	$(PYTHON) scripts/write_azoth_readmes.py --azoth-root $(AZOTH_ROOT)
	@# Mirror staged bundle into deploy dir, deleting anything in the deploy
	@# tree that isn't in the staged copy. Per-route slot contents (e.g. an
	@# old `models/` dir from a prior multi-seed deploy) get cleaned even
	@# when the new bundle ships a single-seed `model.txt` for that route.
	@# The lock prevents concurrent deploys from deleting each other's rsync
	@# temp files; --delete-before avoids interleaving deletes with writes.
	@# Protect only deploy repo metadata and static root legal/training docs;
	@# the staged runtime/generated-docs payload is otherwise canonical, so
	@# stale training artifacts are removed.
	@# SCAN_NO_UPDATE=1 is load-bearing: atomscan's default-on auto-update
	@# (auto_update::refresh_if_stale) installs into models_repo::install_target(),
	@# which honours SCAN_MODELS_DIR — so pointing the compatibility checks at
	@# $$_STAGE makes the staging dir the updater's install TARGET. Its only guard
	@# is `dir/.git exists` (that is what spares $(AZOTH_DEPLOY_DIR)), and a mktemp
	@# dir has none, so the updater silently replaces the freshly staged bundle with
	@# the published R2 bundle. Every gate after that point then validates the OLD
	@# models and the mirror ships them straight back (2026-08-21: the whole nightly
	@# deployed nothing but the updater's .litmus-models.toml sidecar).
	@_STAGE=$$(mktemp -d) && \
	  _LOCK=$$(dirname "$(AZOTH_DEPLOY_DIR)")/.azoth-deploy.lock && \
	  trap 'rm -rf "$$_STAGE"' EXIT INT TERM && \
	  export MAKEFLAGS= MFLAGS= CARGO_MAKEFLAGS= SCAN_NO_UPDATE=1 && \
	  $(PYTHON) scripts/stage_azoth_runtime_bundle.py "$(AZOTH_ROOT)" "$$_STAGE" && \
	  echo "staged from $$(readlink -f $(AZOTH_ROOT)): $$(find "$$_STAGE" -type f | wc -l) files, $$(du -sh "$$_STAGE" | cut -f1)" && \
	  _STAGED=$$(md5sum "$$_STAGE/config.json" | cut -d' ' -f1) && \
	  echo "staged config.json md5: $$_STAGED" && \
	  cp "$(AZOTH_DIAGNOSTICS)" "$$_STAGE/route_diagnostics.md" && \
	  cp "$(AZOTH_SLICE_METRICS)" "$$_STAGE/slice_metrics.md" && \
	  cp "$(AZOTH_ROUTE_POLICIES_MD)" "$$_STAGE/route_policies.md" && \
	  cp "$(AZOTH_GLOBAL_POLICY_METRICS_MD)" "$$_STAGE/global_policy_metrics.md" && \
	  $(PYTHON) scripts/validate_azoth_bundle.py "$$_STAGE" && \
	  $(PYTHON) scripts/check_azoth_regression.py --staged "$$_STAGE" --deployed "$(AZOTH_DEPLOY_DIR)" --low-water-mark "$(AZOTH_LOW_WATER_MARK_DIR)" $(AZOTH_REGRESSION_SOURCE_ARG) $(AZOTH_REGRESSION_ARGS) && \
	  echo "Preparing litmus before compatibility checks (update + build + update-rules; each non-fatal, all run)..." && \
	  ( cd $(LITMUS_DIR) || { echo "WARN: cannot enter $(LITMUS_DIR); skipping litmus prep"; exit 0; }; \
	    { echo "==> update litmus (git pull)"; git pull --ff-only; } || echo "WARN: litmus update (git pull) failed; continuing with current checkout"; \
	    { echo "==> build litmus (cargo build --release)"; cargo build --release; } || echo "WARN: litmus build failed; continuing (the gating test below will surface a broken build)"; \
	    { echo "==> litmus update-rules"; cargo run --release -- update-rules; } || echo "WARN: litmus update-rules failed; continuing (trait/model rules may be stale)"; \
	    true ) && \
	  echo "Running litmus deployed-ensemble compatibility checks against staged copy..." && \
	  ( cd $(LITMUS_DIR) && SCAN_MODELS_DIR="$$_STAGE" cargo test --release --test scan_no_deadlock -- --ignored ) && \
	  $(PYTHON) scripts/verify_azoth_litmus_runtime.py --litmus-dir $(LITMUS_DIR) --models-dir "$$_STAGE" --required-model az/native --required-model az/elf && \
	  echo "Running litmus validate (fatal feature-layout + benign-corpus gate) against staged copy..." && \
	  ( cd $(LITMUS_DIR) && SCAN_MODELS_DIR="$$_STAGE" cargo run --release -- validate --analyze local --no-update ) && \
	  echo "Checking deploy tuning goal agreement (collimator / bundle / litmus)..." && \
	  $(PYTHON) scripts/check_severity_level_sync.py --staged "$$_STAGE" --litmus-dir $(LITMUS_DIR) && \
	  _NOW=$$(md5sum "$$_STAGE/config.json" | cut -d' ' -f1) && \
	  if [ "$$_NOW" != "$$_STAGED" ]; then \
	    echo "error: staged config.json changed under us ($$_STAGED -> $$_NOW)."; \
	    echo "       Something rewrote $$_STAGE after staging, so the gates above validated"; \
	    echo "       a bundle that is no longer there. Refusing to mirror it."; \
	    exit 1; \
	  fi && \
	  mkdir -p "$(AZOTH_DEPLOY_DIR)" && \
	  _PRE=$$(md5sum "$(AZOTH_DEPLOY_DIR)/config.json" 2>/dev/null | cut -d' ' -f1) && \
	  flock "$$_LOCK" rsync -a --checksum --delete-before --itemize-changes --stats \
	    --filter='protect /.git/***' \
	    --filter='protect /.gitignore' \
	    --filter='protect /LICENSE' \
	    --filter='protect /TRAINING.md' \
	    "$$_STAGE/" "$(AZOTH_DEPLOY_DIR)/" && \
	  _POST=$$(md5sum "$(AZOTH_DEPLOY_DIR)/config.json" 2>/dev/null | cut -d' ' -f1) && \
	  echo "deployed config.json md5: $${_PRE:-<absent>} -> $$_POST" && \
	  if [ "$$_POST" != "$$_STAGED" ]; then \
	    echo "error: deployed config.json ($$_POST) does not match the staged bundle ($$_STAGED)."; \
	    echo "       The mirror did not ship what was staged — check the rsync itemization above."; \
	    exit 1; \
	  fi && \
	  echo "Running litmus default deployed-model check..." && \
	  ( cd $(LITMUS_DIR) && cargo run --release -- --format json --show all scan /bin/ls --analyze local --no-update ) && \
	  echo "Deployed azoth ensemble bundle to $(AZOTH_DEPLOY_DIR)"

# Legacy single-model false/near reports. Source generation is bumped by
# SKIP so triage can page past already-reviewed samples (TOP_ERRORS=250
# SKIP=100 → JSON has 350 rows; triage skips first 100, copies next 250).
false-positives: venv check-db
	$(PYTHON) -u -m collimator false-positives --db $(DB) \
		--model $(OUT_DIR)/$(MODEL_FILE) \
		--spec $(OUT_DIR)/feature_spec.json \
		$(WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		--scores-cache $(THRESHOLD_SCORES) \
		--top-errors $$(( $(TOP_ERRORS) + $(SKIP) )) \
		--output $(OUT_DIR)/false_positives.json

near-false-positives: venv check-db
	$(PYTHON) -u -m collimator near-false-positives --db $(DB) \
		--model $(OUT_DIR)/$(MODEL_FILE) \
		--spec $(OUT_DIR)/feature_spec.json \
		$(WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		--scores-cache $(THRESHOLD_SCORES) \
		--top-errors $$(( $(TOP_ERRORS) + $(SKIP) )) \
		--output $(OUT_DIR)/near_false_positives.json

false-negatives: venv check-db
	$(PYTHON) -u -m collimator false-negatives --db $(DB) \
		--model $(OUT_DIR)/$(MODEL_FILE) \
		--spec $(OUT_DIR)/feature_spec.json \
		$(WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		--scores-cache $(THRESHOLD_SCORES) \
		--top-errors $$(( $(TOP_ERRORS) + $(SKIP) )) \
		--output $(OUT_DIR)/false_negatives.json

near-false-negatives: venv check-db
	$(PYTHON) -u -m collimator near-false-negatives --db $(DB) \
		--model $(OUT_DIR)/$(MODEL_FILE) \
		--spec $(OUT_DIR)/feature_spec.json \
		$(WORKERS_ARG) \
		$(THRESHOLD_MAX_ID_ARG) \
		--scores-cache $(THRESHOLD_SCORES) \
		--top-errors $$(( $(TOP_ERRORS) + $(SKIP) )) \
		--output $(OUT_DIR)/near_false_negatives.json

false-positives-archive: false-positives
	$(PYTHON) scripts/archive_error_samples.py \
		--report $(OUT_DIR)/false_positives.json \
		--output $(FALSE_POSITIVES_ARCHIVE) \
		--samples-dir $(SAMPLES_DIR) \
		--kind false-positives \
		--top $(TOP_ERRORS)

false-negatives-archive: false-negatives
	$(PYTHON) scripts/archive_error_samples.py \
		--report $(OUT_DIR)/false_negatives.json \
		--output $(FALSE_NEGATIVES_ARCHIVE) \
		--samples-dir $(SAMPLES_DIR) \
		--kind false-negatives \
		--top $(TOP_ERRORS)

near-false-positives-archive: near-false-positives
	$(PYTHON) scripts/archive_error_samples.py \
		--report $(OUT_DIR)/near_false_positives.json \
		--output $(NEAR_FALSE_POSITIVES_ARCHIVE) \
		--samples-dir $(SAMPLES_DIR) \
		--kind near-false-positives \
		--top $(TOP_ERRORS)

near-false-negatives-archive: near-false-negatives
	$(PYTHON) scripts/archive_error_samples.py \
		--report $(OUT_DIR)/near_false_negatives.json \
		--output $(NEAR_FALSE_NEGATIVES_ARCHIVE) \
		--samples-dir $(SAMPLES_DIR) \
		--kind near-false-negatives \
		--top $(TOP_ERRORS)

# mislabeled-triage: unified deploy-aware error triage.
#
# Uses the score table + route policies (no model rescore) to pull
# benigns flagged AND malware missed by any combination of scopes,
# deduped by sample. Output: one directory with two subtrees ready
# to feed cleave. Layout:
#   $(MIS_TRIAGE_DIR)/false-positives/...
#   $(MIS_TRIAGE_DIR)/false-negatives/...
#
# Usage:
#   make mislabeled-triage                                                  # default: SCOPE=ensemble,specialists,filegroups LEVEL=3 SEVERITY=hostile
#   make mislabeled-triage SCOPE=ensemble                                   # what litmus would flag in production
#   make mislabeled-triage SCOPE=route:filetypes/jpeg LEVEL=3
#   make mislabeled-triage SCOPE=specialists LEVEL=5 SEVERITY=suspicious
#
# Granular variants surface only one side of the pool:
#   make false-positives-triage SCOPE=ensemble
#   make false-negatives-triage SCOPE=ensemble
#
# Default SCOPE combines ensemble+specialists+filegroups so the pool
# captures every error mode worth triaging in one pass — they overlap
# heavily and dedup-by-row keeps the list manageable.
# COMMA / SPACE: make doesn't let you put a literal comma in a function
# expansion since it's the argument separator, so we capture it here.
# Must be defined BEFORE MIS_TAG because := evaluates immediately.
COMMA := ,
EMPTY :=
SPACE := $(EMPTY) $(EMPTY)

SCOPE ?= ensemble,specialists,filegroups
LEVEL ?= 3
SEVERITY ?= hostile
MIS_TAG := $(subst $(SPACE),_,$(subst $(COMMA),-,$(subst :,-,$(subst /,-,$(SCOPE)))))-L$(LEVEL)-$(SEVERITY)
FP_REPORT ?= $(AZOTH_ROOT)/false_positives_$(MIS_TAG).json
FN_REPORT ?= $(AZOTH_ROOT)/false_negatives_$(MIS_TAG).json
FP_TRIAGE_DIR ?= /var/tmp/false-positives-$(MIS_TAG)
FN_TRIAGE_DIR ?= /var/tmp/false-negatives-$(MIS_TAG)
MIS_TRIAGE_DIR ?= /var/tmp/mislabeled-$(MIS_TAG)

.PHONY: mislabeled-fp-report mislabeled-fn-report \
        false-positives-triage false-negatives-triage \
        mislabeled-triage

mislabeled-fp-report: venv
	$(PYTHON) scripts/mislabeled_by_scope.py \
		--kind false-positives \
		--score-table $(AZOTH_ROOT)/score_table.npz \
		--route-policies $(AZOTH_ROOT)/route_policies.json \
		--scope $(SCOPE) --level $(LEVEL) --severity $(SEVERITY) \
		--skip $(SKIP) --top-errors $(TOP_ERRORS) --db $(DB) \
		--output $(FP_REPORT)

mislabeled-fn-report: venv
	$(PYTHON) scripts/mislabeled_by_scope.py \
		--kind false-negatives \
		--score-table $(AZOTH_ROOT)/score_table.npz \
		--route-policies $(AZOTH_ROOT)/route_policies.json \
		--scope $(SCOPE) --level $(LEVEL) --severity $(SEVERITY) \
		--skip $(SKIP) --top-errors $(TOP_ERRORS) --db $(DB) \
		--output $(FN_REPORT)

false-positives-triage: mislabeled-fp-report
	$(PYTHON) scripts/triage_error_samples.py \
		--report $(FP_REPORT) \
		--output-dir $(FP_TRIAGE_DIR) \
		--samples-dir $(SAMPLES_DIR) \
		--kind false-positives --top $(TOP_ERRORS)
	@echo "report:   $(FP_REPORT)"
	@echo "samples:  $(FP_TRIAGE_DIR)"

false-negatives-triage: mislabeled-fn-report
	$(PYTHON) scripts/triage_error_samples.py \
		--report $(FN_REPORT) \
		--output-dir $(FN_TRIAGE_DIR) \
		--samples-dir $(SAMPLES_DIR) \
		--kind false-negatives --top $(TOP_ERRORS)
	@echo "report:   $(FN_REPORT)"
	@echo "samples:  $(FN_TRIAGE_DIR)"

mislabeled-triage: mislabeled-fp-report mislabeled-fn-report
	$(PYTHON) scripts/triage_error_samples.py \
		--report $(FP_REPORT) \
		--output-dir $(MIS_TRIAGE_DIR)/false-positives \
		--samples-dir $(SAMPLES_DIR) \
		--kind false-positives --top $(TOP_ERRORS)
	$(PYTHON) scripts/triage_error_samples.py \
		--report $(FN_REPORT) \
		--output-dir $(MIS_TRIAGE_DIR)/false-negatives \
		--samples-dir $(SAMPLES_DIR) \
		--kind false-negatives --top $(TOP_ERRORS)
	@echo "scope:    $(SCOPE) | level: L$(LEVEL) | severity: $(SEVERITY)"
	@echo "reports:  $(FP_REPORT) + $(FN_REPORT)"
	@echo "samples:  $(MIS_TRIAGE_DIR)/{false-positives,false-negatives}/"

# triage: per-filetype mislabeled-data dump for hand review.
#
# For every filetype, lands the top $(TRIAGE_TOP) most-confidently-wrong
# samples on each side into its own subdir:
#   $(TRIAGE_DIR)/mislabeled-good/<filetype>/   (false positives: labeled
#       good, the ensemble flags as bad — candidates that may really be bad)
#   $(TRIAGE_DIR)/mislabeled-bad/<filetype>/    (false negatives: labeled
#       bad, the ensemble scores benign — candidates that may really be good)
#
# Over-fetch: the reports rank $(TRIAGE_FETCH) candidates per filetype, then
# triage copies until $(TRIAGE_TOP) land per filetype — samples pruned from
# /data/samples are pulled from hopper (GET /api/file/{sha}); the extra
# candidates cover any that are gone from both.
#
# Also creates two empty output bins for the sorter to move confirmed
# mislabels into:
#   $(TRIAGE_DIR)/misplaced-good/   $(TRIAGE_DIR)/misplaced-bad/
#
# Needs a scored training bundle ($(AZOTH_ROOT)/score_table.npz +
# route_policies.json). Set AZOTH_ROOT to an existing validated candidate to
# review it without rebuilding or installing. Runtime-only bundles lack scores.
TRIAGE_DIR        ?= /var/tmp/triage
TRIAGE_TOP        ?= $(if $(TOP),$(TOP),40)
TRIAGE_FETCH      ?= $(if $(TOP),$(shell echo $$(( $(TOP) * 2 ))),80)
TRIAGE_SCOPE      ?= ensemble,specialists,filegroups
# Optional comma-separated filter, e.g. TRIAGE_FILETYPES=html,npm,tar.
TRIAGE_FILETYPES  ?=
TRIAGE_FILETYPES_ARG = $(if $(TRIAGE_FILETYPES),--filetypes "$(TRIAGE_FILETYPES)",)
# No cutoff: every filetype yields its top-N most label-discordant files,
# ranked by margin. TRIAGE_LEVEL is only the reference the margin is measured
# against (L50: one calibration FP plus 49 extrapolated FP per 100M), NOT a membership
# gate — so all filetypes surface, not just those crossing a fixed threshold.
TRIAGE_LEVEL      ?= 50
TRIAGE_SEVERITY   ?= hostile
TRIAGE_FP_REPORT  ?= $(TRIAGE_DIR)/mislabeled-good.report.json
TRIAGE_FN_REPORT  ?= $(TRIAGE_DIR)/mislabeled-bad.report.json
# Resolve row_id -> sha256/path against the authoritative hopper DB, NOT the
# local replica ($(DB)). The replica is built for bulk training reads and is
# periodically torn down and rebuilt; mid-rebuild it returns no rows, which
# silently yields empty paths and copies nothing. Triage's lookup is tiny
# (only the selected top-N ids) so hitting the real DB is cheap. Password
# comes from ~/.pgpass (hopper-db:5432:hopper:hopper).
TRIAGE_DB         ?= postgres://hopper@hopper-db:5432/hopper

.PHONY: triage triage-good triage-bad
# triage runs both sides sequentially (FP then FN) via sub-make so a heavy
# FP archive pull can't run concurrently with the FN pull. The sides are also
# standalone targets — re-run just one (e.g. `make triage-bad`) without redoing
# the other, e.g. after hopper sheds load mid-run.
triage: venv
	$(MAKE) triage-good
	$(MAKE) triage-bad
	@echo "scope:           $(TRIAGE_SCOPE) | level: L$(TRIAGE_LEVEL) | severity: $(TRIAGE_SEVERITY)"
	@echo "mislabeled-good: $(TRIAGE_DIR)/mislabeled-good/<filetype>/  (labeled good, flagged bad)"
	@echo "mislabeled-bad:  $(TRIAGE_DIR)/mislabeled-bad/<filetype>/   (labeled bad, scored benign)"
	@echo "sort into:       $(TRIAGE_DIR)/misplaced-good/  $(TRIAGE_DIR)/misplaced-bad/"

# triage-good: per filetype, the top-$(TRIAGE_TOP) benigns the ensemble most
# strongly flags as bad (most-likely-mislabeled good files).
triage-good: venv
	@mkdir -p $(TRIAGE_DIR)/misplaced-good
	$(PYTHON) scripts/mislabeled_by_scope.py \
		--kind false-positives \
		--score-table $(AZOTH_ROOT)/score_table.npz \
		--route-policies $(AZOTH_ROOT)/route_policies.json \
		--scope $(TRIAGE_SCOPE) --level $(TRIAGE_LEVEL) --severity $(TRIAGE_SEVERITY) \
		$(TRIAGE_FILETYPES_ARG) \
		--no-cutoff --per-filetype-top $(TRIAGE_FETCH) --db $(TRIAGE_DB) \
		--output $(TRIAGE_FP_REPORT)
	$(PYTHON) scripts/triage_error_samples.py \
		--report $(TRIAGE_FP_REPORT) \
		--output-dir $(TRIAGE_DIR)/mislabeled-good \
		--samples-dir $(SAMPLES_DIR) \
		--kind false-positives \
		--group-by-filetype --top $(TRIAGE_TOP)

# triage-bad: per filetype, the top-$(TRIAGE_TOP) malware the ensemble most
# strongly scores benign (most-likely-mislabeled bad files).
triage-bad: venv
	@mkdir -p $(TRIAGE_DIR)/misplaced-bad
	$(PYTHON) scripts/mislabeled_by_scope.py \
		--kind false-negatives \
		--score-table $(AZOTH_ROOT)/score_table.npz \
		--route-policies $(AZOTH_ROOT)/route_policies.json \
		--scope $(TRIAGE_SCOPE) --level $(TRIAGE_LEVEL) --severity $(TRIAGE_SEVERITY) \
		$(TRIAGE_FILETYPES_ARG) \
		--no-cutoff --per-filetype-top $(TRIAGE_FETCH) --db $(TRIAGE_DB) \
		--output $(TRIAGE_FN_REPORT)
	$(PYTHON) scripts/triage_error_samples.py \
		--report $(TRIAGE_FN_REPORT) \
		--output-dir $(TRIAGE_DIR)/mislabeled-bad \
		--samples-dir $(SAMPLES_DIR) \
		--kind false-negatives \
		--group-by-filetype --top $(TRIAGE_TOP)

near-false-positives-triage: near-false-positives
	$(PYTHON) scripts/triage_error_samples.py \
		--report $(OUT_DIR)/near_false_positives.json \
		--output-dir $(NEAR_FALSE_POSITIVES_TRIAGE_DIR) \
		--samples-dir $(SAMPLES_DIR) \
		--kind near-false-positives \
		--skip $(SKIP) \
		--top $(TOP_ERRORS)
	@echo "samples staged in $(NEAR_FALSE_POSITIVES_TRIAGE_DIR)"

benchmark: venv check-db
	$(PYTHON) -m collimator benchmark --db $(DB) $(WORKERS_ARG) \
		$(if $(wildcard $(OUT_DIR)/$(MODEL_FILE)),--model $(OUT_DIR)/$(MODEL_FILE),) \
		$(if $(wildcard $(OUT_DIR)/feature_spec.json),--spec $(OUT_DIR)/feature_spec.json,)

build-splits: venv check-db
	$(PYTHON) -m collimator build-splits --db $(DB)

# One feature environment for discovery and experiments; new knobs belong here.
EXPERIMENT_FEATURE_ENV = \
    COLLIMATOR_ENSEMBLE_SEEDS=$(EXP_ENSEMBLE_SEEDS) \
    COLLIMATOR_NUM_THREADS=$(EXP_LGBM_THREADS) \
    COLLIMATOR_ALLOWED_FEATURES_FILE=$(EXP_ALLOWED_FEATURES_FILE) \
    COLLIMATOR_SILENT_PACKER_SIGNAL=$(EXP_SILENT_PACKER_SIGNAL) \
    COLLIMATOR_MTIME_KURTOSIS=$(EXP_MTIME_KURTOSIS) \
    COLLIMATOR_AIR_GAP_SIGNAL=$(EXP_AIR_GAP_SIGNAL) \
    COLLIMATOR_EXTREME_FEATURES=$(EXP_EXTREME_FEATURES) \
    COLLIMATOR_ANACHRONISTIC_INJECTION=$(EXP_ANACHRONISTIC_INJECTION) \
    COLLIMATOR_CODE_ENTROPY_SPIKE=$(EXP_CODE_ENTROPY_SPIKE) \
    COLLIMATOR_FOREIGN_BINARY_SIGNAL=$(EXP_FOREIGN_BINARY_SIGNAL) \
    COLLIMATOR_EXTENSION_MISMATCH_SIGNAL=$(EXP_EXTENSION_MISMATCH_SIGNAL) \
    COLLIMATOR_HOSTILE_FINDING_DENSITY=$(EXP_HOSTILE_FINDING_DENSITY) \
    COLLIMATOR_HOSTILE_DEPTH_WEIGHT=$(EXP_HOSTILE_DEPTH_WEIGHT) \
    COLLIMATOR_FILETYPE_INTERACTIONS=$(EXP_FILETYPE_INTERACTIONS) \
    COLLIMATOR_FORMAT_HINTS=$(EXP_FORMAT_HINTS) \
    COLLIMATOR_BLINDFOLD=$(EXP_BLINDFOLD) \
    COLLIMATOR_SCORE_WEIGHTED_TRAITS=$(EXP_SCORE_WEIGHTED_TRAITS) \
    COLLIMATOR_SOFT_PRESENCE=$(EXP_SOFT_PRESENCE) \
    COLLIMATOR_REPETITION_PENALTY_FEATURES=$(EXP_REPETITION_PENALTY_FEATURES) \
    COLLIMATOR_FILE_SEVERITY_DISTRIBUTION=$(EXP_FILE_SEVERITY_DISTRIBUTION) \
    COLLIMATOR_SEVERITY_FRACTION_FEATURES=$(EXP_SEVERITY_FRACTION_FEATURES) \
    COLLIMATOR_HOSTILE_WEIGHTED_DENSITY=$(EXP_HOSTILE_WEIGHTED_DENSITY) \
    COLLIMATOR_HOSTILE_ESCALATION_FEATURES=$(EXP_HOSTILE_ESCALATION_FEATURES) \
    COLLIMATOR_SUSPICIOUS_BREADTH_DENSITY=$(EXP_SUSPICIOUS_BREADTH_DENSITY) \
    COLLIMATOR_STRUCT_FILE_RISK_COVERAGE=$(EXP_STRUCT_FILE_RISK_COVERAGE) \
    COLLIMATOR_TOP_K_RISK_FILES=$(EXP_TOP_K_RISK_FILES) \
    COLLIMATOR_DISABLE_FEATURE_GROUPS=$(EXP_DISABLE_FEATURE_GROUPS) \
    COLLIMATOR_PACKAGED_CAPABILITY_MODE=$(EXP_PACKAGED_CAPABILITY_MODE) \
    COLLIMATOR_MIN_SAMPLE_SCORE=$(EXP_MIN_SAMPLE_SCORE) \
    COLLIMATOR_TAXONOMY_FEATURES=$(EXP_TAXONOMY_FEATURES) \
    COLLIMATOR_EXTENDED_METRICS=$(EXP_EXTENDED_METRICS) \
    COLLIMATOR_METRIC_MIN_FREQ_PCT=$(EXP_METRIC_MIN_FREQ_PCT) \
    COLLIMATOR_EMBER_LITE_FEATURES=$(EXP_EMBER_LITE_FEATURES) \
    COLLIMATOR_BIGRAM_MAX=$(EXP_BIGRAM_MAX) \
    COLLIMATOR_BIGRAM_MIN_FREQ=$(EXP_BIGRAM_MIN_FREQ) \
    COLLIMATOR_TRIGRAM_MAX=$(EXP_TRIGRAM_MAX) \
    COLLIMATOR_TRIGRAM_MAX_BENIGN_FRAC=$(EXP_TRIGRAM_MAX_BENIGN_FRAC) \
    COLLIMATOR_ATTACK_FEATURES=$(EXP_ATTACK_FEATURES) \
    COLLIMATOR_CONFIDENCE_WEIGHTED_NGRAMS=$(EXP_CONFIDENCE_WEIGHTED_NGRAMS) \
    COLLIMATOR_OBJECTIVE_TRIGRAMS=$(EXP_OBJECTIVE_TRIGRAMS) \
    COLLIMATOR_SUSPICIOUS_TRIGRAMS=$(EXP_SUSPICIOUS_TRIGRAMS) \
    COLLIMATOR_ATTACK_NGRAMS=$(EXP_ATTACK_NGRAMS) \
    COLLIMATOR_CRIT_CATEGORY_NGRAMS=$(EXP_CRIT_CATEGORY_NGRAMS) \
    COLLIMATOR_TRAIT_PATH_DEPTH=$(EXP_TRAIT_PATH_DEPTH) \
    COLLIMATOR_TIERED_CRIT_BIGRAMS=$(EXP_TIERED_CRIT_BIGRAMS) \
    COLLIMATOR_TIERED_BIGRAM_MIN_CRIT=$(EXP_TIERED_BIGRAM_MIN_CRIT) \
    COLLIMATOR_TIERED_BIGRAM_MAX=$(EXP_TIERED_BIGRAM_MAX) \
    COLLIMATOR_TIERED_BIGRAM_MIN_FREQ=$(EXP_TIERED_BIGRAM_MIN_FREQ) \
    COLLIMATOR_TIERED_CRIT_TRIGRAMS=$(EXP_TIERED_CRIT_TRIGRAMS) \
    COLLIMATOR_TIERED_TRIGRAM_MIN_CRIT=$(EXP_TIERED_TRIGRAM_MIN_CRIT) \
    COLLIMATOR_TIERED_TRIGRAM_MAX=$(EXP_TIERED_TRIGRAM_MAX) \
    COLLIMATOR_TIERED_TRIGRAM_MIN_FREQ=$(EXP_TIERED_TRIGRAM_MIN_FREQ) \
    COLLIMATOR_SYMBOL_VOCAB=$(EXP_SYMBOL_VOCAB) \
    COLLIMATOR_RULE_KEYWORDS=$(EXP_RULE_KEYWORDS) \
    COLLIMATOR_DESCRIPTION_KEYWORDS=$(EXP_DESCRIPTION_KEYWORDS) \
    COLLIMATOR_FULL_RULE_KEYWORDS=$(EXP_FULL_RULE_KEYWORDS) \
    COLLIMATOR_RULE_DESCRIPTION_KEYWORDS=$(EXP_RULE_DESCRIPTION_KEYWORDS) \
    COLLIMATOR_CONTEXT_KEYWORDS=$(EXP_CONTEXT_KEYWORDS) \
    COLLIMATOR_ARCHIVE_MEMBER_CONTEXT=$(EXP_ARCHIVE_MEMBER_CONTEXT) \
    COLLIMATOR_KEYWORD_VOCAB_MAX=$(EXP_KEYWORD_VOCAB_MAX) \
    COLLIMATOR_KEYWORD_MIN_FREQ=$(EXP_KEYWORD_MIN_FREQ) \
    COLLIMATOR_KEYWORD_SELECTION=$(EXP_KEYWORD_SELECTION) \
    COLLIMATOR_KEYWORD_CANDIDATE_MAX=$(EXP_KEYWORD_CANDIDATE_MAX) \
    COLLIMATOR_RULE_KEYWORD_VOCAB_MAX=$(EXP_RULE_KEYWORD_VOCAB_MAX) \
    COLLIMATOR_RULE_KEYWORD_MIN_FREQ=$(EXP_RULE_KEYWORD_MIN_FREQ) \
    COLLIMATOR_RULE_KEYWORD_SELECTION=$(EXP_RULE_KEYWORD_SELECTION) \
    COLLIMATOR_DESCRIPTION_KEYWORD_VOCAB_MAX=$(EXP_DESCRIPTION_KEYWORD_VOCAB_MAX) \
    COLLIMATOR_DESCRIPTION_KEYWORD_MIN_FREQ=$(EXP_DESCRIPTION_KEYWORD_MIN_FREQ) \
    COLLIMATOR_DESCRIPTION_KEYWORD_SELECTION=$(EXP_DESCRIPTION_KEYWORD_SELECTION) \
    COLLIMATOR_FULL_RULE_KEYWORD_VOCAB_MAX=$(EXP_FULL_RULE_KEYWORD_VOCAB_MAX) \
    COLLIMATOR_FULL_RULE_KEYWORD_MIN_FREQ=$(EXP_FULL_RULE_KEYWORD_MIN_FREQ) \
    COLLIMATOR_FULL_RULE_KEYWORD_SELECTION=$(EXP_FULL_RULE_KEYWORD_SELECTION) \
    COLLIMATOR_RULE_DESCRIPTION_KEYWORD_VOCAB_MAX=$(EXP_RULE_DESCRIPTION_KEYWORD_VOCAB_MAX) \
    COLLIMATOR_RULE_DESCRIPTION_KEYWORD_MIN_FREQ=$(EXP_RULE_DESCRIPTION_KEYWORD_MIN_FREQ) \
    COLLIMATOR_RULE_DESCRIPTION_KEYWORD_SELECTION=$(EXP_RULE_DESCRIPTION_KEYWORD_SELECTION) \
    COLLIMATOR_CONTEXT_KEYWORD_VOCAB_MAX=$(EXP_CONTEXT_KEYWORD_VOCAB_MAX) \
    COLLIMATOR_CONTEXT_KEYWORD_MIN_FREQ=$(EXP_CONTEXT_KEYWORD_MIN_FREQ) \
    COLLIMATOR_CONTEXT_KEYWORD_SELECTION=$(EXP_CONTEXT_KEYWORD_SELECTION) \
    COLLIMATOR_SYMBOL_VOCAB_MAX=$(EXP_SYMBOL_VOCAB_MAX) \
    COLLIMATOR_SYMBOL_MIN_FREQ=$(EXP_SYMBOL_MIN_FREQ) \
    COLLIMATOR_KV_VOCAB=$(EXP_KV_VOCAB) \
    COLLIMATOR_KV_VOCAB_MAX=$(EXP_KV_VOCAB_MAX) \
    COLLIMATOR_KV_MIN_FREQ=$(EXP_KV_MIN_FREQ) \
    COLLIMATOR_KV_SHAPE_FEATURES=$(EXP_KV_SHAPE_FEATURES) \
    COLLIMATOR_ATTACK_CODE_NGRAMS=$(EXP_ATTACK_CODE_NGRAMS) \
    COLLIMATOR_PE_FORMAT_FLAGS=$(EXP_PE_FORMAT_FLAGS) \
    COLLIMATOR_PE_TEMPORAL_ANOMALY=$(EXP_PE_TEMPORAL_ANOMALY) \
    COLLIMATOR_TEXT_METRICS_FULL=$(EXP_TEXT_METRICS_FULL) \
    COLLIMATOR_OVERLAY_SIGNAL=$(EXP_OVERLAY_SIGNAL) \
    COLLIMATOR_METRIC_RATIO_FEATURES=$(EXP_METRIC_RATIO_FEATURES) \
    COLLIMATOR_SIZE_NORMALIZED_METRICS=$(EXP_SIZE_NORMALIZED_METRICS) \
    COLLIMATOR_NONSTANDARD_SECTION_SIGNAL=$(EXP_NONSTANDARD_SECTION_SIGNAL) \
    COLLIMATOR_LINE_LENGTH_BUCKETS=$(EXP_LINE_LENGTH_BUCKETS) \
    COLLIMATOR_EXTENDED_METRICS_INCLUDE=$(EXP_EXTENDED_METRICS_INCLUDE) \
    COLLIMATOR_TOP_K_RISK_FILES_MIN_CRIT=$(EXP_TOP_K_RISK_FILES_MIN_CRIT) \
    COLLIMATOR_METRIC_CORRELATION_PAIRS=$(EXP_METRIC_CORRELATION_PAIRS) \
    COLLIMATOR_KV_VALUE_SPLIT=$(EXP_KV_VALUE_SPLIT) \
    COLLIMATOR_SYMBOL_BIGRAMS=$(EXP_SYMBOL_BIGRAMS) \
    COLLIMATOR_SYMBOL_BIGRAM_MAX=$(EXP_SYMBOL_BIGRAM_MAX) \
    COLLIMATOR_SYMBOL_MIN_FREQ_BIGRAM=$(EXP_SYMBOL_MIN_FREQ_BIGRAM) \
    COLLIMATOR_SYMBOL_TRIGRAMS=$(EXP_SYMBOL_TRIGRAMS) \
    COLLIMATOR_SYMBOL_TRIGRAM_MAX=$(EXP_SYMBOL_TRIGRAM_MAX) \
    COLLIMATOR_SYMBOL_MIN_FREQ_TRIGRAM=$(EXP_SYMBOL_MIN_FREQ_TRIGRAM) \
    COLLIMATOR_TRIGRAM_MIN_FREQ=$(EXP_TRIGRAM_MIN_FREQ) \
    COLLIMATOR_TIERED_CRIT_QUADGRAMS=$(EXP_TIERED_CRIT_QUADGRAMS) \
    COLLIMATOR_TIERED_QUADGRAM_MIN_CRIT=$(EXP_TIERED_QUADGRAM_MIN_CRIT) \
    COLLIMATOR_TIERED_QUADGRAM_MAX=$(EXP_TIERED_QUADGRAM_MAX) \
    COLLIMATOR_TIERED_QUADGRAM_MIN_FREQ=$(EXP_TIERED_QUADGRAM_MIN_FREQ) \
    COLLIMATOR_MBC_ID_VOCAB=$(EXP_MBC_ID_VOCAB) \
    COLLIMATOR_TRAIT_CONFIDENCE_MOMENTS=$(EXP_TRAIT_CONFIDENCE_MOMENTS) \
    COLLIMATOR_TRAIT_ID_LEXICAL_DISTANCE=$(EXP_TRAIT_ID_LEXICAL_DISTANCE) \
    COLLIMATOR_DOCUMENT_OBFUSCATION_FEATURES=$(EXP_DOCUMENT_OBFUSCATION_FEATURES) \
    COLLIMATOR_TIERED_BIGRAM_BRANCH_MIN_CRIT=$(EXP_TIERED_BIGRAM_BRANCH_MIN_CRIT) \
    COLLIMATOR_EXPERIMENT_TAG=$(EXP_TAG)

experiment: venv check-db
	@mkdir -p $(EXP_LOG_DIR)
	$(EXPERIMENT_FEATURE_ENV) \
	$(PYTHON) -u -m collimator experiment --db $(DB) --output $(EXP_OUT_DIR) --model-name $(MODEL) --learner $(LEARNER) $(EXP_WORKERS_ARG) --seed $(SEED) \
		--experiment-idea $(EXP_IDEA) --route $(EXP_ROUTE) $(EXP_RERUN_ARG) \
		--train-samples $(EXP_TRAIN_SAMPLES) --max-test-samples $(EXP_MAX_TEST_SAMPLES) \
		--total-limit $(EXP_TOTAL_LIMIT) \
		$(if $(EXP_MAX_ID),--max-id $(EXP_MAX_ID),) \
		$(EXP_REFRESH_CACHE_SNAPSHOT_ARG) \
		--n-folds $(EXP_FOLDS) --holdout-fraction $(EXP_HOLDOUT_FRACTION) \
		--n-estimators $(EXP_ESTIMATORS) --max-depth $(EXP_MAX_DEPTH) \
		--learning-rate $(EXP_LEARNING_RATE) --early-stopping-rounds $(EXP_EARLY_STOPPING) \
		$(if $(EXP_NUM_LEAVES),--num-leaves $(EXP_NUM_LEAVES),) \
		$(if $(EXP_MIN_CHILD_SAMPLES),--min-child-samples $(EXP_MIN_CHILD_SAMPLES),) \
		$(if $(EXP_MIN_CHILD_WEIGHT),--min-child-weight $(EXP_MIN_CHILD_WEIGHT),) \
		--colsample-bytree $(EXP_COLSAMPLE_BYTREE) --subsample $(EXP_SUBSAMPLE) \
		--gamma $(EXP_GAMMA) --reg-alpha $(EXP_REG_ALPHA) --reg-lambda $(EXP_REG_LAMBDA) \
		$(if $(DEVICE),--device $(DEVICE),) \
		$(if $(DROP_FEATURE_PREFIXES),--drop-feature-prefixes $(DROP_FEATURE_PREFIXES),) \
		$(if $(EXP_MONOTONE_JSON),--monotone-json '$(EXP_MONOTONE_JSON)',) \
		--min-malware-score $(EXP_MIN_MALWARE_SCORE) \
		--beta $(EXP_BETA) --threshold-mode $(EXP_THRESHOLD_MODE) \
		$(if $(EXP_THRESHOLD_FPR_TARGET),--threshold-fpr-target $(EXP_THRESHOLD_FPR_TARGET),) \
		--hard-negative-fraction $(EXP_HARD_NEGATIVE_FRACTION) --hard-negative-weight $(EXP_HARD_NEGATIVE_WEIGHT) \
		--scale-pos-weight-mult $(EXP_SCALE_POS_WEIGHT_MULT) \
		--max-delta-step $(EXP_MAX_DELTA_STEP) \
		--boosting-type $(EXP_BOOSTING_TYPE) \
		$(if $(filter 1 true yes,$(EXP_EXTRA_TREES)),--extra-trees,) \
		--seed-search-k $(EXP_SEED_SEARCH_K) \
		$(if $(filter 1 true yes,$(EXP_SAVE_ALL_SEEDS)),--save-all-seeds,) \
		$(if $(filter 1 true yes,$(EXP_TEST_NATURAL_PREVALENCE)),--test-natural-prevalence,) \
		$(foreach w,$(subst $(_comma), ,$(EXP_BENIGN_FILETYPE_WEIGHT)),--benign-filetype-weight $(w)) \
		$(if $(EXP_CACHE_DIR),--cache-dir $(EXP_CACHE_DIR),) \
		2>&1 | tee "$(EXP_LOG_DIR)/$$(date +%Y-%m-%dT%H-%M-%S)-experiment$(EXP_TAG).log"

# Leave-one-group-out ablation. Same env-resolution pattern: replay the
# deployed model's feature env so ablation results reflect behavior at the
# operating point the production pipeline actually ships. Hyperparameters
# are drawn from the same source — no need for a parallel TRAIN_* knobs
# table here.
ablate: venv check-db
	$(PYTHON) scripts/azoth_train_best.py \
		--runs-dir "$(AZOTH_AUTOCOLLIE_RUNS_DIR)" \
		--route general \
		--exec $(PYTHON) -m collimator ablate --db $(DB) $(WORKERS_ARG) --seed $(SEED) \
			--model-name $(MODEL) --learner $(LEARNER) \
			$(if $(DEVICE),--device $(DEVICE),) \
			--n-folds $(or $(ABLATE_FOLDS),2) \
			$(if $(ABLATE_CACHE_DIR),--cache-dir $(ABLATE_CACHE_DIR),) \
			$(if $(ABLATE_MAX_ID),--max-id $(ABLATE_MAX_ID),) \
			$(if $(ABLATE_SAMPLES),--train-samples $(ABLATE_SAMPLES),) \
			$(if $(ABLATE_TEST_SAMPLES),--max-test-samples $(ABLATE_TEST_SAMPLES),) \
			$(if $(ABLATE_GROUPS),--groups $(ABLATE_GROUPS),) \
			$(if $(ABLATE_OUTPUT),--output $(ABLATE_OUTPUT),) \
			2>&1 | tee "$(LOG_DIR)/$$(date +%Y-%m-%dT%H-%M-%S)-ablation$(EXP_TAG).log"

ablation: ablate

demo-db: venv
	$(PYTHON) -m collimator demo-db --output $(DEMO_DB) --seed $(SEED)

scan: venv
ifndef FILE
	$(error FILE is required. Usage: make scan FILE=/path/to/binary)
endif
	$(PYTHON) -m collimator scan $(FILE) --model $(OUT_DIR)/$(MODEL_FILE) --spec $(OUT_DIR)/feature_spec.json --cleave $(CLEAVE) $(if $(DB),--db $(DB),)

test: venv test-scan-parity
	$(VENV_DIR)/bin/pip install pytest
	$(PYTHON) -m pytest tests/ -v

# Cross-language calculation gate: discover features, train/export tiny Azoth
# models, and compare every input column and probability with Scan. No DB.
# PARITY_BUNDLE optionally checks an existing leaf bundle as well; its saved
# feature environment is required so Python reproduces its training options.
SCAN_DIR ?= ../scan
.PHONY: test-scan-parity
test-scan-parity: venv
	$(PYTHON) scripts/check_scan_parity.py --scan-dir "$(SCAN_DIR)" $(if $(PARITY_BUNDLE),--bundle "$(PARITY_BUNDLE)" --feature-env "$(PARITY_CONFIG)",)

lint: venv
	$(VENV_DIR)/bin/pip install ruff mypy
	$(VENV_DIR)/bin/ruff check src/ tests/
	$(VENV_DIR)/bin/mypy src/collimator/

XDG_DATA_HOME ?= $(HOME)/.local/share
LIGHTGBM_ARS_DIR ?= ../lightgbm-ars

LITMUS_DIR ?= ../scan

# Azoth (LightGBM → ONNX ensemble) is the only deploy path. The legacy
# single-model deploy and its xgboost-ars cross-language parity gate were
# retired along with the xgboost-ars crate.
deploy: azoth-deploy

.PHONY: verify-litmus
verify-litmus:
	@test -d $(LITMUS_DIR) || { echo "error: $(LITMUS_DIR) does not exist"; exit 1; }
	@test -f $(OUT_DIR)/extraction_fixture.json || { echo "error: $(OUT_DIR)/extraction_fixture.json not found — run make azoth-fast-train first"; exit 1; }
	@test ! -f $(OUT_DIR)/threshold_tuning.json || $(PYTHON) scripts/build_litmus_config.py --threshold-tuning $(OUT_DIR)/threshold_tuning.json --output $(OUT_DIR)/config.json
	@echo "Running litmus feature-extraction parity tests..."
	@mkdir -p $(LITMUS_DIR)/tests/fixtures
	cp $(OUT_DIR)/extraction_fixture.json $(LITMUS_DIR)/tests/fixtures/extraction_fixture.json
	cd $(LITMUS_DIR) && SCAN_MODELS_DIR=$(abspath $(OUT_DIR)) cargo test --release --test extraction_parity
	@echo "litmus: extraction parity tests passed"

AUTOCOLLIE_DIR ?= ../autocollie
AUTOCOLLIE_BIN := $(AUTOCOLLIE_DIR)/bin/autocollie
# Shared proposal endpoint; command-line/environment overrides remain supported.
export AUTOCOLLIE_OPENAI_BASE_URL ?= http://elrond:8000/v1
EXPERIMENTS ?= 12
AUTOCOLLIE_LLM_TIMEOUT ?= 15m
AUTOCOLLIE_SCREEN_TIMEOUT ?= 90m
# Parallel route processing for autocollie auto/loop modes. Two routes
# run cycles concurrently by default; promotes serialize internally.
# EXP_WORKERS and EXP_LGBM_THREADS are auto-derived from MAX_CPU_THREADS /
# ROUTE_CONCURRENCY in autocollie so concurrent screens don't fight for
# the host's full CPU. Set ROUTE_CONCURRENCY=1 to force sequential.
# Route cycles include LLM proposals and DB extraction as well as fitting.
# Calibration's CPU-derived parallelism can flood the shared LLM with prompts.
# Keep the default bounded independently; operators may explicitly raise it.
ROUTE_CONCURRENCY ?= 2
MAX_CPU_THREADS ?= $(NPROC)
# Pass extraction limits only when explicitly supplied. The default WORKERS
# expands to the whole host; autocollie divides its own CPU budget per job.
AUTOCOLLIE_WORKER_ARG = $(if $(filter command line environment override,$(origin EXP_WORKERS)),EXP_WORKERS=$(EXP_WORKERS),$(if $(filter command line environment override,$(origin WORKERS)),EXP_WORKERS=$(WORKERS),))
# Autocollie defaults to the local Hopper replica. Using the bare `hopper`
# hostname can resolve through public DNS and burn screen slots on timeouts.
AUTOCOLLIE_DB ?= postgres://hopper@localhost:5432/hopper
# ROUTES is comma-separated, e.g. ROUTES=filetypes/javascript,filegroups/scripts
# ROUTE (singular) is accepted as a convenience.
ROUTES ?= $(ROUTE)

.PHONY: autocollie autocollie-loop autocollie-build autocollie-dryrun autocollie-screen autocollie-full-train autocollie-confirm autocollie-promote autocollie-replay autocollie-replay-favorites azoth-augment-small-routes

# azoth-augment-small-routes — post-hoc threshold re-search for routes whose
# default-level policy is `no_policy` because their own benign pool is too
# small to estimate the tail rate reliably at the saved default level. Loads each specialist, scores
# its filegroup peers' benigns, and re-runs the threshold search against the
# augmented pool.  Optional --use-credible-bound applies Beta-Binomial
# smoothing for routes with very small pools.
#
# Costs real inference time (default ~100k benigns × N routes); not in the
# default azoth-deploy chain.  Run after a fresh azoth-validate to recover
# deployable thresholds on small-corpus routes.
#
# Usage:
#   make azoth-augment-small-routes
#   make azoth-augment-small-routes ONLY_ROUTE=filetypes/macho
#   make azoth-augment-small-routes USE_CREDIBLE_BOUND=1
ONLY_ROUTE ?=
USE_CREDIBLE_BOUND ?=
MAX_POOL_BENIGNS ?= 100000
azoth-augment-small-routes: venv check-db
	$(PYTHON) scripts/azoth_augment_small_route_policies.py \
		--azoth-root $(AZOTH_ROOT) \
		--db $(DB) \
		--max-pool-benigns $(MAX_POOL_BENIGNS) \
		--workers $(or $(WORKERS),$(NPROC)) \
		$(if $(filter 1 true yes,$(USE_CREDIBLE_BOUND)),--use-credible-bound,) \
		$(if $(ONLY_ROUTE),--only-routes $(ONLY_ROUTE),)
	$(PYTHON) scripts/azoth_policy_global_metrics.py \
		--config $(AZOTH_CONFIG) \
		--policy $(AZOTH_ROUTE_POLICIES) \
		--score-table $(AZOTH_SCORE_TABLE) \
		--output $(AZOTH_GLOBAL_POLICY_METRICS) \
		--markdown $(AZOTH_GLOBAL_POLICY_METRICS_MD) \
		$(AZOTH_BUDGET_GATE_ARGS)
	$(PYTHON) scripts/compute_routed_metrics.py --azoth-root $(AZOTH_ROOT) --db $(DB) $(AZOTH_PREVIOUS_BUNDLE_ARG) $(AZOTH_ROUTED_METRICS_ARGS)
	$(PYTHON) scripts/write_azoth_readmes.py --azoth-root $(AZOTH_ROOT)

# Autocollie targets all default DB to AUTOCOLLIE_DB (local replica). User can
# still override with `make autocollie DB=...`; command-line vars beat
# target-specific assignments in GNU make.
autocollie autocollie-screen autocollie-full-train autocollie-confirm autocollie-promote autocollie-loop autocollie-dryrun: DB = $(AUTOCOLLIE_DB)

autocollie-build:
	@test -d $(AUTOCOLLIE_DIR) || { echo "error: $(AUTOCOLLIE_DIR) does not exist"; exit 1; }
	@mkdir -p $(AUTOCOLLIE_DIR)/bin
	go -C $(AUTOCOLLIE_DIR) build -o bin/autocollie ./cmd/autocollie

autocollie-dryrun: autocollie-build
	@test -n "$(ROUTES)" || { echo "error: set ROUTES=route1,route2 (or ROUTE=route)"; exit 1; }
	$(AUTOCOLLIE_BIN) dryrun \
		--collimator $(CURDIR) \
		--autocollie $(abspath $(AUTOCOLLIE_DIR)) \
		--routes $(ROUTES) \
		$(if $(TRY),--try-knobs $(TRY),) \
		--experiments $(EXPERIMENTS)

autocollie-screen: ROUTE_CONCURRENCY = 2
autocollie-screen: venv check-db autocollie-build
	@test -n "$(ROUTES)" || { echo "error: set ROUTES=route1,route2 (or ROUTE=route)"; exit 1; }
	$(AUTOCOLLIE_BIN) screen \
		--collimator $(CURDIR) \
		--autocollie $(abspath $(AUTOCOLLIE_DIR)) \
		--routes $(ROUTES) \
		$(if $(TRY),--try-knobs $(TRY),) \
		--experiments $(EXPERIMENTS) \
		--llm-timeout $(AUTOCOLLIE_LLM_TIMEOUT) \
		--screen-timeout $(AUTOCOLLIE_SCREEN_TIMEOUT) \
		--min-route-malware $(AZOTH_SPECIALIST_MIN_BAD) \
		--min-route-benign $(AZOTH_SPECIALIST_MIN_GOOD) \
		--route-concurrency $(ROUTE_CONCURRENCY) \
		--max-cpu-threads $(MAX_CPU_THREADS) \
		--make-args "DB=$(DB) EXP_LGBM_THREADS=$(EXP_LGBM_THREADS) EXP_ESTIMATORS=$(or $(EXP_ESTIMATORS_DEFAULT),250)$(if $(DEVICE), DEVICE=$(DEVICE),)"

# Run two new proposals plus the inherited candidate per route, then consume
# the saved per-route results in one full bundle build. Make stops on a failed
# screen command, just like `autocollie screen ... && make azoth-full-train`.
# Share the memory budget during concurrent screens; the subsequent full train
# gets the normal single-job memory budget.
# Usage: make autocollie-full-train ROUTES=general,filegroups/archive MAX_CPU_THREADS=64
autocollie-full-train: EXPERIMENTS = 2
autocollie-full-train: ROUTE_CONCURRENCY = 2
autocollie-full-train:
	COLLIMATOR_MEM_SHARES=$(ROUTE_CONCURRENCY) $(MAKE) autocollie-screen \
		EXPERIMENTS=$(EXPERIMENTS) ROUTE_CONCURRENCY=$(ROUTE_CONCURRENCY)
	$(MAKE) azoth-full-train AZOTH_AUTOCOLLIE_RUNS_DIR="$(AZOTH_AUTOCOLLIE_RUNS_DIR)"

# Confirm a screening winner by re-running with a different seed.
# Usage: make autocollie-confirm KEY=<16-hex experiment_key> [SEED=43]
CONFIRM_SEED ?= 43
# Phase-1 favorites report: scan autocollie's screen-run JSONs for
# candidates that look promising under CURRENT gate logic but were never
# promoted (rejected by older gates, broke at confirm/promote, etc.).
# Read-only — no retraining. Use this to spot candidates worth re-running
# via `make autocollie-replay KEY=...` (fresh screening before current gates).
#
# Usage:
#   make autocollie-favorites-report                                   # top 10 per route
#   make autocollie-favorites-report TOP=5
#   make autocollie-favorites-report ROUTES=filetypes/pdf
#   make autocollie-favorites-report ROUTES=filetypes/pe,filegroups/source
#   make autocollie-favorites-report REQUIRE_APPLES=1                   # apples-to-apples only
# ROUTE (singular) is accepted as a legacy alias for ROUTES.
autocollie-favorites-report: venv
	$(PYTHON) scripts/autocollie_favorites.py \
		--runs-dir $(AZOTH_AUTOCOLLIE_RUNS_DIR) \
		--baselines $(AZOTH_ROOT)/.autocollie/promoted_baselines.json \
		--top $(or $(TOP),10) \
		$(if $(or $(ROUTES),$(ROUTE)),--route $(or $(ROUTES),$(ROUTE)),) \
		$(if $(REQUIRE_APPLES),--require-apples,)

# Phase-2 auto-replay. Picks top-N candidates per route (globally
# ranked by replay_score: apples-to-apples first, then biggest d_r3,
# then largest n_ben_holdout), filters out anything that recently
# failed (cooldown window), then refreshes screening and runs confirm + promote
# once per key, under one PostgreSQL snapshot per candidate.
# Each outcome is recorded in out/autocollie/replay_history.json so the
# favorites report can show "last attempted YYYY-MM-DD: <reason>" and
# the cooldown skips recently-failed candidates next time around.
#
# Usage:
#   make autocollie-replay-favorites                                   # top 1 per route, all routes
#   make autocollie-replay-favorites TOP=3
#   make autocollie-replay-favorites ROUTES=filegroups/source
#   make autocollie-replay-favorites ROUTES=filetypes/pe,filetypes/macho
#   make autocollie-replay-favorites COOLDOWN=0                        # ignore retry history
# ROUTE (singular) is accepted as a legacy alias for ROUTES. The plural
# matches the syntax `make autocollie-loop ROUTES=...` already uses.
autocollie-replay-favorites: venv autocollie-build
	@set -e; \
	plan=$$(mktemp); \
	trap 'rm -f "$$plan"' EXIT; \
	$(PYTHON) scripts/autocollie_favorites.py \
		--runs-dir $(AZOTH_AUTOCOLLIE_RUNS_DIR) \
		--baselines $(AZOTH_ROOT)/.autocollie/promoted_baselines.json \
		--top $(or $(TOP),1) \
		--replay-cooldown $(or $(COOLDOWN),2.0) \
		$(if $(or $(ROUTES),$(ROUTE)),--route $(or $(ROUTES),$(ROUTE)),) \
		--replay-plan > "$$plan"; \
	count=$$(wc -l < "$$plan" | tr -d ' '); \
	echo "▶ replay-favorites: $$count candidate(s) queued"; \
	if [ "$$count" -eq 0 ]; then \
	  echo "no candidates eligible (try TOP=N, ROUTE=, or COOLDOWN=0)"; \
	  exit 0; \
	fi; \
	mkdir -p out/autocollie/replays; \
	while IFS="	" read key route idea; do \
	  echo ""; \
	  echo "▶▶▶ replay candidate: $$route  idea=$$idea  key=$$key"; \
	  echo ""; \
	  replay_log=out/autocollie/replays/$$(date -u +%Y%m%dT%H%M%S)-$$key.log; \
	  if ! $(MAKE) --no-print-directory autocollie-replay KEY=$$key 2>&1 | tee "$$replay_log"; then \
	    echo "  replay failed; later stages were not attempted after the error"; \
	  fi; \
	  $(PYTHON) scripts/autocollie_record_replay.py \
	    --auto-detect \
	    --key $$key --route "$$route" \
	    --confirm-log "$$replay_log" --promote-log "$$replay_log" \
	    || echo "warn: record_replay failed for $$key"; \
	done < "$$plan"; \
	echo ""; \
	echo "🏁 replay-favorites complete"

autocollie-confirm: venv check-db autocollie-build
	@test -n "$(KEY)" || { echo "error: set KEY=<16-hex experiment_key>"; exit 1; }
	$(AUTOCOLLIE_BIN) confirm \
		--collimator $(CURDIR) \
		--autocollie $(abspath $(AUTOCOLLIE_DIR)) \
		--key $(KEY) \
		--seed $(CONFIRM_SEED) \
		--route-concurrency $(ROUTE_CONCURRENCY) \
		--max-cpu-threads $(MAX_CPU_THREADS) \
		--make-args "DB=$(DB) $(AUTOCOLLIE_WORKER_ARG) EXP_ESTIMATORS=$(or $(EXP_ESTIMATORS_DEFAULT),250)$(if $(DEVICE), DEVICE=$(DEVICE),)"

# Promote a winner: confirm (different seed) -> full-train (inflated profile)
# -> compare. On pass, writes a report telling the user to run `make azoth-deploy`.
# Never deploys itself. Usage: make autocollie-promote KEY=<16-hex experiment_key>
autocollie-promote: venv check-db autocollie-build
	@test -n "$(KEY)" || { echo "error: set KEY=<16-hex experiment_key>"; exit 1; }
	$(AUTOCOLLIE_BIN) promote \
		--collimator $(CURDIR) \
		--autocollie $(abspath $(AUTOCOLLIE_DIR)) \
		--key $(KEY) \
		--seed $(CONFIRM_SEED) \
		--screen-timeout $(AUTOCOLLIE_SCREEN_TIMEOUT) \
		--promote-timeout 180m \
		--route-concurrency $(ROUTE_CONCURRENCY) \
		--max-cpu-threads $(MAX_CPU_THREADS) \
		--make-args "DB=$(DB) $(AUTOCOLLIE_WORKER_ARG) EXP_ESTIMATORS=$(or $(EXP_ESTIMATORS_DEFAULT),250)$(if $(DEVICE), DEVICE=$(DEVICE),)"

# Recover historical knobs, then measure them on current data before gates.
# REPLAY_SCREEN_ONLY=1 stops after the fresh screen for a bounded live check.
# Resolve the baseline for replay and sweep: candidate cloning uses cp -a,
# which would copy AZOTH_ROOT's symlink and let candidate writes alter the run.
autocollie-replay: venv check-db autocollie-build
	@test -n "$(KEY)" || { echo "error: set KEY=<16-hex historical experiment_key>"; exit 1; }
	$(AUTOCOLLIE_BIN) replay \
		--collimator $(CURDIR) \
		--autocollie $(abspath $(AUTOCOLLIE_DIR)) \
		--baseline-azoth-root $(or $(realpath $(AZOTH_ROOT)),$(abspath $(AZOTH_ROOT))) \
		--key $(KEY) \
		$(if $(filter 1 true yes,$(REPLAY_SCREEN_ONLY)),--screen-only,) \
		--seed $(CONFIRM_SEED) \
		--screen-timeout $(AUTOCOLLIE_SCREEN_TIMEOUT) \
		--promote-timeout 180m \
		--route-concurrency $(ROUTE_CONCURRENCY) \
		--max-cpu-threads $(MAX_CPU_THREADS) \
		--make-args "DB=$(DB) $(AUTOCOLLIE_WORKER_ARG) EXP_ESTIMATORS=$(or $(EXP_ESTIMATORS_DEFAULT),250)$(if $(DEVICE), DEVICE=$(DEVICE),)"

# The full hands-off ladder: screen N specs per route -> if any winner beats
# the route's currently trained Azoth specialist in $(AZOTH_ROOT), automatically promote it (confirm + full-train
# + holdout comparison). Writes per-route summaries and a deploy-or-not
# report on each promotion. Never deploys itself.
# Usage: make autocollie ROUTES=filetypes/python,filetypes/rust [EXPERIMENTS=10] [PASSES=1]
#        make autocollie ROUTES=filetypes/                       (overnight)
#        make autocollie ROUTES=filetypes/python PASSES=0        (loop until Ctrl-C)
PASSES ?= 1
AUTO_ROUTES ?=
SHUFFLE_ROUTES ?=
autocollie: venv check-db autocollie-build
	@test -n "$(ROUTES)$(AUTO_ROUTES)$(TOP)$(SHUFFLE_ROUTES)" || { echo "error: set ROUTES=route1,route2 (or AUTO_ROUTES=N / TOP=N for top-N weakest, or SHUFFLE_ROUTES=1 for every known route). For a directed knob sweep: TRY=<knob> TOP=N (e.g. TRY=severity_fractions TOP=3)"; exit 1; }
	$(AUTOCOLLIE_BIN) auto \
		--collimator $(CURDIR) \
		--autocollie $(abspath $(AUTOCOLLIE_DIR)) \
		--baseline-azoth-root $(or $(realpath $(AZOTH_ROOT)),$(abspath $(AZOTH_ROOT))) \
		$(if $(ROUTES),--routes $(ROUTES),) \
		$(if $(AUTO_ROUTES),--auto-routes $(AUTO_ROUTES),) \
		$(if $(TOP),--auto-routes $(TOP),) \
		$(if $(TRY),--try-knobs $(TRY),) \
		$(if $(filter 1 true yes,$(SHUFFLE_ROUTES)),--shuffle-routes,) \
		--experiments $(EXPERIMENTS) \
		--passes $(PASSES) \
		--seed $(CONFIRM_SEED) \
		--llm-timeout $(AUTOCOLLIE_LLM_TIMEOUT) \
		--screen-timeout $(AUTOCOLLIE_SCREEN_TIMEOUT) \
		--promote-timeout 180m \
		--min-route-malware $(AZOTH_SPECIALIST_MIN_BAD) \
		--min-route-benign $(AZOTH_SPECIALIST_MIN_GOOD) \
		--route-concurrency $(ROUTE_CONCURRENCY) \
		--max-cpu-threads $(MAX_CPU_THREADS) \
		--make-args "DB=$(DB) $(AUTOCOLLIE_WORKER_ARG) EXP_ESTIMATORS=$(or $(EXP_ESTIMATORS_DEFAULT),250)$(if $(DEVICE), DEVICE=$(DEVICE),)"

# autocollie-loop is the same target with PASSES=0 — loops the screen+promote
# ladder over the route list until Ctrl-C.
#
# Three ways to use it:
#   make autocollie-loop ROUTES=filetypes/python EXPERIMENTS=10
#     — fixed route list, loops indefinitely (same order each pass)
#   make autocollie-loop AUTO_ROUTES=3 EXPERIMENTS=5
#     — picks the 3 weakest routes each pass by recall@L50 headroom
#       (self-balancing — concentrates on routes with most to gain)
#   make autocollie-loop SHUFFLE_ROUTES=1 EXPERIMENTS=10
#     — walks every known route (general + all filegroups + all filetypes)
#       in random order, re-shuffling each pass. Try 10 things per route, then
#       move on. Best for "throw a bunch of stuff at the wall overnight."
autocollie-loop: PASSES=0
autocollie-loop: autocollie

.PHONY: nightly nightly-sweep install-nightly uninstall-nightly nightly-logs nightly-sweep-logs nightly-status

# The unattended pipeline is two preemptible jobs (scripts/nightly.sh). A
# publish-train runs 8-34h, so a single daily unit kept overrunning its own next
# firing and flock'ing whole nights away; splitting lets the sweep use the hours
# the train doesn't. They never overlap — the train stops the sweep unit on the
# way in and starts it again on the way out (a full train peaks near 38G; an
# autocollie experiment alongside it is what drove the 2026-07-06/07/08 OOMs).
#
# nightly: the train — repin -> azoth-publish-train (trains + regression-gates +
#   OOF-deploys into ../azoth) -> commit & push the azoth bundle -> hand the box
#   back to the sweep. Safe to run by hand; an flock guard prevents it from
#   colliding with the scheduled run. Output is tee'd to out/nightly/ and (when
#   run under the timer) captured in journald.
nightly:
	scripts/nightly.sh train

# nightly-sweep: the sweep — an indefinite shuffled autocollie walk over every
#   known route (general + all filegroups + all filetypes), NIGHTLY_EXPERIMENTS
#   (default 1) experiment each, looping until the next train preempts it
#   (NIGHTLY_PASSES=1 for a single pass). Normally started by the train, not by
#   hand — run `systemctl --user start azoth-autocollie` if you want it under
#   the unit (and so preemptible) rather than tied to your shell.
nightly-sweep:
	scripts/nightly.sh autocollie

# install-nightly: install the systemd --user units — a daily 21:00 timer for the
# train plus the timer-less sweep unit it drives. Override the schedule with
# NIGHTLY_ONCALENDAR (systemd OnCalendar syntax), e.g.
#   make install-nightly NIGHTLY_ONCALENDAR='*-*-* 03:00:00'
install-nightly:
	NIGHTLY_ONCALENDAR="$(NIGHTLY_ONCALENDAR)" scripts/install_nightly.sh

uninstall-nightly:
	scripts/install_nightly.sh --remove

# nightly-logs / nightly-sweep-logs: follow the running / most recent train or
# sweep. The files are the durable source (journald rotates too fast on this
# box — see nightly.sh).
nightly-logs:
	tail -n 100 -F out/nightly/latest.log

nightly-sweep-logs:
	tail -n 100 -F out/nightly/latest-sweep.log

# nightly-status: show the timer schedule and both jobs' current state.
nightly-status:
	@systemctl --user list-timers azoth-nightly.timer --no-pager 2>/dev/null || true
	@echo
	@systemctl --user status azoth-nightly.service --no-pager 2>/dev/null || true
	@echo
	@systemctl --user status azoth-autocollie.service --no-pager 2>/dev/null || true

clean:
	rm -rf $(OUT_DIR)

help:
	@echo "Collimator Training Pipeline"
	@echo ""
	@echo "Usage: make <target> [DB=postgres://...]"
	@echo ""
	@echo "Targets:"
	@echo "  train              Train production model on full dataset"
	@echo "  experiment         Run fast subsampled experiment"
	@echo "  evaluate           Run evaluation on external test set"
	@echo "  explain            Generate SHAP importance analysis"
	@echo "  inspect            Show feature breakdown for a sample (SAMPLE=sha256)"
	@echo "  errors             Show top false positives/negatives"
	@echo "  traits             Dump all unique traits seen in DB"
	@echo "  thresholds         Tune severity thresholds on the full corpus"
	@echo "  thresholds-refresh Rebuild cached threshold scores, then tune thresholds"
	@echo "  azoth-diagnostics  Report routed ensemble marginal value by model route"
	@echo "  azoth-policies     Search best calibrated route policy per filetype"
	@echo "  elf-route-optimization Test ELF route OR/replacement/acquittal experiments"
	@echo "  false-positives    Show false positives grouped by severity level"
	@echo "  false-negatives    Show false negatives grouped by severity level"
	@echo "  near-false-positives Show benign rows newly flagged by twice-looser level 9"
	@echo "  near-false-negatives Show bad rows newly caught by twice-looser level 9"
	@echo "  false-positives-archive Package top false positives into a tgz"
	@echo "  false-negatives-archive Package top false negatives into a tgz"
	@echo "  near-false-positives-archive Package top near false positives into a tgz"
	@echo "  near-false-negatives-archive Package top near false negatives into a tgz"
	@echo "  benchmark          Measure feature extraction & inference latency"
	@echo "  build-splits       Pre-compute data splits in DB"
	@echo "  autocollie-build   Build the autocollie Go orchestrator (../autocollie)"
	@echo "  autocollie-dryrun  Generate+validate experiment specs via LLM (no run)"
	@echo "  autocollie-screen  Generate, validate, and run experiment specs via LLM"
	@echo "  autocollie-full-train  Screen each route, then build the full bundle using the run history"
	@echo "  autocollie-confirm Re-run KEY=<16hex> with a different seed (SEED=43 default)"
	@echo "  autocollie-promote Confirm + full-train + compare; writes a deploy-or-not report"
	@echo "  autocollie         Full hands-off ladder: screen + auto-promote per route"
	@echo "  autocollie-loop    Same as autocollie with PASSES=0 (loop until Ctrl-C)"
	@echo "  azoth-backtest     Out-of-time backtest of a route operating point (ROUTE=pe)"
	@echo "  azoth-validate     Run azoth-deploy gates against AZOTH_ROOT without copying (AZOTH_SKIP_LITMUS_VALIDATE=1 skips litmus)"
	@echo "  azoth-prune-feature-cache  Drop route-feature caches older than AZOTH_KEEP_CACHE_DAYS (default 2)"
	@echo "  demo-db            Create a small SQLite DB for testing"
	@echo "  test               Run unit tests"
	@echo "  deploy             Alias for azoth-deploy (deploy the routed ensemble)"
	@echo ""
	@echo "Options:"
	@echo "  DB=url             Hopper database DSN"
	@echo "  MODEL=name         Model artifact namespace (default: litmus-xg; e.g. azoth)"
	@echo "  LEARNER=name       Learner implementation (default: inferred from MODEL)"
	@echo "  DROP_FEATURE_PREFIXES=a,b  Drop feature prefixes for train/experiment candidates"
	@echo "  OUT_DIR=path       Output directory (default: out/models/<target-model>)"
	@echo "  DEVICE=name        Training device override (e.g. cuda for azoth)"
	@echo "  WORKERS=n          Parallel extraction workers (default: auto = physical cores)"
	@echo "  EXP_TRAIN_SAMPLES=n   Experiment train rows (default: 150000)"
	@echo "  EXP_MAX_TEST_SAMPLES=n  Cap test set via reservoir (0=full, default: 40000)"
	@echo "  EXP_FOLDS=n           Experiment CV folds (default: 0)"
	@echo "  EXP_ESTIMATORS=n      Experiment max trees (default: 180)"
	@echo "  EXP_IDEA=name        Human-readable experiment idea label"
	@echo "  EXP_ROUTE=route      Experiment route (default: general)"
	@echo "  EXP_RERUN=1          Force rerun of an existing canonical experiment"
	@echo "  ROUTES=a,b           Comma-separated routes for autocollie (or ROUTE=)"
	@echo "                       Trailing slash expands a prefix from prior runs:"
	@echo "                       ROUTES=filetypes/   -> all filetypes/* ever run"
	@echo "                       ROUTES=filegroups/  -> all filegroups/*"
	@echo "  EXPERIMENTS=n        Specs requested per autocollie cycle (default: 12)"
	@echo "  PASSES=n             Times to loop the autocollie route list (0=until Ctrl-C)"
	@echo "  EXP_ESTIMATORS_DEFAULT=n  Floor for screen/confirm/promote estimators (default: 250)"
	@echo "  AZOTH_SPECIALIST_TRAIN_OVERRIDE='route:field=value ...'"
	@echo "                       Per-route specialist TrainConfig overrides"