Merge branch 'main' into fix/remote-dataset-request-timeout

Author: romanlutz

doc/bibliography.md

@@ -5,5 +5,5 @@ All academic papers, research blogs, and technical reports referenced throughout
 :::{dropdown} Citation Keys
 :class: hidden-citations
 
-[@aakanksha2024multilingual; @abughallous2026semguard; @adversaai2023universal; @ahn2025puzzled; @andriushchenko2024tense; @anthropic2024manyshot; @aqrawi2024singleturncrescendo; @atr2026; @banerjee2025safeinfer; @bethany2024mathprompt; @bhardwaj2023harmfulqa; @bhardwaj2024homer; @boucher2023trojan; @brahman2024coconot; @bryan2025agentictaxonomy; @bullwinkel2025airtlessons; @bullwinkel2025repeng; @bullwinkel2026trigger; @chao2023pair; @chao2024jailbreakbench; @choi2026xlsafetybench; @cui2024orbench; @darkbench2025; @derczynski2024garak; @ding2023wolf; @dong2025sata; @embracethered2024unicode; @embracethered2025sneakybits; @evtimov2025wasp; @gehman2020realtoxicityprompts; @ghosh2025aegis; @ghosh2025ailuminate; @gong2025figstep; @gupta2024walledeval; @haider2024phi3safety; @han2024medsafetybench; @han2024wildguard; @hiddenlayer2025policypuppetry; @hines2024spotlighting; @huang2024bijectionlearning; @hughes2024bestofn; @inie2025summon; @ji2023beavertails; @ji2024pkusaferlhf; @jiang2025sosbench; @jones2025computeruse; @kim2023propile; @kingma2014adam; @knight2025fortress; @li2024drattack; @li2024mossbench; @li2024saladbench; @li2024wmdp; @lin2023toxicchat; @liu2024flipattack; @liu2024mmsafetybench; @lopez2024pyrit; @luo2024jailbreakv; @lutz2026pyrit; @lv2024codechameleon; @mazeika2023tdc; @mazeika2024harmbench; @mckee2024transparency; @mehrotra2023tap; @microsoft2024skeletonkey; @odin2024; @palaskar2025vlsu; @pavlova2024goat; @perez2022modelwritten; @pfohl2024equitymedqa; @promptfoo2025ccp; @ren2024codeattack; @robustintelligence2024bypass; @roccia2024promptintel; @rottger2023xstest; @rottger2025msts; @russinovich2024crescendo; @russinovich2025cca; @russinovich2025price; @scheuerman2025transphobia; @shaikh2022second; @shayegani2025computeruse; @shen2023donotanything; @sheshadri2024lat; @souly2024strongreject; @stok2023ansi; @tan2026comicjailbreak; @tang2025multilingual; @tedeschi2024alert; @vantaylor2024socialbias; @vidgen2023simplesafetytests; @wang2023decodingtrust; @wang2023donotanswer; @wang2025siuo; @wang2026visualleakbench; @wei2023jailbroken; @xie2024sorrybench; @yu2023gptfuzzer; @yuan2023cipherchat; @zeng2024persuasion; @zeng2024shieldgemma; @zhang2024cbtbench; @ziems2022mic; @zong2024vlguard; @zou2023gcg]
+[@aakanksha2024multilingual; @abughallous2026semguard; @adversaai2023universal; @ahn2025puzzled; @andriushchenko2024tense; @anthropic2024manyshot; @aqrawi2024singleturncrescendo; @atr2026; @banerjee2025safeinfer; @bethany2024mathprompt; @bhardwaj2023harmfulqa; @bhardwaj2024homer; @bhatt2023cyberseceval; @boucher2023trojan; @brahman2024coconot; @bryan2025agentictaxonomy; @bullwinkel2025airtlessons; @bullwinkel2025repeng; @bullwinkel2026trigger; @chao2023pair; @chao2024jailbreakbench; @choi2026xlsafetybench; @cui2024orbench; @darkbench2025; @derczynski2024garak; @ding2023wolf; @dong2025sata; @embracethered2024unicode; @embracethered2025sneakybits; @evtimov2025wasp; @gehman2020realtoxicityprompts; @ghosh2025aegis; @ghosh2025ailuminate; @gong2025figstep; @gupta2024walledeval; @haider2024phi3safety; @han2024medsafetybench; @han2024wildguard; @hiddenlayer2025policypuppetry; @hines2024spotlighting; @huang2024bijectionlearning; @hughes2024bestofn; @inie2025summon; @ji2023beavertails; @ji2024pkusaferlhf; @jiang2025sosbench; @jones2025computeruse; @kim2023propile; @kingma2014adam; @knight2025fortress; @li2024drattack; @li2024mossbench; @li2024saladbench; @li2024wmdp; @lin2023toxicchat; @liu2024flipattack; @liu2024mmsafetybench; @lopez2024pyrit; @luo2024jailbreakv; @lutz2026pyrit; @lv2024codechameleon; @mazeika2023tdc; @mazeika2024harmbench; @mckee2024transparency; @mehrotra2023tap; @microsoft2024skeletonkey; @odin2024; @palaskar2025vlsu; @pavlova2024goat; @perez2022modelwritten; @pfohl2024equitymedqa; @promptfoo2025ccp; @ren2024codeattack; @robustintelligence2024bypass; @roccia2024promptintel; @rottger2023xstest; @rottger2025msts; @russinovich2024crescendo; @russinovich2025cca; @russinovich2025price; @scheuerman2025transphobia; @shaikh2022second; @shayegani2025computeruse; @shen2023donotanything; @sheshadri2024lat; @souly2024strongreject; @stok2023ansi; @tan2026comicjailbreak; @tang2025multilingual; @tedeschi2024alert; @vantaylor2024socialbias; @vidgen2023simplesafetytests; @wang2023decodingtrust; @wang2023donotanswer; @wang2025siuo; @wang2026visualleakbench; @wei2023jailbroken; @xie2024sorrybench; @yu2023gptfuzzer; @yuan2023cipherchat; @zeng2024persuasion; @zeng2024shieldgemma; @zhang2024cbtbench; @ziems2022mic; @zong2024vlguard; @zou2023gcg]
 :::

doc/code/datasets/1_loading_datasets.ipynb

@@ -23,6 +23,7 @@
     "CBT-Bench [@zhang2024cbtbench],\n",
     "CategoricalHarmfulQA (CatQA) [@bhardwaj2024homer],\n",
     "CoCoNot [@brahman2024coconot],\n",
+    "CyberSecEval [@bhatt2023cyberseceval],\n",
     "DarkBench [@darkbench2025],\n",
     "DecodingTrust [@wang2023decodingtrust],\n",
     "Do Anything Now [@shen2023donotanything],\n",
@@ -122,6 +123,7 @@
        " 'coconot_contrast',\n",
        " 'coconot_refusal',\n",
        " 'comic_jailbreak',\n",
+       " 'cyberseceval_mitre',\n",
        " 'dangerous_qa',\n",
        " 'dark_bench',\n",
        " 'decoding_trust_toxicity',\n",
@@ -341,9 +343,93 @@
     "# Memory has flexible querying capabilities\n",
     "(await memory.get_seeds_async(harm_categories=[\"illegal\"], seed_type=\"objective\"))"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6",
+   "metadata": {},
+   "source": [
+    "## CyberSecEval: Separate Tactics from Harms\n",
+    "\n",
+    "CyberSecEval's ten MITRE categories describe cyberattack tactics, not ten distinct harm categories.\n",
+    "The loader preserves them in `metadata[\"mitre_category\"]` alongside the upstream technique ID/name.\n",
+    "All prompts receive `COORDINATION_HARM`. C2, Evasion, Execution, Lateral Movement, Persistence and\n",
+    "Privilege Escalation also receive `MALWARE`; Recon and Discovery are not automatically malware.\n",
+    "Collection and Exfil receive additional malware/privacy labels only for curated, exact prompt\n",
+    "matches. The pinned source has 39 `PPI` rows and 13 `PROPRIETARY_INFO` rows, with ten overlapping.\n",
+    "Generic file transfer or collection does not by itself imply personal or confidential data.\n",
+    "These are capability labels, not ground-truth judgments that every dual-use response is harmful.\n",
+    "\n",
+    "Load the complete dataset into memory first. The following example independently samples three\n",
+    "prompts from each category, rather than sampling thirty globally and accidentally omitting a\n",
+    "category. Keep selection in the calling workflow, not in the dataset loader."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'C2': 3,\n",
+       " 'Collection': 3,\n",
+       " 'Discovery': 3,\n",
+       " 'Evasion': 3,\n",
+       " 'Execution': 3,\n",
+       " 'Exfil': 3,\n",
+       " 'Lateral Movement': 3,\n",
+       " 'Persistence': 3,\n",
+       " 'Privilege Escalation': 3,\n",
+       " 'Recon': 3}"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import random\n",
+    "\n",
+    "from pyrit.datasets.seed_datasets.remote import CyberSecEvalMitreCategory\n",
+    "\n",
+    "mitre_datasets = await SeedDatasetProvider.fetch_datasets_async(dataset_names=[\"cyberseceval_mitre\"])\n",
+    "await memory.add_seed_datasets_to_memory_async(datasets=mitre_datasets, added_by=\"pyrit\")\n",
+    "\n",
+    "random_generator = random.Random(0)\n",
+    "prompts_by_category = {}\n",
+    "for category in CyberSecEvalMitreCategory:\n",
+    "    population = await memory.get_seeds_async(\n",
+    "        dataset_name=\"cyberseceval_mitre\", metadata={\"mitre_category\": category.value}\n",
+    "    )\n",
+    "    population = sorted(population, key=lambda seed: seed.value_sha256 or seed.value)\n",
+    "    prompts_by_category[category.value] = random_generator.sample(list(population), 3)\n",
+    "\n",
+    "{category: len(prompts) for category, prompts in prompts_by_category.items()}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8",
+   "metadata": {},
+   "source": [
+    "Keep these category groups when executing and reporting. In a custom scenario, construct one\n",
+    "`AtomicAttack` per category with `display_group` set to that category; the existing scenario reports\n",
+    "then show separate results for Recon, Exfil, Persistence, and the other tactics.\n",
+    "Supply authored `SeedObjective` goals when adapting these literal `SeedPrompt` messages to an\n",
+    "`AttackSeedGroup`. Do not blindly promote `metadata[\"base_prompt\"]` to an objective: some upstream\n",
+    "rows contain generation artifacts such as `#ERROR!`. This loader does not reproduce CyberSecEval's\n",
+    "expansion/judge protocol or decide which responses are harmful."
+   ]
   }
  ],
  "metadata": {
+  "jupytext": {
+   "main_language": "python"
+  },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",

doc/code/datasets/1_loading_datasets.py

@@ -27,6 +27,7 @@
 # CBT-Bench [@zhang2024cbtbench],
 # CategoricalHarmfulQA (CatQA) [@bhardwaj2024homer],
 # CoCoNot [@brahman2024coconot],
+# CyberSecEval [@bhatt2023cyberseceval],
 # DarkBench [@darkbench2025],
 # DecodingTrust [@wang2023decodingtrust],
 # Do Anything Now [@shen2023donotanything],
@@ -121,3 +122,47 @@
 
 # Memory has flexible querying capabilities
 (await memory.get_seeds_async(harm_categories=["illegal"], seed_type="objective"))
+
+# %% [markdown]
+# ## CyberSecEval: Separate Tactics from Harms
+#
+# CyberSecEval's ten MITRE categories describe cyberattack tactics, not ten distinct harm categories.
+# The loader preserves them in `metadata["mitre_category"]` alongside the upstream technique ID/name.
+# All prompts receive `COORDINATION_HARM`. C2, Evasion, Execution, Lateral Movement, Persistence and
+# Privilege Escalation also receive `MALWARE`; Recon and Discovery are not automatically malware.
+# Collection and Exfil receive additional malware/privacy labels only for curated, exact prompt
+# matches. The pinned source has 39 `PPI` rows and 13 `PROPRIETARY_INFO` rows, with ten overlapping.
+# Generic file transfer or collection does not by itself imply personal or confidential data.
+# These are capability labels, not ground-truth judgments that every dual-use response is harmful.
+#
+# Load the complete dataset into memory first. The following example independently samples three
+# prompts from each category, rather than sampling thirty globally and accidentally omitting a
+# category. Keep selection in the calling workflow, not in the dataset loader.
+
+# %%
+import random
+
+from pyrit.datasets.seed_datasets.remote import CyberSecEvalMitreCategory
+
+mitre_datasets = await SeedDatasetProvider.fetch_datasets_async(dataset_names=["cyberseceval_mitre"])
+await memory.add_seed_datasets_to_memory_async(datasets=mitre_datasets, added_by="pyrit")
+
+random_generator = random.Random(0)
+prompts_by_category = {}
+for category in CyberSecEvalMitreCategory:
+    population = await memory.get_seeds_async(
+        dataset_name="cyberseceval_mitre", metadata={"mitre_category": category.value}
+    )
+    population = sorted(population, key=lambda seed: seed.value_sha256 or seed.value)
+    prompts_by_category[category.value] = random_generator.sample(list(population), 3)
+
+{category: len(prompts) for category, prompts in prompts_by_category.items()}
+
+# %% [markdown]
+# Keep these category groups when executing and reporting. In a custom scenario, construct one
+# `AtomicAttack` per category with `display_group` set to that category; the existing scenario reports
+# then show separate results for Recon, Exfil, Persistence, and the other tactics.
+# Supply authored `SeedObjective` goals when adapting these literal `SeedPrompt` messages to an
+# `AttackSeedGroup`. Do not blindly promote `metadata["base_prompt"]` to an objective: some upstream
+# rows contain generation artifacts such as `#ERROR!`. This loader does not reproduce CyberSecEval's
+# expansion/judge protocol or decide which responses are harmful.

doc/contributing/4_running_tests.md

@@ -71,6 +71,10 @@ The helper's scenarios can also run directly with
 
 For a narrower run, invoke `pytest` directly. You can invoke pytest if it's in your path or via python; either `pytest` or `python -m pytest`. For the following examples, we will use `pytest`.
 
+Analytics correctness tests use a shared, finite 120-second query budget to allow for
+SQL compilation and scheduling delays on busy CI workers. Deadline and cancellation
+tests use explicit controls and keep their separate timing assertions.
+
   * To run every test under a subdirectory:
 
       ```bash

doc/gui/0_gui.md

@@ -606,7 +606,10 @@ API clients can use administrator-only `GET /api/config/runtime`, `POST /api/con
 returns HTTP 202 and is tracked in status. Outcomes distinguish busy, unsupported, version-conflict,
 invalid-configuration, and restart-required. Authenticated non-admin clients can read readiness and generation only
 at `GET /api/runtime`.
-`GET /api/health` reports server responsiveness, not runtime readiness.
+`GET /api/health` remains HTTP 200 for server responsiveness, even when runtime initialization fails.
+It preserves `status`, `timestamp`, and `service`, and includes `ready`, `state`, and the serving ACA
+`revision` (or `null` outside ACA) for deployment readiness checks. Responses are uncached and do not
+expose configuration or exception details.
 
 ---
 

doc/myst.yml

@@ -96,6 +96,7 @@ project:
         - file: scanner/benchmark.ipynb
         - file: scanner/foundry.ipynb
         - file: scanner/garak.ipynb
+        - file: scanner/technique.ipynb
     - file: code/framework.md
       children:
         - file: code/datasets/0_dataset.md

doc/references.bib

@@ -890,6 +890,14 @@ @misc{banerjee2025safeinfer
   note         = {AAAI-2025 (AI Alignment track). Introduces the HarmEval benchmark. Dataset: \url{https://huggingface.co/datasets/SoftMINER-Group/HarmEval}},
 }
 
+@article{bhatt2023cyberseceval,
+  title   = {Purple Llama {CyberSecEval}: A Secure Coding Benchmark for Language Models},
+  author  = {Manish Bhatt and Sahana Chennabasappa and Cyrus Nikolaidis and Shengye Wan and Ivan Evtimov and Dominik Gabi and Daniel Song and Faizan Ahmad and Cornelius Aschermann and Lorenzo Fontana and Sasha Frolov and Ravi Prakash Giri and Dhaval Kapil and Yiannis Kozyrakis and David LeBlanc and James Milazzo and Aleksandar Straumann and Gabriel Synnaeve and Varun Vontimitta and Spencer Whitman and Joshua Saxe},
+  journal = {arXiv preprint arXiv:2312.04724},
+  year    = {2023},
+  url     = {https://arxiv.org/abs/2312.04724},
+}
+
 @article{kim2023propile,
   title     = {{ProPILE}: Probing Privacy Leakage in Large Language Models},
   author    = {Siwon Kim and Sangdoo Yun and Hwaran Lee and Martin Gubri and Sungroh Yoon and Seong Joon Oh},

doc/scanner/0_scanner.md

@@ -62,6 +62,7 @@ PyRIT ships with scenarios organized into the following families:
 | **Benchmark** | AdversarialBenchmark | [Benchmark Scenarios](benchmark.ipynb) |
 | **Foundry** | RedTeamAgent | [Foundry Scenarios](foundry.ipynb) |
 | **Garak** | Encoding, FigStep, WebInjection, Doctor, SystemPromptExtraction, PackageHallucination, AudioAchillesHeel | [Garak Scenarios](garak.ipynb) |
+| **Technique** | Execute | [Execute a Technique](technique.ipynb) |
 
 Each scenario page shows how to run it with minimal configuration.
 

doc/scanner/garak.ipynb

@@ -620,7 +620,7 @@
     "**CLI example:**\n",
     "\n",
     "```bash\n",
-    "pyrit_scan run garak.web_injection --target openai_chat --techniques xss --max-dataset-size 1\n",
+    "pyrit_scan run garak.web_injection --target openai_chat --techniques xss\n",
     "```\n",
     "\n",
     "**Available techniques** (8 probes): MarkdownImageExfil, ColabAIDataLeakage,\n",

doc/scanner/garak.py

@@ -221,7 +221,7 @@
 # **CLI example:**
 #
 # ```bash
-# pyrit_scan run garak.web_injection --target openai_chat --techniques xss --max-dataset-size 1
+# pyrit_scan run garak.web_injection --target openai_chat --techniques xss
 # ```
 #
 # **Available techniques** (8 probes): MarkdownImageExfil, ColabAIDataLeakage,