Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions pyrit/memory/attack_analytics_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -869,7 +869,9 @@ def _converter_source(self, *, direction: AttackAnalyticsConverterDirection, emb
A retained converter list takes precedence: migration backfills can have
fewer edges than recorded members when a child has no usable hash.
Edges remain a fallback for identifiers without a retained converter list.
A recorded [] means no converters, unlike absent identifier metadata.
A recorded [] means no converters, unlike absent identifier metadata. A recorded
attack identifier with no list or edges also means no converters, because
identifiers omit an empty converter list.

Args:
direction (AttackAnalyticsConverterDirection): Request or response pipeline;
Expand Down Expand Up @@ -930,6 +932,11 @@ def _converter_source(self, *, direction: AttackAnalyticsConverterDirection, emb
self._from = self._from.outerjoin(arrays, arrays.c.attack_identifier_hash == self.attack.c.hash)
path = f"children.{direction.value}_converters"
retained = JsonContainer(self.attack.c.identifier_json, f"$.{path}")
# Identifiers leave out an empty converter list, so a recorded attack with no
# converter list or edges ran without converters, which is not missing metadata.
recorded_without_converters = case(
(self.attack.c.hash.isnot(None), literal("[]", UnicodeText())), else_=null()
)
self._converter_arrays[direction] = (
func.coalesce(
retained,
Expand All @@ -939,8 +946,9 @@ def _converter_source(self, *, direction: AttackAnalyticsConverterDirection, emb
),
JsonContainer(self.root.c.atomic_attack_identifier, f"$.children.attack.{path}"),
arrays.c.names,
recorded_without_converters,
),
func.coalesce(retained, arrays.c.names),
func.coalesce(retained, arrays.c.names, recorded_without_converters),
)
value = self._converter_arrays[direction][0 if embedded else 1]
return _Source(value=value, label=value, array=True, converters=True, insensitive=True)
Expand Down
42 changes: 42 additions & 0 deletions tests/unit/memory/test_attack_analytics.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
from sqlalchemy.sql.functions import Function
from sqlalchemy.sql.selectable import CTE

from pyrit.converter import Base64Converter
from pyrit.exceptions.analytics_exception import AnalyticsDataException, AnalyticsTimeoutException
from pyrit.executor.attack import AttackConverterConfig, PromptSendingAttack
from pyrit.memory import MemoryInterface, SQLiteMemory
from pyrit.memory.analytics_sql import (
JsonArrayEmpty,
Expand Down Expand Up @@ -53,6 +55,8 @@
ConverterIdentifier,
TargetIdentifier,
)
from pyrit.prompt_normalizer import ConverterConfiguration
from unit.mocks import MockPromptTarget

if TYPE_CHECKING:
from pathlib import Path
Expand Down Expand Up @@ -400,6 +404,44 @@ async def test_known_empty_converters_differ_from_missing_identifier(sqlite_inst
assert {group.option.key.kind.value for group in report.groups} == {"missing", "no_converters"}


@pytest.mark.parametrize("direction", ["request", "response"])
async def test_real_attacks_without_converters_are_no_converters_async(
*, direction: str, sqlite_instance: SQLiteMemory
) -> None:
"""Identifiers omit an empty converter list, so a real attack must still land in no_converters."""
plain = await PromptSendingAttack(objective_target=MockPromptTarget()).execute_async(objective="plain")
converted = await PromptSendingAttack(
objective_target=MockPromptTarget(),
attack_converter_config=AttackConverterConfig(
request_converters=ConverterConfiguration.from_converters(converters=[Base64Converter()]),
response_converters=ConverterConfiguration.from_converters(converters=[Base64Converter()]),
),
).execute_async(objective="converted")
unidentified = make_result(index=3)
unidentified.atomic_attack_identifier = None
await sqlite_instance.add_attack_results_to_memory_async(attack_results=[unidentified])
dimension = AttackAnalyticsDimension(name="converter_type", converter_direction=direction)
reader = AttackAnalyticsReader(memory=sqlite_instance)

report = await reader.report_async(query=AttackAnalyticsQuery(group_by=dimension), control=control())
filtered = await reader.report_async(
query=AttackAnalyticsQuery(
filters=AttackAnalyticsFilters.model_validate(
{"dimensions": [{"dimension": dimension.model_dump(), "values": [{"kind": "no_converters"}]}]}
)
),
control=control(),
)
without_converters = await sqlite_instance.get_attack_results_async(has_converters=False)

assert {
(group.option.key.kind.value, group.option.key.value): sum(group.counts.values()) for group in report.groups
} == {("no_converters", None): 1, ("value", "base64converter"): 1, ("missing", None): 1}
assert sum(filtered.counts.values()) == 1
assert plain.attack_result_id in {result.attack_result_id for result in without_converters}
assert converted.attack_result_id not in {result.attack_result_id for result in without_converters}


async def test_literal_label_paths_and_facet_search(sqlite_instance: SQLiteMemory) -> None:
await sqlite_instance.add_attack_results_to_memory_async(
attack_results=[
Expand Down
Loading