DmesgPlugin update

nodescraper/plugins/inband/dmesg/dmesg_analyzer.py

@@ -38,8 +38,8 @@
 from .mce_utils import (
     compile_mce_ce_status_regex,
     compile_mce_uc_status_regex,
-    ignored_mce_block_line_indices,
-    mce_block_all_line_indices,
+    mce_known_regex_skip_line_indices,
+    mce_unknown_suppress_line_indices,
     parse_correctable_mce_counts,
     parse_uncorrectable_mce_counts,
     trim_mce_status_match_content,
@@ -718,8 +718,8 @@ def analyze_data(
             dmesg_content = data.dmesg_content
 
         ignore_match_rules, ignore_mce_banks = parse_ignore_match_rules(args.ignore_match_rules)
-        ignored_mce_block_lines = ignored_mce_block_line_indices(dmesg_content, ignore_mce_banks)
-        mce_block_lines = mce_block_all_line_indices(dmesg_content)
+        known_skip_lines = mce_known_regex_skip_line_indices(dmesg_content, ignore_mce_banks)
+        unknown_skip_lines = mce_unknown_suppress_line_indices(dmesg_content)
 
         known_err_events = self.check_all_regexes(
             content=dmesg_content,
@@ -728,7 +728,7 @@ def analyze_data(
             num_timestamps=args.num_timestamps,
             interval_to_collapse_event=args.interval_to_collapse_event,
             ignore_match_rules=ignore_match_rules,
-            skip_line_indices=ignored_mce_block_lines,
+            skip_line_indices=known_skip_lines,
         )
         for event in known_err_events:
             if event.description in ("MCE Corrected Error", "MCE Uncorrected Error"):
@@ -764,7 +764,7 @@ def analyze_data(
                 num_timestamps=args.num_timestamps,
                 interval_to_collapse_event=args.interval_to_collapse_event,
                 ignore_match_rules=ignore_match_rules,
-                skip_line_indices=mce_block_lines,
+                skip_line_indices=unknown_skip_lines,
             )
 
             for err_event in err_events:

nodescraper/plugins/inband/dmesg/mce_utils.py

@@ -35,29 +35,6 @@
 _MCE_STATUS_START_RE = re.compile(r"\bMC\d+_STATUS\[", re.IGNORECASE)
 _MCE_DETAIL_LINE_RE = re.compile(r"\[Hardware Error\]:")
 
-_CORRECTABLE_SUMMARY_RE = re.compile(
-    r"(?P<count>\d+)\s+correctable hardware errors detected in total in (?P<block>\w+) block"
-    r"(?:\s+on\s+(?P<cpu>CPU:?\d+))?",
-    re.IGNORECASE,
-)
-
-_UNCORRECTABLE_SUMMARY_RE = re.compile(
-    r"(?P<count>\d+)\s+uncorrectable hardware errors detected in (?P<block>\w+) block",
-    re.IGNORECASE,
-)
-
-_GPU_CORRECTABLE_RE = re.compile(
-    r"amdgpu\s+(?P<bdf>[\w:.]+):.*?(?P<count>\d+)\s+correctable hardware errors detected in total in "
-    r"(?P<block>\w+) block",
-    re.IGNORECASE,
-)
-
-_GPU_UNCORRECTABLE_RE = re.compile(
-    r"amdgpu\s+(?P<bdf>[\w:.]+):.*?(?P<count>\d+)\s+uncorrectable hardware errors detected in "
-    r"(?P<block>\w+) block",
-    re.IGNORECASE,
-)
-
 _MCE_CE_STATUS_RE = re.compile(
     r"\[Hardware Error\]:.*?(?P<cpu>CPU:?\d+).*?MC\d+_STATUS\[[^\]]*\|CE\|[^\]]*\]",
     re.IGNORECASE,
@@ -78,6 +55,21 @@
     re.IGNORECASE,
 )
 
+# MCE incident rows only
+_MCE_INCIDENT_LINE_RE = re.compile(
+    r"\[Hardware Error\]:\s*(?:"
+    r"(?:Corrected error|Uncorrected error|Machine check events logged)|"
+    r"Machine Check:|"
+    r"CPU:?\d|"
+    r"MC\d+_STATUS|"
+    r"PPIN:|"
+    r"IPID:|"
+    r"Syndrome:|"
+    r"cache level:"
+    r")",
+    re.IGNORECASE,
+)
+
 
 def compile_mce_ce_status_regex() -> re.Pattern[str]:
     """Return a single-line regex for corrected MCn_STATUS hardware error rows."""
@@ -116,33 +108,6 @@ def _add_count(counts: dict[str, int], part: str, amount: int) -> None:
     counts[part] = counts.get(part, 0) + amount
 
 
-def _part_label(
-    *,
-    cpu: Optional[str] = None,
-    block: Optional[str] = None,
-    bdf: Optional[str] = None,
-    gpu_index: Optional[int] = None,
-) -> str:
-    if bdf is not None:
-        block_suffix = f"/{block}" if block else ""
-        if gpu_index is not None:
-            return f"GPU{gpu_index}{block_suffix}"
-        return f"GPU {bdf}{block_suffix}"
-    if cpu and block:
-        return f"{cpu}/{block}"
-    if cpu:
-        return cpu
-    if block:
-        return block
-    return "unknown"
-
-
-def _gpu_index_for_bdf(bdf: str, bdf_order: list[str]) -> int:
-    if bdf not in bdf_order:
-        bdf_order.append(bdf)
-    return bdf_order.index(bdf)
-
-
 def _is_mce_primary_starter(line: str) -> bool:
     return _MCE_PRIMARY_START_RE.search(line) is not None
 
@@ -160,6 +125,10 @@ def _is_mce_detail_line(line: str) -> bool:
     return _MCE_DETAIL_LINE_RE.search(line) is not None
 
 
+def _is_mce_incident_line(line: str) -> bool:
+    return _MCE_INCIDENT_LINE_RE.search(line) is not None
+
+
 def _mce_detail_line_indices_in_range(lines: Sequence[str], start: int, end: int) -> set[int]:
     return {index for index in range(start, end) if _is_mce_detail_line(lines[index])}
 
@@ -177,8 +146,8 @@ def _is_mce_status_only_starter(line: str) -> bool:
     return not _is_mce_primary_starter(line) and _MCE_STATUS_START_RE.search(line) is not None
 
 
-def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool:
-    """Return True when another [Hardware Error]: line appears before the next incident."""
+def _has_mce_incident_line_ahead(lines: Sequence[str], start_index: int) -> bool:
+    """Return True when another MCE incident [Hardware Error]: row appears before the next block."""
     for idx in range(start_index + 1, len(lines)):
         line = lines[idx]
         if not line.strip():
@@ -187,20 +156,83 @@ def _has_mce_detail_line_ahead(lines: Sequence[str], start_index: int) -> bool:
             return False
         if _is_mce_status_only_starter(line):
             return False
-        if _is_mce_detail_line(line):
+        if _is_mce_incident_line(line):
             return True
+        if _is_mce_detail_line(line):
+            return False
     return False
 
 
+def mce_defining_status_line_indices(content: str) -> frozenset[int]:
+    """Return line indices for MCn_STATUS rows that define a corrected or uncorrected MCE."""
+    lines = content.splitlines()
+    indices: set[int] = set()
+    for index, line in enumerate(lines):
+        if _MCE_CE_STATUS_LINE_RE.search(line) or _MCE_UC_STATUS_LINE_RE.search(line):
+            indices.add(index)
+    return frozenset(indices)
+
+
+def mce_hardware_error_line_indices(content: str) -> frozenset[int]:
+    """Return every line index containing [Hardware Error]:."""
+    lines = content.splitlines()
+    return frozenset(index for index, line in enumerate(lines) if _is_mce_detail_line(line))
+
+
+def mce_non_status_hardware_error_line_indices(content: str) -> frozenset[int]:
+    """Return [Hardware Error]: detail lines that are not defining MCn_STATUS CE/UC rows."""
+    defining = mce_defining_status_line_indices(content)
+    return frozenset(
+        index for index in mce_hardware_error_line_indices(content) if index not in defining
+    )
+
+
+def _primary_starters_in_mce_status_blocks(
+    lines: Sequence[str], defining: frozenset[int]
+) -> frozenset[int]:
+    """Return primary MCE header lines that belong to blocks with MCn_STATUS CE/UC rows."""
+    primary_starters = {index for index, line in enumerate(lines) if _is_mce_primary_starter(line)}
+    suppressed: set[int] = set()
+    for start, end in iter_hardware_error_block_ranges(lines):
+        if any(index in defining for index in range(start, end)):
+            suppressed.update(index for index in range(start, end) if index in primary_starters)
+    return frozenset(suppressed)
+
+
+def mce_known_regex_skip_line_indices(
+    content: str,
+    ignore_banks: Optional[FrozenSet[int]] = None,
+) -> frozenset[int]:
+    """Skip non-defining MCE detail lines and ignored-bank incidents during known regex scan."""
+    ignored = ignore_banks or frozenset()
+    lines = content.splitlines()
+    defining = mce_defining_status_line_indices(content)
+    primary_starters = frozenset(
+        index for index, line in enumerate(lines) if _is_mce_primary_starter(line)
+    )
+    primary_in_status_blocks = _primary_starters_in_mce_status_blocks(lines, defining)
+    skipped = set(hardware_error_block_line_indices(content)) - defining - primary_starters
+    skipped.update(primary_in_status_blocks)
+    skipped.update(ignored_mce_block_line_indices(content, ignored))
+    return frozenset(skipped)
+
+
+def mce_unknown_suppress_line_indices(content: str) -> frozenset[int]:
+    """Suppress MCE block context and every [Hardware Error]: line from unknown scan."""
+    suppressed = set(mce_block_all_line_indices(content))
+    suppressed.update(mce_hardware_error_line_indices(content))
+    return frozenset(suppressed)
+
+
 def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, int]]:
     """Return (start, end) line index ranges for MCE incident blocks.
 
     A block begins at a primary MCE header (Corrected/Uncorrected/Machine check logged)
     or at the first MCn_STATUS line when not already inside a block. The block then
     includes subsequent lines until the next primary header, a blank line before a bare
-    MCn_STATUS starter, trailing non-MCE lines with no further [Hardware Error]: detail,
-    or EOF. Blank lines, warn/err noise, and other non-MCE dmesg lines between detail
-    entries belong to the same block.
+    MCn_STATUS starter, a non-MCE [Hardware Error]: row, trailing lines with no further
+    MCE incident detail, or EOF. Blank lines, warn/err noise, and other non-MCE dmesg
+    lines between MCE incident entries belong to the same block.
     """
     blocks: list[tuple[int, int]] = []
     index = 0
@@ -216,9 +248,11 @@ def iter_hardware_error_block_ranges(lines: Sequence[str]) -> list[tuple[int, in
                 next_line = _next_non_blank_line_index(lines, index)
                 if next_line is not None and _is_mce_status_only_starter(lines[next_line]):
                     break
+            elif _is_mce_detail_line(lines[index]) and not _is_mce_incident_line(lines[index]):
+                break
             elif _is_mce_block_starter(lines[index], in_block=True):
                 break
-            elif not _is_mce_detail_line(lines[index]) and not _has_mce_detail_line_ahead(
+            elif not _is_mce_incident_line(lines[index]) and not _has_mce_incident_line_ahead(
                 lines, index
             ):
                 break
@@ -264,39 +298,14 @@ def parse_correctable_mce_counts(
     content: str,
     ignore_banks: Optional[FrozenSet[int]] = None,
 ) -> dict[str, int]:
-    """Count correctable MCE / RAS hardware errors per component from dmesg text.
-
-    Handles summary lines (for example ``mce: 3 correctable ... on CPU1``),
-    amdgpu block summaries, and per-event ``MCn_STATUS[|CE|]`` hardware error lines.
-    """
+    """Count correctable MCE hardware errors per CPU from MCn_STATUS[|CE|] rows."""
     counts: dict[str, int] = {}
-    gpu_bdf_order: list[str] = []
     ignored = ignore_banks or frozenset()
     ignored_block_lines = ignored_mce_block_line_indices(content, ignored)
 
     for line_no, line in enumerate(content.splitlines()):
         if line_no in ignored_block_lines:
             continue
-        gpu_match = _GPU_CORRECTABLE_RE.search(line)
-        if gpu_match:
-            bdf = gpu_match.group("bdf")
-            part = _part_label(
-                bdf=bdf,
-                block=gpu_match.group("block"),
-                gpu_index=_gpu_index_for_bdf(bdf, gpu_bdf_order),
-            )
-            _add_count(counts, part, int(gpu_match.group("count")))
-            continue
-
-        summary_match = _CORRECTABLE_SUMMARY_RE.search(line)
-        if summary_match:
-            cpu = summary_match.group("cpu")
-            part = _part_label(
-                cpu=_normalize_cpu_label(cpu) if cpu else None,
-                block=summary_match.group("block"),
-            )
-            _add_count(counts, part, int(summary_match.group("count")))
-            continue
 
         status_match = _MCE_CE_STATUS_RE.search(line)
         if status_match:
@@ -314,31 +323,14 @@ def parse_uncorrectable_mce_counts(
     content: str,
     ignore_banks: Optional[FrozenSet[int]] = None,
 ) -> dict[str, int]:
-    """Count uncorrectable MCE / RAS hardware errors per component from dmesg text."""
+    """Count uncorrectable MCE hardware errors per CPU from MCn_STATUS[|UC|] rows."""
     counts: dict[str, int] = {}
-    gpu_bdf_order: list[str] = []
     ignored = ignore_banks or frozenset()
     ignored_block_lines = ignored_mce_block_line_indices(content, ignored)
 
     for line_no, line in enumerate(content.splitlines()):
         if line_no in ignored_block_lines:
             continue
-        gpu_match = _GPU_UNCORRECTABLE_RE.search(line)
-        if gpu_match:
-            bdf = gpu_match.group("bdf")
-            part = _part_label(
-                bdf=bdf,
-                block=gpu_match.group("block"),
-                gpu_index=_gpu_index_for_bdf(bdf, gpu_bdf_order),
-            )
-            _add_count(counts, part, int(gpu_match.group("count")))
-            continue
-
-        summary_match = _UNCORRECTABLE_SUMMARY_RE.search(line)
-        if summary_match:
-            part = _part_label(block=summary_match.group("block"))
-            _add_count(counts, part, int(summary_match.group("count")))
-            continue
 
         status_match = _MCE_UC_STATUS_RE.search(line)
         if status_match: