diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index a8ba6d6660..e53f9c2e5a 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -12,6 +12,7 @@ use tracedecay_code_extraction::incremental::ParseLimits; use tracedecay_code_index::{ capabilities::expected_seal_digest, chunks::{CodeIndexImportEvidenceV1, ExtractionAdmittedCodeSearchChunkV1, content_digest}, + clones::{CloneBodyEligibilityV1, CloneBodyOccurrenceV1}, graph_projection::{ CODE_GRAPH_PROJECTOR_REVISION, CodeGraphProjectionError, build_published_code_graph_manifest_checked, code_graph_projection_identity, @@ -3087,33 +3088,19 @@ fn cross_file_resolution_is_width_invariant() { parallel_equivalence::assert_cross_file_resolution_is_width_invariant(); } -fn partitioned_codec_request(beta_value: u64, sealed_at: i64) -> CodeIndexBuildRequestV1 { - let sources = [ - ( - "file.partitioned.alpha", - "src/alpha.rs", - "pub struct Alpha;\nimpl Alpha { pub fn call(&self) -> u64 { crate::beta::beta() } }\n", - ), - ( - "file.partitioned.beta", - "src/beta.rs", - if beta_value == 1 { - "pub fn beta() -> u64 { 1 }\n" - } else { - "pub fn beta() -> u64 { 2 }\n" - }, - ), - ( - "file.partitioned.unresolved", - "src/unresolved.rs", - "pub fn unresolved() { missing_external(); }\n", - ), - ]; +/// A build request over `(file_occurrence, logical_path, source)` Rust files +/// for the partitioned-codec fixtures; `changed_files` names the paths the +/// increment re-extracts. +fn partitioned_request( + sources: &[(&str, &str, &str)], + changed_files: BTreeSet, + sealed_at: i64, +) -> CodeIndexBuildRequestV1 { let mut identity = Sha256::new(); let mut files = Vec::new(); let mut captured_files = Vec::new(); let mut receipts = Vec::new(); - for (index, (occurrence, logical_path, source)) in sources.into_iter().enumerate() { + for (index, &(occurrence, logical_path, source)) in sources.iter().enumerate() { identity.update(logical_path.as_bytes()); identity.update([0]); identity.update(source.as_bytes()); @@ -3147,11 +3134,7 @@ fn partitioned_codec_request(beta_value: u64, sealed_at: i64) -> CodeIndexBuildR files, }, captured_files, - changed_files: if beta_value == 1 { - BTreeSet::new() - } else { - BTreeSet::from(["src/beta.rs".to_owned()]) - }, + changed_files, invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { @@ -3163,6 +3146,68 @@ fn partitioned_codec_request(beta_value: u64, sealed_at: i64) -> CodeIndexBuildR } } +fn partitioned_codec_request(beta_value: u64, sealed_at: i64) -> CodeIndexBuildRequestV1 { + let sources = [ + ( + "file.partitioned.alpha", + "src/alpha.rs", + "pub struct Alpha;\nimpl Alpha { pub fn call(&self) -> u64 { crate::beta::beta() } }\n", + ), + ( + "file.partitioned.beta", + "src/beta.rs", + if beta_value == 1 { + "pub fn beta() -> u64 { 1 }\n" + } else { + "pub fn beta() -> u64 { 2 }\n" + }, + ), + ( + "file.partitioned.unresolved", + "src/unresolved.rs", + "pub fn unresolved() { missing_external(); }\n", + ), + ]; + let changed_files = if beta_value == 1 { + BTreeSet::new() + } else { + BTreeSet::from(["src/beta.rs".to_owned()]) + }; + partitioned_request(&sources, changed_files, sealed_at) +} + +/// A segment publisher that stores every published segment under its digest +/// (evidence pages assembled into one pack under the commit digest) and counts +/// the file segments the encoder actually published, as opposed to reused. +fn collect_published_segments<'a>( + segments: &'a mut BTreeMap>, + published_files: &'a mut usize, +) -> impl FnMut(SealedGenerationSegmentPublicationV1<'_>) -> Result<(), CodeIndexProductionErrorV1> + 'a +{ + let mut evidence_pack = Vec::new(); + move |publication| { + match publication { + SealedGenerationSegmentPublicationV1::File { digest, bytes } => { + *published_files += 1; + segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + } + SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { + evidence_pack.extend_from_slice(bytes); + } + SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { + segment_digest, + .. + } => { + segments.insert( + segment_digest.as_str().to_owned(), + std::mem::take(&mut evidence_pack), + ); + } + } + Ok(()) + } +} + fn partitioned_codec_fixture() -> ( CodeIndexPublishedGenerationV1, Vec, @@ -3175,56 +3220,22 @@ fn partitioned_codec_fixture() -> ( .build_and_publish(partitioned_codec_request(1, 1_100_000), &ActiveControl) .expect("partitioned parent generation"); let mut segments = BTreeMap::new(); - let mut parent_evidence_pack = Vec::new(); + let mut parent_file_segments = 0; let parent_manifest = first - .encode_partitioned_sealed(|publication| { - match publication { - SealedGenerationSegmentPublicationV1::File { digest, bytes } => { - segments.insert(digest.as_str().to_owned(), bytes.to_vec()); - } - SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { - parent_evidence_pack.extend_from_slice(bytes); - } - SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { - segment_digest, - .. - } => { - segments.insert( - segment_digest.as_str().to_owned(), - std::mem::take(&mut parent_evidence_pack), - ); - } - } - Ok(()) - }) + .encode_partitioned_sealed(collect_published_segments( + &mut segments, + &mut parent_file_segments, + )) .expect("partitioned parent encoding"); let second = owner .build_and_publish(partitioned_codec_request(2, 1_200_000), &ActiveControl) .expect("partitioned child generation"); let mut child_file_segments = 0; - let mut child_evidence_pack = Vec::new(); let manifest = second - .encode_partitioned_sealed_with_parent(Some(&parent_manifest), |publication| { - match publication { - SealedGenerationSegmentPublicationV1::File { digest, bytes } => { - child_file_segments += 1; - segments.insert(digest.as_str().to_owned(), bytes.to_vec()); - } - SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { - child_evidence_pack.extend_from_slice(bytes); - } - SealedGenerationSegmentPublicationV1::GenerationEvidenceCommit { - segment_digest, - .. - } => { - segments.insert( - segment_digest.as_str().to_owned(), - std::mem::take(&mut child_evidence_pack), - ); - } - } - Ok(()) - }) + .encode_partitioned_sealed_with_parent( + Some(&parent_manifest), + collect_published_segments(&mut segments, &mut child_file_segments), + ) .expect("partitioned child encoding"); assert!( child_file_segments < second.snapshot().files.len(), @@ -4230,6 +4241,238 @@ fn partitioned_encode_publishes_only_the_edited_file_segment() { assert_reused_segment_descriptors_stable(&parent_manifest, &child_manifest); } +/// The carried-forward file of the increment wedge: several clone bodies whose +/// sorted-by-id order differs from source order, with `walk_one` above +/// `MIN_AUTOMATIC_CLONE_BODY_TOKENS_V1` (eligible) and the rest below it +/// (excluded as too small), so both eligibility classes ride the reused segment. +const INCREMENT_WEDGE_CARRIED_SOURCE: &str = concat!( + "pub fn walk_one(items: &[u32]) -> u32 {\n", + " let mut total = 0;\n", + " for item in items.iter().copied() {\n", + " if item % 2 == 0 { total += item; } else { total += item * 3 + 1; }\n", + " }\n", + " total.wrapping_mul(7).wrapping_add(11)\n", + "}\n", + "pub fn walk_two(items: &[u32]) -> u32 { items.iter().copied().filter(|i| i % 3 == 0).sum() }\n", + "pub fn walk_three(items: &[u32]) -> u32 { items.iter().copied().filter(|i| i % 5 == 0).sum() }\n", + "pub fn walk_four(items: &[u32]) -> u32 { items.iter().copied().filter(|i| i % 7 == 0).sum() }\n", + "pub fn walk_five(items: &[u32]) -> u32 { items.iter().copied().filter(|i| i % 11 == 0).sum() }\n", +); + +/// The one-edit increment the daemon publishes: one re-extracted file beside +/// a carried-forward file whose segment the child reuses from the parent. +fn increment_wedge_request(edited_value: u64, sealed_at: i64) -> CodeIndexBuildRequestV1 { + let edited = if edited_value == 1 { + "pub fn edited() -> u64 { 1 }\n" + } else { + "pub fn edited() -> u64 { 2 }\npub fn appended_marker() -> u64 { 1103 }\n" + }; + let sources = [ + ( + "file.wedge.carried", + "src/carried.rs", + INCREMENT_WEDGE_CARRIED_SOURCE, + ), + ("file.wedge.edited", "src/edited.rs", edited), + ]; + let changed_files = if edited_value == 1 { + BTreeSet::new() + } else { + BTreeSet::from(["src/edited.rs".to_owned()]) + }; + partitioned_request(&sources, changed_files, sealed_at) +} + +/// Every clone body occurrence per file path, in the order the file carries +/// them, read from the generation's sealed JSON. +fn sealed_clone_bindings( + generation: &CodeIndexPublishedGenerationV1, +) -> BTreeMap> { + let sealed = generation.encode_sealed().expect("generation seals"); + let envelope: serde_json::Value = serde_json::from_slice(&sealed).expect("sealed JSON"); + envelope["generation"]["files"] + .as_array() + .expect("sealed files") + .iter() + .map(|file| { + let bodies = file["artifacts"]["clone_bodies"] + .as_array() + .expect("clone bodies") + .iter() + .map(|body| { + serde_json::from_value(body["occurrence"].clone()) + .expect("sealed clone body occurrence") + }) + .collect(); + ( + file["authority"]["logical_path"] + .as_str() + .expect("file logical path") + .to_owned(), + bodies, + ) + }) + .collect() +} + +/// The daemon publishes a one-file increment with parent-segment reuse and +/// then projects its text through `open_partitioned_sealed`, admitting every +/// restored file under `CodeFileIndexArtifactsV1::validate`. A carried-forward +/// file's clone bodies must come back through that sealed path bound to the +/// same file symbols, in the same strict order, with the same payload digests +/// the in-memory generation published; otherwise the text projection refuses +/// the generation and the index never serves the edit. +#[test] +fn carried_forward_clone_bodies_admit_through_the_reused_sealed_segment() { + let store = SharedPublicationStore::default(); + let mut owner = CodeIndexProductionOwnerV1::new(config(), store, ApplyingProjectionSink) + .expect("increment owner"); + let parent = owner + .build_and_publish(increment_wedge_request(1, 1_100_000), &ActiveControl) + .expect("parent generation"); + let mut segments = BTreeMap::new(); + let mut parent_file_segments = 0; + let parent_manifest = parent + .encode_partitioned_sealed(collect_published_segments( + &mut segments, + &mut parent_file_segments, + )) + .expect("parent encoding"); + assert_eq!( + parent_file_segments, + parent.snapshot().files.len(), + "a parentless encoding publishes every file segment" + ); + let child = owner + .build_and_publish(increment_wedge_request(2, 1_200_000), &ActiveControl) + .expect("child generation"); + let mut child_file_segments = 0; + let child_manifest = child + .encode_partitioned_sealed_with_parent( + Some(&parent_manifest), + collect_published_segments(&mut segments, &mut child_file_segments), + ) + .expect("child encoding"); + // Counted at the publisher, not inferred from store growth: a re-encoded + // carried file hashes to its parent's digest (the encoder writes the + // generation as a marker), so only the publication itself proves the + // carried file took the `Reused(descriptor)` path this test exercises. + assert_eq!( + child_file_segments, 1, + "only the edited file is re-encoded; the carried file reuses its parent segment" + ); + let parent_identities = + CodeIndexPublishedGenerationV1::partitioned_segment_identities(&parent_manifest) + .expect("parent identities parse") + .expect("current partitioned manifest"); + let child_identities = + CodeIndexPublishedGenerationV1::partitioned_segment_identities(&child_manifest) + .expect("child identities parse") + .expect("current partitioned manifest"); + let carried_from_parent = child_identities + .iter() + .filter(|identity| { + parent_identities + .iter() + .any(|parent| parent.digest == identity.digest) + }) + .count(); + assert_eq!( + carried_from_parent, 1, + "the carried file's segment is the parent's content address" + ); + + let expected = sealed_clone_bindings(&child); + let carried = expected + .get("src/carried.rs") + .expect("carried file is in the child generation"); + assert!( + carried.len() >= 2, + "the carried file needs several clone bodies to pin their order: {carried:#?}" + ); + assert!( + carried + .iter() + .any(|body| body.eligibility == CloneBodyEligibilityV1::Eligible), + "the carried file needs an eligible clone body: {carried:#?}" + ); + assert!( + carried.iter().any(|body| matches!( + body.eligibility, + CloneBodyEligibilityV1::ExcludedTooSmall { .. } + )), + "the carried file needs a too-small clone body: {carried:#?}" + ); + + let envelope: serde_json::Value = + serde_json::from_slice(&child_manifest).expect("child manifest JSON"); + let state_digest = id::( + envelope["state_digest"] + .as_str() + .expect("child state digest"), + ); + let segments = Arc::new(segments); + let mut source = VerifiedSealedLexicalPageSourceV1::open_partitioned_sealed( + Cursor::new(Vec::::new()), + &child_manifest, + state_digest, + move |digest, _, buffer| { + let bytes = segments.get(digest.as_str()).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract("published segment is missing".to_owned()) + })?; + buffer.clear(); + buffer.extend_from_slice(bytes); + Ok(()) + }, + 64, + 1 << 20, + ) + .expect("child manifest opens through the daemon's text projection path") + .expect("current partitioned manifest"); + + let mut restored: BTreeMap> = BTreeMap::new(); + let receipt = loop { + match source + .next_page(&ActiveControl) + .expect("every carried and re-extracted file admits under sealed validation") + { + VerifiedSealedLexicalPageReadV1::Page(page) => { + for body in page.clone_bodies() { + restored + .entry(body.occurrence.path.clone()) + .or_default() + .push(body.occurrence.clone()); + } + } + VerifiedSealedLexicalPageReadV1::Complete(receipt) => break receipt, + } + }; + let expected_total = expected.values().map(Vec::len).sum::(); + assert_eq!(receipt.total_clone_bodies(), expected_total as u64); + assert_eq!( + restored, expected, + "sealed decode must yield the clone binding the in-memory generation published" + ); + // Segment reuse is sound only because a carried-forward file's clone + // binding is generation-independent: the parent's segment bytes restore + // under the child's manifest to exactly the child's in-memory binding, + // which differs from the parent's only in the generation it now serves. + let rebound_from_parent = sealed_clone_bindings(&parent) + .remove("src/carried.rs") + .expect("carried file is in the parent generation") + .into_iter() + .map(|mut occurrence| { + occurrence.source_generation = child.manifest().generation_id.clone(); + occurrence.snapshot_digest = child.manifest().snapshot_digest.clone(); + occurrence + }) + .collect::>(); + assert_eq!( + &rebound_from_parent, carried, + "a carried-forward file keeps its parent clone binding across generations" + ); +} + // --------------------------------------------------------------------------- // Peak-RSS bound for the pre-paging (legacy) generation restore. //