diff --git a/src/plan/lxr/gc_work/mature_sweeping.rs b/src/plan/lxr/gc_work/mature_sweeping.rs index dcf3705a10..a9d5472118 100644 --- a/src/plan/lxr/gc_work/mature_sweeping.rs +++ b/src/plan/lxr/gc_work/mature_sweeping.rs @@ -60,9 +60,12 @@ impl SweepDeadCycles { let c = self.rc.count_by_address(cur_cursor); if c != 0 { // Safety: cur_cursor is either a valid object reference, or a straddle line - let o = unsafe { ObjectReference::from_raw_address_unchecked(cur_cursor) }; + // But we don't know o is valid now + let o = unsafe { rc::ObjectEnvelope::from_start::(cur_cursor) } + .to_object_reference(); + // TODO: This check is unsound. But if we only use side metadata, it is fine. if !immix_space.is_marked(o) { - if Line::is_aligned(o.to_raw_address()) { + if Line::is_aligned(cur_cursor) { if c == 1 && self.rc.object_is_in_straddle_line_no_rc_check(o) { // this is a straddle line, skip continue; diff --git a/src/plan/lxr/global.rs b/src/plan/lxr/global.rs index 690eefc844..5f6db4c64a 100644 --- a/src/plan/lxr/global.rs +++ b/src/plan/lxr/global.rs @@ -772,6 +772,12 @@ impl LXR { self.immix_space.in_space(o) && Block::in_defrag_block(o) } + /// Whether LXR maintains a reference count for `o`. Only the Immix and large object spaces are reference counted. + #[allow(dead_code)] + pub fn is_rc_object(&self, o: ObjectReference) -> bool { + self.immix_space.in_space(o) || self.common.los.in_space(o) + } + pub fn address_in_defrag(&self, a: Address) -> bool { self.immix_space.address_in_space(a) && Block::address_in_defrag_block(a) } diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index 22656fe9d3..7e5acbf852 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -24,7 +24,7 @@ use crate::util::metadata::vo_bit; use crate::util::metadata::{self, MetadataSpec}; use crate::util::object_enum::ObjectEnumerator; use crate::util::object_forwarding; -use crate::util::rc::RefCountHelper; +use crate::util::rc::{ObjectEnvelope, RefCountHelper}; use crate::util::{copy::*, epilogue, object_enum}; use crate::util::{Address, ObjectReference}; use crate::vm::*; @@ -817,12 +817,8 @@ impl ImmixSpace { object: ObjectReference, ) -> ObjectReference { if self.attempt_mark(object) { - let addr = object.to_raw_address().as_usize(); - let straddle = if (addr & 0b11110000) == 0 { - self.rc.object_is_in_straddle_line_no_rc_check(object) - } else { - false - }; + let straddle = Line::is_aligned(ObjectEnvelope::of(object).start::()) + && self.rc.object_is_in_straddle_line_no_rc_check(object); if !straddle { queue.enqueue(object); } @@ -1057,7 +1053,7 @@ impl ImmixSpace { ) .expect("to-space overflow"); // Transfer RC count - if new.get_size::() > Line::BYTES { + if ObjectEnvelope::needs_straddle_marks::(new.get_size::()) { self.rc.mark_straddle_object(new); } self.rc.set(new, self.rc.count(object)); diff --git a/src/util/alloc/immix_allocator.rs b/src/util/alloc/immix_allocator.rs index 2658a9cbae..3a7638fd88 100644 --- a/src/util/alloc/immix_allocator.rs +++ b/src/util/alloc/immix_allocator.rs @@ -11,6 +11,7 @@ use crate::util::alloc::allocator::get_maximum_aligned_size; use crate::util::alloc::Allocator; use crate::util::linear_scan::Region; use crate::util::opaque_pointer::VMThread; +use crate::util::rc::ObjectEnvelope; use crate::util::rust_util::unlikely; use crate::util::Address; use crate::vm::*; @@ -191,6 +192,16 @@ impl ImmixAllocator { self.space } + /// The first address in `block` an object may be allocated at. + fn block_usable_start(&self, block: Block) -> Address { + if self.space.rc_enabled { + // For LXR, we need to reserve the leading slack bytes of a block for the envelope reservation. + block.start() + ObjectEnvelope::slack::() + } else { + block.start() + } + } + /// Large-object (larger than a line) bump allocation. fn overflow_alloc(&mut self, size: usize, align: usize, offset: usize) -> Address { trace!("{:?}: overflow_alloc", self.tls); @@ -242,7 +253,9 @@ impl ImmixAllocator { self.immix_space().get_next_available_lines(self.copy, line) { // Find recyclable lines. Update the bump allocation cursor and limit. - self.bump_pointer.cursor = start_line.start(); + self.bump_pointer.cursor = start_line + .start() + .max(self.block_usable_start(line.block())); self.bump_pointer.limit = end_line.start(); trace!( "{:?}: acquire_recyclable_lines -> {:?} [{:?}, {:?}) {:?}", @@ -257,7 +270,7 @@ impl ImmixAllocator { self.bump_pointer.limit - self.bump_pointer.cursor, ); debug_assert!( - align_allocation_no_fill::(self.bump_pointer.cursor, align, offset) + size + align_allocation_no_fill::(start_line.start(), align, offset) + size <= self.bump_pointer.limit ); let block = line.block(); @@ -334,10 +347,10 @@ impl ImmixAllocator { } } if self.request_for_large { - self.large_bump_pointer.cursor = block.start(); + self.large_bump_pointer.cursor = self.block_usable_start(block); self.large_bump_pointer.limit = block.end(); } else { - self.bump_pointer.cursor = block.start(); + self.bump_pointer.cursor = self.block_usable_start(block); self.bump_pointer.limit = block.end(); } self.alloc(size, align, offset) diff --git a/src/util/alloc/large_object_allocator.rs b/src/util/alloc/large_object_allocator.rs index 7a03056c9f..dc8d6738dc 100644 --- a/src/util/alloc/large_object_allocator.rs +++ b/src/util/alloc/large_object_allocator.rs @@ -4,6 +4,7 @@ use crate::policy::largeobjectspace::LargeObjectSpace; use crate::policy::space::Space; use crate::util::alloc::{allocator, Allocator}; use crate::util::opaque_pointer::*; +use crate::util::rc::ObjectEnvelope; use crate::util::Address; use crate::vm::VMBinding; @@ -49,7 +50,8 @@ impl Allocator for LargeObjectAllocator { } fn alloc_slow_once(&mut self, size: usize, align: usize, _offset: usize) -> Address { - let maxbytes = allocator::get_maximum_aligned_size::(size, align); + let reserved = self.envelope_reservation(); + let maxbytes = allocator::get_maximum_aligned_size::(size, align) + reserved; let pages = crate::util::conversions::bytes_to_pages_up(maxbytes); if self.handle_obvious_oom_request( @@ -59,12 +61,29 @@ impl Allocator for LargeObjectAllocator { return Address::ZERO; } - self.space - .allocate_pages(self.tls, pages, self.get_context().get_alloc_options()) + let cell = + self.space + .allocate_pages(self.tls, pages, self.get_context().get_alloc_options()); + // A zero cell means the VM is out of memory; leave it zero so the caller still sees it. + if cell.is_zero() { + cell + } else { + cell + reserved + } } } impl LargeObjectAllocator { + /// Bytes left unused at the front of every cell, so that an object's *envelope* start can + /// never precede the cell it belongs to. See [`ObjectEnvelope`]. + fn envelope_reservation(&self) -> usize { + if self.space.rc_enabled { + ObjectEnvelope::slack::() + } else { + 0 + } + } + pub(crate) fn new( tls: VMThread, space: &'static LargeObjectSpace, diff --git a/src/util/rc.rs b/src/util/rc.rs index daf33acf52..4c2772b58e 100644 --- a/src/util/rc.rs +++ b/src/util/rc.rs @@ -1,6 +1,7 @@ use std::marker::PhantomData; use std::sync::atomic::{AtomicU32, AtomicUsize}; +use crate::util::heap::chunk_map::Chunk; use crate::util::linear_scan::Region; use crate::util::{metadata::side_metadata::address_to_meta_address, Address}; use crate::{ @@ -33,6 +34,138 @@ pub const RC_STRADDLE_LINES: SideMetadataSpec = /// Side metadata spec for the per-object reference count table. pub const RC_TABLE: SideMetadataSpec = crate::util::metadata::side_metadata::spec_defs::RC_TABLE; +/// The start of an object's *envelope*: the address where LXR stores the object's reference count and +/// straddle marks on, in place of either the object reference or the real object start. +/// +/// The envelope is guaranteed to cover the entire memory the object occupies. But unlike computing the object +/// start (which may be at a variable offset from the object reference), the envelope start can be computed +/// from object reference with constants only. This makes it more efficient to access. +/// +/// ```text +/// |<--- UPPER --->| +/// |<-slack->| +/// | |<-L->| +/// | | | | +/// envelope [######################################] +/// object, offset = UPPER [===============o============] +/// object, offset = LOWER [=====o======================] +/// ^ ^ ^ +/// | | ref (L = LOWER; the only address we know) +/// | latest possible object start = ref - LOWER +/// earliest possible object start = ref - UPPER = envelope start +/// ``` +/// +/// All the RC metadata is stored at the envelope start. +/// +/// To avoid the envelope start being outside of the block where the object is in, we reserve the leading 'slack' +/// bytes of each block for the envelope reservation. See `ImmixAllocator::block_usable_start` and `LargeObjectAllcator::envelope_reservation`. +/// +/// Currently `ObjectEnvelope` is only used for the LXR plan for RC-related metadata. But it can be used for other purposes, +/// and in other plans. +#[repr(transparent)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ObjectEnvelope(ObjectReference); + +impl ObjectEnvelope { + /// The width of the bracket the envelope puts around an object, `UPPER - LOWER`: how far its + /// start can precede the object's real start. + pub const fn slack() -> usize { + (VM::VMObjectModel::OBJECT_REF_OFFSET_UPPER_BOUND + - VM::VMObjectModel::OBJECT_REF_OFFSET_LOWER_BOUND) as usize + } + + /// The constant distance between an object's reference address and its envelope start. + const fn offset() -> usize { + VM::VMObjectModel::OBJECT_REF_OFFSET_UPPER_BOUND as usize + } + + /// Whether an object of `size` bytes needs straddle marks, i.e. whether its envelope can + /// touch a line other than the one holding its reference count and the one after it. + pub fn needs_straddle_marks(size: usize) -> bool { + size + Self::slack::() > Line::BYTES + } + + /// The envelope of object `o`. + /// + /// An envelope *is* an object, viewed through the bounds: it holds the object reference and + /// derives its addresses from that, so there is nothing to get wrong and no way to build one + /// that does not belong to an object. + pub fn of(o: ObjectReference) -> Self { + Self(o) + } + + /// The envelope whose start is `a`. `a` must be an envelope start, i.e. an address that + /// [`Self::start`] produced; the reference-count table only ever holds counts at such + /// addresses. + /// + /// # Safety + /// + /// The caller must guarantee that `a` is an envelope start, because this reconstructs the + /// object reference `a` was derived from. + pub unsafe fn from_start(a: Address) -> Self { + Self(ObjectReference::from_raw_address_unchecked( + a + Self::offset::(), + )) + } + + /// The object this envelope belongs to. + pub fn to_object_reference(self) -> ObjectReference { + self.0 + } + + /// The envelope's start address, i.e. the address the object's reference count is stored at. + /// A plain constant subtraction from the reference address. + pub fn start(self) -> Address { + debug_assert!( + VM::VMObjectModel::OBJECT_REF_OFFSET_UPPER_BOUND + >= VM::VMObjectModel::OBJECT_REF_OFFSET_LOWER_BOUND, + "OBJECT_REF_OFFSET_UPPER_BOUND must not be below the lower bound" + ); + let a = self.0.to_raw_address(); + let envelope = a - Self::offset::(); + // The reservations every reference-counted space makes at the front of a region are what + // keep this true. If it ever fires, either an allocation path is handing out the leading + // `slack` bytes of a region, or a count is being read for an object in a space that + // reserves nothing because it is not reference counted. + debug_assert!( + envelope >= Chunk::from_unaligned_address(a).start(), + "envelope start {} for {} escaped its chunk; the region it was allocated in did not \ + reserve ObjectEnvelope::slack, or {} is not in a reference-counted space", + envelope, + self.0, + self.0, + ); + envelope + } + + /// The envelope's exclusive end for an object of `size` bytes: the latest address the object + /// could possibly end at, i.e. `ref - LOWER + size`. + pub fn end(self) -> Address { + self.end_with_size::(VM::VMObjectModel::get_current_size(self.0)) + } + + pub fn end_with_size(self, size: usize) -> Address { + self.start::() + size + Self::slack::() + } + + /// The first region of type `R` the envelope overlaps: the one holding its start. + pub fn first_overlapping_region(self) -> R { + R::from_unaligned_address(self.start::()) + } + + pub fn last_overlapping_region(self) -> R { + // `end_with_size` is exclusive, so the envelope's final byte is the one before it. + R::from_unaligned_address(self.end::() - 1usize) + } + + /// The last region of type `R` the envelope overlaps: the one holding its final byte, for an + /// object of `size` bytes. + pub fn last_overlapping_region_with_size(self, size: usize) -> R { + // `end_with_size` is exclusive, so the envelope's final byte is the one before it. + R::from_unaligned_address(self.end_with_size::(size) - 1usize) + } +} + static INC_BUFFER_SIZE: AtomicUsize = AtomicUsize::new(0); static TOTAL_INCS_PACKETS: AtomicU32 = AtomicU32::new(0); @@ -84,6 +217,11 @@ impl RefCountHelper { INC_BUFFER_SIZE.store(0, Ordering::Relaxed) } + /// The address object `o`'s reference count is stored at: its envelope start + fn rc_slot(&self, o: ObjectReference) -> Address { + ObjectEnvelope::of(o).start::() + } + /// Atomically updates the reference count of object `o` by applying `f` to its current /// value, following the same semantics as `AtomicU8::fetch_update`. pub fn fetch_update( @@ -91,7 +229,7 @@ impl RefCountHelper { o: ObjectReference, f: impl FnMut(u8) -> Option, ) -> Result { - RC_TABLE.fetch_update_atomic(o.to_raw_address(), Ordering::Relaxed, Ordering::Relaxed, f) + RC_TABLE.fetch_update_atomic(self.rc_slot(o), Ordering::Relaxed, Ordering::Relaxed, f) } /// Returns `true` if object `o`'s reference count has saturated at `MAX_REF_COUNT` (sticky). @@ -154,13 +292,13 @@ impl RefCountHelper { /// Atomically sets object `o`'s reference count to `count`. pub fn set(&self, o: ObjectReference, count: u8) { - RC_TABLE.store_atomic(o.to_raw_address(), count, Ordering::Relaxed) + RC_TABLE.store_atomic(self.rc_slot(o), count, Ordering::Relaxed) } /// Sets object `o`'s reference count to `count` using a non-atomic store, for use where the /// caller can guarantee there is no concurrent access. pub fn set_relaxed(&self, o: ObjectReference, count: u8) { - unsafe { RC_TABLE.store(o.to_raw_address(), count) } + unsafe { RC_TABLE.store(self.rc_slot(o), count) } } /// Sets the reference count for the line containing object `o` to `count` using a non-atomic store, @@ -171,7 +309,7 @@ impl RefCountHelper { /// Returns object `o`'s current reference count. pub fn count(&self, o: ObjectReference) -> u8 { - RC_TABLE.load_atomic(o.to_raw_address(), Ordering::Relaxed) + RC_TABLE.load_atomic(self.rc_slot(o), Ordering::Relaxed) } /// Returns the reference count stored in the RC table at address `addr`. If this @@ -184,7 +322,7 @@ impl RefCountHelper { /// Returns `true` if the RC table entry at `o`'s address is zero. Used for both individual /// objects and line-granularity entries (e.g. straddle line markers), which share the same table. pub fn object_or_line_is_dead(&self, o: ObjectReference) -> bool { - RC_TABLE.load_byte(o.to_raw_address()) == 0 + RC_TABLE.load_byte(self.rc_slot(o)) == 0 } /// Returns a slice view over the raw RC table memory covering block `b`, reinterpreted as an @@ -205,14 +343,14 @@ impl RefCountHelper { /// Returns `true` if object `o`'s reference count is zero. #[allow(unused)] pub fn is_dead(&self, o: ObjectReference) -> bool { - let v: u8 = RC_TABLE.load_atomic(o.to_raw_address(), Ordering::Relaxed); + let v: u8 = RC_TABLE.load_atomic(self.rc_slot(o), Ordering::Relaxed); v == 0 } /// Returns `true` if object `o`'s reference count is zero (dead) or has saturated at /// `MAX_REF_COUNT` (sticky). pub fn is_dead_or_stuck(&self, o: ObjectReference) -> bool { - let v: u8 = RC_TABLE.load_atomic(o.to_raw_address(), Ordering::Relaxed); + let v: u8 = RC_TABLE.load_atomic(self.rc_slot(o), Ordering::Relaxed); v == 0 || v == MAX_REF_COUNT } @@ -220,26 +358,44 @@ impl RefCountHelper { pub fn object_is_in_straddle_line_no_rc_check(&self, o: ObjectReference) -> bool { // This directly reads line-granularity straddle line metadata with an unaligned address. // It is still correct, but may break side metadata assertions. - unsafe { RC_STRADDLE_LINES.load::(o.to_raw_address()) != 0 } + unsafe { RC_STRADDLE_LINES.load::(self.rc_slot(o)) != 0 } } /// Returns `true` if address `a` falls within a live object whose containing line is marked /// as a straddle line. pub fn object_is_in_straddle_line(&self, o: ObjectReference) -> bool { - let line = Line::from_unaligned_address(o.to_raw_address()); + let line: Line = ObjectEnvelope::of(o).first_overlapping_region::(); self.count(o) != 0 && unsafe { RC_STRADDLE_LINES.load::(line.start()) != 0 } } + /// The `[start, end)` range of lines that need a synthetic straddle mark for object `o`. + fn straddle_line_range(&self, o: ObjectReference, size: usize) -> (Line, Line) { + use crate::util::linear_scan::UnstraddlableRegion; + let envelope = ObjectEnvelope::of(o); + // Mark every line the envelope overlaps except the first, whose count the object already + // carries, and except the last, which the hole finder's skip-the-first-line-of-a-hole + // margin covers. The range is end-exclusive, so naming the last overlapped line as the + // bound drops exactly that one. + let start_line = envelope.first_overlapping_region::().next(); + let end_line: Line = envelope.last_overlapping_region_with_size::(size); + let block_end_line = Block::containing(o).end_line(); + let end_line = if end_line > block_end_line { + block_end_line + } else { + end_line + }; + // A clamp can put the end at or before the start when the object sits at the very end of + // its block; an empty range is correct there, so normalise rather than wrap around. + if end_line < start_line { + (start_line, start_line) + } else { + (start_line, end_line) + } + } + fn mark_straddle_object_with_size(&self, o: ObjectReference, size: usize) { - debug_assert!(size > Line::BYTES); - let start = o.to_object_start::(); - let end = start + size; - let start_line = Line::from_unaligned_address(start).next(); - let end_line = Line::from_unaligned_address(end); - // Note that `end_line` may be the last line overlapping with `o`. - // In that case, `end_line` will not be marked. - // It is OK because when searching for available lines (`rc_get_next_available_lines`), - // it always skips the first line in a hole. + debug_assert!(ObjectEnvelope::needs_straddle_marks::(size)); + let (start_line, end_line) = self.straddle_line_range(o, size); let mut line = start_line; while line != end_line { unsafe { RC_STRADDLE_LINES.store(line.start(), 1u8) }; @@ -260,15 +416,8 @@ impl RefCountHelper { pub fn unmark_straddle_object(&self, o: ObjectReference) { // debug_assert!(crate::args::RC_NURSERY_EVACUATION); let size = VM::VMObjectModel::get_current_size(o); - if size > Line::BYTES { - let start = o.to_object_start::(); - let end = start + size; - let start_line = Line::from_unaligned_address(start).next(); - let end_line = Line::from_unaligned_address(end); - // Note that `end_line` may be the last line overlapping with `o`. - // In that case, `end_line` will not be marked. - // It is OK because when searching for available lines (`rc_get_next_available_lines`), - // it always skips the first line in a hole. + if ObjectEnvelope::needs_straddle_marks::(size) { + let (start_line, end_line) = self.straddle_line_range(o, size); let mut line = start_line; while line != end_line { self.set_line_relaxed(line, 0); @@ -282,8 +431,9 @@ impl RefCountHelper { /// count of zero, used to verify that a reclaimed object has been fully cleared. pub fn assert_zero_ref_count(&self, o: ObjectReference) { let size = VM::VMObjectModel::get_current_size(o); + let start = ObjectEnvelope::of(o).start::(); for i in (0..size).step_by(MIN_OBJECT_SIZE) { - let a = o.to_raw_address() + i; + let a = start + i; assert_eq!(0, self.count_by_address(a)); } } @@ -292,7 +442,7 @@ impl RefCountHelper { /// spans more than one line, deriving its size from the VM binding. pub fn promote(&self, o: ObjectReference) { let size = o.get_size::(); - if size > Line::BYTES { + if ObjectEnvelope::needs_straddle_marks::(size) { self.mark_straddle_object_with_size(o, size); } } @@ -300,10 +450,35 @@ impl RefCountHelper { /// Same as `promote`, but with the object's size supplied by the caller instead of being /// queried from the VM binding. pub fn promote_with_size(&self, o: ObjectReference, size: usize) { - if size > Line::BYTES { + #[cfg(debug_assertions)] + debug_assert!(self.envelope_is_well_formed(o, size)); + if ObjectEnvelope::needs_straddle_marks::(size) { self.mark_straddle_object_with_size(o, size); } } + + /// The two properties envelope keying relies on, checked per object in debug builds. Only + /// meaningful for an object in an Immix space, which is the only caller. + #[cfg(debug_assertions)] + fn envelope_is_well_formed(&self, o: ObjectReference, size: usize) -> bool { + let envelope = ObjectEnvelope::of(o).start::(); + let start = o.to_object_start::(); + // 1. The envelope covers the object's head, and does not run out of the object's own + // block. Escaping the block would put the count in a neighbour's reference-count + // table, where `Block::clear_rc_table` would bzero it out from under a live object. + // `ImmixAllocator` leaves `ObjectEnvelope::slack` bytes free at each block start so + // this holds; if it fails, some allocation path is missing that skip. + let head_ok = envelope <= start + && envelope >= Block::from_unaligned_address(o.to_raw_address()).start(); + // 2. The envelope reaches at or past the object's true end, so marking covers every + // line the object occupies except at most the last -- and the hole finder's + // skip-the-first-line-of-a-hole margin covers exactly one. This is what the envelope + // being the *union* of the object's possible positions buys: with a coarse offset + // bound the shortfall of a slid-down envelope would span whole lines, leaving two or + // more of them uncounted, which one margin cannot absorb. + let tail_ok = start + size <= ObjectEnvelope::of(o).end_with_size::(size); + head_ok && tail_ok + } } impl Clone for RefCountHelper { diff --git a/src/vm/object_model.rs b/src/vm/object_model.rs index dfae85fa74..14e4beaa1b 100644 --- a/src/vm/object_model.rs +++ b/src/vm/object_model.rs @@ -471,6 +471,18 @@ pub trait ObjectModel { /// * object ref >= object_start + OBJECT_REF_OFFSET_LOWER_BOUND const OBJECT_REF_OFFSET_LOWER_BOUND: isize; + /// The counterpart to [`Self::OBJECT_REF_OFFSET_LOWER_BOUND`]: the *largest* possible offset + /// between an object's allocation result (object_start) and the raw address of its object + /// reference. As with the lower bound, the offset need not be the same for every object -- + /// this is the largest value it can take for any of them, across every space. + /// + /// Defaults to [`Self::OBJECT_REF_OFFSET_LOWER_BOUND`], i.e. a fixed offset, which holds for + /// every binding that does not override this. + /// + /// We should have the invariant: + /// * object ref <= object_start + OBJECT_REF_OFFSET_UPPER_BOUND + const OBJECT_REF_OFFSET_UPPER_BOUND: isize = Self::OBJECT_REF_OFFSET_LOWER_BOUND; + /// Return the lowest address of the storage associated with an object. This should be /// the address that a binding gets by an allocation call ([`crate::memory_manager::alloc`]). ///