#![allow(dead_code)]
use {
crate::{
bucket_storage::{BucketCapacity, BucketOccupied, BucketStorage, Capacity, IncludeHeader},
RefCount,
},
bv::BitVec,
modular_bitfield::prelude::*,
num_enum::FromPrimitive,
solana_sdk::{clock::Slot, pubkey::Pubkey},
std::{fmt::Debug, marker::PhantomData},
};
const OCCUPIED_OCCUPIED: u8 = 1;
const OCCUPIED_FREE: u8 = 0;
impl BucketCapacity for BucketWithHeader {
fn capacity(&self) -> u64 {
self.capacity_pow2.capacity()
}
fn capacity_pow2(&self) -> u8 {
self.capacity_pow2.capacity_pow2()
}
}
#[derive(Copy, Clone)]
#[repr(C)]
struct DataBucketRefCountOccupiedHeader {
packed_ref_count: PackedRefCount,
}
#[derive(Copy, Clone)]
#[repr(C)]
pub struct BucketWithHeader {
capacity_pow2: Capacity,
}
impl BucketOccupied for BucketWithHeader {
fn occupy(&mut self, element: &mut [u8], ix: usize) {
assert!(self.is_free(element, ix));
let entry: &mut DataBucketRefCountOccupiedHeader =
BucketStorage::<BucketWithHeader>::get_mut_from_parts(element);
entry.packed_ref_count.set_occupied(OCCUPIED_OCCUPIED);
}
fn free(&mut self, element: &mut [u8], ix: usize) {
assert!(!self.is_free(element, ix));
let entry: &mut DataBucketRefCountOccupiedHeader =
BucketStorage::<BucketWithHeader>::get_mut_from_parts(element);
entry.packed_ref_count.set_occupied(OCCUPIED_FREE);
}
fn is_free(&self, element: &[u8], _ix: usize) -> bool {
let entry: &DataBucketRefCountOccupiedHeader =
BucketStorage::<BucketWithHeader>::get_from_parts(element);
entry.packed_ref_count.occupied() == OCCUPIED_FREE
}
fn offset_to_first_data() -> usize {
std::mem::size_of::<DataBucketRefCountOccupiedHeader>()
}
fn new(capacity: Capacity) -> Self {
assert!(matches!(capacity, Capacity::Pow2(_)));
Self {
capacity_pow2: capacity,
}
}
}
#[derive(Debug)]
pub struct IndexBucketUsingBitVecBits<T: 'static> {
pub enum_tag: BitVec,
capacity: u64,
_phantom: PhantomData<&'static T>,
}
impl<T: Copy + 'static> IndexBucketUsingBitVecBits<T> {
fn set_bits(&mut self, ix: u64, first: bool, second: bool) {
self.enum_tag.set(ix * 2, first);
self.enum_tag.set(ix * 2 + 1, second);
}
fn get_bits(&self, ix: u64) -> (bool, bool) {
(self.enum_tag.get(ix * 2), self.enum_tag.get(ix * 2 + 1))
}
fn set_enum_tag(&mut self, ix: u64, value: OccupiedEnumTag) {
let value = value as u8;
self.set_bits(ix, (value & 2) == 2, (value & 1) == 1);
}
fn get_enum_tag(&self, ix: u64) -> OccupiedEnumTag {
let (first, second) = self.get_bits(ix);
let tag = (first as u8 * 2) + second as u8;
num_enum::FromPrimitive::from_primitive(tag)
}
}
impl<T: Copy + 'static> BucketOccupied for IndexBucketUsingBitVecBits<T> {
fn occupy(&mut self, element: &mut [u8], ix: usize) {
assert!(self.is_free(element, ix));
self.set_enum_tag(ix as u64, OccupiedEnumTag::ZeroSlots);
}
fn free(&mut self, element: &mut [u8], ix: usize) {
assert!(!self.is_free(element, ix));
self.set_enum_tag(ix as u64, OccupiedEnumTag::Free);
}
fn is_free(&self, _element: &[u8], ix: usize) -> bool {
self.get_enum_tag(ix as u64) == OccupiedEnumTag::Free
}
fn offset_to_first_data() -> usize {
0
}
fn new(capacity: Capacity) -> Self {
Self {
enum_tag: BitVec::new_fill(false, capacity.capacity() * 2),
capacity: capacity.capacity(),
_phantom: PhantomData,
}
}
fn copying_entry(
&mut self,
_element_new: &mut [u8],
ix_new: usize,
other: &Self,
_element_old: &[u8],
ix_old: usize,
) {
self.set_enum_tag(ix_new as u64, other.get_enum_tag(ix_old as u64));
}
}
impl<T> BucketCapacity for IndexBucketUsingBitVecBits<T> {
fn capacity(&self) -> u64 {
self.capacity
}
}
pub type DataBucket = BucketWithHeader;
pub type IndexBucket<T> = IndexBucketUsingBitVecBits<T>;
pub struct IndexEntryPlaceInBucket<T: 'static> {
pub ix: u64,
_phantom: PhantomData<&'static T>,
}
#[repr(C)]
#[derive(Copy, Clone)]
pub struct IndexEntry<T: Clone + Copy> {
pub(crate) key: Pubkey, contents: SingleElementOrMultipleSlots<T>,
}
pub(crate) const MAX_LEGAL_REFCOUNT: RefCount = RefCount::MAX >> 1;
#[bitfield(bits = 64)]
#[repr(C)]
#[derive(Debug, Default, Copy, Clone, Eq, PartialEq)]
pub(crate) struct PackedRefCount {
pub(crate) occupied: B1,
pub(crate) ref_count: B63,
}
#[repr(C)]
#[derive(Debug, Default, Copy, Clone, Eq, PartialEq)]
pub(crate) struct MultipleSlots {
storage_cap_and_offset: PackedStorage,
num_slots: Slot,
}
impl MultipleSlots {
pub(crate) fn set_storage_capacity_when_created_pow2(
&mut self,
storage_capacity_when_created_pow2: u8,
) {
self.storage_cap_and_offset
.set_capacity_when_created_pow2(storage_capacity_when_created_pow2)
}
pub(crate) fn set_storage_offset(&mut self, storage_offset: u64) {
self.storage_cap_and_offset
.set_offset_checked(storage_offset)
.expect("New storage offset must fit into 7 bytes!")
}
fn storage_capacity_when_created_pow2(&self) -> u8 {
self.storage_cap_and_offset.capacity_when_created_pow2()
}
fn storage_offset(&self) -> u64 {
self.storage_cap_and_offset.offset()
}
pub(crate) fn num_slots(&self) -> Slot {
self.num_slots
}
pub(crate) fn set_num_slots(&mut self, num_slots: Slot) {
self.num_slots = num_slots;
}
pub(crate) fn data_bucket_ix(&self) -> u64 {
Self::data_bucket_from_num_slots(self.num_slots())
}
pub(crate) fn data_bucket_from_num_slots(num_slots: Slot) -> u64 {
if num_slots == 0 {
0
} else {
(Slot::BITS - (num_slots - 1).leading_zeros()) as u64
}
}
pub(crate) fn data_loc(&self, storage: &BucketStorage<DataBucket>) -> u64 {
self.storage_offset()
<< (storage.contents.capacity_pow2() - self.storage_capacity_when_created_pow2())
}
pub fn set_ref_count(
data_bucket: &mut BucketStorage<DataBucket>,
data_ix: u64,
ref_count: RefCount,
) {
data_bucket
.get_header_mut::<DataBucketRefCountOccupiedHeader>(data_ix)
.packed_ref_count
.set_ref_count(ref_count);
}
pub fn ref_count(data_bucket: &BucketStorage<DataBucket>, data_ix: u64) -> RefCount {
data_bucket
.get_header::<DataBucketRefCountOccupiedHeader>(data_ix)
.packed_ref_count
.ref_count()
}
}
#[repr(C)]
#[derive(Copy, Clone)]
pub(crate) union SingleElementOrMultipleSlots<T: Clone + Copy> {
pub(crate) single_element: T,
pub(crate) multiple_slots: MultipleSlots,
}
#[derive(PartialEq, FromPrimitive)]
#[repr(u8)]
enum OccupiedEnumTag {
#[default]
Free = 0,
ZeroSlots = 1,
OneSlotInIndex = 2,
MultipleSlots = 3,
}
#[repr(u8)]
#[derive(Debug, Eq, PartialEq)]
pub(crate) enum OccupiedEnum<'a, T> {
Free = OccupiedEnumTag::Free as u8,
ZeroSlots = OccupiedEnumTag::ZeroSlots as u8,
OneSlotInIndex(&'a T) = OccupiedEnumTag::OneSlotInIndex as u8,
MultipleSlots(&'a MultipleSlots) = OccupiedEnumTag::MultipleSlots as u8,
}
#[bitfield(bits = 64)]
#[repr(C)]
#[derive(Debug, Default, Copy, Clone, Eq, PartialEq)]
struct PackedStorage {
capacity_when_created_pow2: B8,
offset: B56,
}
impl<T: Copy + 'static> IndexEntryPlaceInBucket<T> {
pub(crate) fn get_slot_count_enum<'a>(
&self,
index_bucket: &'a BucketStorage<IndexBucket<T>>,
) -> OccupiedEnum<'a, T> {
let enum_tag = index_bucket.contents.get_enum_tag(self.ix);
let index_entry = index_bucket.get::<IndexEntry<T>>(self.ix);
match enum_tag {
OccupiedEnumTag::Free => OccupiedEnum::Free,
OccupiedEnumTag::ZeroSlots => OccupiedEnum::ZeroSlots,
OccupiedEnumTag::OneSlotInIndex => unsafe {
OccupiedEnum::OneSlotInIndex(&index_entry.contents.single_element)
},
OccupiedEnumTag::MultipleSlots => unsafe {
OccupiedEnum::MultipleSlots(&index_entry.contents.multiple_slots)
},
}
}
pub(crate) fn get_multiple_slots_mut<'a>(
&self,
index_bucket: &'a mut BucketStorage<IndexBucket<T>>,
) -> Option<&'a mut MultipleSlots> {
let enum_tag = index_bucket.contents.get_enum_tag(self.ix);
unsafe {
match enum_tag {
OccupiedEnumTag::MultipleSlots => {
let index_entry = index_bucket.get_mut::<IndexEntry<T>>(self.ix);
Some(&mut index_entry.contents.multiple_slots)
}
_ => None,
}
}
}
pub(crate) fn set_slot_count_enum_value<'a>(
&self,
index_bucket: &'a mut BucketStorage<IndexBucket<T>>,
value: OccupiedEnum<'a, T>,
) {
let tag = match value {
OccupiedEnum::Free => OccupiedEnumTag::Free,
OccupiedEnum::ZeroSlots => OccupiedEnumTag::ZeroSlots,
OccupiedEnum::OneSlotInIndex(single_element) => {
let index_entry = index_bucket.get_mut::<IndexEntry<T>>(self.ix);
index_entry.contents.single_element = *single_element;
OccupiedEnumTag::OneSlotInIndex
}
OccupiedEnum::MultipleSlots(multiple_slots) => {
let index_entry = index_bucket.get_mut::<IndexEntry<T>>(self.ix);
index_entry.contents.multiple_slots = *multiple_slots;
OccupiedEnumTag::MultipleSlots
}
};
index_bucket.contents.set_enum_tag(self.ix, tag);
}
pub fn init(&self, index_bucket: &mut BucketStorage<IndexBucket<T>>, pubkey: &Pubkey) {
self.set_slot_count_enum_value(index_bucket, OccupiedEnum::ZeroSlots);
let index_entry = index_bucket.get_mut::<IndexEntry<T>>(self.ix);
index_entry.key = *pubkey;
}
pub(crate) fn read_value<'a>(
&self,
index_bucket: &'a BucketStorage<IndexBucket<T>>,
data_buckets: &'a [BucketStorage<DataBucket>],
) -> (&'a [T], RefCount) {
let mut ref_count = 1;
let slot_list = match self.get_slot_count_enum(index_bucket) {
OccupiedEnum::ZeroSlots => {
&[]
}
OccupiedEnum::OneSlotInIndex(single_element) => {
std::slice::from_ref(single_element)
}
OccupiedEnum::MultipleSlots(multiple_slots) => {
let data_bucket_ix =
MultipleSlots::data_bucket_from_num_slots(multiple_slots.num_slots);
let data_bucket = &data_buckets[data_bucket_ix as usize];
let loc = multiple_slots.data_loc(data_bucket);
assert!(!data_bucket.is_free(loc));
ref_count = MultipleSlots::ref_count(data_bucket, loc);
data_bucket.get_cell_slice::<T>(
loc,
multiple_slots.num_slots,
IncludeHeader::NoHeader,
)
}
_ => {
panic!("trying to read data from a free entry");
}
};
(slot_list, ref_count)
}
pub fn new(ix: u64) -> Self {
Self {
ix,
_phantom: PhantomData,
}
}
pub fn key<'a>(&self, index_bucket: &'a BucketStorage<IndexBucket<T>>) -> &'a Pubkey {
let entry: &IndexEntry<T> = index_bucket.get(self.ix);
&entry.key
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_api() {
for offset in [0, 1, u32::MAX as u64] {
let mut multiple_slots = MultipleSlots::default();
if offset != 0 {
multiple_slots.set_storage_offset(offset);
}
assert_eq!(multiple_slots.storage_offset(), offset);
assert_eq!(multiple_slots.storage_capacity_when_created_pow2(), 0);
for pow in [1, 255, 0] {
multiple_slots.set_storage_capacity_when_created_pow2(pow);
assert_eq!(multiple_slots.storage_offset(), offset);
assert_eq!(multiple_slots.storage_capacity_when_created_pow2(), pow);
}
}
}
#[test]
fn test_size() {
assert_eq!(std::mem::size_of::<PackedStorage>(), 1 + 7);
assert_eq!(std::mem::size_of::<IndexEntry<u64>>(), 32 + 8 + 8);
}
#[test]
#[should_panic(expected = "New storage offset must fit into 7 bytes!")]
fn test_set_storage_offset_value_too_large() {
let too_big = 1 << 56;
let mut multiple_slots = MultipleSlots::default();
multiple_slots.set_storage_offset(too_big);
}
#[test]
fn test_data_bucket_from_num_slots() {
for n in 0..512 {
assert_eq!(
MultipleSlots::data_bucket_from_num_slots(n),
(n as f64).log2().ceil() as u64
);
}
assert_eq!(
MultipleSlots::data_bucket_from_num_slots(u32::MAX as u64),
32
);
assert_eq!(
MultipleSlots::data_bucket_from_num_slots(u32::MAX as u64 + 1),
32
);
assert_eq!(
MultipleSlots::data_bucket_from_num_slots(u32::MAX as u64 + 2),
33
);
}
}