Finish rewrite, strongly type IDs, parallelize hypertrees

This commit is contained in:
asonix 2023-07-01 21:54:07 -05:00
parent a40fb7b041
commit 9a03220bb3
5 changed files with 1063 additions and 914 deletions

2
.gitignore vendored
View file

@ -2,4 +2,4 @@
/Cargo.lock
/.envrc
/.direnv
/db.redb
/vectordb

View file

@ -3,9 +3,10 @@ use std::time::Instant;
use rand::{thread_rng, Rng};
fn main() {
let db = vectordb::VectorDb::open("./db.redb", String::from("moments"), 4).expect("Launched");
let db =
vectordb::VectorDb::<7, 2>::open("./vectordb", String::from("moments")).expect("Launched");
let total_chunks = 8;
let total_chunks = 32;
let chunk_size = 1024 * 32;
for _ in 0..total_chunks {
@ -17,8 +18,11 @@ fn main() {
let now = Instant::now();
db.insert_many_vectors(&vectors).expect("Inserted vectors");
println!("Inserting vectors - Took {:?}", now.elapsed());
std::thread::sleep(std::time::Duration::from_secs(5));
}
std::thread::sleep(std::time::Duration::from_secs(30));
let existing_vector = thread_rng().gen::<[f32; 7]>().into();
db.insert_vector(&existing_vector).expect("Insert works");

View file

@ -1,49 +0,0 @@
pub(crate) struct FlattenResults<I, J> {
iterator: I,
nested: Option<J>,
}
pub(crate) trait IteratorExt<J, E>: std::iter::Iterator<Item = Result<J, E>> {
fn flatten_results(self) -> FlattenResults<Self, J>
where
Self: Sized;
}
impl<I, J, T, E> IteratorExt<J, E> for I
where
I: std::iter::Iterator<Item = Result<J, E>>,
J: std::iter::Iterator<Item = Result<T, E>>,
{
fn flatten_results(self) -> FlattenResults<Self, J> {
FlattenResults {
iterator: self,
nested: None,
}
}
}
impl<I, J, T, E> std::iter::Iterator for FlattenResults<I, J>
where
I: std::iter::Iterator<Item = Result<J, E>>,
J: std::iter::Iterator<Item = Result<T, E>>,
{
type Item = Result<T, E>;
fn next(&mut self) -> Option<Self::Item> {
if let Some(nested) = self.nested.as_mut() {
if let Some(res) = nested.next() {
return Some(res);
}
self.nested.take();
}
match self.iterator.next() {
Some(Ok(nested)) => {
self.nested = Some(nested);
self.next()
}
Some(Err(e)) => Some(Err(e)),
None => None,
}
}
}

File diff suppressed because it is too large Load diff

1200
src/lib.rs

File diff suppressed because it is too large Load diff