Finish rewrite, strongly type IDs, parallelize hypertrees
This commit is contained in:
parent
a40fb7b041
commit
9a03220bb3
5 changed files with 1063 additions and 914 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -2,4 +2,4 @@
|
|||
/Cargo.lock
|
||||
/.envrc
|
||||
/.direnv
|
||||
/db.redb
|
||||
/vectordb
|
||||
|
|
|
@ -3,9 +3,10 @@ use std::time::Instant;
|
|||
use rand::{thread_rng, Rng};
|
||||
|
||||
fn main() {
|
||||
let db = vectordb::VectorDb::open("./db.redb", String::from("moments"), 4).expect("Launched");
|
||||
let db =
|
||||
vectordb::VectorDb::<7, 2>::open("./vectordb", String::from("moments")).expect("Launched");
|
||||
|
||||
let total_chunks = 8;
|
||||
let total_chunks = 32;
|
||||
let chunk_size = 1024 * 32;
|
||||
|
||||
for _ in 0..total_chunks {
|
||||
|
@ -17,8 +18,11 @@ fn main() {
|
|||
let now = Instant::now();
|
||||
db.insert_many_vectors(&vectors).expect("Inserted vectors");
|
||||
println!("Inserting vectors - Took {:?}", now.elapsed());
|
||||
std::thread::sleep(std::time::Duration::from_secs(5));
|
||||
}
|
||||
|
||||
std::thread::sleep(std::time::Duration::from_secs(30));
|
||||
|
||||
let existing_vector = thread_rng().gen::<[f32; 7]>().into();
|
||||
|
||||
db.insert_vector(&existing_vector).expect("Insert works");
|
||||
|
|
|
@ -1,49 +0,0 @@
|
|||
pub(crate) struct FlattenResults<I, J> {
|
||||
iterator: I,
|
||||
nested: Option<J>,
|
||||
}
|
||||
|
||||
pub(crate) trait IteratorExt<J, E>: std::iter::Iterator<Item = Result<J, E>> {
|
||||
fn flatten_results(self) -> FlattenResults<Self, J>
|
||||
where
|
||||
Self: Sized;
|
||||
}
|
||||
|
||||
impl<I, J, T, E> IteratorExt<J, E> for I
|
||||
where
|
||||
I: std::iter::Iterator<Item = Result<J, E>>,
|
||||
J: std::iter::Iterator<Item = Result<T, E>>,
|
||||
{
|
||||
fn flatten_results(self) -> FlattenResults<Self, J> {
|
||||
FlattenResults {
|
||||
iterator: self,
|
||||
nested: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, J, T, E> std::iter::Iterator for FlattenResults<I, J>
|
||||
where
|
||||
I: std::iter::Iterator<Item = Result<J, E>>,
|
||||
J: std::iter::Iterator<Item = Result<T, E>>,
|
||||
{
|
||||
type Item = Result<T, E>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if let Some(nested) = self.nested.as_mut() {
|
||||
if let Some(res) = nested.next() {
|
||||
return Some(res);
|
||||
}
|
||||
self.nested.take();
|
||||
}
|
||||
|
||||
match self.iterator.next() {
|
||||
Some(Ok(nested)) => {
|
||||
self.nested = Some(nested);
|
||||
self.next()
|
||||
}
|
||||
Some(Err(e)) => Some(Err(e)),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
}
|
718
src/hypertree.rs
718
src/hypertree.rs
File diff suppressed because it is too large
Load diff
1200
src/lib.rs
1200
src/lib.rs
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue