2022-03-26 21:49:23 +00:00
|
|
|
use crate::{
|
|
|
|
repo::{Repo, SettingsRepo},
|
|
|
|
store::Store,
|
|
|
|
};
|
2021-10-28 04:06:03 +00:00
|
|
|
use actix_web::web::Bytes;
|
2021-10-31 17:35:11 +00:00
|
|
|
use futures_util::stream::Stream;
|
2021-10-28 04:06:03 +00:00
|
|
|
use s3::{
|
2021-11-02 22:21:00 +00:00
|
|
|
client::Client, command::Command, creds::Credentials, request_trait::Request, Bucket, Region,
|
2021-10-28 04:06:03 +00:00
|
|
|
};
|
|
|
|
use std::{
|
|
|
|
pin::Pin,
|
|
|
|
string::FromUtf8Error,
|
|
|
|
task::{Context, Poll},
|
|
|
|
};
|
|
|
|
use storage_path_generator::{Generator, Path};
|
2021-10-31 17:35:11 +00:00
|
|
|
use tokio::io::{AsyncRead, AsyncWrite};
|
2021-10-28 04:06:03 +00:00
|
|
|
|
|
|
|
mod object_id;
|
|
|
|
pub(crate) use object_id::ObjectId;
|
|
|
|
|
|
|
|
// - Settings Tree
|
|
|
|
// - last-path -> last generated path
|
|
|
|
|
|
|
|
const GENERATOR_KEY: &[u8] = b"last-path";
|
|
|
|
|
|
|
|
#[derive(Debug, thiserror::Error)]
|
|
|
|
pub(crate) enum ObjectError {
|
2022-03-26 21:49:23 +00:00
|
|
|
#[error("Failed to generate path")]
|
2021-10-28 04:06:03 +00:00
|
|
|
PathGenerator(#[from] storage_path_generator::PathError),
|
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
#[error("Failed to interact with sled repo")]
|
|
|
|
Sled(#[from] crate::repo::sled::Error),
|
2021-10-28 04:06:03 +00:00
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
#[error("Failed to parse string")]
|
2021-10-28 04:06:03 +00:00
|
|
|
Utf8(#[from] FromUtf8Error),
|
|
|
|
|
|
|
|
#[error("Invalid length")]
|
|
|
|
Length,
|
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
#[error("Storage error")]
|
2021-10-28 04:06:03 +00:00
|
|
|
Anyhow(#[from] anyhow::Error),
|
|
|
|
}
|
|
|
|
|
2021-10-29 02:07:31 +00:00
|
|
|
#[derive(Clone)]
|
2021-10-28 04:06:03 +00:00
|
|
|
pub(crate) struct ObjectStore {
|
|
|
|
path_gen: Generator,
|
2022-03-26 21:49:23 +00:00
|
|
|
repo: Repo,
|
2021-10-28 04:06:03 +00:00
|
|
|
bucket: Bucket,
|
2021-11-02 22:21:00 +00:00
|
|
|
client: reqwest::Client,
|
2021-10-28 04:06:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pin_project_lite::pin_project! {
|
|
|
|
struct IoError<S> {
|
|
|
|
#[pin]
|
|
|
|
inner: S,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[async_trait::async_trait(?Send)]
|
|
|
|
impl Store for ObjectStore {
|
|
|
|
type Error = ObjectError;
|
|
|
|
type Identifier = ObjectId;
|
|
|
|
type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
|
|
|
|
|
2021-10-29 01:59:11 +00:00
|
|
|
#[tracing::instrument(skip(reader))]
|
2021-10-28 04:06:03 +00:00
|
|
|
async fn save_async_read<Reader>(
|
|
|
|
&self,
|
|
|
|
reader: &mut Reader,
|
|
|
|
) -> Result<Self::Identifier, Self::Error>
|
|
|
|
where
|
|
|
|
Reader: AsyncRead + Unpin,
|
|
|
|
{
|
2022-03-26 21:49:23 +00:00
|
|
|
let path = self.next_file().await?;
|
2021-10-28 04:06:03 +00:00
|
|
|
|
2021-11-02 22:21:00 +00:00
|
|
|
self.bucket
|
|
|
|
.put_object_stream(&self.client, reader, &path)
|
|
|
|
.await?;
|
2021-10-28 04:06:03 +00:00
|
|
|
|
|
|
|
Ok(ObjectId::from_string(path))
|
|
|
|
}
|
|
|
|
|
2021-10-29 01:59:11 +00:00
|
|
|
#[tracing::instrument(skip(bytes))]
|
2022-03-26 21:49:23 +00:00
|
|
|
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
|
|
|
|
let path = self.next_file().await?;
|
2021-10-28 04:06:03 +00:00
|
|
|
|
2021-11-02 22:21:00 +00:00
|
|
|
self.bucket.put_object(&self.client, &path, &bytes).await?;
|
2021-10-28 04:06:03 +00:00
|
|
|
|
|
|
|
Ok(ObjectId::from_string(path))
|
|
|
|
}
|
|
|
|
|
2021-10-29 01:59:11 +00:00
|
|
|
#[tracing::instrument]
|
2021-10-28 04:06:03 +00:00
|
|
|
async fn to_stream(
|
|
|
|
&self,
|
|
|
|
identifier: &Self::Identifier,
|
|
|
|
from_start: Option<u64>,
|
|
|
|
len: Option<u64>,
|
|
|
|
) -> Result<Self::Stream, Self::Error> {
|
|
|
|
let path = identifier.as_str();
|
|
|
|
|
|
|
|
let start = from_start.unwrap_or(0);
|
|
|
|
let end = len.map(|len| start + len);
|
|
|
|
|
2021-11-02 22:21:00 +00:00
|
|
|
let request = Client::request(
|
|
|
|
&self.client,
|
|
|
|
&self.bucket,
|
|
|
|
path,
|
|
|
|
Command::GetObjectRange { start, end },
|
|
|
|
);
|
2021-10-28 04:06:03 +00:00
|
|
|
|
|
|
|
let response = request.response().await?;
|
|
|
|
|
|
|
|
Ok(Box::pin(io_error(response.bytes_stream())))
|
|
|
|
}
|
|
|
|
|
2021-10-29 01:59:11 +00:00
|
|
|
#[tracing::instrument(skip(writer))]
|
2021-10-28 04:06:03 +00:00
|
|
|
async fn read_into<Writer>(
|
|
|
|
&self,
|
|
|
|
identifier: &Self::Identifier,
|
|
|
|
writer: &mut Writer,
|
|
|
|
) -> Result<(), std::io::Error>
|
|
|
|
where
|
2021-10-31 17:35:11 +00:00
|
|
|
Writer: AsyncWrite + Send + Unpin,
|
2021-10-28 04:06:03 +00:00
|
|
|
{
|
2021-10-31 17:35:11 +00:00
|
|
|
let path = identifier.as_str();
|
2021-10-28 04:06:03 +00:00
|
|
|
|
2021-10-31 17:35:11 +00:00
|
|
|
self.bucket
|
2021-11-02 22:21:00 +00:00
|
|
|
.get_object_stream(&self.client, path, writer)
|
2021-10-31 17:35:11 +00:00
|
|
|
.await
|
|
|
|
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, Self::Error::from(e)))?;
|
2021-10-28 04:06:03 +00:00
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2021-10-29 01:59:11 +00:00
|
|
|
#[tracing::instrument]
|
2021-10-28 04:06:03 +00:00
|
|
|
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Self::Error> {
|
|
|
|
let path = identifier.as_str();
|
|
|
|
|
2021-11-02 22:21:00 +00:00
|
|
|
let (head, _) = self.bucket.head_object(&self.client, path).await?;
|
2021-10-28 04:06:03 +00:00
|
|
|
let length = head.content_length.ok_or(ObjectError::Length)?;
|
|
|
|
|
|
|
|
Ok(length as u64)
|
|
|
|
}
|
|
|
|
|
2021-10-29 01:59:11 +00:00
|
|
|
#[tracing::instrument]
|
2021-10-28 04:06:03 +00:00
|
|
|
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error> {
|
|
|
|
let path = identifier.as_str();
|
|
|
|
|
2021-11-02 22:21:00 +00:00
|
|
|
self.bucket.delete_object(&self.client, path).await?;
|
2021-10-28 04:06:03 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl ObjectStore {
|
2021-11-23 22:31:15 +00:00
|
|
|
#[allow(clippy::too_many_arguments)]
|
2022-03-26 21:49:23 +00:00
|
|
|
pub(crate) async fn build(
|
2021-10-28 04:06:03 +00:00
|
|
|
bucket_name: &str,
|
|
|
|
region: Region,
|
|
|
|
access_key: Option<String>,
|
|
|
|
secret_key: Option<String>,
|
|
|
|
security_token: Option<String>,
|
|
|
|
session_token: Option<String>,
|
2022-03-26 21:49:23 +00:00
|
|
|
repo: Repo,
|
2021-11-02 22:21:00 +00:00
|
|
|
client: reqwest::Client,
|
|
|
|
) -> Result<ObjectStore, ObjectError> {
|
2022-03-26 21:49:23 +00:00
|
|
|
let path_gen = init_generator(&repo).await?;
|
2021-10-28 04:06:03 +00:00
|
|
|
|
|
|
|
Ok(ObjectStore {
|
|
|
|
path_gen,
|
2022-03-26 21:49:23 +00:00
|
|
|
repo,
|
2021-10-29 01:59:11 +00:00
|
|
|
bucket: Bucket::new_with_path_style(
|
2021-10-28 04:06:03 +00:00
|
|
|
bucket_name,
|
2021-10-29 01:59:11 +00:00
|
|
|
match region {
|
|
|
|
Region::Custom { endpoint, .. } => Region::Custom {
|
|
|
|
region: String::from(""),
|
|
|
|
endpoint,
|
|
|
|
},
|
|
|
|
region => region,
|
|
|
|
},
|
2021-10-28 04:06:03 +00:00
|
|
|
Credentials {
|
|
|
|
access_key,
|
|
|
|
secret_key,
|
|
|
|
security_token,
|
|
|
|
session_token,
|
|
|
|
},
|
|
|
|
)?,
|
2021-11-02 22:21:00 +00:00
|
|
|
client,
|
2021-10-28 04:06:03 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
async fn next_directory(&self) -> Result<Path, ObjectError> {
|
2021-10-28 04:06:03 +00:00
|
|
|
let path = self.path_gen.next();
|
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
match self.repo {
|
|
|
|
Repo::Sled(ref sled_repo) => {
|
|
|
|
sled_repo
|
|
|
|
.set(GENERATOR_KEY, path.to_be_bytes().into())
|
|
|
|
.await?;
|
|
|
|
}
|
|
|
|
}
|
2021-10-28 04:06:03 +00:00
|
|
|
|
|
|
|
Ok(path)
|
|
|
|
}
|
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
async fn next_file(&self) -> Result<String, ObjectError> {
|
|
|
|
let path = self.next_directory().await?.to_strings().join("/");
|
|
|
|
let filename = uuid::Uuid::new_v4().to_string();
|
2021-10-28 04:06:03 +00:00
|
|
|
|
2021-10-29 01:59:11 +00:00
|
|
|
Ok(format!("{}/{}", path, filename))
|
2021-10-28 04:06:03 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-26 21:49:23 +00:00
|
|
|
async fn init_generator(repo: &Repo) -> Result<Generator, ObjectError> {
|
|
|
|
match repo {
|
|
|
|
Repo::Sled(sled_repo) => {
|
|
|
|
if let Some(ivec) = sled_repo.get(GENERATOR_KEY).await? {
|
|
|
|
Ok(Generator::from_existing(
|
|
|
|
storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
|
|
|
|
))
|
|
|
|
} else {
|
|
|
|
Ok(Generator::new())
|
|
|
|
}
|
|
|
|
}
|
2021-10-28 04:06:03 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn io_error<S, T, E>(stream: S) -> impl Stream<Item = std::io::Result<T>>
|
|
|
|
where
|
|
|
|
S: Stream<Item = Result<T, E>>,
|
|
|
|
E: Into<Box<dyn std::error::Error + Send + Sync>>,
|
|
|
|
{
|
|
|
|
IoError { inner: stream }
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<S, T, E> Stream for IoError<S>
|
|
|
|
where
|
|
|
|
S: Stream<Item = Result<T, E>>,
|
|
|
|
E: Into<Box<dyn std::error::Error + Send + Sync>>,
|
|
|
|
{
|
|
|
|
type Item = std::io::Result<T>;
|
|
|
|
|
|
|
|
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
|
|
|
let this = self.as_mut().project();
|
|
|
|
|
|
|
|
this.inner.poll_next(cx).map(|opt| {
|
|
|
|
opt.map(|res| res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)))
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
2021-10-29 02:07:31 +00:00
|
|
|
|
|
|
|
impl std::fmt::Debug for ObjectStore {
|
|
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
|
|
f.debug_struct("ObjectStore")
|
|
|
|
.field("path_gen", &self.path_gen)
|
|
|
|
.field("bucket", &self.bucket.name)
|
|
|
|
.field("region", &self.bucket.region)
|
|
|
|
.finish()
|
|
|
|
}
|
|
|
|
}
|