asonix/pict-rs
1
1
Fork 1
Code Issues 2 Pull requests Actions Projects Releases 133 Wiki Activity

Update to actix-web 4.0.0-beta.3

Browse source
This commit is contained in:
asonix 2021-02-10 16:57:42 -06:00
parent d51a73cee9
commit 79b26a1dda
8 changed files with 376 additions and 748 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

927
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

19
Cargo.toml
View file

@ -1,7 +1,7 @@
[package]
name = "pict-rs"
description = "A simple image hosting service"
version = "0.3.0-alpha.6"
version = "0.3.0-alpha.7"
authors = ["asonix <asonix@asonix.dog>"]
license = "AGPL-3.0"
readme = "README.md"
@ -11,24 +11,23 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
actix-form-data = "0.5.0"
actix-fs = { git = "https://git.asonix.dog/asonix/actix-fs", branch = "main" }
actix-rt = "1.1.1"
actix-web = { version = "3.0.1", default-features = false, features = ["rustls"] }
actix-form-data = "0.6.0-beta.1"
actix-fs = { git = "https://git.asonix.dog/asonix/actix-fs", branch = "asonix/actix-rt-2" }
actix-rt = "2.0.2"
actix-web = { version = "4.0.0-beta.3", default-features = false, features = ["rustls", "compress"] }
anyhow = "1.0"
async-stream = "0.3.0"
base64 = "0.13.0"
bytes = "0.5"
futures = "0.3.4"
magick_rust = { version = "0.14.0", git = "https://github.com/nlfiedler/magick-rust" }
mime = "0.3.1"
once_cell = "1.4.0"
rand = "0.7.3"
rand = "0.8.0"
rexiv2 = "0.9.1"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
sha2 = "0.9.0"
sled = { version = "0.34.4" }
sled = { version = "0.34.6" }
structopt = "0.3.14"
thiserror = "1.0"
time = { version = "0.2.23", features = ["serde"] }
@ -44,3 +43,7 @@ features = ["codec", "filter", "device", "format", "resampling", "postprocessing
[dependencies.ffmpeg-sys-next]
version = "4.3.5"
git = "https://github.com/baadc0de/rust-ffmpeg-sys"
[patch.crates-io]
ffmpeg-sys-next = { git = "https://github.com/baadc0de/rust-ffmpeg-sys", branch = "master" }

15
src/error.rs
View file

@ -100,15 +100,9 @@ impl From<actix_form_data::Error> for UploadError {
}
}
impl<T> From<actix_web::error::BlockingError<T>> for UploadError
where
T: Into<UploadError> + std::fmt::Debug,
{
fn from(e: actix_web::error::BlockingError<T>) -> Self {
match e {
actix_web::error::BlockingError::Error(e) => e.into(),
_ => UploadError::Canceled,
}
impl From<actix_web::error::BlockingError> for UploadError {
fn from(_: actix_web::error::BlockingError) -> Self {
UploadError::Canceled
}
}
@ -128,6 +122,7 @@ impl ResponseError for UploadError {
}
fn error_response(&self) -> HttpResponse {
HttpResponse::build(self.status_code()).json(serde_json::json!({ "msg": self.to_string() }))
HttpResponse::build(self.status_code())
.json(&serde_json::json!({ "msg": self.to_string() }))
}
}

39
src/main.rs
View file

@ -7,7 +7,6 @@ use actix_web::{
middleware::{Compress, Logger},
web, App, HttpResponse, HttpServer,
};
use bytes::Bytes;
use futures::stream::{once, Stream, TryStreamExt};
use once_cell::sync::Lazy;
use std::{
@ -50,6 +49,7 @@ static TMP_DIR: Lazy<PathBuf> = Lazy::new(|| {
let tmp_nonce = Alphanumeric
.sample_iter(&mut rng)
.take(7)
.map(char::from)
.collect::<String>();
let mut path = std::env::temp_dir();
@ -93,7 +93,7 @@ async fn safe_create_parent(path: PathBuf) -> Result<(), UploadError> {
// Try writing to a file
#[instrument(skip(bytes))]
async fn safe_save_file(path: PathBuf, bytes: bytes::Bytes) -> Result<(), UploadError> {
async fn safe_save_file(path: PathBuf, bytes: web::Bytes) -> Result<(), UploadError> {
if let Some(path) = path.parent() {
// create the directory for the file
debug!("Creating directory {:?}", path);
@ -132,7 +132,11 @@ pub(crate) fn tmp_file() -> PathBuf {
let limit: usize = 10;
let rng = rand::thread_rng();
let s: String = Alphanumeric.sample_iter(rng).take(limit).collect();
let s: String = Alphanumeric
.sample_iter(rng)
.take(limit)
.map(char::from)
.collect();
let name = format!("{}.tmp", s);
@ -203,7 +207,7 @@ async fn upload(
}
}
Ok(HttpResponse::Created().json(serde_json::json!({
Ok(HttpResponse::Created().json(&serde_json::json!({
"msg": "ok",
"files": files
})))
@ -250,7 +254,7 @@ async fn download(
new_details
};
Ok(HttpResponse::Created().json(serde_json::json!({
Ok(HttpResponse::Created().json(&serde_json::json!({
"msg": "ok",
"files": [{
"file": alias,
@ -334,7 +338,7 @@ async fn process_details(
let details = details.ok_or(UploadError::NoFiles)?;
Ok(HttpResponse::Ok().json(details))
Ok(HttpResponse::Ok().json(&details))
}
/// Process files
@ -447,7 +451,7 @@ async fn process(
let range = range_header.ranges().next().unwrap();
let mut builder = HttpResponse::PartialContent();
builder.set(range.to_content_range(img_bytes.len() as u64));
builder.insert_header(range.to_content_range(img_bytes.len() as u64));
(builder, range.chop_bytes(img_bytes))
} else {
return Err(UploadError::Range);
@ -499,7 +503,7 @@ async fn details(
new_details
};
Ok(HttpResponse::Ok().json(details))
Ok(HttpResponse::Ok().json(&details))
}
/// Serve files
@ -550,7 +554,7 @@ async fn ranged_file_resp(
let range = range_header.ranges().next().unwrap();
let mut builder = HttpResponse::PartialContent();
builder.set(range.to_content_range(meta.len()));
builder.insert_header(range.to_content_range(meta.len()));
(builder, range.chop_file(file).await?)
} else {
@ -562,7 +566,8 @@ async fn ranged_file_resp(
let stream = actix_fs::read_to_stream(path)
.await?
.map_err(UploadError::from);
let stream: Pin<Box<dyn Stream<Item = Result<Bytes, UploadError>>>> = Box::pin(stream);
let stream: Pin<Box<dyn Stream<Item = Result<web::Bytes, UploadError>>>> =
Box::pin(stream);
(HttpResponse::Ok(), stream)
}
};
@ -585,18 +590,18 @@ fn srv_response<S, E>(
modified: SystemTime,
) -> HttpResponse
where
S: Stream<Item = Result<bytes::Bytes, E>> + Unpin + 'static,
S: Stream<Item = Result<web::Bytes, E>> + Unpin + 'static,
E: 'static,
actix_web::Error: From<E>,
{
builder
.set(LastModified(modified.into()))
.set(CacheControl(vec![
.insert_header(LastModified(modified.into()))
.insert_header(CacheControl(vec![
CacheDirective::Public,
CacheDirective::MaxAge(expires),
CacheDirective::Extension("immutable".to_owned(), None),
]))
.set_header(ACCEPT_RANGES, "bytes")
.insert_header((ACCEPT_RANGES, "bytes"))
.content_type(ext.to_string())
.streaming(stream)
}
@ -623,7 +628,7 @@ async fn purge(
.await?;
}
Ok(HttpResponse::Ok().json(serde_json::json!({
Ok(HttpResponse::Ok().json(&serde_json::json!({
"msg": "ok",
"aliases": aliases
})))
@ -638,7 +643,7 @@ async fn aliases(
FileOrAlias::Alias { alias } => upload_manager.aliases_by_alias(alias).await?,
};
Ok(HttpResponse::Ok().json(serde_json::json!({
Ok(HttpResponse::Ok().json(&serde_json::json!({
"msg": "ok",
"aliases": aliases,
})))
@ -655,7 +660,7 @@ async fn filename_by_alias(
) -> Result<HttpResponse, UploadError> {
let filename = upload_manager.from_alias(query.into_inner().alias).await?;
Ok(HttpResponse::Ok().json(serde_json::json!({
Ok(HttpResponse::Ok().json(&serde_json::json!({
"msg": "ok",
"filename": filename,
})))

31
src/middleware.rs
View file

@ -26,16 +26,16 @@ impl ResponseError for ApiError {
}
fn error_response(&self) -> HttpResponse {
HttpResponse::build(self.status_code()).json(serde_json::json!({ "msg": self.to_string() }))
HttpResponse::build(self.status_code())
.json(&serde_json::json!({ "msg": self.to_string() }))
}
}
impl<S> Transform<S> for Tracing
impl<S, Request> Transform<S, Request> for Tracing
where
S: Service,
S: Service<Request>,
S::Future: 'static,
{
type Request = S::Request;
type Response = S::Response;
type Error = S::Error;
type InitError = ();
@ -47,21 +47,20 @@ where
}
}
impl<S> Service for TracingMiddleware<S>
impl<S, Request> Service<Request> for TracingMiddleware<S>
where
S: Service,
S: Service<Request>,
S::Future: 'static,
{
type Request = S::Request;
type Response = S::Response;
type Error = S::Error;
type Future = Instrumented<S::Future>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
fn poll_ready(&self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, req: S::Request) -> Self::Future {
fn call(&self, req: Request) -> Self::Future {
let uuid = Uuid::new_v4();
self.inner
@ -70,12 +69,11 @@ where
}
}
impl<S> Transform<S> for Internal
impl<S> Transform<S, ServiceRequest> for Internal
where
S: Service<Request = ServiceRequest, Error = actix_web::Error>,
S: Service<ServiceRequest, Error = actix_web::Error>,
S::Future: 'static,
{
type Request = S::Request;
type Response = S::Response;
type Error = S::Error;
type InitError = ();
@ -87,21 +85,20 @@ where
}
}
impl<S> Service for InternalMiddleware<S>
impl<S> Service<ServiceRequest> for InternalMiddleware<S>
where
S: Service<Request = ServiceRequest, Error = actix_web::Error>,
S: Service<ServiceRequest, Error = actix_web::Error>,
S::Future: 'static,
{
type Request = S::Request;
type Response = S::Response;
type Error = S::Error;
type Future = LocalBoxFuture<'static, Result<S::Response, S::Error>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
fn poll_ready(&self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.1.poll_ready(cx)
}
fn call(&mut self, req: S::Request) -> Self::Future {
fn call(&self, req: ServiceRequest) -> Self::Future {
if let Some(value) = req.headers().get("x-api-token") {
if value.to_str().is_ok() && value.to_str().ok() == self.0.as_deref() {
let fut = self.1.call(req);

9
src/processor.rs
View file

@ -4,7 +4,6 @@ use crate::{
validate::{ptos, Op},
};
use actix_web::web;
use bytes::Bytes;
use magick_rust::MagickWand;
use std::path::PathBuf;
use tracing::{debug, error, instrument, Span};
@ -372,7 +371,7 @@ pub(crate) async fn prepare_image(
transcode(orig_path, tmpfile, Target::Jpeg).map_err(UploadError::Transcode)
})
.await;
.await?;
if let Err(e) = res {
error!("transcode error: {:?}", e);
@ -395,7 +394,7 @@ pub(crate) async fn process_image(
original_file: PathBuf,
chain: ProcessChain,
format: Format,
) -> Result<Bytes, UploadError> {
) -> Result<web::Bytes, UploadError> {
let original_path_str = ptos(&original_file)?;
let span = Span::current();
@ -414,9 +413,9 @@ pub(crate) async fn process_image(
let vec = wand.op(|w| w.write_image_blob(format.to_magick_format()))?;
drop(entered);
Ok(Bytes::from(vec)) as Result<Bytes, UploadError>
Ok(web::Bytes::from(vec)) as Result<web::Bytes, UploadError>
})
.await?;
.await??;
Ok(bytes)
}

82
src/upload_manager.rs
View file

@ -44,7 +44,7 @@ impl std::fmt::Debug for UploadManager {
}
}
type UploadStream<E> = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, E>>>>;
type UploadStream<E> = Pin<Box<dyn Stream<Item = Result<web::Bytes, E>>>>;
#[derive(Clone)]
pub(crate) struct Serde<T> {
@ -134,7 +134,7 @@ impl Details {
})
})
})
.await?;
.await??;
Ok(Details::now(width as usize, height as usize, mime_type))
}
@ -213,7 +213,7 @@ impl UploadManager {
) -> Result<Self, UploadError> {
let root_clone = root_dir.clone();
// sled automatically creates it's own directories
let db = web::block(move || LatestDb::exists(root_clone).migrate()).await?;
let db = web::block(move || LatestDb::exists(root_clone).migrate()).await??;
root_dir.push("files");
@ -245,13 +245,13 @@ impl UploadManager {
let fname_tree = self.inner.filename_tree.clone();
debug!("Getting hash");
let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes()))
.await?
.await??
.ok_or(UploadError::MissingFilename)?;
let key = variant_key(&hash, &path_string);
let main_tree = self.inner.main_tree.clone();
debug!("Storing variant");
web::block(move || main_tree.insert(key, path_string.as_bytes())).await?;
web::block(move || main_tree.insert(key, path_string.as_bytes())).await??;
debug!("Stored variant");
Ok(())
@ -268,13 +268,13 @@ impl UploadManager {
let fname_tree = self.inner.filename_tree.clone();
debug!("Getting hash");
let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes()))
.await?
.await??
.ok_or(UploadError::MissingFilename)?;
let key = variant_details_key(&hash, &path_string);
let main_tree = self.inner.main_tree.clone();
debug!("Getting details");
let opt = match web::block(move || main_tree.get(key)).await? {
let opt = match web::block(move || main_tree.get(key)).await?? {
Some(ivec) => match serde_json::from_slice(&ivec) {
Ok(details) => Some(details),
Err(_) => None,
@ -297,14 +297,14 @@ impl UploadManager {
let fname_tree = self.inner.filename_tree.clone();
debug!("Getting hash");
let hash: sled::IVec = web::block(move || fname_tree.get(filename.as_bytes()))
.await?
.await??
.ok_or(UploadError::MissingFilename)?;
let key = variant_details_key(&hash, &path_string);
let main_tree = self.inner.main_tree.clone();
let details_value = serde_json::to_string(details)?;
debug!("Storing details");
web::block(move || main_tree.insert(key, details_value.as_bytes())).await?;
web::block(move || main_tree.insert(key, details_value.as_bytes())).await??;
debug!("Stored details");
Ok(())
@ -317,7 +317,7 @@ impl UploadManager {
) -> Result<Vec<String>, UploadError> {
let fname_tree = self.inner.filename_tree.clone();
let hash = web::block(move || fname_tree.get(filename.as_bytes()))
.await?
.await??
.ok_or(UploadError::MissingAlias)?;
self.aliases_by_hash(&hash).await
@ -327,7 +327,7 @@ impl UploadManager {
pub(crate) async fn aliases_by_alias(&self, alias: String) -> Result<Vec<String>, UploadError> {
let alias_tree = self.inner.alias_tree.clone();
let hash = web::block(move || alias_tree.get(alias.as_bytes()))
.await?
.await??
.ok_or(UploadError::MissingFilename)?;
self.aliases_by_hash(&hash).await
@ -342,7 +342,7 @@ impl UploadManager {
.values()
.collect::<Result<Vec<_>, _>>()
})
.await?;
.await??;
debug!("Got {} aliases for hash", aliases.len());
let aliases = aliases
@ -362,7 +362,7 @@ impl UploadManager {
let token_key = delete_key(&alias);
let alias_tree = self.inner.alias_tree.clone();
let token = web::block(move || alias_tree.get(token_key.as_bytes()))
.await?
.await??
.ok_or(UploadError::MissingAlias)?;
self.delete(alias, String::from_utf8(token.to_vec())?).await
@ -415,7 +415,7 @@ impl UploadManager {
Ok(hash)
})
})
.await?;
.await??;
// -- CHECK IF ANY OTHER ALIASES EXIST --
let main_tree = self.inner.main_tree.clone();
@ -424,7 +424,7 @@ impl UploadManager {
let any_aliases = web::block(move || {
Ok(main_tree.range(start..end).next().is_some()) as Result<bool, UploadError>
})
.await?;
.await??;
// Bail if there are existing aliases
if any_aliases {
@ -437,7 +437,7 @@ impl UploadManager {
let hash2 = hash.clone();
debug!("Deleting hash -> filename mapping");
let filename = web::block(move || main_tree.remove(&hash2))
.await?
.await??
.ok_or(UploadError::MissingFile)?;
// -- DELETE FILES --
@ -468,7 +468,11 @@ impl UploadManager {
debug!("Generating delete token");
use rand::distributions::{Alphanumeric, Distribution};
let rng = rand::thread_rng();
let s: String = Alphanumeric.sample_iter(rng).take(10).collect();
let s: String = Alphanumeric
.sample_iter(rng)
.take(10)
.map(char::from)
.collect();
let delete_token = s.clone();
debug!("Saving delete token");
@ -481,7 +485,7 @@ impl UploadManager {
Some(s.as_bytes()),
)
})
.await?;
.await??;
if let Err(sled::CompareAndSwapError {
current: Some(ivec),
@ -579,13 +583,13 @@ impl UploadManager {
let tree = self.inner.alias_tree.clone();
debug!("Getting hash from alias");
let hash = web::block(move || tree.get(alias.as_bytes()))
.await?
.await??
.ok_or(UploadError::MissingAlias)?;
let main_tree = self.inner.main_tree.clone();
debug!("Getting filename from hash");
let filename = web::block(move || main_tree.get(hash))
.await?
.await??
.ok_or(UploadError::MissingFile)?;
let filename = String::from_utf8(filename.to_vec())?;
@ -610,7 +614,7 @@ impl UploadManager {
let fname_tree = self.inner.filename_tree.clone();
debug!("Deleting filename -> hash mapping");
let hash = web::block(move || fname_tree.remove(filename))
.await?
.await??
.ok_or(UploadError::MissingFile)?;
let (start, end) = variant_key_bounds(&hash);
@ -624,13 +628,13 @@ impl UploadManager {
Ok(keys) as Result<Vec<sled::IVec>, UploadError>
})
.await?;
.await??;
debug!("{} files prepared for deletion", keys.len());
for key in keys {
let main_tree = self.inner.main_tree.clone();
if let Some(path) = web::block(move || main_tree.remove(key)).await? {
if let Some(path) = web::block(move || main_tree.remove(key)).await?? {
let s = String::from_utf8_lossy(&path);
debug!("Deleting {}", s);
// ignore json objects
@ -685,12 +689,12 @@ impl UploadManager {
hasher.update(&bytes);
Ok(hasher) as Result<_, UploadError>
})
.await?;
.await??;
}
let hash =
web::block(move || Ok(hasher.finalize_reset().to_vec()) as Result<_, UploadError>)
.await?;
.await??;
Ok(Hash::new(hash))
}
@ -715,7 +719,7 @@ impl UploadManager {
Some(filename2.as_bytes()),
)
})
.await?;
.await??;
if let Err(sled::CompareAndSwapError {
current: Some(ivec),
@ -730,7 +734,7 @@ impl UploadManager {
let fname_tree = self.inner.filename_tree.clone();
let filename2 = filename.clone();
debug!("Saving filename -> hash relation");
web::block(move || fname_tree.insert(filename2, hash.inner)).await?;
web::block(move || fname_tree.insert(filename2, hash.inner)).await??;
Ok((Dup::New, filename))
}
@ -741,11 +745,15 @@ impl UploadManager {
let image_dir = self.image_dir();
use rand::distributions::{Alphanumeric, Distribution};
let mut limit: usize = 10;
let rng = rand::thread_rng();
let mut rng = rand::thread_rng();
loop {
debug!("Filename generation loop");
let mut path = image_dir.clone();
let s: String = Alphanumeric.sample_iter(rng).take(limit).collect();
let s: String = Alphanumeric
.sample_iter(&mut rng)
.take(limit)
.map(char::from)
.collect();
let filename = file_name(s, content_type.clone())?;
@ -799,7 +807,7 @@ impl UploadManager {
loop {
debug!("hash -> alias save loop");
let db = self.inner.db.clone();
let id = web::block(move || db.generate_id()).await?.to_string();
let id = web::block(move || db.generate_id()).await??.to_string();
let key = alias_key(&hash.inner, &id);
let main_tree = self.inner.main_tree.clone();
@ -808,13 +816,13 @@ impl UploadManager {
let res = web::block(move || {
main_tree.compare_and_swap(key, None as Option<sled::IVec>, Some(alias2.as_bytes()))
})
.await?;
.await??;
if res.is_ok() {
let alias_tree = self.inner.alias_tree.clone();
let key = alias_id_key(&alias);
debug!("Saving alias -> id mapping");
web::block(move || alias_tree.insert(key.as_bytes(), id.as_bytes())).await?;
web::block(move || alias_tree.insert(key.as_bytes(), id.as_bytes())).await??;
break;
}
@ -834,10 +842,14 @@ impl UploadManager {
) -> Result<String, UploadError> {
use rand::distributions::{Alphanumeric, Distribution};
let mut limit: usize = 10;
let rng = rand::thread_rng();
let mut rng = rand::thread_rng();
loop {
debug!("Alias gen loop");
let s: String = Alphanumeric.sample_iter(rng).take(limit).collect();
let s: String = Alphanumeric
.sample_iter(&mut rng)
.take(limit)
.map(char::from)
.collect();
let alias = file_name(s, content_type.clone())?;
let res = self.save_alias(hash, &alias).await?;
@ -866,7 +878,7 @@ impl UploadManager {
let res = web::block(move || {
tree.compare_and_swap(alias.as_bytes(), None as Option<sled::IVec>, Some(vec))
})
.await?;
.await??;
if res.is_err() {
warn!("Duplicate alias");

2
src/validate/mod.rs
View file

@ -199,7 +199,7 @@ pub(crate) async fn validate_image(
drop(entered);
Ok(content_type) as Result<mime::Mime, UploadError>
})
.await?;
.await??;
Ok(content_type)
}