From fc1ae4be4961a08df6721591e2df63dbfdabfff4 Mon Sep 17 00:00:00 2001 From: asonix Date: Sun, 7 Jun 2020 10:59:58 -0500 Subject: [PATCH] Add endpoint for downloading remote images --- Cargo.lock | 121 +++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 3 +- README.md | 2 + src/error.rs | 18 +++++++ src/main.rs | 57 ++++++++++++++++++-- src/upload_manager.rs | 40 +++++++------- 6 files changed, 216 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a4de9f..a39a8d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,8 +30,11 @@ dependencies = [ "futures-util", "http", "log", + "rustls", + "tokio-rustls", "trust-dns-proto", "trust-dns-resolver", + "webpki", ] [[package]] @@ -73,8 +76,9 @@ dependencies = [ "actix-rt", "actix-service", "actix-threadpool", + "actix-tls", "actix-utils", - "base64", + "base64 0.12.1", "bitflags", "brotli2", "bytes", @@ -236,6 +240,10 @@ dependencies = [ "either", "futures", "log", + "rustls", + "tokio-rustls", + "webpki", + "webpki-roots", ] [[package]] @@ -286,6 +294,7 @@ dependencies = [ "mime", "pin-project", "regex", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -389,7 +398,7 @@ dependencies = [ "actix-http", "actix-rt", "actix-service", - "base64", + "base64 0.12.1", "bytes", "derive_more", "futures-core", @@ -397,6 +406,7 @@ dependencies = [ "mime", "percent-encoding", "rand", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -421,6 +431,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + [[package]] name = "base64" version = "0.12.1" @@ -1047,6 +1063,15 @@ dependencies = [ "rayon", ] +[[package]] +name = "js-sys" +version = "0.3.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -1330,6 +1355,7 @@ dependencies = [ "log", "mime", "rand", + "serde", "serde_json", "sha2", "sled", @@ -1548,6 +1574,21 @@ dependencies = [ "quick-error", ] +[[package]] +name = "ring" +version = "0.16.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06b3fefa4f12272808f809a0af618501fdaba41a58963c5fb72238ab0be09603" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.8", +] + [[package]] name = "rustc-demangle" version = "0.1.16" @@ -1563,6 +1604,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustls" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" +dependencies = [ + "base64 0.11.0", + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "ryu" version = "1.0.5" @@ -1581,6 +1635,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "semver" version = "0.9.0" @@ -1720,6 +1784,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "standback" version = "0.2.9" @@ -1962,6 +2032,18 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "tokio-rustls" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4" +dependencies = [ + "futures-core", + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-util" version = "0.2.0" @@ -2088,6 +2170,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.1.1" @@ -2171,6 +2259,35 @@ version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd" +[[package]] +name = "web-sys" +version = "0.3.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eff4b7516a57307f9349c64bf34caa34b940b66fed4b2fb3136cb7386e5739" +dependencies = [ + "webpki", +] + [[package]] name = "widestring" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 83d7105..995213b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ edition = "2018" actix-form-data = { git = "https://git.asonix.dog/Aardwolf/actix-form-data" } actix-fs = { git = "https://git.asonix.dog/asonix/actix-fs" } actix-rt = "1.1.1" -actix-web = "3.0.0-alpha.2" +actix-web = { version = "3.0.0-alpha.2", features = ["rustls"] } anyhow = "1.0" bytes = "0.5" env_logger = "0.7" @@ -23,6 +23,7 @@ image = "0.23.4" log = "0.4" mime = "0.3.1" rand = "0.7.3" +serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.8.2" sled = "0.32.0-rc1" diff --git a/README.md b/README.md index af48938..1a6998b 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,8 @@ pict-rs offers four endpoints: "msg": "ok" } ``` +- `GET /image/download?url=...` Download an image from a remote server, returning the same JSON + payload as the `POST` endpoint - `GET /image/{file}` for getting a full-resolution image. `file` here is the `file` key from the `/image` endpoint's JSON - `GET /image/{size}/{file}` where `size` is a positive integer. This endpoint is for accessing diff --git a/src/error.rs b/src/error.rs index 46060d1..d53b3bd 100644 --- a/src/error.rs +++ b/src/error.rs @@ -43,6 +43,24 @@ pub enum UploadError { #[error("Uploaded content could not be validated as an image")] InvalidImage(image::error::ImageError), + + #[error("Unsupported image format")] + UnsupportedFormat, + + #[error("Unable to download image, bad response {0}")] + Download(actix_web::http::StatusCode), + + #[error("Unable to download image, {0}")] + Payload(#[from] actix_web::client::PayloadError), + + #[error("Unable to send request, {0}")] + SendRequest(String), +} + +impl From for UploadError { + fn from(e: actix_web::client::SendRequestError) -> Self { + UploadError::SendRequest(e.to_string()) + } } impl From> for UploadError { diff --git a/src/main.rs b/src/main.rs index e00011e..38e0f1c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ use actix_form_data::{Field, Form, Value}; use actix_web::{ + client::Client, guard, http::header::{CacheControl, CacheDirective}, middleware::{Compress, Logger}, @@ -100,12 +101,43 @@ async fn upload( let delete_token = manager.delete_token(saved_as.to_owned()).await?; files.push(serde_json::json!({ "file": saved_as, - "delete_token": delete_token, + "delete_token": delete_token })); } } - Ok(HttpResponse::Created().json(serde_json::json!({ "msg": "ok", "files": files }))) + Ok(HttpResponse::Created().json(serde_json::json!({ + "msg": "ok", + "files": files + }))) +} + +/// download an image from a URL +async fn download( + client: web::Data, + manager: web::Data, + query: web::Query, +) -> Result { + let mut res = client.get(&query.url).send().await?; + + if !res.status().is_success() { + return Err(UploadError::Download(res.status())); + } + + let fut = res.body().limit(40 * MEGABYTES); + + let stream = Box::pin(futures::stream::once(fut)); + + let alias = manager.upload(stream).await?; + let delete_token = manager.delete_token(alias.clone()).await?; + + Ok(HttpResponse::Created().json(serde_json::json!({ + "msg": "ok", + "files": [{ + "file": alias, + "delete_token": delete_token + }] + }))) } async fn delete( @@ -234,6 +266,11 @@ where .streaming(stream.err_into()) } +#[derive(Debug, serde::Deserialize)] +struct UrlQuery { + url: String, +} + #[actix_rt::main] async fn main() -> Result<(), anyhow::Error> { let config = Config::from_args(); @@ -250,18 +287,29 @@ async fn main() -> Result<(), anyhow::Error> { .max_file_size(40 * MEGABYTES) .field( "images", - Field::array(Field::file(move |filename, content_type, stream| { + Field::array(Field::file(move |_, _, stream| { let manager = manager2.clone(); - async move { manager.upload(filename, content_type, stream).await } + async move { + manager.upload(stream).await.map(|alias| { + let mut path = PathBuf::new(); + path.push(alias); + Some(path) + }) + } })), ); HttpServer::new(move || { + let client = Client::build() + .header("User-Agent", "pict-rs v0.1.0-master") + .finish(); + App::new() .wrap(Logger::default()) .wrap(Compress::default()) .data(manager.clone()) + .data(client) .service( web::scope("/image") .service( @@ -270,6 +318,7 @@ async fn main() -> Result<(), anyhow::Error> { .wrap(form.clone()) .route(web::post().to(upload)), ) + .service(web::resource("/download").route(web::get().to(download))) .service(web::resource("/{filename}").route(web::get().to(serve))) .service( web::resource("/delete/{delete_token}/{filename}") diff --git a/src/upload_manager.rs b/src/upload_manager.rs index a300439..406ee82 100644 --- a/src/upload_manager.rs +++ b/src/upload_manager.rs @@ -18,7 +18,7 @@ struct UploadManagerInner { db: sled::Db, } -type UploadStream = Pin>>>; +type UploadStream = Pin>>>; enum Dup { Exists, @@ -176,16 +176,10 @@ impl UploadManager { } /// Upload the file, discarding bytes if it's already present, or saving if it's new - pub(crate) async fn upload( - &self, - _filename: String, - content_type: mime::Mime, - mut stream: UploadStream, - ) -> Result, UploadError> { - if ACCEPTED_MIMES.iter().all(|valid| *valid != content_type) { - return Err(UploadError::ContentType(content_type)); - } - + pub(crate) async fn upload(&self, mut stream: UploadStream) -> Result + where + UploadError: From, + { let (img, format) = { // -- READ IN BYTES FROM CLIENT -- let mut bytes = bytes::BytesMut::new(); @@ -208,7 +202,11 @@ impl UploadManager { .format .as_ref() .map(|f| (f.to_image_format(), f.to_mime())) - .unwrap_or((format, content_type)); + .unwrap_or((format.clone(), valid_format(format)?)); + + if ACCEPTED_MIMES.iter().all(|valid| *valid != content_type) { + return Err(UploadError::ContentType(content_type)); + } let bytes: bytes::Bytes = { let mut bytes = std::io::Cursor::new(vec![]); @@ -226,9 +224,7 @@ impl UploadManager { // bail early with alias to existing file if this is a duplicate if dup.exists() { - let mut path = PathBuf::new(); - path.push(alias); - return Ok(Some(path)); + return Ok(alias); } // -- WRITE NEW FILE -- @@ -238,9 +234,7 @@ impl UploadManager { safe_save_file(real_path, bytes).await?; // Return alias to file - let mut path = PathBuf::new(); - path.push(alias); - Ok(Some(path)) + Ok(alias) } pub(crate) async fn from_alias(&self, alias: String) -> Result { @@ -440,3 +434,13 @@ fn alias_id_key(alias: &str) -> String { fn delete_key(alias: &str) -> String { format!("{}/delete", alias) } + +fn valid_format(format: image::ImageFormat) -> Result { + match format { + image::ImageFormat::Jpeg => Ok(mime::IMAGE_JPEG), + image::ImageFormat::Png => Ok(mime::IMAGE_PNG), + image::ImageFormat::Gif => Ok(mime::IMAGE_GIF), + image::ImageFormat::Bmp => Ok(mime::IMAGE_BMP), + _ => Err(UploadError::UnsupportedFormat), + } +}