asonix/pict-rs
1
1
Fork 1
Code Issues 2 Pull requests 3 Actions Projects Releases 133 Wiki Activity

Add endpoint for downloading remote images

Browse source
This commit is contained in:
asonix 2020-06-07 10:59:58 -05:00
parent 1b526f4c1f
commit fc1ae4be49
6 changed files with 216 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

121
Cargo.lock generated
View file

@ -30,8 +30,11 @@ dependencies = [
"futures-util",
"http",
"log",
"rustls",
"tokio-rustls",
"trust-dns-proto",
"trust-dns-resolver",
"webpki",
]
[[package]]
@ -73,8 +76,9 @@ dependencies = [
"actix-rt",
"actix-service",
"actix-threadpool",
"actix-tls",
"actix-utils",
"base64",
"base64 0.12.1",
"bitflags",
"brotli2",
"bytes",
@ -236,6 +240,10 @@ dependencies = [
"either",
"futures",
"log",
"rustls",
"tokio-rustls",
"webpki",
"webpki-roots",
]
[[package]]
@ -286,6 +294,7 @@ dependencies = [
"mime",
"pin-project",
"regex",
"rustls",
"serde",
"serde_json",
"serde_urlencoded",
@ -389,7 +398,7 @@ dependencies = [
"actix-http",
"actix-rt",
"actix-service",
"base64",
"base64 0.12.1",
"bytes",
"derive_more",
"futures-core",
@ -397,6 +406,7 @@ dependencies = [
"mime",
"percent-encoding",
"rand",
"rustls",
"serde",
"serde_json",
"serde_urlencoded",
@ -421,6 +431,12 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1"
[[package]]
name = "base64"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
[[package]]
name = "base64"
version = "0.12.1"
@ -1047,6 +1063,15 @@ dependencies = [
"rayon",
]
[[package]]
name = "js-sys"
version = "0.3.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "kernel32-sys"
version = "0.2.2"
@ -1330,6 +1355,7 @@ dependencies = [
"log",
"mime",
"rand",
"serde",
"serde_json",
"sha2",
"sled",
@ -1548,6 +1574,21 @@ dependencies = [
"quick-error",
]
[[package]]
name = "ring"
version = "0.16.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06b3fefa4f12272808f809a0af618501fdaba41a58963c5fb72238ab0be09603"
dependencies = [
"cc",
"libc",
"once_cell",
"spin",
"untrusted",
"web-sys",
"winapi 0.3.8",
]
[[package]]
name = "rustc-demangle"
version = "0.1.16"
@ -1563,6 +1604,19 @@ dependencies = [
"semver",
]
[[package]]
name = "rustls"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1"
dependencies = [
"base64 0.11.0",
"log",
"ring",
"sct",
"webpki",
]
[[package]]
name = "ryu"
version = "1.0.5"
@ -1581,6 +1635,16 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "sct"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "semver"
version = "0.9.0"
@ -1720,6 +1784,12 @@ dependencies = [
"winapi 0.3.8",
]
[[package]]
name = "spin"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "standback"
version = "0.2.9"
@ -1962,6 +2032,18 @@ dependencies = [
"winapi 0.3.8",
]
[[package]]
name = "tokio-rustls"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4"
dependencies = [
"futures-core",
"rustls",
"tokio",
"webpki",
]
[[package]]
name = "tokio-util"
version = "0.2.0"
@ -2088,6 +2170,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
[[package]]
name = "untrusted"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "url"
version = "2.1.1"
@ -2171,6 +2259,35 @@ version = "0.2.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd"
[[package]]
name = "web-sys"
version = "0.3.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki"
version = "0.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "webpki-roots"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8eff4b7516a57307f9349c64bf34caa34b940b66fed4b2fb3136cb7386e5739"
dependencies = [
"webpki",
]
[[package]]
name = "widestring"
version = "0.4.0"

3
Cargo.toml
View file

@ -14,7 +14,7 @@ edition = "2018"
actix-form-data = { git = "https://git.asonix.dog/Aardwolf/actix-form-data" }
actix-fs = { git = "https://git.asonix.dog/asonix/actix-fs" }
actix-rt = "1.1.1"
actix-web = "3.0.0-alpha.2"
actix-web = { version = "3.0.0-alpha.2", features = ["rustls"] }
anyhow = "1.0"
bytes = "0.5"
env_logger = "0.7"
@ -23,6 +23,7 @@ image = "0.23.4"
log = "0.4"
mime = "0.3.1"
rand = "0.7.3"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
sha2 = "0.8.2"
sled = "0.32.0-rc1"

2
README.md
View file

@ -55,6 +55,8 @@ pict-rs offers four endpoints:
"msg": "ok"
}
```
- `GET /image/download?url=...` Download an image from a remote server, returning the same JSON
payload as the `POST` endpoint
- `GET /image/{file}` for getting a full-resolution image. `file` here is the `file` key from the
`/image` endpoint's JSON
- `GET /image/{size}/{file}` where `size` is a positive integer. This endpoint is for accessing

18
src/error.rs
View file

@ -43,6 +43,24 @@ pub enum UploadError {
#[error("Uploaded content could not be validated as an image")]
InvalidImage(image::error::ImageError),
#[error("Unsupported image format")]
UnsupportedFormat,
#[error("Unable to download image, bad response {0}")]
Download(actix_web::http::StatusCode),
#[error("Unable to download image, {0}")]
Payload(#[from] actix_web::client::PayloadError),
#[error("Unable to send request, {0}")]
SendRequest(String),
}
impl From<actix_web::client::SendRequestError> for UploadError {
fn from(e: actix_web::client::SendRequestError) -> Self {
UploadError::SendRequest(e.to_string())
}
}
impl From<sled::transaction::TransactionError<UploadError>> for UploadError {

57
src/main.rs
View file

@ -1,5 +1,6 @@
use actix_form_data::{Field, Form, Value};
use actix_web::{
client::Client,
guard,
http::header::{CacheControl, CacheDirective},
middleware::{Compress, Logger},
@ -100,12 +101,43 @@ async fn upload(
let delete_token = manager.delete_token(saved_as.to_owned()).await?;
files.push(serde_json::json!({
"file": saved_as,
"delete_token": delete_token,
"delete_token": delete_token
}));
}
}
Ok(HttpResponse::Created().json(serde_json::json!({ "msg": "ok", "files": files })))
Ok(HttpResponse::Created().json(serde_json::json!({
"msg": "ok",
"files": files
})))
}
/// download an image from a URL
async fn download(
client: web::Data<Client>,
manager: web::Data<UploadManager>,
query: web::Query<UrlQuery>,
) -> Result<HttpResponse, UploadError> {
let mut res = client.get(&query.url).send().await?;
if !res.status().is_success() {
return Err(UploadError::Download(res.status()));
}
let fut = res.body().limit(40 * MEGABYTES);
let stream = Box::pin(futures::stream::once(fut));
let alias = manager.upload(stream).await?;
let delete_token = manager.delete_token(alias.clone()).await?;
Ok(HttpResponse::Created().json(serde_json::json!({
"msg": "ok",
"files": [{
"file": alias,
"delete_token": delete_token
}]
})))
}
async fn delete(
@ -234,6 +266,11 @@ where
.streaming(stream.err_into())
}
#[derive(Debug, serde::Deserialize)]
struct UrlQuery {
url: String,
}
#[actix_rt::main]
async fn main() -> Result<(), anyhow::Error> {
let config = Config::from_args();
@ -250,18 +287,29 @@ async fn main() -> Result<(), anyhow::Error> {
.max_file_size(40 * MEGABYTES)
.field(
"images",
Field::array(Field::file(move |filename, content_type, stream| {
Field::array(Field::file(move |_, _, stream| {
let manager = manager2.clone();
async move { manager.upload(filename, content_type, stream).await }
async move {
manager.upload(stream).await.map(|alias| {
let mut path = PathBuf::new();
path.push(alias);
Some(path)
})
}
})),
);
HttpServer::new(move || {
let client = Client::build()
.header("User-Agent", "pict-rs v0.1.0-master")
.finish();
App::new()
.wrap(Logger::default())
.wrap(Compress::default())
.data(manager.clone())
.data(client)
.service(
web::scope("/image")
.service(
@ -270,6 +318,7 @@ async fn main() -> Result<(), anyhow::Error> {
.wrap(form.clone())
.route(web::post().to(upload)),
)
.service(web::resource("/download").route(web::get().to(download)))
.service(web::resource("/{filename}").route(web::get().to(serve)))
.service(
web::resource("/delete/{delete_token}/{filename}")

40
src/upload_manager.rs
View file

@ -18,7 +18,7 @@ struct UploadManagerInner {
db: sled::Db,
}
type UploadStream = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, actix_form_data::Error>>>>;
type UploadStream<E> = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, E>>>>;
enum Dup {
Exists,
@ -176,16 +176,10 @@ impl UploadManager {
}
/// Upload the file, discarding bytes if it's already present, or saving if it's new
pub(crate) async fn upload(
&self,
_filename: String,
content_type: mime::Mime,
mut stream: UploadStream,
) -> Result<Option<PathBuf>, UploadError> {
if ACCEPTED_MIMES.iter().all(|valid| *valid != content_type) {
return Err(UploadError::ContentType(content_type));
}
pub(crate) async fn upload<E>(&self, mut stream: UploadStream<E>) -> Result<String, UploadError>
where
UploadError: From<E>,
{
let (img, format) = {
// -- READ IN BYTES FROM CLIENT --
let mut bytes = bytes::BytesMut::new();
@ -208,7 +202,11 @@ impl UploadManager {
.format
.as_ref()
.map(|f| (f.to_image_format(), f.to_mime()))
.unwrap_or((format, content_type));
.unwrap_or((format.clone(), valid_format(format)?));
if ACCEPTED_MIMES.iter().all(|valid| *valid != content_type) {
return Err(UploadError::ContentType(content_type));
}
let bytes: bytes::Bytes = {
let mut bytes = std::io::Cursor::new(vec![]);
@ -226,9 +224,7 @@ impl UploadManager {
// bail early with alias to existing file if this is a duplicate
if dup.exists() {
let mut path = PathBuf::new();
path.push(alias);
return Ok(Some(path));
return Ok(alias);
}
// -- WRITE NEW FILE --
@ -238,9 +234,7 @@ impl UploadManager {
safe_save_file(real_path, bytes).await?;
// Return alias to file
let mut path = PathBuf::new();
path.push(alias);
Ok(Some(path))
Ok(alias)
}
pub(crate) async fn from_alias(&self, alias: String) -> Result<String, UploadError> {
@ -440,3 +434,13 @@ fn alias_id_key(alias: &str) -> String {
fn delete_key(alias: &str) -> String {
format!("{}/delete", alias)
}
fn valid_format(format: image::ImageFormat) -> Result<mime::Mime, UploadError> {
match format {
image::ImageFormat::Jpeg => Ok(mime::IMAGE_JPEG),
image::ImageFormat::Png => Ok(mime::IMAGE_PNG),
image::ImageFormat::Gif => Ok(mime::IMAGE_GIF),
image::ImageFormat::Bmp => Ok(mime::IMAGE_BMP),
_ => Err(UploadError::UnsupportedFormat),
}
}