From 154914e61a90e092fdddcaab138f5c95e22110af Mon Sep 17 00:00:00 2001 From: asonix Date: Sun, 14 Jun 2020 21:41:45 -0500 Subject: [PATCH] Use rexiv2 for metadata removal --- Cargo.lock | 28 +++++ Cargo.toml | 1 + docker/dev/Dockerfile | 110 ++++++++++++++++---- docker/prod/Dockerfile.amd64 | 40 ++++++-- docker/prod/Dockerfile.arm32v7 | 40 ++++++-- docker/prod/Dockerfile.arm64v8 | 40 ++++++-- src/config.rs | 16 --- src/error.rs | 9 +- src/main.rs | 16 ++- src/upload_manager.rs | 125 +++++++++++++++++------ src/validate.rs | 181 ++++++++++++++++++++------------- 11 files changed, 436 insertions(+), 170 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 271f268..ba2d726 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -920,6 +920,16 @@ dependencies = [ "wasi", ] +[[package]] +name = "gexiv2-sys" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc8f7e79962171a99792ff6895fac7abe89380c02f9abf9dc73c88f2e56697c" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "gif" version = "0.10.3" @@ -1378,6 +1388,7 @@ dependencies = [ "mime", "once_cell", "rand", + "rexiv2", "serde", "serde_json", "sha2", @@ -1422,6 +1433,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" + [[package]] name = "png" version = "0.16.5" @@ -1611,6 +1628,17 @@ dependencies = [ "quick-error", ] +[[package]] +name = "rexiv2" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982534bb5ab05ec02973487cb3338392dc68fdc8e189fdcf268ef8c95a78cfa8" +dependencies = [ + "gexiv2-sys", + "libc", + "num-rational", +] + [[package]] name = "ring" version = "0.16.14" diff --git a/Cargo.toml b/Cargo.toml index ee55b21..0c46769 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ image = "0.23.4" mime = "0.3.1" once_cell = "1.4.0" rand = "0.7.3" +rexiv2 = "0.9.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.9.0" diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index abe394b..5ae9e29 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,30 +1,106 @@ -# Build -FROM ekidd/rust-musl-builder:1.44.0 as rust +FROM rustembedded/cross:x86_64-unknown-linux-gnu AS x86_64-builder + +ARG UID=1000 +ARG GID=1000 + +ENV TOOLCHAIN=stable +ENV TARGET=x86_64-unknown-linux-gnu +ENV TOOL=x86_64-linux-gnu + +RUN \ + apt-get update && \ + apt-get upgrade -y + +RUN \ + addgroup --gid "${GID}" build && \ + adduser \ + --disabled-password \ + --gecos "" \ + --ingroup build \ + --uid "${UID}" \ + --home /opt/build \ + build + +ADD https://sh.rustup.rs /opt/build/rustup.sh + +RUN \ + chown -R build:build /opt/build + +USER build +WORKDIR /opt/build + +ENV PATH=$PATH:/opt/build/.cargo/bin + +RUN \ + chmod +x rustup.sh && \ + ./rustup.sh --default-toolchain $TOOLCHAIN --profile minimal -y && \ + rustup target add $TARGET + +FROM x86_64-builder as builder + +USER root + +RUN \ + dpkg --add-architecture amd64 && \ + apt-get update && \ + apt-get -y install libgexiv2-dev:amd64 + +USER build +ENV USER=build # Cache deps -WORKDIR /app -RUN sudo chown -R rust:rust . -RUN USER=root cargo new server -WORKDIR /app/server +RUN \ + cargo new repo + +WORKDIR /opt/build/repo + COPY Cargo.toml Cargo.lock ./ -RUN sudo chown -R rust:rust . -RUN mkdir -p ./src/bin \ - && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs -RUN cargo build --release -RUN rm -f ./target/x86_64-unknown-linux-musl/release/deps/pict_rs* + +USER root +RUN \ + chown -R build:build ./ + +USER build + +RUN \ + mkdir -p ./src && \ + echo 'fn main() { println!("Dummy") }' > ./src/main.rs && \ + cargo build --release && \ + rm -rf ./src + COPY src ./src/ +USER root +RUN \ + chown -R build:build ./src && \ + rm -r ./target/release/deps/pict_rs-* + +USER build + # Build for release RUN cargo build --frozen --release -FROM alpine:3.11 +FROM ubuntu:20.04 + +ARG UID=1000 +ARG GID=1000 + +RUN apt-get update \ + && apt-get install -y libgexiv2-2 # Copy resources -COPY --from=rust /app/server/target/x86_64-unknown-linux-musl/release/pict-rs /app/pict-rs +COPY --from=builder /opt/build/repo/target/release/pict-rs /usr/bin/pict-rs -RUN addgroup -g 1000 pictrs -RUN adduser -D -s /bin/sh -u 1000 -G pictrs pictrs -RUN chown pictrs:pictrs /app/pict-rs +RUN \ + addgroup -gid "${GID}" pictrs && \ + adduser \ + --disabled-password \ + --gecos "" \ + --ingroup pictrs \ + --uid "${UID}" \ + --home /opt/pictrs \ + pictrs +WORKDIR /opt/pictrs USER pictrs EXPOSE 8080 -CMD ["/app/pict-rs"] +CMD ["/usr/bin/pict-rs"] diff --git a/docker/prod/Dockerfile.amd64 b/docker/prod/Dockerfile.amd64 index 06f8771..8d227f8 100644 --- a/docker/prod/Dockerfile.amd64 +++ b/docker/prod/Dockerfile.amd64 @@ -1,11 +1,11 @@ -FROM rustembedded/cross:x86_64-unknown-linux-musl AS x86_64-builder +FROM rustembedded/cross:x86_64-unknown-linux-gnu AS x86_64-builder ARG UID=991 ARG GID=991 ENV TOOLCHAIN=stable -ENV TARGET=x86_64-unknown-linux-musl -ENV TOOL=x86_64-linux-musl +ENV TARGET=x86_64-unknown-linux-gnu +ENV TOOL=x86_64-linux-gnu RUN \ apt-get update && \ @@ -29,7 +29,7 @@ RUN \ USER build WORKDIR /opt/build -ENV PATH=/opt/build/.cargo/bin:/usr/local/musl/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV PATH=$PATH:/opt/build/.cargo/bin RUN \ chmod +x rustup.sh && \ @@ -38,10 +38,22 @@ RUN \ FROM x86_64-builder as builder +USER root + +RUN \ + dpkg --add-architecture amd64 && \ + apt-get update && \ + apt-get -y install libgexiv2-dev:amd64 + +USER build + ARG TAG=master ARG REPOSITORY=https://git.asonix.dog/asonix/pict-rs ARG BINARY=pict-rs +ENV PKG_CONFIG_ALLOW_CROSS=1 +ENV PKG_CONFIG_PATH=/usr/lib/$TOOL/pkgconfig:/usr/lib/$PKGCONFIG + RUN \ git clone -b $TAG $REPOSITORY repo @@ -51,18 +63,26 @@ RUN \ cargo build --release --target $TARGET && \ $TOOL-strip target/$TARGET/release/$BINARY -FROM amd64/alpine:3.11 +FROM amd64/ubuntu:20.04 ARG UID=991 ARG GID=991 ARG BINARY=pict-rs -COPY --from=builder /opt/build/repo/target/x86_64-unknown-linux-musl/release/$BINARY /usr/bin/$BINARY +COPY --from=builder /opt/build/repo/target/x86_64-unknown-linux-gnu/release/$BINARY /usr/bin/$BINARY RUN \ - apk add tini && \ - addgroup -g $GID pictrs && \ - adduser -D -G pictrs -u $UID -g "" -h /opt/pictrs pictrs + apt-get update && \ + apt-get -y upgrade && \ + apt-get -y install tini libgexiv2-2 && \ + addgroup --gid $GID pictrs && \ + adduser \ + --disabled-password \ + --gecos "" \ + --ingroup pictrs \ + --uid $UID \ + --home /opt/pictrs \ + pictrs RUN \ chown -R pictrs:pictrs /mnt @@ -70,5 +90,5 @@ RUN \ VOLUME /mnt WORKDIR /opt/pictrs USER pictrs -ENTRYPOINT ["/sbin/tini", "--"] +ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/bin/pict-rs", "-p", "/mnt", "-a", "0.0.0.0:8080", "-w", "thumbnail"] diff --git a/docker/prod/Dockerfile.arm32v7 b/docker/prod/Dockerfile.arm32v7 index 142fdde..21638af 100644 --- a/docker/prod/Dockerfile.arm32v7 +++ b/docker/prod/Dockerfile.arm32v7 @@ -1,11 +1,11 @@ -FROM rustembedded/cross:arm-unknown-linux-musleabihf AS arm32v7-builder +FROM rustembedded/cross:arm-unknown-linux-gnueabihf AS arm32v7-builder ARG UID=991 ARG GID=991 ENV TOOLCHAIN=stable -ENV TARGET=arm-unknown-linux-musleabihf -ENV TOOL=arm-linux-musleabihf +ENV TARGET=arm-unknown-linux-gnueabihf +ENV TOOL=arm-linux-gnueabihf RUN \ apt-get update && \ @@ -29,7 +29,7 @@ RUN \ USER build WORKDIR /opt/build -ENV PATH=/opt/build/.cargo/bin:/usr/local/musl/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV PATH=$PATH:/opt/build/.cargo/bin RUN \ chmod +x rustup.sh && \ @@ -38,10 +38,22 @@ RUN \ FROM arm32v7-builder as builder +USER root + +RUN \ + dpkg --add-architecture armhf && \ + apt-get update && \ + apt-get -y install libgexiv2-dev:armhf + +USER build + ARG TAG=master ARG REPOSITORY=https://git.asonix.dog/asonix/pict-rs ARG BINARY=pict-rs +ENV PKG_CONFIG_ALLOW_CROSS=1 +ENV PKG_CONFIG_PATH=/usr/lib/$TOOL/pkgconfig:/usr/lib/$PKGCONFIG + RUN \ git clone -b $TAG $REPOSITORY repo @@ -51,18 +63,26 @@ RUN \ cargo build --release --target $TARGET && \ $TOOL-strip target/$TARGET/release/$BINARY -FROM arm32v7/alpine:3.11 +FROM arm32v7/ubuntu:20.04 ARG UID=991 ARG GID=991 ARG BINARY=pict-rs -COPY --from=builder /opt/build/repo/target/arm-unknown-linux-musleabihf/release/$BINARY /usr/bin/$BINARY +COPY --from=builder /opt/build/repo/target/arm-unknown-linux-gnueabihf/release/$BINARY /usr/bin/$BINARY RUN \ - apk add tini && \ - addgroup -g $GID pictrs && \ - adduser -D -G pictrs -u $UID -g "" -h /opt/pictrs pictrs + apt-get update && \ + apt-get -y upgrade && \ + apt-get -y install tini libgexiv2-2 && \ + addgroup --gid $GID pictrs && \ + adduser \ + --disabled-password \ + --gecos "" \ + --ingroup pictrs \ + --uid $UID \ + --home /opt/pictrs \ + pictrs RUN \ chown -R pictrs:pictrs /mnt @@ -70,5 +90,5 @@ RUN \ VOLUME /mnt WORKDIR /opt/pictrs USER pictrs -ENTRYPOINT ["/sbin/tini", "--"] +ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/bin/pict-rs", "-p", "/mnt", "-a", "0.0.0.0:8080", "-w", "thumbnail"] diff --git a/docker/prod/Dockerfile.arm64v8 b/docker/prod/Dockerfile.arm64v8 index 7367ceb..bf48528 100644 --- a/docker/prod/Dockerfile.arm64v8 +++ b/docker/prod/Dockerfile.arm64v8 @@ -1,11 +1,11 @@ -FROM rustembedded/cross:aarch64-unknown-linux-musl AS aarch64-builder +FROM rustembedded/cross:aarch64-unknown-linux-gnu AS aarch64-builder ARG UID=991 ARG GID=991 ENV TOOLCHAIN=stable -ENV TARGET=aarch64-unknown-linux-musl -ENV TOOL=aarch64-linux-musl +ENV TARGET=aarch64-unknown-linux-gnu +ENV TOOL=aarch64-linux-gnu RUN \ apt-get update && \ @@ -29,7 +29,7 @@ RUN \ USER build WORKDIR /opt/build -ENV PATH=/opt/build/.cargo/bin:/usr/local/musl/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV PATH=$PATH:/opt/build/.cargo/bin RUN \ chmod +x rustup.sh && \ @@ -38,10 +38,22 @@ RUN \ FROM aarch64-builder as builder +USER root + +RUN \ + dpkg --add-architecture arm64 && \ + apt-get update && \ + apt-get -y install libgexiv2-dev:arm64 + +USER build + ARG TAG=master ARG REPOSITORY=https://git.asonix.dog/asonix/pict-rs ARG BINARY=pict-rs +ENV PKG_CONFIG_ALLOW_CROSS=1 +ENV PKG_CONFIG_PATH=/usr/lib/$TOOL/pkgconfig:/usr/lib/$PKGCONFIG + RUN \ git clone -b $TAG $REPOSITORY repo @@ -51,18 +63,26 @@ RUN \ cargo build --release --target $TARGET && \ $TOOL-strip target/$TARGET/release/$BINARY -FROM arm64v8/alpine:3.11 +FROM arm64v8/ubuntu:20.04 ARG UID=991 ARG GID=991 ARG BINARY=pict-rs -COPY --from=builder /opt/build/repo/target/aarch64-unknown-linux-musl/release/$BINARY /usr/bin/$BINARY +COPY --from=builder /opt/build/repo/target/aarch64-unknown-linux-gnu/release/$BINARY /usr/bin/$BINARY RUN \ - apk add tini && \ - addgroup -g $GID pictrs && \ - adduser -D -G pictrs -u $UID -g "" -h /opt/pictrs pictrs + apt-get update && \ + apt-get -y upgrade && \ + apt-get -y install tini libgexiv2-2 && \ + addgroup --gid $GID pictrs && \ + adduser \ + --disabled-password \ + --gecos "" \ + --ingroup pictrs \ + --uid $UID \ + --home /opt/pictrs \ + pictrs RUN \ chown -R pictrs:pictrs /mnt @@ -70,5 +90,5 @@ RUN \ VOLUME /mnt WORKDIR /opt/pictrs USER pictrs -ENTRYPOINT ["/sbin/tini", "--"] +ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/bin/pict-rs", "-p", "/mnt", "-a", "0.0.0.0:8080", "-w", "thumbnail"] diff --git a/src/config.rs b/src/config.rs index 30434ac..c6501e4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -90,22 +90,6 @@ pub(crate) enum Format { Png, } -impl Format { - pub(crate) fn to_image_format(&self) -> image::ImageFormat { - match self { - Format::Jpeg => image::ImageFormat::Jpeg, - Format::Png => image::ImageFormat::Png, - } - } - - pub(crate) fn to_mime(&self) -> mime::Mime { - match self { - Format::Jpeg => mime::IMAGE_JPEG, - Format::Png => mime::IMAGE_PNG, - } - } -} - impl std::str::FromStr for Format { type Err = FormatError; diff --git a/src/error.rs b/src/error.rs index d0bc1dd..676b33b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -39,9 +39,6 @@ pub(crate) enum UploadError { #[error("Provided token did not match expected token")] InvalidToken, - #[error("Uploaded content could not be validated as an image")] - InvalidImage(image::error::ImageError), - #[error("Unsupported image format")] UnsupportedFormat, @@ -65,6 +62,12 @@ pub(crate) enum UploadError { #[error("Error validating Gif file, {0}")] Gif(#[from] GifError), + + #[error("Tried to create file, but file already exists")] + FileExists, + + #[error("File metadata could not be parsed, {0}")] + Validate(#[from] rexiv2::Rexiv2Error), } impl From for UploadError { diff --git a/src/main.rs b/src/main.rs index 24a4fe8..b7589f7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ use actix_web::{ web, App, HttpResponse, HttpServer, }; use futures::stream::{Stream, TryStreamExt}; +use image::{ImageFormat, ImageOutputFormat}; use once_cell::sync::Lazy; use std::{collections::HashSet, path::PathBuf}; use structopt::StructOpt; @@ -150,6 +151,7 @@ async fn download( }))) } +/// Delete aliases and files #[instrument(skip(manager))] async fn delete( manager: web::Data, @@ -162,6 +164,16 @@ async fn delete( Ok(HttpResponse::NoContent().finish()) } +fn convert_format(format: ImageFormat) -> Result { + match format { + ImageFormat::Jpeg => Ok(ImageOutputFormat::Jpeg(100)), + ImageFormat::Png => Ok(ImageOutputFormat::Png), + ImageFormat::Gif => Ok(ImageOutputFormat::Gif), + ImageFormat::Bmp => Ok(ImageOutputFormat::Bmp), + _ => Err(UploadError::UnsupportedFormat), + } +} + /// Serve files #[instrument(skip(manager, whitelist))] async fn serve( @@ -220,8 +232,8 @@ async fn serve( debug!("Exporting image"); let img_bytes: bytes::Bytes = web::block(move || { let mut bytes = std::io::Cursor::new(vec![]); - img.write_to(&mut bytes, format)?; - Ok(bytes::Bytes::from(bytes.into_inner())) as Result<_, image::error::ImageError> + img.write_to(&mut bytes, convert_format(format)?)?; + Ok(bytes::Bytes::from(bytes.into_inner())) as Result<_, UploadError> }) .await?; diff --git a/src/upload_manager.rs b/src/upload_manager.rs index 72c6387..6272d0d 100644 --- a/src/upload_manager.rs +++ b/src/upload_manager.rs @@ -1,6 +1,6 @@ -use crate::{config::Format, error::UploadError, safe_save_file, to_ext, validate::validate_image}; +use crate::{config::Format, error::UploadError, to_ext, validate::validate_image}; use actix_web::web; -use futures::stream::{Stream, StreamExt}; +use futures::stream::{Stream, StreamExt, TryStreamExt}; use sha2::Digest; use std::{path::PathBuf, pin::Pin, sync::Arc}; use tracing::{debug, error, info, instrument, warn, Span}; @@ -272,29 +272,32 @@ impl UploadManager { ) -> Result where UploadError: From, + E: Unpin, { + // -- READ IN BYTES FROM CLIENT -- debug!("Reading stream"); - let bytes = read_stream(stream).await?; + let tmpfile = tmp_file(); + safe_save_stream(tmpfile.clone(), stream).await?; - let (bytes, content_type) = if validate { + let content_type = if validate { debug!("Validating image"); let format = self.inner.format.clone(); - validate_image(bytes, format).await? + validate_image(tmpfile.clone(), format).await? } else { - (bytes, content_type) + content_type }; // -- DUPLICATE CHECKS -- // Cloning bytes is fine because it's actually a pointer debug!("Hashing bytes"); - let hash = self.hash(bytes.clone()).await?; + let hash = self.hash(tmpfile.clone()).await?; debug!("Storing alias"); self.add_existing_alias(&hash, &alias).await?; debug!("Saving file"); - self.save_upload(bytes, hash, content_type).await?; + self.save_upload(tmpfile, hash, content_type).await?; // Return alias to file Ok(alias) @@ -305,27 +308,29 @@ impl UploadManager { pub(crate) async fn upload(&self, stream: UploadStream) -> Result where UploadError: From, + E: Unpin, { // -- READ IN BYTES FROM CLIENT -- debug!("Reading stream"); - let bytes = read_stream(stream).await?; + let tmpfile = tmp_file(); + safe_save_stream(tmpfile.clone(), stream).await?; // -- VALIDATE IMAGE -- debug!("Validating image"); let format = self.inner.format.clone(); - let (bytes, content_type) = validate_image(bytes, format).await?; + let content_type = validate_image(tmpfile.clone(), format).await?; // -- DUPLICATE CHECKS -- // Cloning bytes is fine because it's actually a pointer debug!("Hashing bytes"); - let hash = self.hash(bytes.clone()).await?; + let hash = self.hash(tmpfile.clone()).await?; debug!("Adding alias"); let alias = self.add_alias(&hash, content_type.clone()).await?; debug!("Saving file"); - self.save_upload(bytes, hash, content_type).await?; + self.save_upload(tmpfile, hash, content_type).await?; // Return alias to file Ok(alias) @@ -405,7 +410,7 @@ impl UploadManager { // check duplicates & store image if new async fn save_upload( &self, - bytes: bytes::Bytes, + tmpfile: PathBuf, hash: Hash, content_type: mime::Mime, ) -> Result<(), UploadError> { @@ -421,19 +426,29 @@ impl UploadManager { let mut real_path = self.image_dir(); real_path.push(name); - safe_save_file(real_path, bytes).await?; + safe_move_file(tmpfile, real_path).await?; Ok(()) } // produce a sh256sum of the uploaded file - async fn hash(&self, bytes: bytes::Bytes) -> Result { + async fn hash(&self, tmpfile: PathBuf) -> Result { let mut hasher = self.inner.hasher.clone(); - let hash = web::block(move || { - hasher.update(&bytes); - Ok(hasher.finalize_reset().to_vec()) as Result<_, UploadError> - }) - .await?; + + let mut stream = actix_fs::read_to_stream(tmpfile).await?; + + while let Some(res) = stream.next().await { + let bytes = res?; + hasher = web::block(move || { + hasher.update(&bytes); + Ok(hasher) as Result<_, UploadError> + }) + .await?; + } + + let hash = + web::block(move || Ok(hasher.finalize_reset().to_vec()) as Result<_, UploadError>) + .await?; Ok(Hash::new(hash)) } @@ -620,20 +635,68 @@ impl UploadManager { } } -#[instrument(skip(stream))] -async fn read_stream(mut stream: UploadStream) -> Result -where - UploadError: From, -{ - let mut bytes = bytes::BytesMut::new(); +pub(crate) fn tmp_file() -> PathBuf { + use rand::distributions::{Alphanumeric, Distribution}; + let limit: usize = 10; + let rng = rand::thread_rng(); - while let Some(res) = stream.next().await { - let new = res?; - debug!("Extending with {} bytes", new.len()); - bytes.extend(new); + let s: String = Alphanumeric.sample_iter(rng).take(limit).collect(); + + let name = format!("{}.tmp", s); + + let mut path = std::env::temp_dir(); + path.push("pict-rs"); + path.push(&name); + + path +} + +#[instrument] +async fn safe_move_file(from: PathBuf, to: PathBuf) -> Result<(), UploadError> { + if let Some(path) = to.parent() { + debug!("Creating directory {:?}", path); + actix_fs::create_dir_all(path.to_owned()).await?; } - Ok(bytes.freeze()) + debug!("Checking if {:?} already exists", to); + if let Err(e) = actix_fs::metadata(to.clone()).await { + if e.kind() != Some(std::io::ErrorKind::NotFound) { + return Err(e.into()); + } + } else { + return Err(UploadError::FileExists); + } + + debug!("Moving {:?} to {:?}", from, to); + actix_fs::rename(from, to).await?; + Ok(()) +} + +#[instrument(skip(stream))] +async fn safe_save_stream(to: PathBuf, stream: UploadStream) -> Result<(), UploadError> +where + UploadError: From, + E: Unpin, +{ + if let Some(path) = to.parent() { + debug!("Creating directory {:?}", path); + actix_fs::create_dir_all(path.to_owned()).await?; + } + + debug!("Checking if {:?} alreayd exists", to); + if let Err(e) = actix_fs::metadata(to.clone()).await { + if e.kind() != Some(std::io::ErrorKind::NotFound) { + return Err(e.into()); + } + } else { + return Err(UploadError::FileExists); + } + + debug!("Writing stream to {:?}", to); + let stream = stream.err_into::(); + actix_fs::write_stream(to, stream).await?; + + Ok(()) } async fn remove_path(path: sled::IVec) -> Result<(), UploadError> { diff --git a/src/validate.rs b/src/validate.rs index c709e9e..acf7223 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -1,9 +1,13 @@ -use crate::{config::Format, error::UploadError}; +use crate::{config::Format, error::UploadError, upload_manager::tmp_file}; use actix_web::web; -use bytes::Bytes; -use image::{ImageDecoder, ImageEncoder, ImageFormat}; -use std::io::Cursor; -use tracing::{debug, instrument, Span}; +use image::{io::Reader, ImageDecoder, ImageEncoder, ImageFormat}; +use rexiv2::{MediaType, Metadata}; +use std::{ + fs::File, + io::{BufReader, BufWriter, Write}, + path::PathBuf, +}; +use tracing::{debug, instrument, trace, Span}; #[derive(Debug, thiserror::Error)] pub(crate) enum GifError { @@ -15,86 +19,119 @@ pub(crate) enum GifError { } // import & export image using the image crate -#[instrument(skip(bytes, prescribed_format))] +#[instrument] pub(crate) async fn validate_image( - bytes: Bytes, + tmpfile: PathBuf, prescribed_format: Option, -) -> Result<(Bytes, mime::Mime), UploadError> { +) -> Result { let span = Span::current(); - let tup = web::block(move || { + let content_type = web::block(move || { let entered = span.enter(); - if let Some(prescribed) = prescribed_format { - debug!("Load from memory"); - let img = image::load_from_memory(&bytes).map_err(UploadError::InvalidImage)?; - debug!("Loaded"); - let mime = prescribed.to_mime(); + let meta = Metadata::new_from_path(&tmpfile)?; - debug!("Writing"); - let mut bytes = Cursor::new(vec![]); - img.write_to(&mut bytes, prescribed.to_image_format())?; - debug!("Written"); - return Ok((Bytes::from(bytes.into_inner()), mime)); - } + let content_type = match (prescribed_format, meta.get_media_type()?) { + (_, MediaType::Gif) => { + let newfile = tmp_file(); + validate_gif(&tmpfile, &newfile)?; - let format = image::guess_format(&bytes).map_err(UploadError::InvalidImage)?; + mime::IMAGE_GIF + } + (Some(Format::Jpeg), MediaType::Jpeg) | (None, MediaType::Jpeg) => { + validate(&tmpfile, ImageFormat::Jpeg)?; - debug!("Validating {:?}", format); - let res = match format { - ImageFormat::Png => Ok((validate_png(bytes)?, mime::IMAGE_PNG)), - ImageFormat::Jpeg => Ok((validate_jpg(bytes)?, mime::IMAGE_JPEG)), - ImageFormat::Bmp => Ok((validate_bmp(bytes)?, mime::IMAGE_BMP)), - ImageFormat::Gif => Ok((validate_gif(bytes)?, mime::IMAGE_GIF)), - _ => Err(UploadError::UnsupportedFormat), + meta.clear(); + meta.save_to_file(&tmpfile)?; + + mime::IMAGE_JPEG + } + (Some(Format::Png), MediaType::Png) | (None, MediaType::Png) => { + validate(&tmpfile, ImageFormat::Png)?; + + meta.clear(); + meta.save_to_file(&tmpfile)?; + + mime::IMAGE_PNG + } + (Some(Format::Jpeg), _) => { + let newfile = tmp_file(); + convert(&tmpfile, &newfile, ImageFormat::Jpeg)?; + + mime::IMAGE_JPEG + } + (Some(Format::Png), _) => { + let newfile = tmp_file(); + convert(&tmpfile, &newfile, ImageFormat::Png)?; + + mime::IMAGE_PNG + } + (_, MediaType::Bmp) => { + let newfile = tmp_file(); + validate_bmp(&tmpfile, &newfile)?; + + mime::IMAGE_BMP + } + _ => return Err(UploadError::UnsupportedFormat), }; - debug!("Validated"); + drop(entered); - res + Ok(content_type) as Result }) .await?; - Ok(tup) + Ok(content_type) } -#[instrument(skip(bytes))] -fn validate_png(bytes: Bytes) -> Result { - let decoder = image::png::PngDecoder::new(Cursor::new(&bytes))?; +#[instrument] +fn convert(from: &PathBuf, to: &PathBuf, format: ImageFormat) -> Result<(), UploadError> { + debug!("Converting"); + let reader = Reader::new(BufReader::new(File::open(from)?)).with_guessed_format()?; - let mut bytes = Cursor::new(vec![]); - let encoder = image::png::PNGEncoder::new(&mut bytes); + if reader.format() != Some(format) { + return Err(UploadError::UnsupportedFormat); + } + + let img = reader.decode()?; + + img.save_with_format(to, format)?; + std::fs::rename(to, from)?; + Ok(()) +} + +#[instrument] +fn validate(path: &PathBuf, format: ImageFormat) -> Result<(), UploadError> { + debug!("Validating"); + let reader = Reader::new(BufReader::new(File::open(path)?)).with_guessed_format()?; + + if reader.format() != Some(format) { + return Err(UploadError::UnsupportedFormat); + } + + reader.decode()?; + Ok(()) +} + +#[instrument] +fn validate_bmp(from: &PathBuf, to: &PathBuf) -> Result<(), UploadError> { + debug!("Transmuting BMP"); + let decoder = image::bmp::BmpDecoder::new(BufReader::new(File::open(from)?))?; + + let mut writer = BufWriter::new(File::create(to)?); + let encoder = image::bmp::BMPEncoder::new(&mut writer); validate_still_image(decoder, encoder)?; - Ok(Bytes::from(bytes.into_inner())) + writer.flush()?; + std::fs::rename(to, from)?; + Ok(()) } -#[instrument(skip(bytes))] -fn validate_jpg(bytes: Bytes) -> Result { - let decoder = image::jpeg::JpegDecoder::new(Cursor::new(&bytes))?; - - let mut bytes = Cursor::new(vec![]); - let encoder = image::jpeg::JPEGEncoder::new(&mut bytes); - validate_still_image(decoder, encoder)?; - - Ok(Bytes::from(bytes.into_inner())) -} - -#[instrument(skip(bytes))] -fn validate_bmp(bytes: Bytes) -> Result { - let decoder = image::bmp::BmpDecoder::new(Cursor::new(&bytes))?; - - let mut bytes = Cursor::new(vec![]); - let encoder = image::bmp::BMPEncoder::new(&mut bytes); - validate_still_image(decoder, encoder)?; - - Ok(Bytes::from(bytes.into_inner())) -} - -#[instrument(skip(bytes))] -fn validate_gif(bytes: Bytes) -> Result { +#[instrument] +fn validate_gif(from: &PathBuf, to: &PathBuf) -> Result<(), GifError> { + debug!("Transmuting GIF"); use gif::{Parameter, SetParameter}; - let mut decoder = gif::Decoder::new(Cursor::new(&bytes)); + let mut decoder = gif::Decoder::new(BufReader::new(File::open(from)?)); decoder.set(gif::ColorOutput::Indexed); @@ -104,19 +141,21 @@ fn validate_gif(bytes: Bytes) -> Result { let height = reader.height(); let global_palette = reader.global_palette().unwrap_or(&[]); - let mut bytes = Cursor::new(vec![]); - { - let mut encoder = gif::Encoder::new(&mut bytes, width, height, global_palette)?; + let mut writer = BufWriter::new(File::create(to)?); + let mut encoder = gif::Encoder::new(&mut writer, width, height, global_palette)?; - gif::Repeat::Infinite.set_param(&mut encoder)?; + gif::Repeat::Infinite.set_param(&mut encoder)?; - while let Some(frame) = reader.read_next_frame()? { - debug!("Writing frame"); - encoder.write_frame(frame)?; - } + while let Some(frame) = reader.read_next_frame()? { + trace!("Writing frame"); + encoder.write_frame(frame)?; } - Ok(Bytes::from(bytes.into_inner())) + drop(encoder); + writer.flush()?; + + std::fs::rename(to, from)?; + Ok(()) } fn validate_still_image<'a, D, E>(decoder: D, encoder: E) -> Result<(), UploadError>