From 366db1e1fe4344780226ca0ca9f34e6dbdeaefe0 Mon Sep 17 00:00:00 2001 From: "Aode (lion)" Date: Thu, 28 Oct 2021 20:59:11 -0500 Subject: [PATCH] Make object storage with minio work --- .gitignore | 1 + docker/object-storage/Dockerfile | 23 ++++++++++++++++ docker/object-storage/dev.sh | 12 ++++++++ docker/object-storage/docker-compose.yml | 35 ++++++++++++++++++++++++ docker/object-storage/pict-rs.toml | 9 ++++++ pict-rs.toml | 4 +-- src/config.rs | 2 +- src/store/file_store.rs | 6 ++++ src/store/object_store.rs | 18 ++++++++++-- 9 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 docker/object-storage/Dockerfile create mode 100755 docker/object-storage/dev.sh create mode 100644 docker/object-storage/docker-compose.yml create mode 100644 docker/object-storage/pict-rs.toml diff --git a/.gitignore b/.gitignore index 82d1dc9..c807c0b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /data /docker/dev/volumes /client-examples/javascript/node_modules +/docker/object-storage/storage diff --git a/docker/object-storage/Dockerfile b/docker/object-storage/Dockerfile new file mode 100644 index 0000000..5980a60 --- /dev/null +++ b/docker/object-storage/Dockerfile @@ -0,0 +1,23 @@ +FROM archlinux:latest + +ARG UID=1000 +ARG GID=1000 + +RUN \ + pacman -Syu --noconfirm \ + perl-image-exiftool \ + imagemagick \ + ffmpeg && \ + groupadd -g 1000 app && \ + useradd -m \ + -d /opt/app \ + -u $UID \ + -g $GID \ + app + +COPY ./pict-rs.toml /etc/pict-rs.toml + +ENV PATH=$PATH:/usr/bin/vendor_perl + +WORKDIR /opt/app +USER app diff --git a/docker/object-storage/dev.sh b/docker/object-storage/dev.sh new file mode 100755 index 0000000..42261b6 --- /dev/null +++ b/docker/object-storage/dev.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +ARCH=${1:-amd64} + +export USER_ID=$(id -u) +export GROUP_ID=$(id -g) + +sudo docker-compose build --pull +sudo docker-compose up -d minio +sudo docker-compose up -d pictrs_proxy +sudo docker-compose run --service-ports --use-aliases pictrs +sudo docker-compose down diff --git a/docker/object-storage/docker-compose.yml b/docker/object-storage/docker-compose.yml new file mode 100644 index 0000000..7ae3693 --- /dev/null +++ b/docker/object-storage/docker-compose.yml @@ -0,0 +1,35 @@ +version: '3.3' + +services: + pictrs: + build: + context: . + dockerfile: ./Dockerfile + args: + UID: "${USER_ID:-1000}" + GID: "${GROUP_ID:-1000}" + ports: + - "8080:8080" + links: + - "minio:pict-rs.minio" + stdin_open: true + tty: true + volumes: + - ./storage/pict-rs:/mnt + - ../../:/opt/app + + pictrs_proxy: + image: asonix/pictrs-proxy:latest + ports: + - "8081:8081" + environment: + - PICTRS_PROXY_UPSTREAM=http://pictrs:8080 + + minio: + image: quay.io/minio/minio + command: server /mnt --console-address ":9001" + ports: + - "9000:9000" + - "9001:9001" + volumes: + - ./storage/minio:/mnt diff --git a/docker/object-storage/pict-rs.toml b/docker/object-storage/pict-rs.toml new file mode 100644 index 0000000..29c3c9d --- /dev/null +++ b/docker/object-storage/pict-rs.toml @@ -0,0 +1,9 @@ +path = '/mnt' +addr = '0.0.0.0:8080' + +[store] +type = 's3_store' +bucket_name = 'pict-rs' +region = 'http://minio:9000' +access_key = '09ODZ3BGBISV4U92JLIM' +secret_key = 'j35YE9RrxhBP0dpiD5mmdXRXvPkEJR4k6zK12q3o' diff --git a/pict-rs.toml b/pict-rs.toml index dd5b1e0..ff62f7b 100644 --- a/pict-rs.toml +++ b/pict-rs.toml @@ -41,8 +41,8 @@ max_image_width = 10000 # in Pixels # default: 10,000 max_image_height = 10000 # in Pixels # environment variable: PICTRS_MAX_IMAGE_AREA -# default: 40,000 -max_image_area = 40000 # in Pixels +# default: 40,000,000 +max_image_area = 40000000 # in Pixels ## Optional: skip image validation on the import endpoint # environment variable: PICTRS_SKIP_VALIDATE_IMPORTS diff --git a/src/config.rs b/src/config.rs index 101567c..f958f62 100644 --- a/src/config.rs +++ b/src/config.rs @@ -182,7 +182,7 @@ impl Defaults { max_file_size: 40, max_image_width: 10_000, max_image_height: 10_000, - max_image_area: 40_000, + max_image_area: 40_000_000, store: Store::FileStore { path: None }, } } diff --git a/src/store/file_store.rs b/src/store/file_store.rs index 6766244..c0937cc 100644 --- a/src/store/file_store.rs +++ b/src/store/file_store.rs @@ -54,6 +54,7 @@ impl Store for FileStore { type Identifier = FileId; type Stream = Pin>>>; + #[tracing::instrument(skip(reader))] async fn save_async_read( &self, reader: &mut Reader, @@ -71,6 +72,7 @@ impl Store for FileStore { self.file_id_from_path(path) } + #[tracing::instrument(skip(bytes))] async fn save_bytes(&self, bytes: Bytes) -> Result { let path = self.next_file()?; @@ -82,6 +84,7 @@ impl Store for FileStore { self.file_id_from_path(path) } + #[tracing::instrument] async fn to_stream( &self, identifier: &Self::Identifier, @@ -98,6 +101,7 @@ impl Store for FileStore { Ok(Box::pin(stream)) } + #[tracing::instrument(skip(writer))] async fn read_into( &self, identifier: &Self::Identifier, @@ -113,6 +117,7 @@ impl Store for FileStore { Ok(()) } + #[tracing::instrument] async fn len(&self, identifier: &Self::Identifier) -> Result { let path = self.path_from_file_id(identifier); @@ -121,6 +126,7 @@ impl Store for FileStore { Ok(len) } + #[tracing::instrument] async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error> { let path = self.path_from_file_id(identifier); diff --git a/src/store/object_store.rs b/src/store/object_store.rs index 067ea5a..c8718cb 100644 --- a/src/store/object_store.rs +++ b/src/store/object_store.rs @@ -59,6 +59,7 @@ impl Store for ObjectStore { type Identifier = ObjectId; type Stream = Pin>>>; + #[tracing::instrument(skip(reader))] async fn save_async_read( &self, reader: &mut Reader, @@ -73,6 +74,7 @@ impl Store for ObjectStore { Ok(ObjectId::from_string(path)) } + #[tracing::instrument(skip(bytes))] async fn save_bytes(&self, bytes: Bytes) -> Result { let path = self.next_file()?; @@ -81,6 +83,7 @@ impl Store for ObjectStore { Ok(ObjectId::from_string(path)) } + #[tracing::instrument] async fn to_stream( &self, identifier: &Self::Identifier, @@ -99,6 +102,7 @@ impl Store for ObjectStore { Ok(Box::pin(io_error(response.bytes_stream()))) } + #[tracing::instrument(skip(writer))] async fn read_into( &self, identifier: &Self::Identifier, @@ -121,6 +125,7 @@ impl Store for ObjectStore { Ok(()) } + #[tracing::instrument] async fn len(&self, identifier: &Self::Identifier) -> Result { let path = identifier.as_str(); @@ -130,6 +135,7 @@ impl Store for ObjectStore { Ok(length as u64) } + #[tracing::instrument] async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error> { let path = identifier.as_str(); @@ -155,9 +161,15 @@ impl ObjectStore { Ok(ObjectStore { path_gen, settings_tree, - bucket: Bucket::new( + bucket: Bucket::new_with_path_style( bucket_name, - region, + match region { + Region::Custom { endpoint, .. } => Region::Custom { + region: String::from(""), + endpoint, + }, + region => region, + }, Credentials { access_key, secret_key, @@ -181,7 +193,7 @@ impl ObjectStore { let filename = Uuid::new_v4().to_string(); let path = self.next_directory()?.to_strings().join("/"); - Ok(format!("/{}/{}", path, filename)) + Ok(format!("{}/{}", path, filename)) } }