Bump version

.gitignore

@@ -3,3 +3,6 @@
 /docker/dev/volumes
 /client-examples/javascript/node_modules
 /docker/object-storage/storage
+/.envrc
+/.direnv
+/.ash_history

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pict-rs"
 description = "A simple image hosting service"
-version = "0.3.0"
+version = "0.3.3"
 authors = ["asonix <asonix@asonix.dog>"]
 license = "AGPL-3.0"
 readme = "README.md"
@@ -27,7 +27,7 @@ actix-web = { version = "4.0.0", default-features = false }
 anyhow = "1.0"
 async-trait = "0.1.51"
 awc = { version = "3.0.0", default-features = false, features = ["rustls"] }
-base64 = "0.13.0"
+base64 = "0.21.0"
 config = "0.13.0"
 console-subscriber = "0.1"
 dashmap = "5.1.0"
@@ -35,14 +35,14 @@ futures-util = "0.3.17"
 mime = "0.3.1"
 num_cpus = "1.13"
 once_cell = "1.4.0"
-opentelemetry = { version = "0.17", features = ["rt-tokio"] }
-opentelemetry-otlp = "0.10"
+opentelemetry = { version = "0.18", features = ["rt-tokio"] }
+opentelemetry-otlp = "0.11"
 pin-project-lite = "0.2.7"
 reqwest = { version = "0.11.5", default-features = false, features = [
   "rustls-tls",
   "stream",
 ], optional = true }
-rust-s3 = { version = "0.29.0", default-features = false, features = [
+rust-s3 = { version = "0.31.0", default-features = false, features = [
   "fail-on-err",
   "with-reqwest",
 ], optional = true, git = "https://github.com/asonix/rust-s3", branch = "asonix/generic-client" }
@@ -55,27 +55,27 @@ structopt = "0.3.14"
 thiserror = "1.0"
 time = { version = "0.3.0", features = ["serde"] }
 tokio = { version = "1", features = ["full", "tracing"] }
-tokio-uring = { version = "0.3", optional = true, features = ["bytes"] }
+tokio-uring = { version = "0.4", optional = true, features = ["bytes"] }
 tokio-util = { version = "0.7", default-features = false, features = ["codec"] }
 tracing = "0.1.15"
 tracing-error = "0.2.0"
 tracing-futures = "0.2.4"
 tracing-log = "0.1.2"
-tracing-opentelemetry = "0.17"
+tracing-opentelemetry = "0.18"
 tracing-subscriber = { version = "0.3.0", features = [
   "env-filter",
   "fmt",
   "tracing-log",
 ] }
 url = { version = "2.2", features = ["serde"] }
-uuid = { version = "0.8.2", features = ["v4", "serde"] }
+uuid = { version = "1.1.2", features = ["v4", "serde"] }
 
 [dependencies.tracing-actix-web]
-version = "0.5.0"
+version = "0.7.0"
 default-features = false
-features = ["emit_event_on_error", "opentelemetry_0_17"]
+features = ["emit_event_on_error", "opentelemetry_0_18"]
 
 [dependencies.tracing-awc]
-version = "0.1.0"
+version = "0.1.6"
 default-features = false
-features = ["emit_event_on_error", "opentelemetry_0_17"]
+features = ["emit_event_on_error", "opentelemetry_0_18"]

README.md

@@ -9,7 +9,7 @@ _a simple image hosting service_
 ## Usage
 ### Running
 ```
-pict-rs 0.3.0
+pict-rs 0.3.1
 
 USAGE:
     pict-rs [FLAGS] [OPTIONS] [SUBCOMMAND]
@@ -56,7 +56,7 @@ SUBCOMMANDS:
 ```
 
 ```
-pict-rs-file-store 0.3.0
+pict-rs-file-store 0.3.1
 
 USAGE:
     pict-rs file-store [OPTIONS]
@@ -70,7 +70,7 @@ OPTIONS:
 ```
 
 ```
-pict-rs-s3-store 0.3.0
+pict-rs-s3-store 0.3.1
 
 USAGE:
     pict-rs s3-store [OPTIONS] --bucket-name <bucket-name> --region <region>
@@ -239,7 +239,7 @@ pict-rs offers the following endpoints:
 
 
 The following endpoints are protected by an API key via the `X-Api-Token` header, and are disabled
-unless the `--api-key` option is passed to the binary or the PICTRS_API_KEY environment variable is
+unless the `--api-key` option is passed to the binary or the PICTRS__API_KEY environment variable is
 set.
 
 A secure API key can be generated by any password generator.

docker/object-storage/root/usr/lib/ImageMagick-7.1.0/config-Q16HDRI/policy.xml

@@ -15,7 +15,7 @@
     <policy domain="delegate" rights="execute" pattern="ffmpeg" />
     <policy domain="filter" rights="none" pattern="*" />
     <policy domain="module" rights="none" pattern="*" />
-    <policy domain="module" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP,TMP,PAM,VIDEO}" />
+    <policy domain="module" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP,TMP,PAM,PNM,VIDEO}" />
         <!-- indirect reads not permitted -->
     <policy domain="system" name="precision" value="6" />
 </policymap>

docker/prod/root/usr/lib/ImageMagick-7.1.0/config-Q16HDRI/policy.xml

@@ -15,7 +15,7 @@
     <policy domain="delegate" rights="execute" pattern="ffmpeg" />
     <policy domain="filter" rights="none" pattern="*" />
     <policy domain="module" rights="none" pattern="*" />
-    <policy domain="module" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP,TMP,PAM,VIDEO}" />
+    <policy domain="module" rights="read | write" pattern="{GIF,JPEG,PNG,WEBP,TMP,PAM,PNM,VIDEO}" />
         <!-- indirect reads not permitted -->
     <policy domain="system" name="precision" value="6" />
 </policymap>

flake.lock

@@ -0,0 +1,61 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1681202837,
+        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1683408522,
+        "narHash": "sha256-9kcPh6Uxo17a3kK3XCHhcWiV1Yu1kYj22RHiymUhMkU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "897876e4c484f1e8f92009fd11b7d988a121a4e7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,50 @@
+{
+  description = "pict-rs";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs = { self, nixpkgs, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = import nixpkgs {
+          inherit system;
+        };
+      in
+      {
+        packages = rec {
+          pict-rs = pkgs.callPackage ./pict-rs.nix {
+            inherit (pkgs.darwin.apple_sdk.frameworks) Security;
+          };
+
+          default = pict-rs;
+        };
+
+        apps = rec {
+          dev = flake-utils.lib.mkApp { drv = self.packages.${system}.pict-rs; };
+          default = dev;
+        };
+
+        devShell = with pkgs; mkShell {
+          nativeBuildInputs = [
+            cargo
+            cargo-outdated
+            cargo-zigbuild
+            clippy
+            imagemagick
+            ffmpeg_5-full
+            gcc
+            imagemagick
+            protobuf
+            rust-analyzer
+            rustc
+            rustfmt
+            taplo
+          ];
+
+          RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}";
+        };
+      });
+}

pict-rs.nix

@@ -0,0 +1,42 @@
+{ exiftool
+, ffmpeg_5-full
+, imagemagick
+, lib
+, makeWrapper
+, nixosTests
+, protobuf
+, rustPlatform
+, Security
+, stdenv
+}:
+
+rustPlatform.buildRustPackage {
+  pname = "pict-rs";
+  version = "0.3.3";
+  src = ./.;
+  cargoLock = {
+    lockFile = ./Cargo.lock;
+    outputHashes = {
+      "aws-creds-0.29.1" = "bwDFmDPThMLrpaB7cAj/2/vJKhbX6/DqgcIRBVKSZhg=";
+    };
+  };
+
+  PROTOC = "${protobuf}/bin/protoc";
+  PROTOC_INCLUDE = "${protobuf}/include";
+
+  nativeBuildInputs = [ makeWrapper ];
+  buildInputs = lib.optionals stdenv.isDarwin [ Security ];
+
+  postInstall = ''
+    wrapProgram $out/bin/pict-rs \
+        --prefix PATH : "${lib.makeBinPath [ imagemagick ffmpeg_5-full exiftool ]}"
+  '';
+
+  passthru.tests = { inherit (nixosTests) pict-rs; };
+
+  meta = with lib; {
+    description = "A simple image hosting service";
+    homepage = "https://git.asonix.dog/asonix/pict-rs";
+    license = with licenses; [ agpl3Plus ];
+  };
+}

pict-rs.toml

@@ -1,14 +1,14 @@
 ## Required: path to store pict-rs database
-# environment variable: PICTRS_PATH
+# environment variable: PICTRS__PATH
 path = './data'
 
 ## Optional: pict-rs binding address
-# environment variable: PICTRS_ADDR
+# environment variable: PICTRS__ADDR
 # default: 0.0.0.0:8080
 addr = '0.0.0.0:8080'
 
 ## Optional: format to transcode all uploaded images
-# environment variable: PICTRS_IMAGE_FORMAT
+# environment variable: PICTRS__IMAGE_FORMAT
 # valid options: 'jpeg', 'png', 'webp'
 # default: empty
 #
@@ -17,7 +17,7 @@ addr = '0.0.0.0:8080'
 image_format = 'jpeg'
 
 ## Optional: permitted image processing filters
-# environment variable: PICTRS_FILTERS
+# environment variable: PICTRS__FILTERS
 # valid options: 'identity', 'thumbnail', 'resize', 'crop', 'blur'
 # default: empty
 #
@@ -25,33 +25,33 @@ image_format = 'jpeg'
 filters = ['identity', 'thumbnail', 'resize', 'crop', 'blur']
 
 ## Optional: image bounds
-# environment variable: PICTRS_MAX_FILE_SIZE
+# environment variable: PICTRS__MAX_FILE_SIZE
 # default: 40
 max_file_size = 40 # in Megabytes
-# environment variable: PICTRS_MAX_IMAGE_WIDTH
+# environment variable: PICTRS__MAX_IMAGE_WIDTH
 # default: 10,000
 max_image_width = 10_000 # in Pixels
-# environment variable: PICTRS_MAX_IMAGE_HEIGHT
+# environment variable: PICTRS__MAX_IMAGE_HEIGHT
 # default: 10,000
 max_image_height = 10_000 # in Pixels
-# environment variable: PICTRS_MAX_IMAGE_AREA
+# environment variable: PICTRS__MAX_IMAGE_AREA
 # default: 40,000,000
 max_image_area = 40_000_000 # in Pixels
 
 ## Optional: skip image validation on the import endpoint
-# environment variable: PICTRS_SKIP_VALIDATE_IMPORTS
+# environment variable: PICTRS__SKIP_VALIDATE_IMPORTS
 # default: false
 skip_validate_imports = false
 
 ## Optional: set sled's cache capacity to a given number of bytes
-# environment variable: PICTRS_SLED_CACHE_CAPACITY
+# environment variable: PICTRS__SLED_CACHE_CAPACITY
 # default: 67_108_864 (1024 * 1024 * 64) e.g. 64MB
 #
 # Increasing this value can improve performance by keeping more of the database in RAM
 sled_cache_capacity = 67_108_864 # in bytes
 
 ## Optional: enable tokio-console and set the event buffer size
-# environment variable: PICTRS_CONSOLE_BUFFER_CAPACITY
+# environment variable: PICTRS__CONSOLE_BUFFER_CAPACITY
 # default: empty
 #
 # NOTE: this is the number of _events_ to buffer, not the number of bytes. In reality, the amount of
@@ -82,14 +82,14 @@ sled_cache_capacity = 67_108_864 # in bytes
 console_buffer_capacity = 102_400 # 102_400 (or 1024 * 100) is the default size used by console-subscriber
 
 ## Optional: shared secret for internal endpoints
-# environment variable: PICTRS_API_KEY
+# environment variable: PICTRS__API_KEY
 # default: empty
 #
 # Not specifying api_key disables internal endpoints
 api_key = 'API_KEY'
 
 ## Optional: url for exporting otlp traces
-# environment variable: PICTRS_OPENTELEMETRY_URL
+# environment variable: PICTRS__OPENTELEMETRY_URL
 # default: empty
 #
 # Not specifying opentelemetry_url means no traces will be exported
@@ -105,11 +105,11 @@ type = "file_store"
 ## Example file store
 # [store]
 #
-# # environment variable: PICTRS_STORE__TYPE
+# # environment variable: PICTRS__STORE__TYPE
 # type = 'file_store'
 #
 # # Optional: file path
-# # environment variable: PICTRS_STORE__PATH
+# # environment variable: PICTRS__STORE__PATH
 # # default: empty
 # #
 # # Not specifying path means pict-rs' top-level `path` config is used
@@ -118,35 +118,35 @@ type = "file_store"
 ## Example s3 store
 # [store]
 #
-# # environment variable: PICTRS_STORE__TYPE
+# # environment variable: PICTRS__STORE__TYPE
 # type = 's3_store'
 #
 # # Required: bucket name
-# # environment variable: PICTRS_STORE__BUCKET_NAME
+# # environment variable: PICTRS__STORE__BUCKET_NAME
 # bucket_name = 'rust_s3'
 #
 # # Required: bucket region
-# # environment variable: PICTRS_STORE__REGION
+# # environment variable: PICTRS__STORE__REGION
 # #
 # # can also be endpoint of local s3 store, e.g. 'http://minio:9000'
 # region = 'eu-central-1'
 #
 # # Optional: bucket access key
-# # environment variable: PICTRS_STORE__ACCESS_KEY
+# # environment variable: PICTRS__STORE__ACCESS_KEY
 # # default: empty
 # access_key = 'ACCESS_KEY'
 #
 # # Optional: bucket secret key
-# # environment variable: PICTRS_STORE__SECRET_KEY
+# # environment variable: PICTRS__STORE__SECRET_KEY
 # # default: empty
 # secret_key = 'SECRET_KEY'
 #
 # # Optional: bucket security token
-# # environment variable: PICTRS_STORE__SECURITY_TOKEN
+# # environment variable: PICTRS__STORE__SECURITY_TOKEN
 # # default: empty
 # security_token = 'SECURITY_TOKEN'
 #
 # # Optional: bucket session token
-# # environment variable: PICTRS_STORE__SESSION_TOKEN
+# # environment variable: PICTRS__STORE__SESSION_TOKEN
 # # default: empty
 # session_token = 'SESSION_TOKEN'

src/concurrent_processor.rs

@@ -114,7 +114,7 @@ impl Drop for CancelToken {
     fn drop(&mut self) {
         if self.receiver.is_none() {
             let completed = PROCESS_MAP.remove(&self.path).is_none();
-            self.span.record("completed", &completed);
+            self.span.record("completed", completed);
         }
     }
 }

src/magick.rs

@@ -180,7 +180,7 @@ where
     parse_details(s)
 }
 
-pub(crate) async fn details_file(path_str: &str) -> Result<Details, Error> {
+async fn details_file(path_str: &str) -> Result<Details, Error> {
     let process = Process::run(
         "magick",
         &["identify", "-ping", "-format", "%w %h | %m\n", path_str],

src/migrate/s034.rs

@@ -46,7 +46,7 @@ impl SledDb for sled034::Db {
     }
 
     fn self_tree(&self) -> &Self::SledTree {
-        &*self
+        self
     }
 }
 

src/process.rs

@@ -212,8 +212,8 @@ where
                     if let Ok(err) = res {
                         let display = format!("{}", err);
                         let debug = format!("{:?}", err);
-                        span.record("exception.message", &display.as_str());
-                        span.record("exception.details", &debug.as_str());
+                        span.record("exception.message", display.as_str());
+                        span.record("exception.details", debug.as_str());
                         return Poll::Ready(Err(err));
                     }
                 }
@@ -223,8 +223,8 @@ where
                 if let Err(err) = &res {
                     let display = format!("{}", err);
                     let debug = format!("{:?}", err);
-                    span.record("exception.message", &display.as_str());
-                    span.record("exception.details", &debug.as_str());
+                    span.record("exception.message", display.as_str());
+                    span.record("exception.details", debug.as_str());
                 }
                 return Poll::Ready(res);
             }

src/range.rs

@@ -17,7 +17,7 @@ pub(crate) fn chop_bytes(
     if let Some((start, end)) = byte_range.to_satisfiable_range(length) {
         // END IS INCLUSIVE
         let end = end as usize + 1;
-        return Ok(once(ready(Ok(bytes.slice(start as usize..end as usize)))));
+        return Ok(once(ready(Ok(bytes.slice(start as usize..end)))));
     }
 
     Err(UploadError::Range.into())

src/store/object_store.rs

@@ -2,7 +2,8 @@ use crate::store::Store;
 use actix_web::web::Bytes;
 use futures_util::stream::Stream;
 use s3::{
-    client::Client, command::Command, creds::Credentials, request_trait::Request, Bucket, Region,
+    client::Client, command::Command, creds::Credentials, error::S3Error, request_trait::Request,
+    Bucket, Region,
 };
 use std::{
     pin::Pin,
@@ -31,6 +32,9 @@ pub(crate) enum ObjectError {
     #[error(transparent)]
     Utf8(#[from] FromUtf8Error),
 
+    #[error(transparent)]
+    Upstream(#[from] S3Error),
+
     #[error("Invalid length")]
     Length,
 
@@ -171,7 +175,7 @@ impl ObjectStore {
         Ok(ObjectStore {
             path_gen,
             settings_tree,
-            bucket: Bucket::new_with_path_style(
+            bucket: Bucket::new(
                 bucket_name,
                 match region {
                     Region::Custom { endpoint, .. } => Region::Custom {
@@ -186,7 +190,8 @@ impl ObjectStore {
                     security_token,
                     session_token,
                 },
-            )?,
+            )?
+            .with_path_style(),
             client,
         })
     }

src/upload_manager/hasher.rs

@@ -1,5 +1,6 @@
 use crate::error::Error;
 use actix_web::web;
+use base64::engine::{Engine, general_purpose::STANDARD};
 use sha2::{digest::FixedOutputReset, Digest};
 use std::{
     pin::Pin,
@@ -79,7 +80,7 @@ where
 
 impl std::fmt::Debug for Hash {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "{}", base64::encode(&self.inner))
+        write!(f, "{}", STANDARD.encode(&self.inner))
     }
 }
 

src/upload_manager/session.rs

@@ -89,7 +89,7 @@ where
                             debug!("Remove hash/id -> alias mapping");
                             let id = String::from_utf8_lossy(&id);
                             let key = alias_key(&hash, &id);
-                            let _ = manager.inner.main_tree.remove(&key);
+                            let _ = manager.inner.main_tree.remove(key);
                         }
 
                         let _ = manager.check_delete_files(store, hash).await;