Optionally support s3-compatible storage (untested)
This commit is contained in:
parent
7aa6c695a0
commit
f9f4fc63d6
|
|
@ -45,8 +45,8 @@ dependencies = [
|
|||
"actix-service",
|
||||
"actix-tls",
|
||||
"actix-utils",
|
||||
"ahash",
|
||||
"base64",
|
||||
"ahash 0.7.6",
|
||||
"base64 0.13.0",
|
||||
"bitflags",
|
||||
"bytes",
|
||||
"bytestring",
|
||||
|
|
@ -113,7 +113,7 @@ dependencies = [
|
|||
"http",
|
||||
"log",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde 1.0.130",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -169,9 +169,9 @@ dependencies = [
|
|||
"futures-core",
|
||||
"http",
|
||||
"log",
|
||||
"tokio-rustls",
|
||||
"tokio-rustls 0.23.0",
|
||||
"tokio-util",
|
||||
"webpki-roots",
|
||||
"webpki-roots 0.22.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -199,7 +199,7 @@ dependencies = [
|
|||
"actix-service",
|
||||
"actix-utils",
|
||||
"actix-web-codegen",
|
||||
"ahash",
|
||||
"ahash 0.7.6",
|
||||
"bytes",
|
||||
"cfg-if",
|
||||
"derive_more",
|
||||
|
|
@ -215,12 +215,12 @@ dependencies = [
|
|||
"paste",
|
||||
"pin-project",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde 1.0.130",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"smallvec",
|
||||
"socket2",
|
||||
"time",
|
||||
"time 0.3.4",
|
||||
"url",
|
||||
]
|
||||
|
||||
|
|
@ -236,6 +236,12 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.7.6"
|
||||
|
|
@ -280,6 +286,12 @@ version = "1.0.44"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
|
||||
|
||||
[[package]]
|
||||
name = "async-stream"
|
||||
version = "0.3.2"
|
||||
|
|
@ -312,6 +324,23 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "attohttpc"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8bda305457262b339322106c776e3fd21df860018e566eb6a5b1aa4b6ae02d"
|
||||
dependencies = [
|
||||
"http",
|
||||
"log",
|
||||
"rustls 0.18.1",
|
||||
"serde 1.0.130",
|
||||
"serde_json",
|
||||
"url",
|
||||
"webpki 0.21.4",
|
||||
"webpki-roots 0.19.0",
|
||||
"wildmatch",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.14"
|
||||
|
|
@ -339,7 +368,7 @@ dependencies = [
|
|||
"actix-http",
|
||||
"actix-rt",
|
||||
"actix-service",
|
||||
"base64",
|
||||
"base64 0.13.0",
|
||||
"bytes",
|
||||
"cfg-if",
|
||||
"derive_more",
|
||||
|
|
@ -350,12 +379,43 @@ dependencies = [
|
|||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rand",
|
||||
"rustls",
|
||||
"serde",
|
||||
"rustls 0.20.0",
|
||||
"serde 1.0.130",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-creds"
|
||||
version = "0.26.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5e1c8f64305d3f3096cb247983a3cae16f8c2960129699bcb70639e31180794"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"attohttpc",
|
||||
"dirs",
|
||||
"rust-ini 0.17.0",
|
||||
"serde 1.0.130",
|
||||
"serde-xml-rs",
|
||||
"serde_derive",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-region"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2884b8f2aaeb4a4bf80b219b4fe1d340139ca9331679c57e0fd4a24f571a78bd"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.13.0"
|
||||
|
|
@ -416,6 +476,19 @@ version = "1.0.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"num-integer",
|
||||
"num-traits 0.2.14",
|
||||
"time 0.1.43",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "2.33.3"
|
||||
|
|
@ -431,6 +504,22 @@ dependencies = [
|
|||
"vec_map",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "config"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b1b9d958c2b1368a663f05538fc1b5975adce1e19f435acceae987aceeeb369"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"nom",
|
||||
"rust-ini 0.13.0",
|
||||
"serde 1.0.130",
|
||||
"serde-hjson",
|
||||
"serde_json",
|
||||
"toml",
|
||||
"yaml-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "convert_case"
|
||||
version = "0.4.0"
|
||||
|
|
@ -488,6 +577,16 @@ dependencies = [
|
|||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-mac"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "4.0.2"
|
||||
|
|
@ -520,6 +619,35 @@ dependencies = [
|
|||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs"
|
||||
version = "4.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
|
||||
dependencies = [
|
||||
"dirs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"redox_users",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dlv-list"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68df3f2b690c1b86e65ef7830956aededf3cb0a16f898f79b9a6f421a7b6211b"
|
||||
dependencies = [
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.6.1"
|
||||
|
|
@ -716,6 +844,15 @@ dependencies = [
|
|||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
|
||||
dependencies = [
|
||||
"ahash 0.4.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.11.2"
|
||||
|
|
@ -740,6 +877,22 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
|
||||
dependencies = [
|
||||
"crypto-mac",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.5"
|
||||
|
|
@ -798,6 +951,21 @@ dependencies = [
|
|||
"want",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-rustls"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"hyper",
|
||||
"log",
|
||||
"rustls 0.19.1",
|
||||
"tokio",
|
||||
"tokio-rustls 0.22.0",
|
||||
"webpki 0.21.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-timeout"
|
||||
version = "0.4.1"
|
||||
|
|
@ -828,7 +996,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"hashbrown",
|
||||
"hashbrown 0.11.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -850,6 +1018,12 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9"
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.10.1"
|
||||
|
|
@ -886,12 +1060,31 @@ version = "1.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "lexical-core"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"ryu",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.105"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "869d572136620d55835903746bcb5cdc54cb2851fd0aeec53220b4bb65ef3013"
|
||||
|
||||
[[package]]
|
||||
name = "linked-hash-map"
|
||||
version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
|
||||
|
||||
[[package]]
|
||||
name = "local-channel"
|
||||
version = "0.1.2"
|
||||
|
|
@ -943,6 +1136,23 @@ version = "0.1.9"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
|
||||
|
||||
[[package]]
|
||||
name = "maybe-async"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6007f9dad048e0a224f27ca599d669fca8cfa0dac804725aab542b2eb032bce6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "md5"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.4.1"
|
||||
|
|
@ -964,6 +1174,15 @@ version = "0.3.16"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
|
||||
|
||||
[[package]]
|
||||
name = "minidom"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e"
|
||||
dependencies = [
|
||||
"quick-xml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.7.14"
|
||||
|
|
@ -992,6 +1211,17 @@ version = "0.8.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "5.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
|
||||
dependencies = [
|
||||
"lexical-core",
|
||||
"memchr",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
version = "0.3.6"
|
||||
|
|
@ -1001,6 +1231,34 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"num-traits 0.2.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.1.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
|
||||
dependencies = [
|
||||
"num-traits 0.2.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.13.0"
|
||||
|
|
@ -1059,6 +1317,16 @@ dependencies = [
|
|||
"tonic-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ordered-multimap"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485"
|
||||
dependencies = [
|
||||
"dlv-list",
|
||||
"hashbrown 0.9.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.2"
|
||||
|
|
@ -1126,7 +1394,8 @@ dependencies = [
|
|||
"anyhow",
|
||||
"async-trait",
|
||||
"awc",
|
||||
"base64",
|
||||
"base64 0.13.0",
|
||||
"config",
|
||||
"dashmap",
|
||||
"futures-util",
|
||||
"mime",
|
||||
|
|
@ -1135,14 +1404,16 @@ dependencies = [
|
|||
"opentelemetry",
|
||||
"opentelemetry-otlp",
|
||||
"pin-project-lite",
|
||||
"serde",
|
||||
"reqwest",
|
||||
"rust-s3",
|
||||
"serde 1.0.130",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"sled",
|
||||
"storage-path-generator",
|
||||
"structopt",
|
||||
"thiserror",
|
||||
"time",
|
||||
"time 0.3.4",
|
||||
"tokio",
|
||||
"tokio-uring",
|
||||
"tokio-util",
|
||||
|
|
@ -1292,6 +1563,15 @@ dependencies = [
|
|||
"prost",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.10"
|
||||
|
|
@ -1350,6 +1630,16 @@ dependencies = [
|
|||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"redox_syscall",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.5.4"
|
||||
|
|
@ -1385,6 +1675,42 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.11.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66d2927ca2f685faf0fc620ac4834690d29e7abb153add10f5812eef20b5e280"
|
||||
dependencies = [
|
||||
"base64 0.13.0",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"ipnet",
|
||||
"js-sys",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustls 0.19.1",
|
||||
"serde 1.0.130",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"tokio",
|
||||
"tokio-rustls 0.22.0",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"webpki-roots 0.21.1",
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.16.20"
|
||||
|
|
@ -1409,6 +1735,54 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-ini"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2"
|
||||
|
||||
[[package]]
|
||||
name = "rust-ini"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63471c4aa97a1cf8332a5f97709a79a4234698de6a1f5087faf66f2dae810e22"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"ordered-multimap",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-s3"
|
||||
version = "0.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2f26775d15f43dc848ef0ec65f83de8775549e486c7a3a576652049a7122d32"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"aws-creds",
|
||||
"aws-region",
|
||||
"base64 0.13.0",
|
||||
"cfg-if",
|
||||
"chrono",
|
||||
"futures",
|
||||
"hex",
|
||||
"hmac",
|
||||
"http",
|
||||
"log",
|
||||
"maybe-async",
|
||||
"md5",
|
||||
"minidom",
|
||||
"percent-encoding",
|
||||
"reqwest",
|
||||
"serde 1.0.130",
|
||||
"serde-xml-rs",
|
||||
"serde_derive",
|
||||
"sha2",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.3.3"
|
||||
|
|
@ -1418,6 +1792,32 @@ dependencies = [
|
|||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81"
|
||||
dependencies = [
|
||||
"base64 0.12.3",
|
||||
"log",
|
||||
"ring",
|
||||
"sct 0.6.1",
|
||||
"webpki 0.21.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.19.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7"
|
||||
dependencies = [
|
||||
"base64 0.13.0",
|
||||
"log",
|
||||
"ring",
|
||||
"sct 0.6.1",
|
||||
"webpki 0.21.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.20.0"
|
||||
|
|
@ -1426,8 +1826,8 @@ checksum = "9b5ac6078ca424dc1d3ae2328526a76787fecc7f8011f520e3276730e711fc95"
|
|||
dependencies = [
|
||||
"log",
|
||||
"ring",
|
||||
"sct",
|
||||
"webpki",
|
||||
"sct 0.7.0",
|
||||
"webpki 0.22.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1448,6 +1848,16 @@ version = "1.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "sct"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sct"
|
||||
version = "0.7.0"
|
||||
|
|
@ -1476,6 +1886,12 @@ dependencies = [
|
|||
"pest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "0.8.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.130"
|
||||
|
|
@ -1485,6 +1901,30 @@ dependencies = [
|
|||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde-hjson"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"num-traits 0.1.43",
|
||||
"regex",
|
||||
"serde 0.8.23",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde-xml-rs"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65162e9059be2f6a3421ebbb4fef3e74b7d9e7c60c50a0e292c6239f19f1edfa"
|
||||
dependencies = [
|
||||
"log",
|
||||
"serde 1.0.130",
|
||||
"thiserror",
|
||||
"xml-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.130"
|
||||
|
|
@ -1504,7 +1944,7 @@ checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8"
|
|||
dependencies = [
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
"serde 1.0.130",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1516,7 +1956,7 @@ dependencies = [
|
|||
"form_urlencoded",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
"serde 1.0.130",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1608,6 +2048,12 @@ version = "0.5.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "storage-path-generator"
|
||||
version = "0.1.0"
|
||||
|
|
@ -1647,6 +2093,12 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.81"
|
||||
|
|
@ -1710,6 +2162,16 @@ dependencies = [
|
|||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.1.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.4"
|
||||
|
|
@ -1718,7 +2180,7 @@ checksum = "99beeb0daeac2bd1e86ac2c21caddecb244b39a093594da1a661ec2060c7aedd"
|
|||
dependencies = [
|
||||
"itoa",
|
||||
"libc",
|
||||
"serde",
|
||||
"serde 1.0.130",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1777,15 +2239,26 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
|
||||
dependencies = [
|
||||
"rustls 0.19.1",
|
||||
"tokio",
|
||||
"webpki 0.21.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d49194a46b06a69f2498a34a595ab4a9c1babd2642ffa3dbccf6c6778d1426f2"
|
||||
dependencies = [
|
||||
"rustls",
|
||||
"rustls 0.20.0",
|
||||
"tokio",
|
||||
"webpki",
|
||||
"webpki 0.22.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1826,6 +2299,15 @@ dependencies = [
|
|||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"
|
||||
dependencies = [
|
||||
"serde 1.0.130",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.5.2"
|
||||
|
|
@ -1834,7 +2316,7 @@ checksum = "796c5e1cd49905e65dd8e700d4cb1dffcbfdb4fc9d017de08c1a537afd83627c"
|
|||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.13.0",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
|
|
@ -1954,7 +2436,7 @@ dependencies = [
|
|||
"futures-core",
|
||||
"mime",
|
||||
"opentelemetry",
|
||||
"serde",
|
||||
"serde 1.0.130",
|
||||
"tracing",
|
||||
"tracing-futures",
|
||||
"tracing-opentelemetry",
|
||||
|
|
@ -2114,6 +2596,7 @@ dependencies = [
|
|||
"idna",
|
||||
"matches",
|
||||
"percent-encoding",
|
||||
"serde 1.0.130",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2123,7 +2606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"serde",
|
||||
"serde 1.0.130",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2179,6 +2662,18 @@ dependencies = [
|
|||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.78"
|
||||
|
|
@ -2218,6 +2713,16 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki"
|
||||
version = "0.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki"
|
||||
version = "0.22.0"
|
||||
|
|
@ -2228,13 +2733,31 @@ dependencies = [
|
|||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8eff4b7516a57307f9349c64bf34caa34b940b66fed4b2fb3136cb7386e5739"
|
||||
dependencies = [
|
||||
"webpki 0.21.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.21.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940"
|
||||
dependencies = [
|
||||
"webpki 0.21.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c475786c6f47219345717a043a37ec04cb4bc185e28853adcc4fa0a947eba630"
|
||||
dependencies = [
|
||||
"webpki",
|
||||
"webpki 0.22.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2248,6 +2771,12 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wildmatch"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f44b95f62d34113cf558c93511ac93027e03e9c29a60dd0fd70e6e025c7270a"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
|
|
@ -2269,3 +2798,27 @@ name = "winapi-x86_64-pc-windows-gnu"
|
|||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xml-rs"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"
|
||||
|
||||
[[package]]
|
||||
name = "yaml-rust"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
|
||||
dependencies = [
|
||||
"linked-hash-map",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ edition = "2021"
|
|||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
[features]
|
||||
default = []
|
||||
object-storage = ["reqwest", "rust-s3"]
|
||||
io-uring = ["actix-rt/io-uring", "actix-server/io-uring", "tokio-uring", "sled/io_uring"]
|
||||
|
||||
[dependencies]
|
||||
|
|
@ -22,6 +23,7 @@ anyhow = "1.0"
|
|||
async-trait = "0.1.51"
|
||||
awc = { version = "3.0.0-beta.9", default-features = false, features = ["rustls"] }
|
||||
base64 = "0.13.0"
|
||||
config = "0.11.0"
|
||||
dashmap = "4.0.2"
|
||||
futures-util = "0.3.17"
|
||||
mime = "0.3.1"
|
||||
|
|
@ -30,6 +32,8 @@ once_cell = "1.4.0"
|
|||
opentelemetry = { version = "0.16", features = ["rt-tokio"] }
|
||||
opentelemetry-otlp = "0.9"
|
||||
pin-project-lite = "0.2.7"
|
||||
reqwest = { version = "0.11.5", default-features = false, features = ["stream"], optional = true}
|
||||
rust-s3 = { version = "0.27.0", default-features = false, features = ["tokio-rustls-tls"], optional = true }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
sha2 = "0.9.0"
|
||||
|
|
@ -47,8 +51,8 @@ tracing-futures = "0.2.4"
|
|||
tracing-log = "0.1.2"
|
||||
tracing-opentelemetry = "0.16"
|
||||
tracing-subscriber = { version = "0.3.0", features = ["env-filter", "fmt", "tracing-log"] }
|
||||
url = "2.2"
|
||||
uuid = { version = "0.8.2", features = ["v4", "serde"]}
|
||||
url = { version = "2.2", features = ["serde"] }
|
||||
uuid = { version = "0.8.2", features = ["v4", "serde"] }
|
||||
|
||||
[dependencies.tracing-actix-web]
|
||||
version = "0.5.0-beta.1"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
## Required: path to store pict-rs database
|
||||
path = './data'
|
||||
|
||||
## Optional: pict-rs binding address
|
||||
# default: 0.0.0.0:8080
|
||||
addr = '0.0.0.0:8080'
|
||||
|
||||
## Optional: format to transcode all uploaded images
|
||||
# valid options: 'jpeg', 'png', 'webp'
|
||||
# default: empty
|
||||
#
|
||||
# Not specifying image-format means images will be stored in their original format
|
||||
# This does not affect gif or mp4 uploads
|
||||
image-format = 'jpeg'
|
||||
|
||||
## Optional: permitted image processing filters
|
||||
# valid options: 'identity', 'thumbnail', 'resize', 'crop', 'blur'
|
||||
# default: empty
|
||||
#
|
||||
# Not specifying filters implies all filters are permitted
|
||||
filters = [
|
||||
'identity',
|
||||
'thumbnail',
|
||||
'resize',
|
||||
'crop',
|
||||
'blur',
|
||||
]
|
||||
|
||||
## Optional: image bounds
|
||||
# default: 40
|
||||
max-file-size = 40 # in Megabytes
|
||||
# default: 10,000
|
||||
max-image-width = 10000 # in Pixels
|
||||
# default: 10,000
|
||||
max-image-height = 10000 # in Pixels
|
||||
# default: 40,000
|
||||
max-image-area = 40000 # in Pixels
|
||||
|
||||
## Optional:
|
||||
# default: false
|
||||
skip-validate-imports = false
|
||||
|
||||
## Optional: url for exporting otlp traces
|
||||
# default: empty
|
||||
#
|
||||
# Not specifying opentelemetry-url means no traces will be exported
|
||||
opentelemetry-url = 'http://localhost:4317/'
|
||||
|
||||
## Optional: store definition
|
||||
# default: empty
|
||||
#
|
||||
# Not specifying a store will default to using a file-store in the same directory provided by the `path` field defined above
|
||||
[store]
|
||||
type = 'file-store'
|
||||
path = './data'
|
||||
|
||||
## Example s3 store
|
||||
# [store]
|
||||
# type = 's3-store'
|
||||
# bucket-name = 'rust-s3'
|
||||
# region = 'eu-central-1' # can also be endpoint of local s3 store
|
||||
# access-key = 'ACCESS_KEY'
|
||||
# secret-key = 'SECRET_KEY'
|
||||
# security-token = 'SECURITY_TOKEN'
|
||||
# session-token = 'SESSION-TOKEN'
|
||||
168
src/config.rs
168
src/config.rs
|
|
@ -1,10 +1,21 @@
|
|||
use std::{collections::HashSet, net::SocketAddr, path::PathBuf};
|
||||
use structopt::StructOpt;
|
||||
use url::Url;
|
||||
|
||||
use crate::magick::ValidInputType;
|
||||
|
||||
#[derive(Clone, Debug, structopt::StructOpt)]
|
||||
pub(crate) struct Config {
|
||||
#[derive(Clone, Debug, StructOpt)]
|
||||
pub(crate) struct Args {
|
||||
#[structopt(short, long, help = "Path to the pict-rs configuration file")]
|
||||
config_file: Option<PathBuf>,
|
||||
|
||||
#[structopt(flatten)]
|
||||
overrides: Overrides,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, serde::Serialize, structopt::StructOpt)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub(crate) struct Overrides {
|
||||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
|
|
@ -12,27 +23,15 @@ pub(crate) struct Config {
|
|||
)]
|
||||
skip_validate_imports: bool,
|
||||
|
||||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
env = "PICTRS_ADDR",
|
||||
default_value = "0.0.0.0:8080",
|
||||
help = "The address and port the server binds to."
|
||||
)]
|
||||
addr: SocketAddr,
|
||||
#[structopt(short, long, help = "The address and port the server binds to.")]
|
||||
addr: Option<SocketAddr>,
|
||||
|
||||
#[structopt(short, long, help = "The path to the data directory, e.g. data/")]
|
||||
path: Option<PathBuf>,
|
||||
|
||||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
env = "PICTRS_PATH",
|
||||
help = "The path to the data directory, e.g. data/"
|
||||
)]
|
||||
path: PathBuf,
|
||||
|
||||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
env = "PICTRS_FORMAT",
|
||||
help = "An optional image format to convert all uploaded files into, supports 'jpg', 'png', and 'webp'"
|
||||
)]
|
||||
image_format: Option<Format>,
|
||||
|
|
@ -40,7 +39,6 @@ pub(crate) struct Config {
|
|||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
env = "PICTRS_ALLOWED_FILTERS",
|
||||
help = "An optional list of filters to permit, supports 'identity', 'thumbnail', 'resize', 'crop', and 'blur'"
|
||||
)]
|
||||
filters: Option<Vec<String>>,
|
||||
|
|
@ -48,31 +46,21 @@ pub(crate) struct Config {
|
|||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
env = "PICTRS_MAX_FILE_SIZE",
|
||||
help = "Specify the maximum allowed uploaded file size (in Megabytes)",
|
||||
default_value = "40"
|
||||
help = "Specify the maximum allowed uploaded file size (in Megabytes)"
|
||||
)]
|
||||
max_file_size: usize,
|
||||
max_file_size: Option<usize>,
|
||||
|
||||
#[structopt(long, help = "Specify the maximum width in pixels allowed on an image")]
|
||||
max_image_width: Option<usize>,
|
||||
|
||||
#[structopt(long, help = "Specify the maximum width in pixels allowed on an image")]
|
||||
max_image_height: Option<usize>,
|
||||
|
||||
#[structopt(long, help = "Specify the maximum area in pixels allowed in an image")]
|
||||
max_image_area: Option<usize>,
|
||||
|
||||
#[structopt(
|
||||
long,
|
||||
env = "PICTRS_MAX_IMAGE_WIDTH",
|
||||
help = "Specify the maximum width in pixels allowed on an image",
|
||||
default_value = "10000"
|
||||
)]
|
||||
max_image_width: usize,
|
||||
|
||||
#[structopt(
|
||||
long,
|
||||
env = "PICTRS_MAX_IMAGE_HEIGHT",
|
||||
help = "Specify the maximum width in pixels allowed on an image",
|
||||
default_value = "10000"
|
||||
)]
|
||||
max_image_height: usize,
|
||||
|
||||
#[structopt(
|
||||
long,
|
||||
env = "PICTRS_API_KEY",
|
||||
help = "An optional string to be checked on requests to privileged endpoints"
|
||||
)]
|
||||
api_key: Option<String>,
|
||||
|
|
@ -80,13 +68,96 @@ pub(crate) struct Config {
|
|||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
env = "PICTRS_OPENTELEMETRY_URL",
|
||||
help = "Enable OpenTelemetry Tracing exports to the given OpenTelemetry collector"
|
||||
)]
|
||||
opentelemetry_url: Option<Url>,
|
||||
|
||||
#[structopt(subcommand)]
|
||||
store: Option<Store>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, structopt::StructOpt)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
#[serde(tag = "type")]
|
||||
pub(crate) enum Store {
|
||||
FileStore {
|
||||
// defaults to {config.path}
|
||||
path: Option<PathBuf>,
|
||||
},
|
||||
#[cfg(feature = "object-storage")]
|
||||
S3Store {
|
||||
bucket_name: String,
|
||||
region: crate::serde_str::Serde<s3::Region>,
|
||||
access_key: Option<String>,
|
||||
secret_key: Option<String>,
|
||||
security_token: Option<String>,
|
||||
session_token: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub(crate) struct Config {
|
||||
skip_validate_imports: bool,
|
||||
addr: SocketAddr,
|
||||
path: PathBuf,
|
||||
image_format: Option<Format>,
|
||||
filters: Option<Vec<String>>,
|
||||
max_file_size: usize,
|
||||
max_image_width: usize,
|
||||
max_image_height: usize,
|
||||
max_image_area: usize,
|
||||
api_key: Option<String>,
|
||||
opentelemetry_url: Option<Url>,
|
||||
store: Store,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub(crate) struct Defaults {
|
||||
skip_validate_imports: bool,
|
||||
addr: SocketAddr,
|
||||
max_file_size: usize,
|
||||
max_image_width: usize,
|
||||
max_image_height: usize,
|
||||
max_image_area: usize,
|
||||
store: Store,
|
||||
}
|
||||
|
||||
impl Defaults {
|
||||
fn new() -> Self {
|
||||
Defaults {
|
||||
skip_validate_imports: false,
|
||||
addr: ([0, 0, 0, 0], 8080).into(),
|
||||
max_file_size: 40,
|
||||
max_image_width: 10_000,
|
||||
max_image_height: 10_000,
|
||||
max_image_area: 40_000,
|
||||
store: Store::FileStore { path: None },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub(crate) fn build() -> anyhow::Result<Self> {
|
||||
let args = Args::from_args();
|
||||
let mut base_config = config::Config::try_from(&Defaults::new())?;
|
||||
|
||||
if let Some(path) = args.config_file {
|
||||
base_config.merge(config::File::from(path))?;
|
||||
};
|
||||
|
||||
base_config.merge(config::Config::try_from(&args.overrides)?)?;
|
||||
|
||||
base_config.merge(config::Environment::with_prefix("PICTRS"))?;
|
||||
|
||||
Ok(base_config.try_into()?)
|
||||
}
|
||||
|
||||
pub(crate) fn store(&self) -> &Store {
|
||||
&self.store
|
||||
}
|
||||
|
||||
pub(crate) fn bind_address(&self) -> SocketAddr {
|
||||
self.addr
|
||||
}
|
||||
|
|
@ -96,7 +167,7 @@ impl Config {
|
|||
}
|
||||
|
||||
pub(crate) fn format(&self) -> Option<Format> {
|
||||
self.image_format.clone()
|
||||
self.image_format
|
||||
}
|
||||
|
||||
pub(crate) fn allowed_filters(&self) -> Option<HashSet<String>> {
|
||||
|
|
@ -119,6 +190,10 @@ impl Config {
|
|||
self.max_image_height
|
||||
}
|
||||
|
||||
pub(crate) fn max_area(&self) -> usize {
|
||||
self.max_image_area
|
||||
}
|
||||
|
||||
pub(crate) fn api_key(&self) -> Option<&str> {
|
||||
self.api_key.as_deref()
|
||||
}
|
||||
|
|
@ -132,7 +207,8 @@ impl Config {
|
|||
#[error("Invalid format supplied, {0}")]
|
||||
pub(crate) struct FormatError(String);
|
||||
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
#[derive(Copy, Clone, Debug, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub(crate) enum Format {
|
||||
Jpeg,
|
||||
Png,
|
||||
|
|
@ -140,7 +216,7 @@ pub(crate) enum Format {
|
|||
}
|
||||
|
||||
impl Format {
|
||||
pub(crate) fn to_magick_format(&self) -> &'static str {
|
||||
pub(crate) fn as_magick_format(&self) -> &'static str {
|
||||
match self {
|
||||
Format::Jpeg => "JPEG",
|
||||
Format::Png => "PNG",
|
||||
|
|
@ -148,7 +224,7 @@ impl Format {
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) fn to_hint(&self) -> Option<ValidInputType> {
|
||||
pub(crate) fn as_hint(&self) -> Option<ValidInputType> {
|
||||
match self {
|
||||
Format::Jpeg => Some(ValidInputType::Jpeg),
|
||||
Format::Png => Some(ValidInputType::Png),
|
||||
|
|
|
|||
|
|
@ -8,13 +8,13 @@ pub(crate) struct Error {
|
|||
|
||||
impl std::fmt::Debug for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}\n", self.kind)
|
||||
writeln!(f, "{}", self.kind)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}\n", self.kind)?;
|
||||
writeln!(f, "{}", self.kind)?;
|
||||
std::fmt::Display::fmt(&self.context, f)
|
||||
}
|
||||
}
|
||||
|
|
@ -72,6 +72,10 @@ pub(crate) enum UploadError {
|
|||
#[error(transparent)]
|
||||
FileStore(#[from] crate::store::file_store::FileError),
|
||||
|
||||
#[cfg(feature = "object-storage")]
|
||||
#[error(transparent)]
|
||||
ObjectStore(#[from] crate::store::object_store::ObjectError),
|
||||
|
||||
#[error("Provided process path is invalid")]
|
||||
ParsePath,
|
||||
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ pub(crate) async fn to_mp4_bytes(
|
|||
"ffmpeg",
|
||||
&[
|
||||
"-i",
|
||||
&input_file_str,
|
||||
input_file_str,
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-vf",
|
||||
|
|
@ -80,7 +80,7 @@ pub(crate) async fn to_mp4_bytes(
|
|||
"h264",
|
||||
"-f",
|
||||
"mp4",
|
||||
&output_file_str,
|
||||
output_file_str,
|
||||
],
|
||||
)?;
|
||||
|
||||
|
|
@ -123,14 +123,14 @@ where
|
|||
"ffmpeg",
|
||||
&[
|
||||
"-i",
|
||||
&input_file_str,
|
||||
input_file_str,
|
||||
"-vframes",
|
||||
"1",
|
||||
"-codec",
|
||||
format.as_codec(),
|
||||
"-f",
|
||||
format.as_format(),
|
||||
&output_file_str,
|
||||
output_file_str,
|
||||
],
|
||||
)?;
|
||||
|
||||
|
|
|
|||
|
|
@ -317,14 +317,14 @@ mod io_uring {
|
|||
let max_size = (size - cursor).min(65_536);
|
||||
let buf = Vec::with_capacity(max_size.try_into().unwrap());
|
||||
|
||||
let (res, mut buf): (_, Vec<u8>) = self.read_at(buf, cursor).await;
|
||||
let (res, buf): (_, Vec<u8>) = self.read_at(buf, cursor).await;
|
||||
let n: usize = res?;
|
||||
|
||||
if n == 0 {
|
||||
return Err(std::io::ErrorKind::UnexpectedEof.into());
|
||||
}
|
||||
|
||||
writer.write_all(&mut buf[0..n]).await?;
|
||||
writer.write_all(&buf[0..n]).await?;
|
||||
|
||||
let n: u64 = n.try_into().unwrap();
|
||||
cursor += n;
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ pub(crate) enum ValidInputType {
|
|||
}
|
||||
|
||||
impl ValidInputType {
|
||||
fn to_str(&self) -> &'static str {
|
||||
fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Mp4 => "MP4",
|
||||
Self::Gif => "GIF",
|
||||
|
|
@ -47,7 +47,7 @@ impl ValidInputType {
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) fn to_ext(&self) -> &'static str {
|
||||
pub(crate) fn as_ext(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Mp4 => ".mp4",
|
||||
Self::Gif => ".gif",
|
||||
|
|
@ -93,7 +93,7 @@ pub(crate) fn convert_bytes_read(
|
|||
"convert",
|
||||
"-",
|
||||
"-strip",
|
||||
format!("{}:-", format.to_magick_format()).as_str(),
|
||||
format!("{}:-", format.as_magick_format()).as_str(),
|
||||
],
|
||||
)?;
|
||||
|
||||
|
|
@ -118,7 +118,7 @@ pub(crate) async fn details_bytes(
|
|||
}
|
||||
|
||||
let last_arg = if let Some(expected_format) = hint {
|
||||
format!("{}:-", expected_format.to_str())
|
||||
format!("{}:-", expected_format.as_str())
|
||||
} else {
|
||||
"-".to_owned()
|
||||
};
|
||||
|
|
@ -160,7 +160,7 @@ where
|
|||
}
|
||||
|
||||
let last_arg = if let Some(expected_format) = hint {
|
||||
format!("{}:-", expected_format.to_str())
|
||||
format!("{}:-", expected_format.as_str())
|
||||
} else {
|
||||
"-".to_owned()
|
||||
};
|
||||
|
|
@ -183,7 +183,7 @@ where
|
|||
pub(crate) async fn details_file(path_str: &str) -> Result<Details, Error> {
|
||||
let process = Process::run(
|
||||
"magick",
|
||||
&["identify", "-ping", "-format", "%w %h | %m\n", &path_str],
|
||||
&["identify", "-ping", "-format", "%w %h | %m\n", path_str],
|
||||
)?;
|
||||
|
||||
let mut reader = process.read().unwrap();
|
||||
|
|
@ -262,7 +262,7 @@ pub(crate) fn process_image_store_read<S: Store>(
|
|||
) -> std::io::Result<impl AsyncRead + Unpin> {
|
||||
let command = "magick";
|
||||
let convert_args = ["convert", "-"];
|
||||
let last_arg = format!("{}:-", format.to_magick_format());
|
||||
let last_arg = format!("{}:-", format.as_magick_format());
|
||||
|
||||
let process = Process::spawn(
|
||||
Command::new(command)
|
||||
|
|
@ -277,7 +277,10 @@ pub(crate) fn process_image_store_read<S: Store>(
|
|||
impl Details {
|
||||
#[instrument(name = "Validating input type")]
|
||||
fn validate_input(&self) -> Result<ValidInputType, Error> {
|
||||
if self.width > crate::CONFIG.max_width() || self.height > crate::CONFIG.max_height() {
|
||||
if self.width > crate::CONFIG.max_width()
|
||||
|| self.height > crate::CONFIG.max_height()
|
||||
|| self.width * self.height > crate::CONFIG.max_area()
|
||||
{
|
||||
return Err(UploadError::Dimensions.into());
|
||||
}
|
||||
|
||||
|
|
|
|||
42
src/main.rs
42
src/main.rs
|
|
@ -15,7 +15,6 @@ use std::{
|
|||
task::{Context, Poll},
|
||||
time::SystemTime,
|
||||
};
|
||||
use structopt::StructOpt;
|
||||
use tokio::{io::AsyncReadExt, sync::Semaphore};
|
||||
use tracing::{debug, error, info, instrument, Span};
|
||||
use tracing_actix_web::TracingLogger;
|
||||
|
|
@ -37,6 +36,7 @@ mod migrate;
|
|||
mod process;
|
||||
mod processor;
|
||||
mod range;
|
||||
mod serde_str;
|
||||
mod store;
|
||||
mod tmp_file;
|
||||
mod upload_manager;
|
||||
|
|
@ -61,7 +61,7 @@ const MINUTES: u32 = 60;
|
|||
const HOURS: u32 = 60 * MINUTES;
|
||||
const DAYS: u32 = 24 * HOURS;
|
||||
|
||||
static CONFIG: Lazy<Config> = Lazy::new(Config::from_args);
|
||||
static CONFIG: Lazy<Config> = Lazy::new(|| Config::build().unwrap());
|
||||
static PROCESS_SEMAPHORE: Lazy<Semaphore> =
|
||||
Lazy::new(|| Semaphore::new(num_cpus::get().saturating_sub(1).max(1)));
|
||||
|
||||
|
|
@ -397,7 +397,7 @@ where
|
|||
|
||||
drop(permit);
|
||||
|
||||
let details = Details::from_bytes(bytes.clone(), format.to_hint()).await?;
|
||||
let details = Details::from_bytes(bytes.clone(), format.as_hint()).await?;
|
||||
|
||||
let save_span = tracing::info_span!(
|
||||
parent: None,
|
||||
|
|
@ -835,10 +835,38 @@ async fn main() -> anyhow::Result<()> {
|
|||
let root_dir = CONFIG.data_dir();
|
||||
let db = LatestDb::exists(root_dir.clone()).migrate()?;
|
||||
|
||||
let store = FileStore::build(root_dir, &db)?;
|
||||
match CONFIG.store() {
|
||||
config::Store::FileStore { path } => {
|
||||
let path = path.to_owned().unwrap_or_else(|| root_dir.clone());
|
||||
|
||||
let manager = UploadManager::new(store, db, CONFIG.format()).await?;
|
||||
let store = FileStore::build(path, &db)?;
|
||||
|
||||
manager.restructure().await?;
|
||||
launch(manager).await
|
||||
let manager = UploadManager::new(store, db, CONFIG.format()).await?;
|
||||
|
||||
manager.restructure().await?;
|
||||
launch(manager).await
|
||||
}
|
||||
#[cfg(feature = "object-storage")]
|
||||
config::Store::S3Store {
|
||||
bucket_name,
|
||||
region,
|
||||
access_key,
|
||||
secret_key,
|
||||
security_token,
|
||||
session_token,
|
||||
} => {
|
||||
let store = crate::store::object_store::ObjectStore::build(
|
||||
bucket_name,
|
||||
(**region).clone(),
|
||||
access_key.clone(),
|
||||
secret_key.clone(),
|
||||
security_token.clone(),
|
||||
session_token.clone(),
|
||||
&db,
|
||||
)?;
|
||||
|
||||
let manager = UploadManager::new(store, db, CONFIG.format()).await?;
|
||||
launch(manager).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ pub(crate) fn build_chain(
|
|||
}
|
||||
|
||||
let (path, args) =
|
||||
args.into_iter()
|
||||
args.iter()
|
||||
.fold(Ok((PathBuf::default(), vec![])), |inner, (name, value)| {
|
||||
if let Ok(inner) = inner {
|
||||
parse!(inner, Identity, name, value);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,73 @@
|
|||
use std::{
|
||||
ops::{Deref, DerefMut},
|
||||
str::FromStr,
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct Serde<T> {
|
||||
inner: T,
|
||||
}
|
||||
|
||||
impl<T> Serde<T> {
|
||||
pub(crate) fn new(inner: T) -> Self {
|
||||
Serde { inner }
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Deref for Serde<T> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> DerefMut for Serde<T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> FromStr for Serde<T>
|
||||
where
|
||||
T: FromStr,
|
||||
{
|
||||
type Err = T::Err;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
Ok(Serde {
|
||||
inner: T::from_str(s)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> serde::Serialize for Serde<T>
|
||||
where
|
||||
T: std::fmt::Display,
|
||||
{
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
let s = self.inner.to_string();
|
||||
serde::Serialize::serialize(s.as_str(), serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de, T> serde::Deserialize<'de> for Serde<T>
|
||||
where
|
||||
T: std::str::FromStr,
|
||||
<T as std::str::FromStr>::Err: std::fmt::Display,
|
||||
{
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let s: String = serde::Deserialize::deserialize(deserializer)?;
|
||||
let inner = s
|
||||
.parse::<T>()
|
||||
.map_err(|e| serde::de::Error::custom(e.to_string()))?;
|
||||
|
||||
Ok(Serde { inner })
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,8 @@ use futures_util::stream::Stream;
|
|||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
|
||||
pub(crate) mod file_store;
|
||||
#[cfg(feature = "object-storage")]
|
||||
pub(crate) mod object_store;
|
||||
|
||||
pub(crate) trait Identifier: Send + Sync + Clone + Debug {
|
||||
type Error: std::error::Error;
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ pub(crate) use file_id::FileId;
|
|||
// - last-path -> last generated path
|
||||
// - fs-restructure-01-complete -> bool
|
||||
|
||||
const GENERATOR_KEY: &'static [u8] = b"last-path";
|
||||
const GENERATOR_KEY: &[u8] = b"last-path";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum FileError {
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ use crate::{
|
|||
};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const RESTRUCTURE_COMPLETE: &'static [u8] = b"fs-restructure-01-complete";
|
||||
const DETAILS: &'static [u8] = b"details";
|
||||
const RESTRUCTURE_COMPLETE: &[u8] = b"fs-restructure-01-complete";
|
||||
const DETAILS: &[u8] = b"details";
|
||||
|
||||
impl UploadManager<FileStore> {
|
||||
#[tracing::instrument(skip(self))]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,220 @@
|
|||
use crate::store::Store;
|
||||
use actix_web::web::Bytes;
|
||||
use futures_util::stream::{Stream, StreamExt};
|
||||
use s3::{
|
||||
command::Command, creds::Credentials, request::Reqwest, request_trait::Request, Bucket, Region,
|
||||
};
|
||||
use std::{
|
||||
pin::Pin,
|
||||
string::FromUtf8Error,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use storage_path_generator::{Generator, Path};
|
||||
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
|
||||
use uuid::Uuid;
|
||||
|
||||
mod object_id;
|
||||
pub(crate) use object_id::ObjectId;
|
||||
|
||||
// - Settings Tree
|
||||
// - last-path -> last generated path
|
||||
|
||||
const GENERATOR_KEY: &[u8] = b"last-path";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum ObjectError {
|
||||
#[error(transparent)]
|
||||
PathGenerator(#[from] storage_path_generator::PathError),
|
||||
|
||||
#[error(transparent)]
|
||||
Sled(#[from] sled::Error),
|
||||
|
||||
#[error(transparent)]
|
||||
Utf8(#[from] FromUtf8Error),
|
||||
|
||||
#[error("Invalid length")]
|
||||
Length,
|
||||
|
||||
#[error("Storage error: {0}")]
|
||||
Anyhow(#[from] anyhow::Error),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ObjectStore {
|
||||
path_gen: Generator,
|
||||
settings_tree: sled::Tree,
|
||||
bucket: Bucket,
|
||||
}
|
||||
|
||||
pin_project_lite::pin_project! {
|
||||
struct IoError<S> {
|
||||
#[pin]
|
||||
inner: S,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait(?Send)]
|
||||
impl Store for ObjectStore {
|
||||
type Error = ObjectError;
|
||||
type Identifier = ObjectId;
|
||||
type Stream = Pin<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
|
||||
|
||||
async fn save_async_read<Reader>(
|
||||
&self,
|
||||
reader: &mut Reader,
|
||||
) -> Result<Self::Identifier, Self::Error>
|
||||
where
|
||||
Reader: AsyncRead + Unpin,
|
||||
{
|
||||
let path = self.next_file()?;
|
||||
|
||||
self.bucket.put_object_stream(reader, &path).await?;
|
||||
|
||||
Ok(ObjectId::from_string(path))
|
||||
}
|
||||
|
||||
async fn save_bytes(&self, bytes: Bytes) -> Result<Self::Identifier, Self::Error> {
|
||||
let path = self.next_file()?;
|
||||
|
||||
self.bucket.put_object(&path, &bytes).await?;
|
||||
|
||||
Ok(ObjectId::from_string(path))
|
||||
}
|
||||
|
||||
async fn to_stream(
|
||||
&self,
|
||||
identifier: &Self::Identifier,
|
||||
from_start: Option<u64>,
|
||||
len: Option<u64>,
|
||||
) -> Result<Self::Stream, Self::Error> {
|
||||
let path = identifier.as_str();
|
||||
|
||||
let start = from_start.unwrap_or(0);
|
||||
let end = len.map(|len| start + len);
|
||||
|
||||
let request = Reqwest::new(&self.bucket, path, Command::GetObjectRange { start, end });
|
||||
|
||||
let response = request.response().await?;
|
||||
|
||||
Ok(Box::pin(io_error(response.bytes_stream())))
|
||||
}
|
||||
|
||||
async fn read_into<Writer>(
|
||||
&self,
|
||||
identifier: &Self::Identifier,
|
||||
writer: &mut Writer,
|
||||
) -> Result<(), std::io::Error>
|
||||
where
|
||||
Writer: AsyncWrite + Unpin,
|
||||
{
|
||||
let mut stream = self
|
||||
.to_stream(identifier, None, None)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
|
||||
|
||||
while let Some(res) = stream.next().await {
|
||||
let mut bytes = res?;
|
||||
|
||||
writer.write_all_buf(&mut bytes).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn len(&self, identifier: &Self::Identifier) -> Result<u64, Self::Error> {
|
||||
let path = identifier.as_str();
|
||||
|
||||
let (head, _) = self.bucket.head_object(path).await?;
|
||||
let length = head.content_length.ok_or(ObjectError::Length)?;
|
||||
|
||||
Ok(length as u64)
|
||||
}
|
||||
|
||||
async fn remove(&self, identifier: &Self::Identifier) -> Result<(), Self::Error> {
|
||||
let path = identifier.as_str();
|
||||
|
||||
self.bucket.delete_object(path).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectStore {
|
||||
pub(crate) fn build(
|
||||
bucket_name: &str,
|
||||
region: Region,
|
||||
access_key: Option<String>,
|
||||
secret_key: Option<String>,
|
||||
security_token: Option<String>,
|
||||
session_token: Option<String>,
|
||||
db: &sled::Db,
|
||||
) -> Result<Self, ObjectError> {
|
||||
let settings_tree = db.open_tree("settings")?;
|
||||
|
||||
let path_gen = init_generator(&settings_tree)?;
|
||||
|
||||
Ok(ObjectStore {
|
||||
path_gen,
|
||||
settings_tree,
|
||||
bucket: Bucket::new(
|
||||
bucket_name,
|
||||
region,
|
||||
Credentials {
|
||||
access_key,
|
||||
secret_key,
|
||||
security_token,
|
||||
session_token,
|
||||
},
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn next_directory(&self) -> Result<Path, ObjectError> {
|
||||
let path = self.path_gen.next();
|
||||
|
||||
self.settings_tree
|
||||
.insert(GENERATOR_KEY, path.to_be_bytes())?;
|
||||
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn next_file(&self) -> Result<String, ObjectError> {
|
||||
let filename = Uuid::new_v4().to_string();
|
||||
let path = self.next_directory()?.to_strings().join("/");
|
||||
|
||||
Ok(format!("/{}/{}", path, filename))
|
||||
}
|
||||
}
|
||||
|
||||
fn init_generator(settings: &sled::Tree) -> Result<Generator, ObjectError> {
|
||||
if let Some(ivec) = settings.get(GENERATOR_KEY)? {
|
||||
Ok(Generator::from_existing(
|
||||
storage_path_generator::Path::from_be_bytes(ivec.to_vec())?,
|
||||
))
|
||||
} else {
|
||||
Ok(Generator::new())
|
||||
}
|
||||
}
|
||||
|
||||
fn io_error<S, T, E>(stream: S) -> impl Stream<Item = std::io::Result<T>>
|
||||
where
|
||||
S: Stream<Item = Result<T, E>>,
|
||||
E: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
IoError { inner: stream }
|
||||
}
|
||||
|
||||
impl<S, T, E> Stream for IoError<S>
|
||||
where
|
||||
S: Stream<Item = Result<T, E>>,
|
||||
E: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
type Item = std::io::Result<T>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let this = self.as_mut().project();
|
||||
|
||||
this.inner.poll_next(cx).map(|opt| {
|
||||
opt.map(|res| res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
use crate::store::{object_store::ObjectError, Identifier};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ObjectId(String);
|
||||
|
||||
impl Identifier for ObjectId {
|
||||
type Error = ObjectError;
|
||||
|
||||
fn to_bytes(&self) -> Result<Vec<u8>, Self::Error> {
|
||||
Ok(self.0.as_bytes().to_vec())
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Result<Self, Self::Error> {
|
||||
Ok(ObjectId(String::from_utf8(bytes)?))
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectId {
|
||||
pub(super) fn from_string(string: String) -> Self {
|
||||
ObjectId(string)
|
||||
}
|
||||
|
||||
pub(super) fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
|
@ -4,15 +4,12 @@ use crate::{
|
|||
ffmpeg::{InputFormat, ThumbnailFormat},
|
||||
magick::{details_hint, ValidInputType},
|
||||
migrate::{alias_id_key, alias_key, alias_key_bounds},
|
||||
serde_str::Serde,
|
||||
store::{Identifier, Store},
|
||||
};
|
||||
use actix_web::web;
|
||||
use sha2::Digest;
|
||||
use std::{
|
||||
ops::{Deref, DerefMut},
|
||||
string::FromUtf8Error,
|
||||
sync::Arc,
|
||||
};
|
||||
use std::{string::FromUtf8Error, sync::Arc};
|
||||
use tracing::{debug, error, info, instrument, warn, Span};
|
||||
use tracing_futures::Instrument;
|
||||
|
||||
|
|
@ -58,11 +55,6 @@ pub(crate) struct UploadManagerInner {
|
|||
db: sled::Db,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct Serde<T> {
|
||||
inner: T,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
|
||||
pub(crate) struct Details {
|
||||
width: usize,
|
||||
|
|
@ -482,7 +474,7 @@ where
|
|||
let identifier = S::Identifier::from_bytes(identifier.to_vec())?;
|
||||
debug!("Deleting {:?}", identifier);
|
||||
if let Err(e) = self.store.remove(&identifier).await {
|
||||
errors.push(e.into());
|
||||
errors.push(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -554,12 +546,6 @@ where
|
|||
}
|
||||
}
|
||||
|
||||
impl<T> Serde<T> {
|
||||
pub(crate) fn new(inner: T) -> Self {
|
||||
Serde { inner }
|
||||
}
|
||||
}
|
||||
|
||||
impl Details {
|
||||
fn is_motion(&self) -> bool {
|
||||
self.content_type.type_() == "video"
|
||||
|
|
@ -608,7 +594,7 @@ impl Details {
|
|||
}
|
||||
|
||||
pub(crate) fn content_type(&self) -> mime::Mime {
|
||||
self.content_type.inner.clone()
|
||||
(*self.content_type).clone()
|
||||
}
|
||||
|
||||
pub(crate) fn system_time(&self) -> std::time::SystemTime {
|
||||
|
|
@ -643,57 +629,12 @@ fn delete_key(alias: &str) -> String {
|
|||
format!("{}/delete", alias)
|
||||
}
|
||||
|
||||
impl<T> Deref for Serde<T> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> DerefMut for Serde<T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> std::fmt::Debug for UploadManager<S> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
f.debug_struct("UploadManager").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> serde::Serialize for Serde<T>
|
||||
where
|
||||
T: std::fmt::Display,
|
||||
{
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
let s = self.inner.to_string();
|
||||
serde::Serialize::serialize(s.as_str(), serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de, T> serde::Deserialize<'de> for Serde<T>
|
||||
where
|
||||
T: std::str::FromStr,
|
||||
<T as std::str::FromStr>::Err: std::fmt::Display,
|
||||
{
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let s: String = serde::Deserialize::deserialize(deserializer)?;
|
||||
let inner = s
|
||||
.parse::<T>()
|
||||
.map_err(|e| serde::de::Error::custom(e.to_string()))?;
|
||||
|
||||
Ok(Serde { inner })
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for FilenameIVec {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
write!(f, "{:?}", String::from_utf8(self.inner.to_vec()))
|
||||
|
|
@ -157,7 +157,7 @@ where
|
|||
debug!("Validating bytes");
|
||||
let (content_type, validated_reader) = crate::validate::validate_image_bytes(
|
||||
bytes_mut.freeze(),
|
||||
self.manager.inner.format.clone(),
|
||||
self.manager.inner.format,
|
||||
validate,
|
||||
)
|
||||
.await?;
|
||||
|
|
@ -199,7 +199,7 @@ where
|
|||
debug!("Validating bytes");
|
||||
let (input_type, validated_reader) = crate::validate::validate_image_bytes(
|
||||
bytes_mut.freeze(),
|
||||
self.manager.inner.format.clone(),
|
||||
self.manager.inner.format,
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
|
|
@ -236,11 +236,11 @@ where
|
|||
if dup.exists() {
|
||||
debug!("Duplicate exists, removing file");
|
||||
|
||||
self.manager.store.remove(&identifier).await?;
|
||||
self.manager.store.remove(identifier).await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.manager.store_identifier(name, &identifier).await?;
|
||||
self.manager.store_identifier(name, identifier).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -412,5 +412,5 @@ where
|
|||
}
|
||||
|
||||
fn file_name(name: Uuid, input_type: ValidInputType) -> String {
|
||||
format!("{}{}", name, input_type.to_ext())
|
||||
format!("{}{}", name, input_type.as_ext())
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue