Compare commits

...

1 commit

Author SHA1 Message Date
asonix 6c3842edfc pict-rs can build on eyra with sled disabled
Some checks failed
continuous-integration/drone/push Build is failing
2023-09-26 17:17:29 -05:00
19 changed files with 388 additions and 40 deletions

258
Cargo.lock generated
View file

@ -210,7 +210,7 @@ version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
dependencies = [
"gimli",
"gimli 0.28.0",
]
[[package]]
@ -454,6 +454,43 @@ dependencies = [
"bytes",
]
[[package]]
name = "c-gull"
version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44589b5130ab40d739906751b9066cc514e83b7f0c7df1e0484c96df34791b85"
dependencies = [
"c-scape",
"errno",
"libc",
"printf-compat",
"rustix",
"sync-resolve",
"tz-rs",
]
[[package]]
name = "c-scape"
version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f19eced9b8932f6d75223bc43263a7fbc92241ae2ff4e1325e893f0bc7f0e261"
dependencies = [
"errno",
"libc",
"libm",
"memoffset",
"origin",
"rand",
"rand_core",
"rand_pcg",
"realpath-ext",
"rustix",
"rustix-dlmalloc",
"rustix-futex-sync",
"rustix-openpty",
"unwinding",
]
[[package]]
name = "cc"
version = "1.0.83"
@ -663,6 +700,22 @@ dependencies = [
"typenum",
]
[[package]]
name = "cstr_core"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd98742e4fdca832d40cab219dc2e3048de17d873248f83f17df47c1bea70956"
dependencies = [
"cty",
"memchr",
]
[[package]]
name = "cty"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
[[package]]
name = "dashmap"
version = "5.5.3"
@ -821,6 +874,36 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd"
dependencies = [
"errno-dragonfly",
"libc",
"windows-sys",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "eyra"
version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a271b0c7bd629bf6d0464c57b0e784c426581c283b2fcd18537c664c7956dcbf"
dependencies = [
"c-gull",
]
[[package]]
name = "eyre"
version = "0.6.8"
@ -1009,6 +1092,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "gimli"
version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
[[package]]
name = "gimli"
version = "0.28.0"
@ -1194,6 +1283,17 @@ dependencies = [
"tokio-io-timeout",
]
[[package]]
name = "idna"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8"
dependencies = [
"matches",
"unicode-bidi",
"unicode-normalization",
]
[[package]]
name = "idna"
version = "0.4.0"
@ -1255,6 +1355,15 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
[[package]]
name = "itertools"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.10.5"
@ -1304,9 +1413,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.147"
version = "0.2.148"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
[[package]]
name = "libm"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"
[[package]]
name = "linked-hash-map"
@ -1314,6 +1429,12 @@ version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "linux-raw-sys"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128"
[[package]]
name = "local-channel"
version = "0.1.3"
@ -1366,6 +1487,12 @@ dependencies = [
"regex-automata 0.1.10",
]
[[package]]
name = "matches"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
[[package]]
name = "matchit"
version = "0.7.2"
@ -1664,6 +1791,21 @@ dependencies = [
"hashbrown 0.12.3",
]
[[package]]
name = "origin"
version = "0.13.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad6f816ea4b0df9f8210ad29b206d856ffdb3d1093fdfc0210e8ca96cd2b2bc1"
dependencies = [
"bitflags 2.4.0",
"linux-raw-sys",
"log",
"memoffset",
"rustix",
"rustix-futex-sync",
"unwinding",
]
[[package]]
name = "overload"
version = "0.1.1"
@ -1826,6 +1968,7 @@ dependencies = [
"diesel",
"diesel-async",
"diesel-derive-enum",
"eyra",
"flume",
"futures-core",
"hex",
@ -1961,6 +2104,18 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "printf-compat"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b002af28ffe3d3d67202ae717810a28125a494d5396debc43de01ee136ac404"
dependencies = [
"bitflags 1.3.2",
"cstr_core",
"cty",
"itertools 0.9.0",
]
[[package]]
name = "proc-macro2"
version = "1.0.66"
@ -1987,7 +2142,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
dependencies = [
"anyhow",
"itertools",
"itertools 0.10.5",
"proc-macro2",
"quote",
"syn 1.0.109",
@ -2077,6 +2232,15 @@ dependencies = [
"getrandom",
]
[[package]]
name = "rand_pcg"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e"
dependencies = [
"rand_core",
]
[[package]]
name = "raw-cpuid"
version = "10.7.0"
@ -2086,6 +2250,17 @@ dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "realpath-ext"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "692f72862a0d532b44a0f4965fb10f17e7659eaedf24d2ce3c989ca778bd092f"
dependencies = [
"bitflags 1.3.2",
"errno",
"libc",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
@ -2333,6 +2508,52 @@ dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "0.38.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f"
dependencies = [
"bitflags 2.4.0",
"errno",
"itoa",
"libc",
"linux-raw-sys",
"once_cell",
"windows-sys",
]
[[package]]
name = "rustix-dlmalloc"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "730c0a5fbe9885450abcfca57da8c5d5a886492d01aa1a78b782fc5ddb2c0931"
dependencies = [
"rustix",
"rustix-futex-sync",
]
[[package]]
name = "rustix-futex-sync"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "778811def1fc5ab594a0474b44ccbd02fe276e4a3f3551d005770108bdc92072"
dependencies = [
"lock_api",
"rustix",
]
[[package]]
name = "rustix-openpty"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a25c3aad9fc1424eb82c88087789a7d938e1829724f3e4043163baf0d13cfc12"
dependencies = [
"errno",
"libc",
"rustix",
]
[[package]]
name = "rustls"
version = "0.21.7"
@ -2702,6 +2923,17 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "sync-resolve"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a89fa9e37fd57511c6c486c4e7a2f38c94a27c130347b4a310425074a4ad7a7"
dependencies = [
"idna 0.2.3",
"log",
"rand",
]
[[package]]
name = "sync_wrapper"
version = "0.1.2"
@ -3137,6 +3369,12 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
[[package]]
name = "tz-rs"
version = "0.6.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4"
[[package]]
name = "ucd-trie"
version = "0.1.6"
@ -3179,6 +3417,16 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "unwinding"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ee215fc0991fb587275819562685254302112d0b7c8d25e635655526c0ecdc6"
dependencies = [
"gimli 0.26.2",
"libc",
]
[[package]]
name = "url"
version = "2.4.1"
@ -3186,7 +3434,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5"
dependencies = [
"form_urlencoded",
"idna",
"idna 0.4.0",
"percent-encoding",
"serde",
]

View file

@ -7,11 +7,14 @@ license = "AGPL-3.0"
readme = "README.md"
repository = "https://git.asonix.dog/asonix/pict-rs"
edition = "2021"
build = "./src/build.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
default = []
io-uring = ["actix-rt/io-uring", "actix-server/io-uring", "tokio-uring", "sled/io_uring"]
default = ["postgres", "sled"]
sled = ["dep:sled"]
postgres = ["dep:barrel", "dep:deadpool", "dep:diesel", "dep:diesel-async", "dep:diesel-derive-enum", "dep:refinery", "dep:tokio-postgres"]
io-uring = ["actix-rt/io-uring", "actix-server/io-uring", "tokio-uring", "sled?/io_uring"]
[dependencies]
actix-form-data = "0.7.0-beta.4"
@ -20,17 +23,18 @@ actix-server = "2.0.0"
actix-web = { version = "4.0.0", default-features = false }
anyhow = "1.0"
async-trait = "0.1.51"
barrel = { version = "0.7.0", features = ["pg"] }
barrel = { version = "0.7.0", features = ["pg"], optional = true }
base64 = "0.21.0"
clap = { version = "4.0.2", features = ["derive"] }
color-eyre = "0.6"
config = "0.13.0"
console-subscriber = "0.1"
dashmap = "5.1.0"
deadpool = { version = "0.9.5", features = ["rt_tokio_1"] }
diesel = { version = "2.1.1", features = ["postgres_backend", "serde_json", "time", "uuid"] }
diesel-async = { version = "0.4.1", features = ["postgres", "deadpool"] }
diesel-derive-enum = { version = "2.1.0", features = ["postgres"] }
deadpool = { version = "0.9.5", features = ["rt_tokio_1"], optional = true }
diesel = { version = "2.1.1", features = ["postgres_backend", "serde_json", "time", "uuid"], optional = true }
diesel-async = { version = "0.4.1", features = ["postgres", "deadpool"], optional = true }
diesel-derive-enum = { version = "2.1.0", features = ["postgres"], optional = true }
eyra = { version = "0.15.3", features = ["log"] }
flume = "0.11.0"
futures-core = "0.3"
hex = "0.4.3"
@ -44,7 +48,7 @@ opentelemetry = { version = "0.20", features = ["rt-tokio"] }
opentelemetry-otlp = "0.13"
pin-project-lite = "0.2.7"
quick-xml = { version = "0.30.0", features = ["serialize"] }
refinery = { version = "0.8.10", features = ["tokio-postgres", "postgres"] }
refinery = { version = "0.8.10", features = ["tokio-postgres", "postgres"], optional = true }
reqwest = { version = "0.11.18", default-features = false, features = ["json", "rustls-tls", "stream"] }
reqwest-middleware = "0.2.2"
reqwest-tracing = { version = "0.4.5" }
@ -55,13 +59,13 @@ serde_cbor = "0.11.2"
serde_json = "1.0"
serde_urlencoded = "0.7.1"
sha2 = "0.10.0"
sled = { version = "0.34.7" }
sled = { version = "0.34.7", optional = true }
storage-path-generator = "0.1.0"
streem = "0.2.0"
thiserror = "1.0"
time = { version = "0.3.0", features = ["serde", "serde-well-known"] }
tokio = { version = "1", features = ["full", "tracing"] }
tokio-postgres = { version = "0.7.10", features = ["with-uuid-1", "with-time-0_3", "with-serde_json-1"] }
tokio-postgres = { version = "0.7.10", features = ["with-uuid-1", "with-time-0_3", "with-serde_json-1"], optional = true }
tokio-uring = { version = "0.4", optional = true, features = ["bytes"] }
tokio-util = { version = "0.7", default-features = false, features = [
"codec",

View file

@ -29,8 +29,6 @@
devShell = with pkgs; mkShell {
nativeBuildInputs = [
cargo
cargo-outdated
clippy
diesel-cli
exiftool
@ -38,10 +36,8 @@
garage
gcc
imagemagick
rustup
protobuf
rust-analyzer
rustc
rustfmt
taplo
];

3
src/build.rs Normal file
View file

@ -0,0 +1,3 @@
fn main() {
println!("cargo:rustc-link-arg=-nostartfiles");
}

View file

@ -11,9 +11,13 @@ use config::Config;
use defaults::Defaults;
pub(crate) use commandline::Operation;
#[cfg(feature = "postgres")]
pub(crate) use file::Postgres;
#[cfg(feature = "sled")]
pub(crate) use file::Sled;
pub(crate) use file::{
Animation, ConfigFile as Configuration, Image, Media, ObjectStorage, OpenTelemetry, Postgres,
Repo, Sled, Store, Tracing, Video,
Animation, ConfigFile as Configuration, Image, Media, ObjectStorage, OpenTelemetry, Repo,
Store, Tracing, Video,
};
pub(crate) use primitives::{Filesystem, LogFormat};

View file

@ -140,6 +140,7 @@ impl Details {
}
}
#[cfg(feature = "sled")]
pub(crate) fn from_parts_full(
format: InternalFormat,
width: u16,

View file

@ -74,6 +74,7 @@ pub(crate) enum UploadError {
#[error("Error in DB")]
Repo(#[from] crate::repo::RepoError),
#[cfg(feature = "sled")]
#[error("Error in old repo")]
OldRepo(#[from] crate::repo_04::RepoError),
@ -158,6 +159,7 @@ impl UploadError {
match self {
Self::Upload(_) => ErrorCode::FILE_UPLOAD_ERROR,
Self::Repo(e) => e.error_code(),
#[cfg(feature = "sled")]
Self::OldRepo(_) => ErrorCode::OLD_REPO_ERROR,
Self::Io(_) => ErrorCode::IO_ERROR,
Self::Validation(e) => e.error_code(),

View file

@ -26,6 +26,7 @@ impl ErrorCode {
pub(crate) const COMMAND_FAILURE: ErrorCode = ErrorCode {
code: "command-failure",
};
#[cfg(feature = "sled")]
pub(crate) const OLD_REPO_ERROR: ErrorCode = ErrorCode {
code: "old-repo-error",
};
@ -55,7 +56,9 @@ impl ErrorCode {
pub(crate) const ALREADY_CLAIMED: ErrorCode = ErrorCode {
code: "already-claimed",
};
#[cfg(feature = "sled")]
pub(crate) const SLED_ERROR: ErrorCode = ErrorCode { code: "sled-error" };
#[cfg(feature = "postgres")]
pub(crate) const POSTGRES_ERROR: ErrorCode = ErrorCode {
code: "postgres-error",
};
@ -66,9 +69,11 @@ impl ErrorCode {
code: "extract-upload-result",
};
pub(crate) const PUSH_JOB: ErrorCode = ErrorCode { code: "push-job" };
#[cfg(feature = "sled")]
pub(crate) const EXTRACT_JOB: ErrorCode = ErrorCode {
code: "extract-job",
};
#[cfg(feature = "sled")]
pub(crate) const CONFLICTED_RECORD: ErrorCode = ErrorCode {
code: "conflicted-record",
};

View file

@ -113,6 +113,7 @@ impl InternalFormat {
}
}
#[cfg(feature = "sled")]
pub(crate) fn maybe_from_media_type(mime: &mime::Mime, has_frames: bool) -> Option<Self> {
match (mime.type_(), mime.subtype().as_str(), has_frames) {
(mime::IMAGE, "apng", _) => Some(Self::Animation(AnimationFormat::Apng)),

View file

@ -24,6 +24,7 @@ mod queue;
mod range;
mod read;
mod repo;
#[cfg(feature = "sled")]
mod repo_04;
mod serde_str;
mod store;
@ -73,12 +74,15 @@ use self::{
middleware::{Deadline, Internal},
migrate_store::migrate_store,
queue::queue_generate,
repo::{sled::SledRepo, Alias, DeleteToken, Hash, Repo, UploadId, UploadResult},
repo::{Alias, DeleteToken, Hash, Repo, UploadId, UploadResult},
serde_str::Serde,
store::{file_store::FileStore, object_store::ObjectStore, Store},
stream::{empty, once},
};
#[cfg(feature = "sled")]
use repo::sled::SledRepo;
pub use self::config::{ConfigSource, PictRsConfiguration};
const MEGABYTES: usize = 1024 * 1024;
@ -1922,6 +1926,7 @@ impl<P: AsRef<Path>, T: serde::Serialize> ConfigSource<P, T> {
}
}
#[cfg(feature = "sled")]
async fn export_handler(repo: web::Data<SledRepo>) -> Result<HttpResponse, Error> {
repo.export().await?;
@ -1930,6 +1935,7 @@ async fn export_handler(repo: web::Data<SledRepo>) -> Result<HttpResponse, Error
})))
}
#[cfg(feature = "sled")]
fn sled_extra_config(sc: &mut web::ServiceConfig, repo: SledRepo) {
sc.app_data(web::Data::new(repo))
.service(web::resource("/export").route(web::post().to(export_handler)));
@ -2062,6 +2068,7 @@ impl PictRsConfiguration {
let store = FileStore::build(path, arc_repo.clone()).await?;
#[cfg(feature = "sled")]
if arc_repo.get("migrate-0.4").await?.is_none() {
if let Some(old_repo) = repo_04::open(&config.old_repo)? {
repo::migrate_04(old_repo, arc_repo.clone(), store.clone(), config.clone())
@ -2073,12 +2080,14 @@ impl PictRsConfiguration {
}
match repo {
#[cfg(feature = "sled")]
Repo::Sled(sled_repo) => {
launch_file_store(arc_repo, store, client, config, move |sc| {
sled_extra_config(sc, sled_repo.clone())
})
.await?;
}
#[cfg(feature = "postgres")]
Repo::Postgres(_) => {
launch_file_store(arc_repo, store, client, config, |_| {}).await?;
}
@ -2118,6 +2127,7 @@ impl PictRsConfiguration {
.await?
.build(client.clone());
#[cfg(feature = "sled")]
if arc_repo.get("migrate-0.4").await?.is_none() {
if let Some(old_repo) = repo_04::open(&config.old_repo)? {
repo::migrate_04(old_repo, arc_repo.clone(), store.clone(), config.clone())
@ -2129,12 +2139,14 @@ impl PictRsConfiguration {
}
match repo {
#[cfg(feature = "sled")]
Repo::Sled(sled_repo) => {
launch_object_store(arc_repo, store, client, config, move |sc| {
sled_extra_config(sc, sled_repo.clone())
})
.await?;
}
#[cfg(feature = "postgres")]
Repo::Postgres(_) => {
launch_object_store(arc_repo, store, client, config, |_| {}).await?;
}

View file

@ -1,3 +1,5 @@
extern crate eyra;
#[actix_rt::main]
async fn main() -> color_eyre::Result<()> {
pict_rs::PictRsConfiguration::build_default()?

View file

@ -16,19 +16,27 @@ use std::{fmt::Debug, sync::Arc};
use url::Url;
use uuid::Uuid;
#[cfg(feature = "postgres")]
pub(crate) mod postgres;
#[cfg(feature = "sled")]
pub(crate) mod sled;
pub(crate) use alias::Alias;
pub(crate) use delete_token::DeleteToken;
pub(crate) use hash::Hash;
pub(crate) use migrate::{migrate_04, migrate_repo};
pub(crate) use migrate::migrate_repo;
#[cfg(feature = "sled")]
pub(crate) use migrate::migrate_04;
pub(crate) type ArcRepo = Arc<dyn FullRepo>;
#[derive(Clone, Debug)]
pub(crate) enum Repo {
#[cfg(feature = "sled")]
Sled(self::sled::SledRepo),
#[cfg(feature = "postgres")]
Postgres(self::postgres::PostgresRepo),
}
@ -64,15 +72,18 @@ pub(crate) enum UploadResult {
#[derive(Debug, thiserror::Error)]
pub(crate) enum RepoError {
#[cfg(feature = "sled")]
#[error("Error in sled")]
SledError(#[from] crate::repo::sled::SledError),
#[cfg(feature = "postgres")]
#[error("Error in postgres")]
PostgresError(#[from] crate::repo::postgres::PostgresError),
#[error("Upload was already claimed")]
AlreadyClaimed,
#[cfg(feature = "sled")]
#[error("Panic in blocking operation")]
Canceled,
}
@ -80,16 +91,22 @@ pub(crate) enum RepoError {
impl RepoError {
pub(crate) const fn error_code(&self) -> ErrorCode {
match self {
#[cfg(feature = "sled")]
Self::SledError(e) => e.error_code(),
#[cfg(feature = "postgres")]
Self::PostgresError(e) => e.error_code(),
Self::AlreadyClaimed => ErrorCode::ALREADY_CLAIMED,
#[cfg(feature = "sled")]
Self::Canceled => ErrorCode::PANIC,
}
}
pub(crate) const fn is_disconnected(&self) -> bool {
match self {
#[cfg(feature = "postgres")]
Self::PostgresError(e) => e.is_disconnected(),
_ => false,
}
}
@ -338,14 +355,17 @@ where
pub(crate) struct JobId(Uuid);
impl JobId {
#[cfg(feature = "sled")]
pub(crate) fn gen() -> Self {
Self(Uuid::now_v7())
}
#[cfg(feature = "sled")]
pub(crate) const fn as_bytes(&self) -> &[u8; 16] {
self.0.as_bytes()
}
#[cfg(feature = "sled")]
pub(crate) const fn from_bytes(bytes: [u8; 16]) -> Self {
Self(Uuid::from_bytes(bytes))
}
@ -791,6 +811,7 @@ impl Repo {
#[tracing::instrument(skip(config))]
pub(crate) async fn open(config: config::Repo) -> color_eyre::Result<Self> {
match config {
#[cfg(feature = "sled")]
config::Repo::Sled(config::Sled {
path,
cache_capacity,
@ -800,17 +821,30 @@ impl Repo {
Ok(Self::Sled(repo))
}
#[cfg(not(feature = "sled"))]
config::Repo::Sled(_) => unimplemented!("pict-rs is not compiled with sled support"),
#[cfg(feature = "postgres")]
config::Repo::Postgres(config::Postgres { url }) => {
let repo = self::postgres::PostgresRepo::connect(url).await?;
Ok(Self::Postgres(repo))
}
#[cfg(not(feature = "postgres"))]
config::Repo::Postgres(_) => {
unimplemented!("pict-rs is not compiled with postgres support")
}
}
}
pub(crate) fn to_arc(&self) -> ArcRepo {
match self {
#[cfg(feature = "sled")]
Self::Sled(sled_repo) => Arc::new(sled_repo.clone()),
#[cfg(feature = "postgres")]
Self::Postgres(postgres_repo) => Arc::new(postgres_repo.clone()),
}
}
@ -834,10 +868,12 @@ impl MaybeUuid {
}
impl UploadId {
#[cfg(feature = "sled")]
pub(crate) fn generate() -> Self {
Self { id: Uuid::new_v4() }
}
#[cfg(feature = "sled")]
pub(crate) fn as_bytes(&self) -> &[u8] {
&self.id.as_bytes()[..]
}

View file

@ -1,15 +1,19 @@
#[cfg(feature = "postgres")]
use diesel::{backend::Backend, sql_types::VarChar, AsExpression, FromSqlRow};
use uuid::Uuid;
use super::MaybeUuid;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, AsExpression, FromSqlRow)]
#[diesel(sql_type = VarChar)]
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[cfg_attr(feature = "postgres", derive(AsExpression, FromSqlRow))]
#[cfg_attr(feature = "postgres", diesel(sql_type = VarChar))]
pub(crate) struct Alias {
id: MaybeUuid,
extension: Option<String>,
}
#[cfg(feature = "postgres")]
impl diesel::serialize::ToSql<VarChar, diesel::pg::Pg> for Alias {
fn to_sql<'b>(
&'b self,
@ -24,6 +28,7 @@ impl diesel::serialize::ToSql<VarChar, diesel::pg::Pg> for Alias {
}
}
#[cfg(feature = "postgres")]
impl<B> diesel::deserialize::FromSql<VarChar, B> for Alias
where
B: Backend,

View file

@ -1,14 +1,18 @@
#[cfg(feature = "postgres")]
use diesel::{backend::Backend, sql_types::VarChar, AsExpression, FromSqlRow};
use uuid::Uuid;
use super::MaybeUuid;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, AsExpression, FromSqlRow)]
#[diesel(sql_type = VarChar)]
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[cfg_attr(feature = "postgres", derive(AsExpression, FromSqlRow))]
#[cfg_attr(feature = "postgres", diesel(sql_type = VarChar))]
pub(crate) struct DeleteToken {
id: MaybeUuid,
}
#[cfg(feature = "postgres")]
impl diesel::serialize::ToSql<VarChar, diesel::pg::Pg> for DeleteToken {
fn to_sql<'b>(
&'b self,
@ -23,6 +27,7 @@ impl diesel::serialize::ToSql<VarChar, diesel::pg::Pg> for DeleteToken {
}
}
#[cfg(feature = "postgres")]
impl<B> diesel::deserialize::FromSql<VarChar, B> for DeleteToken
where
B: Backend,
@ -56,10 +61,12 @@ impl DeleteToken {
}
}
#[cfg(feature = "sled")]
pub(crate) fn to_bytes(&self) -> Vec<u8> {
self.id.as_bytes().to_vec()
}
#[cfg(feature = "sled")]
pub(crate) fn from_slice(bytes: &[u8]) -> Option<Self> {
if let Ok(s) = std::str::from_utf8(bytes) {
Some(DeleteToken::from_existing(s))

View file

@ -1,16 +1,19 @@
#[cfg(feature = "postgres")]
use diesel::{backend::Backend, sql_types::VarChar, AsExpression, FromSqlRow};
use crate::formats::InternalFormat;
use std::sync::Arc;
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, AsExpression, FromSqlRow)]
#[diesel(sql_type = VarChar)]
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[cfg_attr(feature = "postgres", derive(AsExpression, FromSqlRow))]
#[cfg_attr(feature = "postgres", diesel(sql_type = VarChar))]
pub(crate) struct Hash {
hash: Arc<[u8; 32]>,
size: u64,
format: InternalFormat,
}
#[cfg(feature = "postgres")]
impl diesel::serialize::ToSql<VarChar, diesel::pg::Pg> for Hash {
fn to_sql<'b>(
&'b self,
@ -25,6 +28,7 @@ impl diesel::serialize::ToSql<VarChar, diesel::pg::Pg> for Hash {
}
}
#[cfg(feature = "postgres")]
impl<B> diesel::deserialize::FromSql<VarChar, B> for Hash
where
B: Backend,
@ -59,12 +63,14 @@ impl Hash {
hex::encode(self.to_bytes())
}
#[cfg(feature = "postgres")]
pub(crate) fn to_base64(&self) -> String {
use base64::Engine;
base64::engine::general_purpose::STANDARD.encode(self.to_bytes())
}
#[cfg(feature = "postgres")]
pub(crate) fn from_base64(input: String) -> Option<Self> {
use base64::Engine;
@ -87,10 +93,12 @@ impl Hash {
vec
}
#[cfg(feature = "sled")]
pub(super) fn to_ivec(&self) -> sled::IVec {
sled::IVec::from(self.to_bytes())
}
#[cfg(feature = "sled")]
pub(super) fn from_ivec(ivec: sled::IVec) -> Option<Self> {
Self::from_bytes(&ivec)
}

View file

@ -1,21 +1,26 @@
#[cfg(feature = "sled")]
use std::sync::Arc;
use streem::IntoStreamer;
#[cfg(feature = "sled")]
use tokio::task::JoinSet;
use crate::{
config::Configuration,
details::Details,
error::Error,
repo::{ArcRepo, DeleteToken, Hash},
repo_04::{
AliasRepo as _, HashRepo as _, IdentifierRepo as _, SettingsRepo as _,
SledRepo as OldSledRepo,
},
store::Store,
#[cfg(feature = "sled")]
use crate::repo_04::{
AliasRepo as _, HashRepo as _, IdentifierRepo as _, SettingsRepo as _, SledRepo as OldSledRepo,
};
#[cfg(feature = "sled")]
use crate::{config::Configuration, details::Details, store::Store};
use crate::{
error::Error,
repo::{ArcRepo, DeleteToken, Hash},
};
#[cfg(feature = "sled")]
const MIGRATE_CONCURRENCY: usize = 32;
const GENERATOR_KEY: &str = "last-path";
#[tracing::instrument(skip_all)]
@ -66,6 +71,7 @@ pub(crate) async fn migrate_repo(old_repo: ArcRepo, new_repo: ArcRepo) -> Result
Ok(())
}
#[cfg(feature = "sled")]
#[tracing::instrument(skip_all)]
pub(crate) async fn migrate_04<S: Store + 'static>(
old_repo: OldSledRepo,
@ -169,6 +175,7 @@ async fn migrate_hash(old_repo: ArcRepo, new_repo: ArcRepo, hash: Hash) {
}
}
#[cfg(feature = "sled")]
async fn migrate_hash_04<S: Store>(
old_repo: OldSledRepo,
new_repo: ArcRepo,
@ -265,6 +272,7 @@ async fn do_migrate_hash(old_repo: &ArcRepo, new_repo: &ArcRepo, hash: Hash) ->
Ok(())
}
#[cfg(feature = "sled")]
#[tracing::instrument(skip_all)]
async fn do_migrate_hash_04<S: Store>(
old_repo: &OldSledRepo,
@ -326,6 +334,7 @@ async fn do_migrate_hash_04<S: Store>(
Ok(())
}
#[cfg(feature = "sled")]
async fn set_details<S: Store>(
old_repo: &OldSledRepo,
new_repo: &ArcRepo,
@ -342,6 +351,7 @@ async fn set_details<S: Store>(
}
}
#[cfg(feature = "sled")]
#[tracing::instrument(skip_all)]
async fn fetch_or_generate_details<S: Store>(
old_repo: &OldSledRepo,

View file

@ -19,6 +19,7 @@ pub(crate) enum StoreError {
#[error("Error in DB")]
Repo(#[from] crate::repo::RepoError),
#[cfg(feature = "sled")]
#[error("Error in 0.4 DB")]
Repo04(#[from] crate::repo_04::RepoError),
@ -35,6 +36,7 @@ impl StoreError {
Self::FileStore(e) => e.error_code(),
Self::ObjectStore(e) => e.error_code(),
Self::Repo(e) => e.error_code(),
#[cfg(feature = "sled")]
Self::Repo04(_) => ErrorCode::OLD_REPO_ERROR,
Self::FileNotFound(_) | Self::ObjectNotFound(_) => ErrorCode::NOT_FOUND,
}

View file

@ -52,6 +52,7 @@ where
})
}
#[cfg(feature = "sled")]
pub(crate) fn from_iterator<I>(iterator: I, buffer: usize) -> impl Stream<Item = I::Item> + Send
where
I: IntoIterator + Send + 'static,

View file

@ -31,6 +31,7 @@ where
tracing::trace_span!(parent: None, "spawn task").in_scope(|| actix_rt::spawn(future))
}
#[cfg(feature = "sled")]
#[track_caller]
pub(crate) fn spawn_blocking<F, Out>(function: F) -> actix_rt::task::JoinHandle<Out>
where