From 750ce4782e85af1bbe869ed2c055e08498da531f Mon Sep 17 00:00:00 2001 From: "Aode (lion)" Date: Thu, 24 Mar 2022 22:06:29 -0500 Subject: [PATCH] Rework configuration --- Cargo.lock | 107 +++++++++------- Cargo.toml | 2 +- src/config.rs | 302 +++++++++++++++++++++++++++++++------------- src/migrate/repo.rs | 0 src/repo.rs | 2 + src/serde_str.rs | 4 + 6 files changed, 281 insertions(+), 136 deletions(-) create mode 100644 src/migrate/repo.rs diff --git a/Cargo.lock b/Cargo.lock index 15de7f6..a7a97e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -470,17 +470,32 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "2.34.0" +version = "3.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "d8c93436c21e4698bacadf42917db28b23017027a4deccb35dbe47a7e7840123" dependencies = [ - "ansi_term", "atty", "bitflags", + "clap_derive", + "indexmap", + "lazy_static", + "os_str_bytes", "strsim", + "termcolor", "textwrap", - "unicode-width", - "vec_map", +] + +[[package]] +name = "clap_derive" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da95d038ede1a964ce99f49cbe27a7fb538d1da595e4b4f70b8c8f338d17bf16" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -976,6 +991,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1459,6 +1480,15 @@ dependencies = [ "hashbrown 0.12.0", ] +[[package]] +name = "os_str_bytes" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" +dependencies = [ + "memchr", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -1590,6 +1620,7 @@ dependencies = [ "async-trait", "awc", "base64", + "clap", "config", "console-subscriber", "dashmap", @@ -1607,7 +1638,6 @@ dependencies = [ "sha2 0.10.2", "sled", "storage-path-generator", - "structopt", "thiserror", "time", "tokio", @@ -1713,7 +1743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" dependencies = [ "bytes", - "heck", + "heck 0.3.3", "itertools", "lazy_static", "log", @@ -2206,33 +2236,9 @@ dependencies = [ [[package]] name = "strsim" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "structopt" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" -dependencies = [ - "clap", - "lazy_static", - "structopt-derive", -] - -[[package]] -name = "structopt-derive" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" -dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "subtle" @@ -2266,14 +2272,20 @@ dependencies = [ ] [[package]] -name = "textwrap" -version = "0.11.0" +name = "termcolor" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ - "unicode-width", + "winapi-util", ] +[[package]] +name = "textwrap" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" + [[package]] name = "thiserror" version = "1.0.30" @@ -2711,12 +2723,6 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" -[[package]] -name = "unicode-width" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" - [[package]] name = "unicode-xid" version = "0.2.2" @@ -2758,12 +2764,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "version_check" version = "0.9.4" @@ -2914,6 +2914,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index cd13060..c70255e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ anyhow = "1.0" async-trait = "0.1.51" awc = { version = "3.0.0", default-features = false, features = ["rustls"] } base64 = "0.13.0" +clap = { version = "3.1.6", features = ["derive"] } config = "0.12.0" console-subscriber = "0.1" dashmap = "5.1.0" @@ -51,7 +52,6 @@ serde_json = "1.0" sha2 = "0.10.0" sled = { version = "0.34.7" } storage-path-generator = "0.1.0" -structopt = "0.3.14" thiserror = "1.0" time = { version = "0.3.0", features = ["serde"] } tokio = { version = "1", features = ["full", "tracing"] } diff --git a/src/config.rs b/src/config.rs index be1e09c..8ff247f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,18 +1,19 @@ +use crate::serde_str::Serde; +use clap::{ArgEnum, Parser, Subcommand}; use std::{collections::HashSet, net::SocketAddr, path::PathBuf}; -use structopt::StructOpt; use url::Url; use crate::magick::ValidInputType; -#[derive(Clone, Debug, StructOpt)] +#[derive(Clone, Debug, Parser)] pub(crate) struct Args { - #[structopt(short, long, help = "Path to the pict-rs configuration file")] + #[clap(short, long, help = "Path to the pict-rs configuration file")] config_file: Option, - #[structopt(long, help = "Path to a file defining a store migration")] - migrate_file: Option, + #[clap(subcommand)] + command: Command, - #[structopt(flatten)] + #[clap(flatten)] overrides: Overrides, } @@ -20,10 +21,10 @@ fn is_false(b: &bool) -> bool { !b } -#[derive(Clone, Debug, serde::Serialize, structopt::StructOpt)] +#[derive(Clone, Debug, serde::Serialize, Parser)] #[serde(rename_all = "snake_case")] pub(crate) struct Overrides { - #[structopt( + #[clap( short, long, help = "Whether to skip validating images uploaded via the internal import API" @@ -31,15 +32,15 @@ pub(crate) struct Overrides { #[serde(skip_serializing_if = "is_false")] skip_validate_imports: bool, - #[structopt(short, long, help = "The address and port the server binds to.")] + #[clap(short, long, help = "The address and port the server binds to.")] #[serde(skip_serializing_if = "Option::is_none")] addr: Option, - #[structopt(short, long, help = "The path to the data directory, e.g. data/")] + #[clap(short, long, help = "The path to the data directory, e.g. data/")] #[serde(skip_serializing_if = "Option::is_none")] path: Option, - #[structopt( + #[clap( short, long, help = "An optional image format to convert all uploaded files into, supports 'jpg', 'png', and 'webp'" @@ -47,7 +48,7 @@ pub(crate) struct Overrides { #[serde(skip_serializing_if = "Option::is_none")] image_format: Option, - #[structopt( + #[clap( short, long, help = "An optional list of filters to permit, supports 'identity', 'thumbnail', 'resize', 'crop', and 'blur'" @@ -55,7 +56,7 @@ pub(crate) struct Overrides { #[serde(skip_serializing_if = "Option::is_none")] filters: Option>, - #[structopt( + #[clap( short, long, help = "Specify the maximum allowed uploaded file size (in Megabytes)" @@ -63,40 +64,40 @@ pub(crate) struct Overrides { #[serde(skip_serializing_if = "Option::is_none")] max_file_size: Option, - #[structopt(long, help = "Specify the maximum width in pixels allowed on an image")] + #[clap(long, help = "Specify the maximum width in pixels allowed on an image")] #[serde(skip_serializing_if = "Option::is_none")] max_image_width: Option, - #[structopt(long, help = "Specify the maximum width in pixels allowed on an image")] + #[clap(long, help = "Specify the maximum width in pixels allowed on an image")] #[serde(skip_serializing_if = "Option::is_none")] max_image_height: Option, - #[structopt(long, help = "Specify the maximum area in pixels allowed in an image")] + #[clap(long, help = "Specify the maximum area in pixels allowed in an image")] #[serde(skip_serializing_if = "Option::is_none")] max_image_area: Option, - #[structopt( + #[clap( long, help = "Specify the number of bytes sled is allowed to use for it's cache" )] #[serde(skip_serializing_if = "Option::is_none")] sled_cache_capacity: Option, - #[structopt( + #[clap( long, help = "Specify the number of events the console subscriber is allowed to buffer" )] #[serde(skip_serializing_if = "Option::is_none")] console_buffer_capacity: Option, - #[structopt( + #[clap( long, help = "An optional string to be checked on requests to privileged endpoints" )] #[serde(skip_serializing_if = "Option::is_none")] api_key: Option, - #[structopt( + #[clap( short, long, help = "Enable OpenTelemetry Tracing exports to the given OpenTelemetry collector" @@ -104,9 +105,41 @@ pub(crate) struct Overrides { #[serde(skip_serializing_if = "Option::is_none")] opentelemetry_url: Option, - #[structopt(subcommand)] + #[serde(skip_serializing_if = "Option::is_none")] + repo: Option, + + #[clap(flatten)] + sled_repo: SledRepo, + #[serde(skip_serializing_if = "Option::is_none")] store: Option, + + #[clap(flatten)] + filesystem_storage: FilesystemStorage, + + #[clap(flatten)] + object_storage: ObjectStorage, +} + +impl ObjectStorage { + pub(crate) fn required(&self) -> Result { + Ok(RequiredObjectStorage { + bucket_name: self + .s3_store_bucket_name + .as_ref() + .cloned() + .ok_or(RequiredError)?, + region: self + .s3_store_region + .as_ref() + .cloned() + .map(Serde::into_inner) + .ok_or(RequiredError)?, + access_key: self.s3_store_access_key.as_ref().cloned(), + security_token: self.s3_store_security_token.as_ref().cloned(), + session_token: self.s3_store_session_token.as_ref().cloned(), + }) + } } impl Overrides { @@ -124,67 +157,99 @@ impl Overrides { && self.console_buffer_capacity.is_none() && self.api_key.is_none() && self.opentelemetry_url.is_none() + && self.repo.is_none() && self.store.is_none() } } -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "snake_case")] -pub(crate) struct Migrate { - from: Store, - to: Store, -} - -impl Migrate { - pub(crate) fn from(&self) -> &Store { - &self.from - } - - pub(crate) fn to(&self) -> &Store { - &self.to - } -} - -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, structopt::StructOpt)] +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Subcommand)] #[serde(rename_all = "snake_case")] #[serde(tag = "type")] +pub(crate) enum Command { + Run, + MigrateStore { to: Store }, + MigrateRepo { to: Repo }, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Repo { + Sled, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)] +#[serde(rename_all = "snake_case")] +pub(crate) struct SledRepo { + // defaults to {config.path} + #[clap(long, help = "Path in which pict-rs will create it's 'repo' directory")] + #[serde(skip_serializing_if = "Option::is_none")] + sled_repo_path: Option, + + #[clap( + long, + help = "The number of bytes sled is allowed to use for it's in-memory cache" + )] + #[serde(skip_serializing_if = "Option::is_none")] + sled_repo_cache_capacity: Option, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)] +#[serde(rename_all = "snake_case")] pub(crate) enum Store { - FileStore { - // defaults to {config.path} - #[structopt( - long, - help = "Path in which pict-rs will create it's 'files' directory" - )] - #[serde(skip_serializing_if = "Option::is_none")] - path: Option, - }, + Filesystem, #[cfg(feature = "object-storage")] - S3Store { - #[structopt(long, help = "Name of the bucket in which pict-rs will store images")] - bucket_name: String, + ObjectStorage, +} - #[structopt( - long, - help = "Region in which the bucket exists, can be an http endpoint" - )] - region: crate::serde_str::Serde, +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)] +#[serde(rename_all = "snake_case")] +pub(crate) struct FilesystemStorage { + // defaults to {config.path} + #[clap( + long, + help = "Path in which pict-rs will create it's 'files' directory" + )] + #[serde(skip_serializing_if = "Option::is_none")] + filesystem_storage_path: Option, +} - #[serde(skip_serializing_if = "Option::is_none")] - #[structopt(long)] - access_key: Option, +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)] +#[serde(rename_all = "snake_case")] +pub(crate) struct ObjectStorage { + #[serde(skip_serializing_if = "Option::is_none")] + #[clap(long, help = "Name of the bucket in which pict-rs will store images")] + s3_store_bucket_name: Option, - #[structopt(long)] - #[serde(skip_serializing_if = "Option::is_none")] - secret_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[clap( + long, + help = "Region in which the bucket exists, can be an http endpoint" + )] + s3_store_region: Option>, - #[structopt(long)] - #[serde(skip_serializing_if = "Option::is_none")] - security_token: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[clap(long)] + s3_store_access_key: Option, - #[structopt(long)] - #[serde(skip_serializing_if = "Option::is_none")] - session_token: Option, - }, + #[clap(long)] + #[serde(skip_serializing_if = "Option::is_none")] + s3_store_secret_key: Option, + + #[clap(long)] + #[serde(skip_serializing_if = "Option::is_none")] + s3_store_security_token: Option, + + #[clap(long)] + #[serde(skip_serializing_if = "Option::is_none")] + s3_store_session_token: Option, +} + +pub(crate) struct RequiredObjectStorage { + pub(crate) bucket_name: String, + pub(crate) region: s3::Region, + pub(crate) access_key: Option, + pub(crate) security_token: Option, + pub(crate) session_token: Option, } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] @@ -203,7 +268,11 @@ pub(crate) struct Config { console_buffer_capacity: Option, api_key: Option, opentelemetry_url: Option, + repo: Repo, + sled_repo: SledRepo, store: Store, + filesystem_storage: FilesystemStorage, + object_storage: ObjectStorage, } #[derive(serde::Serialize)] @@ -216,9 +285,22 @@ pub(crate) struct Defaults { max_image_height: usize, max_image_area: usize, sled_cache_capacity: u64, + repo: Repo, + sled_repo: SledRepoDefaults, store: Store, + filesystem_store: FilesystemDefaults, } +#[derive(serde::Serialize)] +#[serde(rename_all = "snake_case")] +struct SledRepoDefaults { + sled_repo_cache_capacity: usize, +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "snake_case")] +struct FilesystemDefaults {} + impl Defaults { fn new() -> Self { Defaults { @@ -229,23 +311,19 @@ impl Defaults { max_image_height: 10_000, max_image_area: 40_000_000, sled_cache_capacity: 1024 * 1024 * 64, // 16 times smaller than sled's default of 1GB - store: Store::FileStore { path: None }, + repo: Repo::Sled, + sled_repo: SledRepoDefaults { + sled_repo_cache_capacity: 1024 * 1024 * 64, + }, + store: Store::Filesystem, + filesystem_store: FilesystemDefaults {}, } } } impl Config { pub(crate) fn build() -> anyhow::Result { - let args = Args::from_args(); - - if let Some(path) = args.migrate_file { - let migrate_config = config::Config::builder() - .add_source(config::File::from(path)) - .build()?; - let migrate: Migrate = migrate_config.try_deserialize()?; - - crate::MIGRATE.set(migrate).unwrap(); - } + let args = Args::parse(); let mut base_config = config::Config::builder().add_source(config::Config::try_from(&Defaults::new())?); @@ -254,6 +332,8 @@ impl Config { base_config = base_config.add_source(config::File::from(path)); }; + // TODO: Command parsing + if !args.overrides.is_default() { let merging = config::Config::try_from(&args.overrides)?; @@ -272,6 +352,18 @@ impl Config { &self.store } + pub(crate) fn repo(&self) -> &Repo { + &self.repo + } + + pub(crate) fn object_storage(&self) -> Result { + self.object_storage.required() + } + + pub(crate) fn filesystem_storage_path(&self) -> Option<&PathBuf> { + self.filesystem_storage.filesystem_storage_path.as_ref() + } + pub(crate) fn bind_address(&self) -> SocketAddr { self.addr } @@ -329,7 +421,19 @@ impl Config { #[error("Invalid format supplied, {0}")] pub(crate) struct FormatError(String); -#[derive(Copy, Clone, Debug, serde::Deserialize, serde::Serialize)] +#[derive(Debug, thiserror::Error)] +#[error("Invalid store supplied, {0}")] +pub(crate) struct StoreError(String); + +#[derive(Debug, thiserror::Error)] +#[error("Invalid repo supplied, {0}")] +pub(crate) struct RepoError(String); + +#[derive(Debug, thiserror::Error)] +#[error("Missing required fields")] +pub(crate) struct RequiredError; + +#[derive(Copy, Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)] #[serde(rename_all = "snake_case")] pub(crate) enum Format { Jpeg, @@ -359,11 +463,37 @@ impl std::str::FromStr for Format { type Err = FormatError; fn from_str(s: &str) -> Result { - match s { - "png" => Ok(Format::Png), - "jpg" => Ok(Format::Jpeg), - "webp" => Ok(Format::Webp), - other => Err(FormatError(other.to_string())), + for variant in Self::value_variants() { + if variant.to_possible_value().unwrap().matches(s, false) { + return Ok(*variant); + } } + Err(FormatError(s.into())) + } +} + +impl std::str::FromStr for Store { + type Err = StoreError; + + fn from_str(s: &str) -> Result { + for variant in Self::value_variants() { + if variant.to_possible_value().unwrap().matches(s, false) { + return Ok(*variant); + } + } + Err(StoreError(s.into())) + } +} + +impl std::str::FromStr for Repo { + type Err = RepoError; + + fn from_str(s: &str) -> Result { + for variant in Self::value_variants() { + if variant.to_possible_value().unwrap().matches(s, false) { + return Ok(*variant); + } + } + Err(RepoError(s.into())) } } diff --git a/src/migrate/repo.rs b/src/migrate/repo.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/repo.rs b/src/repo.rs index 25e9be3..87d751a 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -8,9 +8,11 @@ pub(crate) struct Alias { id: Uuid, extension: String, } + pub(crate) struct DeleteToken { id: Uuid, } + pub(crate) struct AlreadyExists; impl Alias { diff --git a/src/serde_str.rs b/src/serde_str.rs index 53c6d1b..c255f5b 100644 --- a/src/serde_str.rs +++ b/src/serde_str.rs @@ -12,6 +12,10 @@ impl Serde { pub(crate) fn new(inner: T) -> Self { Serde { inner } } + + pub(crate) fn into_inner(this: Self) -> T { + this.inner + } } impl Deref for Serde {