Rework configuration
Some checks failed
continuous-integration/drone/push Build is failing

This commit is contained in:
Aode (lion) 2022-03-24 22:06:29 -05:00
parent d6567fbbbd
commit 750ce4782e
6 changed files with 281 additions and 136 deletions

107
Cargo.lock generated
View file

@ -470,17 +470,32 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clap"
version = "2.34.0"
version = "3.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
checksum = "d8c93436c21e4698bacadf42917db28b23017027a4deccb35dbe47a7e7840123"
dependencies = [
"ansi_term",
"atty",
"bitflags",
"clap_derive",
"indexmap",
"lazy_static",
"os_str_bytes",
"strsim",
"termcolor",
"textwrap",
"unicode-width",
"vec_map",
]
[[package]]
name = "clap_derive"
version = "3.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da95d038ede1a964ce99f49cbe27a7fb538d1da595e4b4f70b8c8f338d17bf16"
dependencies = [
"heck 0.4.0",
"proc-macro-error",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@ -976,6 +991,12 @@ dependencies = [
"unicode-segmentation",
]
[[package]]
name = "heck"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
[[package]]
name = "hermit-abi"
version = "0.1.19"
@ -1459,6 +1480,15 @@ dependencies = [
"hashbrown 0.12.0",
]
[[package]]
name = "os_str_bytes"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
dependencies = [
"memchr",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
@ -1590,6 +1620,7 @@ dependencies = [
"async-trait",
"awc",
"base64",
"clap",
"config",
"console-subscriber",
"dashmap",
@ -1607,7 +1638,6 @@ dependencies = [
"sha2 0.10.2",
"sled",
"storage-path-generator",
"structopt",
"thiserror",
"time",
"tokio",
@ -1713,7 +1743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
dependencies = [
"bytes",
"heck",
"heck 0.3.3",
"itertools",
"lazy_static",
"log",
@ -2206,33 +2236,9 @@ dependencies = [
[[package]]
name = "strsim"
version = "0.8.0"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "structopt"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"
dependencies = [
"clap",
"lazy_static",
"structopt-derive",
]
[[package]]
name = "structopt-derive"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
dependencies = [
"heck",
"proc-macro-error",
"proc-macro2",
"quote",
"syn",
]
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "subtle"
@ -2266,14 +2272,20 @@ dependencies = [
]
[[package]]
name = "textwrap"
version = "0.11.0"
name = "termcolor"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
dependencies = [
"unicode-width",
"winapi-util",
]
[[package]]
name = "textwrap"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
[[package]]
name = "thiserror"
version = "1.0.30"
@ -2711,12 +2723,6 @@ version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
[[package]]
name = "unicode-width"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
[[package]]
name = "unicode-xid"
version = "0.2.2"
@ -2758,12 +2764,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "vec_map"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "version_check"
version = "0.9.4"
@ -2914,6 +2914,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"

View file

@ -28,6 +28,7 @@ anyhow = "1.0"
async-trait = "0.1.51"
awc = { version = "3.0.0", default-features = false, features = ["rustls"] }
base64 = "0.13.0"
clap = { version = "3.1.6", features = ["derive"] }
config = "0.12.0"
console-subscriber = "0.1"
dashmap = "5.1.0"
@ -51,7 +52,6 @@ serde_json = "1.0"
sha2 = "0.10.0"
sled = { version = "0.34.7" }
storage-path-generator = "0.1.0"
structopt = "0.3.14"
thiserror = "1.0"
time = { version = "0.3.0", features = ["serde"] }
tokio = { version = "1", features = ["full", "tracing"] }

View file

@ -1,18 +1,19 @@
use crate::serde_str::Serde;
use clap::{ArgEnum, Parser, Subcommand};
use std::{collections::HashSet, net::SocketAddr, path::PathBuf};
use structopt::StructOpt;
use url::Url;
use crate::magick::ValidInputType;
#[derive(Clone, Debug, StructOpt)]
#[derive(Clone, Debug, Parser)]
pub(crate) struct Args {
#[structopt(short, long, help = "Path to the pict-rs configuration file")]
#[clap(short, long, help = "Path to the pict-rs configuration file")]
config_file: Option<PathBuf>,
#[structopt(long, help = "Path to a file defining a store migration")]
migrate_file: Option<PathBuf>,
#[clap(subcommand)]
command: Command,
#[structopt(flatten)]
#[clap(flatten)]
overrides: Overrides,
}
@ -20,10 +21,10 @@ fn is_false(b: &bool) -> bool {
!b
}
#[derive(Clone, Debug, serde::Serialize, structopt::StructOpt)]
#[derive(Clone, Debug, serde::Serialize, Parser)]
#[serde(rename_all = "snake_case")]
pub(crate) struct Overrides {
#[structopt(
#[clap(
short,
long,
help = "Whether to skip validating images uploaded via the internal import API"
@ -31,15 +32,15 @@ pub(crate) struct Overrides {
#[serde(skip_serializing_if = "is_false")]
skip_validate_imports: bool,
#[structopt(short, long, help = "The address and port the server binds to.")]
#[clap(short, long, help = "The address and port the server binds to.")]
#[serde(skip_serializing_if = "Option::is_none")]
addr: Option<SocketAddr>,
#[structopt(short, long, help = "The path to the data directory, e.g. data/")]
#[clap(short, long, help = "The path to the data directory, e.g. data/")]
#[serde(skip_serializing_if = "Option::is_none")]
path: Option<PathBuf>,
#[structopt(
#[clap(
short,
long,
help = "An optional image format to convert all uploaded files into, supports 'jpg', 'png', and 'webp'"
@ -47,7 +48,7 @@ pub(crate) struct Overrides {
#[serde(skip_serializing_if = "Option::is_none")]
image_format: Option<Format>,
#[structopt(
#[clap(
short,
long,
help = "An optional list of filters to permit, supports 'identity', 'thumbnail', 'resize', 'crop', and 'blur'"
@ -55,7 +56,7 @@ pub(crate) struct Overrides {
#[serde(skip_serializing_if = "Option::is_none")]
filters: Option<Vec<String>>,
#[structopt(
#[clap(
short,
long,
help = "Specify the maximum allowed uploaded file size (in Megabytes)"
@ -63,40 +64,40 @@ pub(crate) struct Overrides {
#[serde(skip_serializing_if = "Option::is_none")]
max_file_size: Option<usize>,
#[structopt(long, help = "Specify the maximum width in pixels allowed on an image")]
#[clap(long, help = "Specify the maximum width in pixels allowed on an image")]
#[serde(skip_serializing_if = "Option::is_none")]
max_image_width: Option<usize>,
#[structopt(long, help = "Specify the maximum width in pixels allowed on an image")]
#[clap(long, help = "Specify the maximum width in pixels allowed on an image")]
#[serde(skip_serializing_if = "Option::is_none")]
max_image_height: Option<usize>,
#[structopt(long, help = "Specify the maximum area in pixels allowed in an image")]
#[clap(long, help = "Specify the maximum area in pixels allowed in an image")]
#[serde(skip_serializing_if = "Option::is_none")]
max_image_area: Option<usize>,
#[structopt(
#[clap(
long,
help = "Specify the number of bytes sled is allowed to use for it's cache"
)]
#[serde(skip_serializing_if = "Option::is_none")]
sled_cache_capacity: Option<u64>,
#[structopt(
#[clap(
long,
help = "Specify the number of events the console subscriber is allowed to buffer"
)]
#[serde(skip_serializing_if = "Option::is_none")]
console_buffer_capacity: Option<usize>,
#[structopt(
#[clap(
long,
help = "An optional string to be checked on requests to privileged endpoints"
)]
#[serde(skip_serializing_if = "Option::is_none")]
api_key: Option<String>,
#[structopt(
#[clap(
short,
long,
help = "Enable OpenTelemetry Tracing exports to the given OpenTelemetry collector"
@ -104,9 +105,41 @@ pub(crate) struct Overrides {
#[serde(skip_serializing_if = "Option::is_none")]
opentelemetry_url: Option<Url>,
#[structopt(subcommand)]
#[serde(skip_serializing_if = "Option::is_none")]
repo: Option<Repo>,
#[clap(flatten)]
sled_repo: SledRepo,
#[serde(skip_serializing_if = "Option::is_none")]
store: Option<Store>,
#[clap(flatten)]
filesystem_storage: FilesystemStorage,
#[clap(flatten)]
object_storage: ObjectStorage,
}
impl ObjectStorage {
pub(crate) fn required(&self) -> Result<RequiredObjectStorage, RequiredError> {
Ok(RequiredObjectStorage {
bucket_name: self
.s3_store_bucket_name
.as_ref()
.cloned()
.ok_or(RequiredError)?,
region: self
.s3_store_region
.as_ref()
.cloned()
.map(Serde::into_inner)
.ok_or(RequiredError)?,
access_key: self.s3_store_access_key.as_ref().cloned(),
security_token: self.s3_store_security_token.as_ref().cloned(),
session_token: self.s3_store_session_token.as_ref().cloned(),
})
}
}
impl Overrides {
@ -124,67 +157,99 @@ impl Overrides {
&& self.console_buffer_capacity.is_none()
&& self.api_key.is_none()
&& self.opentelemetry_url.is_none()
&& self.repo.is_none()
&& self.store.is_none()
}
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct Migrate {
from: Store,
to: Store,
}
impl Migrate {
pub(crate) fn from(&self) -> &Store {
&self.from
}
pub(crate) fn to(&self) -> &Store {
&self.to
}
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, structopt::StructOpt)]
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Subcommand)]
#[serde(rename_all = "snake_case")]
#[serde(tag = "type")]
pub(crate) enum Command {
Run,
MigrateStore { to: Store },
MigrateRepo { to: Repo },
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
#[serde(rename_all = "snake_case")]
pub(crate) enum Repo {
Sled,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)]
#[serde(rename_all = "snake_case")]
pub(crate) struct SledRepo {
// defaults to {config.path}
#[clap(long, help = "Path in which pict-rs will create it's 'repo' directory")]
#[serde(skip_serializing_if = "Option::is_none")]
sled_repo_path: Option<PathBuf>,
#[clap(
long,
help = "The number of bytes sled is allowed to use for it's in-memory cache"
)]
#[serde(skip_serializing_if = "Option::is_none")]
sled_repo_cache_capacity: Option<u64>,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
#[serde(rename_all = "snake_case")]
pub(crate) enum Store {
FileStore {
// defaults to {config.path}
#[structopt(
long,
help = "Path in which pict-rs will create it's 'files' directory"
)]
#[serde(skip_serializing_if = "Option::is_none")]
path: Option<PathBuf>,
},
Filesystem,
#[cfg(feature = "object-storage")]
S3Store {
#[structopt(long, help = "Name of the bucket in which pict-rs will store images")]
bucket_name: String,
ObjectStorage,
}
#[structopt(
long,
help = "Region in which the bucket exists, can be an http endpoint"
)]
region: crate::serde_str::Serde<s3::Region>,
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)]
#[serde(rename_all = "snake_case")]
pub(crate) struct FilesystemStorage {
// defaults to {config.path}
#[clap(
long,
help = "Path in which pict-rs will create it's 'files' directory"
)]
#[serde(skip_serializing_if = "Option::is_none")]
filesystem_storage_path: Option<PathBuf>,
}
#[serde(skip_serializing_if = "Option::is_none")]
#[structopt(long)]
access_key: Option<String>,
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Parser)]
#[serde(rename_all = "snake_case")]
pub(crate) struct ObjectStorage {
#[serde(skip_serializing_if = "Option::is_none")]
#[clap(long, help = "Name of the bucket in which pict-rs will store images")]
s3_store_bucket_name: Option<String>,
#[structopt(long)]
#[serde(skip_serializing_if = "Option::is_none")]
secret_key: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[clap(
long,
help = "Region in which the bucket exists, can be an http endpoint"
)]
s3_store_region: Option<Serde<s3::Region>>,
#[structopt(long)]
#[serde(skip_serializing_if = "Option::is_none")]
security_token: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[clap(long)]
s3_store_access_key: Option<String>,
#[structopt(long)]
#[serde(skip_serializing_if = "Option::is_none")]
session_token: Option<String>,
},
#[clap(long)]
#[serde(skip_serializing_if = "Option::is_none")]
s3_store_secret_key: Option<String>,
#[clap(long)]
#[serde(skip_serializing_if = "Option::is_none")]
s3_store_security_token: Option<String>,
#[clap(long)]
#[serde(skip_serializing_if = "Option::is_none")]
s3_store_session_token: Option<String>,
}
pub(crate) struct RequiredObjectStorage {
pub(crate) bucket_name: String,
pub(crate) region: s3::Region,
pub(crate) access_key: Option<String>,
pub(crate) security_token: Option<String>,
pub(crate) session_token: Option<String>,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
@ -203,7 +268,11 @@ pub(crate) struct Config {
console_buffer_capacity: Option<usize>,
api_key: Option<String>,
opentelemetry_url: Option<Url>,
repo: Repo,
sled_repo: SledRepo,
store: Store,
filesystem_storage: FilesystemStorage,
object_storage: ObjectStorage,
}
#[derive(serde::Serialize)]
@ -216,9 +285,22 @@ pub(crate) struct Defaults {
max_image_height: usize,
max_image_area: usize,
sled_cache_capacity: u64,
repo: Repo,
sled_repo: SledRepoDefaults,
store: Store,
filesystem_store: FilesystemDefaults,
}
#[derive(serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct SledRepoDefaults {
sled_repo_cache_capacity: usize,
}
#[derive(serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct FilesystemDefaults {}
impl Defaults {
fn new() -> Self {
Defaults {
@ -229,23 +311,19 @@ impl Defaults {
max_image_height: 10_000,
max_image_area: 40_000_000,
sled_cache_capacity: 1024 * 1024 * 64, // 16 times smaller than sled's default of 1GB
store: Store::FileStore { path: None },
repo: Repo::Sled,
sled_repo: SledRepoDefaults {
sled_repo_cache_capacity: 1024 * 1024 * 64,
},
store: Store::Filesystem,
filesystem_store: FilesystemDefaults {},
}
}
}
impl Config {
pub(crate) fn build() -> anyhow::Result<Self> {
let args = Args::from_args();
if let Some(path) = args.migrate_file {
let migrate_config = config::Config::builder()
.add_source(config::File::from(path))
.build()?;
let migrate: Migrate = migrate_config.try_deserialize()?;
crate::MIGRATE.set(migrate).unwrap();
}
let args = Args::parse();
let mut base_config =
config::Config::builder().add_source(config::Config::try_from(&Defaults::new())?);
@ -254,6 +332,8 @@ impl Config {
base_config = base_config.add_source(config::File::from(path));
};
// TODO: Command parsing
if !args.overrides.is_default() {
let merging = config::Config::try_from(&args.overrides)?;
@ -272,6 +352,18 @@ impl Config {
&self.store
}
pub(crate) fn repo(&self) -> &Repo {
&self.repo
}
pub(crate) fn object_storage(&self) -> Result<RequiredObjectStorage, RequiredError> {
self.object_storage.required()
}
pub(crate) fn filesystem_storage_path(&self) -> Option<&PathBuf> {
self.filesystem_storage.filesystem_storage_path.as_ref()
}
pub(crate) fn bind_address(&self) -> SocketAddr {
self.addr
}
@ -329,7 +421,19 @@ impl Config {
#[error("Invalid format supplied, {0}")]
pub(crate) struct FormatError(String);
#[derive(Copy, Clone, Debug, serde::Deserialize, serde::Serialize)]
#[derive(Debug, thiserror::Error)]
#[error("Invalid store supplied, {0}")]
pub(crate) struct StoreError(String);
#[derive(Debug, thiserror::Error)]
#[error("Invalid repo supplied, {0}")]
pub(crate) struct RepoError(String);
#[derive(Debug, thiserror::Error)]
#[error("Missing required fields")]
pub(crate) struct RequiredError;
#[derive(Copy, Clone, Debug, serde::Deserialize, serde::Serialize, ArgEnum)]
#[serde(rename_all = "snake_case")]
pub(crate) enum Format {
Jpeg,
@ -359,11 +463,37 @@ impl std::str::FromStr for Format {
type Err = FormatError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"png" => Ok(Format::Png),
"jpg" => Ok(Format::Jpeg),
"webp" => Ok(Format::Webp),
other => Err(FormatError(other.to_string())),
for variant in Self::value_variants() {
if variant.to_possible_value().unwrap().matches(s, false) {
return Ok(*variant);
}
}
Err(FormatError(s.into()))
}
}
impl std::str::FromStr for Store {
type Err = StoreError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
for variant in Self::value_variants() {
if variant.to_possible_value().unwrap().matches(s, false) {
return Ok(*variant);
}
}
Err(StoreError(s.into()))
}
}
impl std::str::FromStr for Repo {
type Err = RepoError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
for variant in Self::value_variants() {
if variant.to_possible_value().unwrap().matches(s, false) {
return Ok(*variant);
}
}
Err(RepoError(s.into()))
}
}

0
src/migrate/repo.rs Normal file
View file

View file

@ -8,9 +8,11 @@ pub(crate) struct Alias {
id: Uuid,
extension: String,
}
pub(crate) struct DeleteToken {
id: Uuid,
}
pub(crate) struct AlreadyExists;
impl Alias {

View file

@ -12,6 +12,10 @@ impl<T> Serde<T> {
pub(crate) fn new(inner: T) -> Self {
Serde { inner }
}
pub(crate) fn into_inner(this: Self) -> T {
this.inner
}
}
impl<T> Deref for Serde<T> {