dru5k1/pict-rs
1
0
Fork 0
forked from asonix/pict-rs
Code Pull requests Activity

Enable configuring download client timeout

Browse source
This commit is contained in:
asonix 2023-07-17 13:44:31 -05:00
parent 5e8ab7856d
commit dd1d509bb1
6 changed files with 72 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
defaults.toml
View file

@ -1,7 +1,10 @@
[server]
address = "0.0.0.0:8080"
worker_id = "pict-rs-1"
client_pool_size = 100
[client]
pool_size = 100
timeout = 30
[tracing.logging]
format = "normal"

14
pict-rs.toml
View file

@ -20,8 +20,10 @@ worker_id = 'pict-rs-1'
# Not specifying api_key disables internal endpoints
api_key = 'API_KEY'
## Client configuration
[client]
## Optional: connection pool size for internal http client
# environment variable: PICTRS__SERVER__CLIENT_POOL_SIZE
# environment variable: PICTRS__CLIENT__POOL_SIZE
# default: 100
#
# This number is multiplied the number of cores available to pict-rs. Running on a 2 core machine
@ -31,7 +33,15 @@ api_key = 'API_KEY'
# This number can be lowered to keep pict-rs within ulimit bounds if you encounter errors related to
# "Too many open files". Alternatively, increasing the ulimit of your system can solve this problem
# as well.
client_pool_size = 100
pool_size = 100
## Optional: time (in seconds) the client will wait for a response before giving up
# environment variable: PICTRS__CLIENT__TIMEOUT
# default: 30
#
# This is used for the `/image/download` endpoint when fetching media from another server. It is
# distinct from the object storage client timeout, which can be configured separately
timeout = 30
## Logging configuration

31
src/config/commandline.rs
View file

@ -47,6 +47,7 @@ impl Args {
api_key,
worker_id,
client_pool_size,
client_timeout,
media_preprocess_steps,
media_max_file_size,
media_image_max_width,
@ -76,7 +77,11 @@ impl Args {
address,
api_key,
worker_id,
client_pool_size,
};
let client = Client {
pool_size: client_pool_size,
timeout: client_timeout,
};
let image = Image {
@ -124,6 +129,7 @@ impl Args {
Output {
config_format: ConfigFormat {
server,
client,
old_db,
tracing,
media,
@ -140,6 +146,7 @@ impl Args {
Output {
config_format: ConfigFormat {
server,
client,
old_db,
tracing,
media,
@ -154,6 +161,7 @@ impl Args {
None => Output {
config_format: ConfigFormat {
server,
client,
old_db,
tracing,
media,
@ -171,6 +179,7 @@ impl Args {
store,
}) => {
let server = Server::default();
let client = Client::default();
let media = Media::default();
match store {
@ -178,6 +187,7 @@ impl Args {
MigrateStoreTo::Filesystem(MigrateFilesystemInner { to, repo }) => Output {
config_format: ConfigFormat {
server,
client,
old_db,
tracing,
media,
@ -196,6 +206,7 @@ impl Args {
Output {
config_format: ConfigFormat {
server,
client,
old_db,
tracing,
media,
@ -218,6 +229,7 @@ impl Args {
Output {
config_format: ConfigFormat {
server,
client,
old_db,
tracing,
media,
@ -239,6 +251,7 @@ impl Args {
}) => Output {
config_format: ConfigFormat {
server,
client,
old_db,
tracing,
media,
@ -283,6 +296,7 @@ pub(crate) enum Operation {
#[serde(rename_all = "snake_case")]
pub(super) struct ConfigFormat {
server: Server,
client: Client,
old_db: OldDb,
tracing: Tracing,
media: Media,
@ -301,8 +315,15 @@ struct Server {
worker_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
api_key: Option<String>,
}
#[derive(Debug, Default, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct Client {
#[serde(skip_serializing_if = "Option::is_none")]
client_pool_size: Option<usize>,
pool_size: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
timeout: Option<u64>,
}
#[derive(Debug, Default, serde::Serialize)]
@ -549,6 +570,12 @@ struct Run {
#[arg(long)]
client_pool_size: Option<usize>,
/// How long (in seconds) the internel HTTP client should wait for responses
///
/// This number defaults to 30
#[arg(long)]
client_timeout: Option<u64>,
/// Optional pre-processing steps for uploaded media.
///
/// All still images will be put through these steps before saving

19
src/config/defaults.rs
View file

@ -9,6 +9,7 @@ use std::{net::SocketAddr, path::PathBuf};
#[serde(rename_all = "snake_case")]
pub(crate) struct Defaults {
server: ServerDefaults,
client: ClientDefaults,
tracing: TracingDefaults,
old_db: OldDbDefaults,
media: MediaDefaults,
@ -21,7 +22,13 @@ pub(crate) struct Defaults {
struct ServerDefaults {
address: SocketAddr,
worker_id: String,
client_pool_size: usize,
}
#[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct ClientDefaults {
pool_size: usize,
timeout: u64,
}
#[derive(Clone, Debug, Default, serde::Serialize)]
@ -149,7 +156,15 @@ impl Default for ServerDefaults {
ServerDefaults {
address: "0.0.0.0:8080".parse().expect("Valid address string"),
worker_id: String::from("pict-rs-1"),
client_pool_size: 100,
}
}
}
impl Default for ClientDefaults {
fn default() -> Self {
ClientDefaults {
pool_size: 100,
timeout: 30,
}
}
}

10
src/config/file.rs
View file

@ -12,6 +12,8 @@ use url::Url;
pub(crate) struct ConfigFile {
pub(crate) server: Server,
pub(crate) client: Client,
pub(crate) tracing: Tracing,
pub(crate) old_db: OldDb,
@ -95,9 +97,13 @@ pub(crate) struct Server {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) api_key: Option<String>,
}
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) client_pool_size: Option<usize>,
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub(crate) struct Client {
pub(crate) pool_size: usize,
pub(crate) timeout: u64,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]

8
src/lib.rs
View file

@ -1128,17 +1128,13 @@ fn transform_error(error: actix_form_data::Error) -> actix_web::Error {
}
fn build_client() -> awc::Client {
let connector = CONFIG
.server
.client_pool_size
.map(|size| Connector::new().limit(size))
.unwrap_or_else(Connector::new);
let connector = Connector::new().limit(CONFIG.client.pool_size);
Client::builder()
.connector(connector)
.wrap(Tracing)
.add_default_header(("User-Agent", "pict-rs v0.4.1"))
.timeout(Duration::from_secs(30))
.timeout(Duration::from_secs(CONFIG.client.timeout))
.finish()
}