From da159c69cbbb101f98addd4ab619a4f544aff0f6 Mon Sep 17 00:00:00 2001 From: "Aode (Lion)" Date: Mon, 21 Mar 2022 21:15:43 -0500 Subject: [PATCH] Try setting a lower default cache capacity --- Cargo.lock | 155 ++++++++++++++++++++++++++++---------------- src/config.rs | 12 ++++ src/main.rs | 2 +- src/migrate.rs | 27 +++++--- src/migrate/s034.rs | 13 ++-- 5 files changed, 139 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b1a002..14f6af7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,9 +37,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4fb03b18c988b0944792e2538c7c1a4b32d3019214e2a734846e45207743e" +checksum = "a5885cb81a0d4d0d322864bea1bb6c2a8144626b4fdc625d4c51eba197e7797a" dependencies = [ "actix-codec", "actix-rt", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2a4951893e62e1e53749cbbc20e25cdfb40915ed528e13497480ab027e8bc6" +checksum = "0da34f8e659ea1b077bb4637948b815cd3768ad5a188fdcd74ff4d84240cd824" dependencies = [ "actix-rt", "actix-service", @@ -268,9 +268,9 @@ checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" [[package]] name = "async-stream" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" dependencies = [ "async-stream-impl", "futures-core", @@ -278,9 +278,9 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" dependencies = [ "proc-macro2", "quote", @@ -352,11 +352,11 @@ dependencies = [ [[package]] name = "aws-creds" version = "0.27.1" -source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#09e6d50799650490db5bd5b97e860e9222e11fa8" +source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#943aacf18c4e395ab3c162510cbb12844e2da6c5" dependencies = [ "anyhow", "dirs", - "rust-ini", + "rust-ini 0.18.0", "serde", "serde-xml-rs", "serde_derive", @@ -495,7 +495,7 @@ dependencies = [ "nom", "pathdiff", "ron", - "rust-ini", + "rust-ini 0.17.0", "serde", "serde_json", "toml", @@ -547,9 +547,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "cpufeatures" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" dependencies = [ "libc", ] @@ -565,9 +565,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" dependencies = [ "cfg-if", "crossbeam-utils", @@ -575,10 +575,11 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" dependencies = [ + "autocfg", "cfg-if", "crossbeam-utils", "lazy_static", @@ -588,9 +589,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ "cfg-if", "lazy_static", @@ -618,9 +619,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.1.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0834a35a3fce649144119e18da2a4d8ed12ef3862f47183fd46f625d072d96c" +checksum = "4c8858831f7781322e539ea39e72449c46b059638250c14344fec8d0aa6e539c" dependencies = [ "cfg-if", "num_cpus", @@ -679,9 +680,9 @@ dependencies = [ [[package]] name = "dirs-sys" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", @@ -697,6 +698,12 @@ dependencies = [ "rand", ] +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + [[package]] name = "either" version = "1.6.1" @@ -901,14 +908,14 @@ checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] name = "h2" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f1f717ddc7b2ba36df7e871fd88db79326551d3d6f1fc406fbfd28b582ff8e" +checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b" dependencies = [ "bytes", "fnv", @@ -938,6 +945,15 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +[[package]] +name = "hashbrown" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758" +dependencies = [ + "ahash 0.7.6", +] + [[package]] name = "hdrhistogram" version = "7.5.0" @@ -1169,9 +1185,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.119" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" +checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" [[package]] name = "linked-hash-map" @@ -1292,14 +1308,15 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" dependencies = [ "libc", "log", "miow", "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", "winapi", ] @@ -1320,13 +1337,12 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "nom" -version = "7.1.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" dependencies = [ "memchr", "minimal-lexical", - "version_check", ] [[package]] @@ -1359,9 +1375,9 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15" +checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0" dependencies = [ "libc", ] @@ -1429,10 +1445,20 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485" dependencies = [ - "dlv-list", + "dlv-list 0.2.3", "hashbrown 0.9.1", ] +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list 0.3.0", + "hashbrown 0.12.0", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -1725,9 +1751,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" dependencies = [ "proc-macro2", ] @@ -1773,12 +1799,13 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55" dependencies = [ "getrandom", "redox_syscall", + "thiserror", ] [[package]] @@ -1818,9 +1845,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.9" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525" +checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" dependencies = [ "base64", "bytes", @@ -1897,13 +1924,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63471c4aa97a1cf8332a5f97709a79a4234698de6a1f5087faf66f2dae810e22" dependencies = [ "cfg-if", - "ordered-multimap", + "ordered-multimap 0.3.1", +] + +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if", + "ordered-multimap 0.4.3", ] [[package]] name = "rust-s3" version = "0.29.0" -source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#09e6d50799650490db5bd5b97e860e9222e11fa8" +source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#943aacf18c4e395ab3c162510cbb12844e2da6c5" dependencies = [ "anyhow", "async-trait", @@ -1952,9 +1989,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360" dependencies = [ "base64", ] @@ -2205,9 +2242,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" dependencies = [ "proc-macro2", "quote", @@ -2344,9 +2381,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b" +checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e" dependencies = [ "rustls", "tokio", @@ -2493,9 +2530,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6c650a8ef0cd2dd93736f033d21cbd1224c5a967aa0c258d00fcf7dafef9b9f" +checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" dependencies = [ "cfg-if", "log", @@ -2749,6 +2786,12 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.79" @@ -2846,9 +2889,9 @@ dependencies = [ [[package]] name = "which" -version = "4.2.4" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a5a7e487e921cf220206864a94a89b6c6905bfc19f1057fa26a4cb360e5c1d2" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" dependencies = [ "either", "lazy_static", @@ -2922,9 +2965,9 @@ checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" [[package]] name = "winreg" -version = "0.7.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] diff --git a/src/config.rs b/src/config.rs index 4ece53e..c9285da 100644 --- a/src/config.rs +++ b/src/config.rs @@ -75,6 +75,10 @@ pub(crate) struct Overrides { #[serde(skip_serializing_if = "Option::is_none")] max_image_area: Option, + #[structopt(long, help = "Specify the maximum area in pixels allowed in an image")] + #[serde(skip_serializing_if = "Option::is_none")] + sled_cache_capacity: Option, + #[structopt( long, help = "An optional string to be checked on requests to privileged endpoints" @@ -106,6 +110,7 @@ impl Overrides { && self.max_image_width.is_none() && self.max_image_height.is_none() && self.max_image_area.is_none() + && self.sled_cache_capacity.is_none() && self.api_key.is_none() && self.opentelemetry_url.is_none() && self.store.is_none() @@ -183,6 +188,7 @@ pub(crate) struct Config { max_image_width: usize, max_image_height: usize, max_image_area: usize, + sled_cache_capacity: u64, api_key: Option, opentelemetry_url: Option, store: Store, @@ -197,6 +203,7 @@ pub(crate) struct Defaults { max_image_width: usize, max_image_height: usize, max_image_area: usize, + sled_cache_capacity: u64, store: Store, } @@ -209,6 +216,7 @@ impl Defaults { max_image_width: 10_000, max_image_height: 10_000, max_image_area: 40_000_000, + sled_cache_capacity: 1024 * 1024 * 64, // 16 times smaller than sled's default of 1GB store: Store::FileStore { path: None }, } } @@ -260,6 +268,10 @@ impl Config { self.path.clone() } + pub(crate) fn sled_cache_capacity(&self) -> u64 { + self.sled_cache_capacity + } + pub(crate) fn format(&self) -> Option { self.image_format } diff --git a/src/main.rs b/src/main.rs index 6de2bb5..8bda459 100644 --- a/src/main.rs +++ b/src/main.rs @@ -884,7 +884,7 @@ where async fn main() -> anyhow::Result<()> { init_tracing("pict-rs", CONFIG.opentelemetry_url())?; - let db = LatestDb::exists(CONFIG.data_dir()).migrate()?; + let db = LatestDb::exists(CONFIG.data_dir(), CONFIG.sled_cache_capacity()).migrate()?; let manager = UploadManager::new(db.clone(), CONFIG.format()).await?; diff --git a/src/migrate.rs b/src/migrate.rs index 8ac65d6..90a391d 100644 --- a/src/migrate.rs +++ b/src/migrate.rs @@ -51,21 +51,30 @@ trait SledTree { pub(crate) struct LatestDb { root_dir: PathBuf, version: DbVersion, + cache_capacity: u64, } impl LatestDb { - pub(crate) fn exists(root_dir: PathBuf) -> Self { - let version = DbVersion::exists(root_dir.clone()); + pub(crate) fn exists(root_dir: PathBuf, cache_capacity: u64) -> Self { + let version = DbVersion::exists(root_dir.clone(), cache_capacity); - LatestDb { root_dir, version } + LatestDb { + root_dir, + version, + cache_capacity, + } } pub(crate) fn migrate(self) -> Result { - let LatestDb { root_dir, version } = self; + let LatestDb { + root_dir, + version, + cache_capacity, + } = self; loop { let root_dir2 = root_dir.clone(); - let res = std::panic::catch_unwind(move || version.migrate(root_dir2)); + let res = std::panic::catch_unwind(move || version.migrate(root_dir2, cache_capacity)); if let Ok(res) = res { return res; @@ -81,17 +90,17 @@ enum DbVersion { } impl DbVersion { - fn exists(root: PathBuf) -> Self { - if s034::exists(root.clone()) && !s034::migrating(root) { + fn exists(root: PathBuf, cache_capacity: u64) -> Self { + if s034::exists(root.clone()) && !s034::migrating(root, cache_capacity) { return DbVersion::Sled034; } DbVersion::Fresh } - fn migrate(self, root: PathBuf) -> Result { + fn migrate(self, root: PathBuf, cache_capacity: u64) -> Result { match self { - DbVersion::Sled034 | DbVersion::Fresh => s034::open(root), + DbVersion::Sled034 | DbVersion::Fresh => s034::open(root, cache_capacity), } } } diff --git a/src/migrate/s034.rs b/src/migrate/s034.rs index e24cf60..5638b46 100644 --- a/src/migrate/s034.rs +++ b/src/migrate/s034.rs @@ -14,8 +14,8 @@ pub(crate) fn exists(mut base: PathBuf) -> bool { std::fs::metadata(base).is_ok() } -pub(crate) fn migrating(base: PathBuf) -> bool { - if let Ok(db) = open(base) { +pub(crate) fn migrating(base: PathBuf, cache_capacity: u64) -> bool { + if let Ok(db) = open(base, cache_capacity) { if let Ok(tree) = db.open_tree("migrate") { if let Ok(Some(_)) = tree.get("done") { return false; @@ -26,11 +26,16 @@ pub(crate) fn migrating(base: PathBuf) -> bool { true } -pub(crate) fn open(mut base: PathBuf) -> Result { +pub(crate) fn open(mut base: PathBuf, cache_capacity: u64) -> Result { base.push("sled"); base.push(SLED_034); - Ok(sled034::open(base)?) + let db = sled034::Config::default() + .cache_capacity(cache_capacity) + .path(base) + .open()?; + + Ok(db) } impl SledDb for sled034::Db {