Try setting a lower default cache capacity

Cargo.lock

@@ -37,9 +37,9 @@ dependencies = [
 
 [[package]]
 name = "actix-http"
-version = "3.0.3"
+version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f4fb03b18c988b0944792e2538c7c1a4b32d3019214e2a734846e45207743e"
+checksum = "a5885cb81a0d4d0d322864bea1bb6c2a8144626b4fdc625d4c51eba197e7797a"
 dependencies = [
  "actix-codec",
  "actix-rt",
@@ -125,9 +125,9 @@ dependencies = [
 
 [[package]]
 name = "actix-server"
-version = "2.1.0"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff2a4951893e62e1e53749cbbc20e25cdfb40915ed528e13497480ab027e8bc6"
+checksum = "0da34f8e659ea1b077bb4637948b815cd3768ad5a188fdcd74ff4d84240cd824"
 dependencies = [
  "actix-rt",
  "actix-service",
@@ -268,9 +268,9 @@ checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
 
 [[package]]
 name = "async-stream"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625"
+checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e"
 dependencies = [
  "async-stream-impl",
  "futures-core",
@@ -278,9 +278,9 @@ dependencies = [
 
 [[package]]
 name = "async-stream-impl"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308"
+checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -352,11 +352,11 @@ dependencies = [
 [[package]]
 name = "aws-creds"
 version = "0.27.1"
-source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#09e6d50799650490db5bd5b97e860e9222e11fa8"
+source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#943aacf18c4e395ab3c162510cbb12844e2da6c5"
 dependencies = [
  "anyhow",
  "dirs",
- "rust-ini",
+ "rust-ini 0.18.0",
  "serde",
  "serde-xml-rs",
  "serde_derive",
@@ -495,7 +495,7 @@ dependencies = [
  "nom",
  "pathdiff",
  "ron",
- "rust-ini",
+ "rust-ini 0.17.0",
  "serde",
  "serde_json",
  "toml",
@@ -547,9 +547,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"
+checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
 dependencies = [
  "libc",
 ]
@@ -565,9 +565,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.2"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa"
+checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
 dependencies = [
  "cfg-if",
  "crossbeam-utils",
@@ -575,10 +575,11 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.7"
+version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9"
+checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
 dependencies = [
+ "autocfg",
  "cfg-if",
  "crossbeam-utils",
  "lazy_static",
@@ -588,9 +589,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.7"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6"
+checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
 dependencies = [
  "cfg-if",
  "lazy_static",
@@ -618,9 +619,9 @@ dependencies = [
 
 [[package]]
 name = "dashmap"
-version = "5.1.0"
+version = "5.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0834a35a3fce649144119e18da2a4d8ed12ef3862f47183fd46f625d072d96c"
+checksum = "4c8858831f7781322e539ea39e72449c46b059638250c14344fec8d0aa6e539c"
 dependencies = [
  "cfg-if",
  "num_cpus",
@@ -679,9 +680,9 @@ dependencies = [
 
 [[package]]
 name = "dirs-sys"
-version = "0.3.6"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780"
+checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
 dependencies = [
  "libc",
  "redox_users",
@@ -697,6 +698,12 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "dlv-list"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
+
 [[package]]
 name = "either"
 version = "1.6.1"
@@ -901,14 +908,14 @@ checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
 dependencies = [
  "cfg-if",
  "libc",
- "wasi",
+ "wasi 0.10.2+wasi-snapshot-preview1",
 ]
 
 [[package]]
 name = "h2"
-version = "0.3.11"
+version = "0.3.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9f1f717ddc7b2ba36df7e871fd88db79326551d3d6f1fc406fbfd28b582ff8e"
+checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b"
 dependencies = [
  "bytes",
  "fnv",
@@ -938,6 +945,15 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 
+[[package]]
+name = "hashbrown"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758"
+dependencies = [
+ "ahash 0.7.6",
+]
+
 [[package]]
 name = "hdrhistogram"
 version = "7.5.0"
@@ -1169,9 +1185,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.119"
+version = "0.2.121"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
+checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f"
 
 [[package]]
 name = "linked-hash-map"
@@ -1292,14 +1308,15 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.8.0"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2"
+checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9"
 dependencies = [
  "libc",
  "log",
  "miow",
  "ntapi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
  "winapi",
 ]
 
@@ -1320,13 +1337,12 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 
 [[package]]
 name = "nom"
-version = "7.1.0"
+version = "7.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109"
+checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
 dependencies = [
  "memchr",
  "minimal-lexical",
- "version_check",
 ]
 
 [[package]]
@@ -1359,9 +1375,9 @@ dependencies = [
 
 [[package]]
 name = "num_threads"
-version = "0.1.3"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15"
+checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0"
 dependencies = [
  "libc",
 ]
@@ -1429,10 +1445,20 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485"
 dependencies = [
- "dlv-list",
+ "dlv-list 0.2.3",
  "hashbrown 0.9.1",
 ]
 
+[[package]]
+name = "ordered-multimap"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
+dependencies = [
+ "dlv-list 0.3.0",
+ "hashbrown 0.12.0",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -1725,9 +1751,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.15"
+version = "1.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145"
+checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57"
 dependencies = [
  "proc-macro2",
 ]
@@ -1773,12 +1799,13 @@ dependencies = [
 
 [[package]]
 name = "redox_users"
-version = "0.4.0"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
+checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55"
 dependencies = [
  "getrandom",
  "redox_syscall",
+ "thiserror",
 ]
 
 [[package]]
@@ -1818,9 +1845,9 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.9"
+version = "0.11.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525"
+checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb"
 dependencies = [
  "base64",
  "bytes",
@@ -1897,13 +1924,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "63471c4aa97a1cf8332a5f97709a79a4234698de6a1f5087faf66f2dae810e22"
 dependencies = [
  "cfg-if",
- "ordered-multimap",
+ "ordered-multimap 0.3.1",
+]
+
+[[package]]
+name = "rust-ini"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
+dependencies = [
+ "cfg-if",
+ "ordered-multimap 0.4.3",
 ]
 
 [[package]]
 name = "rust-s3"
 version = "0.29.0"
-source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#09e6d50799650490db5bd5b97e860e9222e11fa8"
+source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#943aacf18c4e395ab3c162510cbb12844e2da6c5"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1952,9 +1989,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pemfile"
-version = "0.2.1"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9"
+checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360"
 dependencies = [
  "base64",
 ]
@@ -2205,9 +2242,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.86"
+version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b"
+checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2344,9 +2381,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-rustls"
-version = "0.23.2"
+version = "0.23.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b"
+checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e"
 dependencies = [
  "rustls",
  "tokio",
@@ -2493,9 +2530,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
 
 [[package]]
 name = "tracing"
-version = "0.1.31"
+version = "0.1.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6c650a8ef0cd2dd93736f033d21cbd1224c5a967aa0c258d00fcf7dafef9b9f"
+checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"
 dependencies = [
  "cfg-if",
  "log",
@@ -2749,6 +2786,12 @@ version = "0.10.2+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
 
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.79"
@@ -2846,9 +2889,9 @@ dependencies = [
 
 [[package]]
 name = "which"
-version = "4.2.4"
+version = "4.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a5a7e487e921cf220206864a94a89b6c6905bfc19f1057fa26a4cb360e5c1d2"
+checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae"
 dependencies = [
  "either",
  "lazy_static",
@@ -2922,9 +2965,9 @@ checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"
 
 [[package]]
 name = "winreg"
-version = "0.7.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
+checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
 dependencies = [
  "winapi",
 ]

src/config.rs

@@ -75,6 +75,10 @@ pub(crate) struct Overrides {
     #[serde(skip_serializing_if = "Option::is_none")]
     max_image_area: Option<usize>,
 
+    #[structopt(long, help = "Specify the maximum area in pixels allowed in an image")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    sled_cache_capacity: Option<u64>,
+
     #[structopt(
         long,
         help = "An optional string to be checked on requests to privileged endpoints"
@@ -106,6 +110,7 @@ impl Overrides {
             && self.max_image_width.is_none()
             && self.max_image_height.is_none()
             && self.max_image_area.is_none()
+            && self.sled_cache_capacity.is_none()
             && self.api_key.is_none()
             && self.opentelemetry_url.is_none()
             && self.store.is_none()
@@ -183,6 +188,7 @@ pub(crate) struct Config {
     max_image_width: usize,
     max_image_height: usize,
     max_image_area: usize,
+    sled_cache_capacity: u64,
     api_key: Option<String>,
     opentelemetry_url: Option<Url>,
     store: Store,
@@ -197,6 +203,7 @@ pub(crate) struct Defaults {
     max_image_width: usize,
     max_image_height: usize,
     max_image_area: usize,
+    sled_cache_capacity: u64,
     store: Store,
 }
 
@@ -209,6 +216,7 @@ impl Defaults {
             max_image_width: 10_000,
             max_image_height: 10_000,
             max_image_area: 40_000_000,
+            sled_cache_capacity: 1024 * 1024 * 64, // 16 times smaller than sled's default of 1GB
             store: Store::FileStore { path: None },
         }
     }
@@ -260,6 +268,10 @@ impl Config {
         self.path.clone()
     }
 
+    pub(crate) fn sled_cache_capacity(&self) -> u64 {
+        self.sled_cache_capacity
+    }
+
     pub(crate) fn format(&self) -> Option<Format> {
         self.image_format
     }

src/main.rs

@@ -884,7 +884,7 @@ where
 async fn main() -> anyhow::Result<()> {
     init_tracing("pict-rs", CONFIG.opentelemetry_url())?;
 
-    let db = LatestDb::exists(CONFIG.data_dir()).migrate()?;
+    let db = LatestDb::exists(CONFIG.data_dir(), CONFIG.sled_cache_capacity()).migrate()?;
 
     let manager = UploadManager::new(db.clone(), CONFIG.format()).await?;
 

src/migrate.rs

@@ -51,21 +51,30 @@ trait SledTree {
 pub(crate) struct LatestDb {
     root_dir: PathBuf,
     version: DbVersion,
+    cache_capacity: u64,
 }
 
 impl LatestDb {
-    pub(crate) fn exists(root_dir: PathBuf) -> Self {
+    pub(crate) fn exists(root_dir: PathBuf, cache_capacity: u64) -> Self {
-        let version = DbVersion::exists(root_dir.clone());
+        let version = DbVersion::exists(root_dir.clone(), cache_capacity);
 
-        LatestDb { root_dir, version }
+        LatestDb {
+            root_dir,
+            version,
+            cache_capacity,
+        }
     }
 
     pub(crate) fn migrate(self) -> Result<sled::Db, UploadError> {
-        let LatestDb { root_dir, version } = self;
+        let LatestDb {
+            root_dir,
+            version,
+            cache_capacity,
+        } = self;
 
         loop {
             let root_dir2 = root_dir.clone();
-            let res = std::panic::catch_unwind(move || version.migrate(root_dir2));
+            let res = std::panic::catch_unwind(move || version.migrate(root_dir2, cache_capacity));
 
             if let Ok(res) = res {
                 return res;
@@ -81,17 +90,17 @@ enum DbVersion {
 }
 
 impl DbVersion {
-    fn exists(root: PathBuf) -> Self {
+    fn exists(root: PathBuf, cache_capacity: u64) -> Self {
-        if s034::exists(root.clone()) && !s034::migrating(root) {
+        if s034::exists(root.clone()) && !s034::migrating(root, cache_capacity) {
             return DbVersion::Sled034;
         }
 
         DbVersion::Fresh
     }
 
-    fn migrate(self, root: PathBuf) -> Result<sled::Db, UploadError> {
+    fn migrate(self, root: PathBuf, cache_capacity: u64) -> Result<sled::Db, UploadError> {
         match self {
-            DbVersion::Sled034 | DbVersion::Fresh => s034::open(root),
+            DbVersion::Sled034 | DbVersion::Fresh => s034::open(root, cache_capacity),
         }
     }
 }

src/migrate/s034.rs

@@ -14,8 +14,8 @@ pub(crate) fn exists(mut base: PathBuf) -> bool {
     std::fs::metadata(base).is_ok()
 }
 
-pub(crate) fn migrating(base: PathBuf) -> bool {
+pub(crate) fn migrating(base: PathBuf, cache_capacity: u64) -> bool {
-    if let Ok(db) = open(base) {
+    if let Ok(db) = open(base, cache_capacity) {
         if let Ok(tree) = db.open_tree("migrate") {
             if let Ok(Some(_)) = tree.get("done") {
                 return false;
@@ -26,11 +26,16 @@ pub(crate) fn migrating(base: PathBuf) -> bool {
     true
 }
 
-pub(crate) fn open(mut base: PathBuf) -> Result<sled034::Db, UploadError> {
+pub(crate) fn open(mut base: PathBuf, cache_capacity: u64) -> Result<sled034::Db, UploadError> {
     base.push("sled");
     base.push(SLED_034);
 
-    Ok(sled034::open(base)?)
+    let db = sled034::Config::default()
+        .cache_capacity(cache_capacity)
+        .path(base)
+        .open()?;
+
+    Ok(db)
 }
 
 impl SledDb for sled034::Db {