Try setting a lower default cache capacity
Some checks reported errors
continuous-integration/drone/push Build was killed

This commit is contained in:
Aode (Lion) 2022-03-21 21:15:43 -05:00
parent f8475a466e
commit da159c69cb
5 changed files with 139 additions and 70 deletions

155
Cargo.lock generated
View file

@ -37,9 +37,9 @@ dependencies = [
[[package]]
name = "actix-http"
version = "3.0.3"
version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86f4fb03b18c988b0944792e2538c7c1a4b32d3019214e2a734846e45207743e"
checksum = "a5885cb81a0d4d0d322864bea1bb6c2a8144626b4fdc625d4c51eba197e7797a"
dependencies = [
"actix-codec",
"actix-rt",
@ -125,9 +125,9 @@ dependencies = [
[[package]]
name = "actix-server"
version = "2.1.0"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff2a4951893e62e1e53749cbbc20e25cdfb40915ed528e13497480ab027e8bc6"
checksum = "0da34f8e659ea1b077bb4637948b815cd3768ad5a188fdcd74ff4d84240cd824"
dependencies = [
"actix-rt",
"actix-service",
@ -268,9 +268,9 @@ checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
[[package]]
name = "async-stream"
version = "0.3.2"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625"
checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e"
dependencies = [
"async-stream-impl",
"futures-core",
@ -278,9 +278,9 @@ dependencies = [
[[package]]
name = "async-stream-impl"
version = "0.3.2"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308"
checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"
dependencies = [
"proc-macro2",
"quote",
@ -352,11 +352,11 @@ dependencies = [
[[package]]
name = "aws-creds"
version = "0.27.1"
source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#09e6d50799650490db5bd5b97e860e9222e11fa8"
source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#943aacf18c4e395ab3c162510cbb12844e2da6c5"
dependencies = [
"anyhow",
"dirs",
"rust-ini",
"rust-ini 0.18.0",
"serde",
"serde-xml-rs",
"serde_derive",
@ -495,7 +495,7 @@ dependencies = [
"nom",
"pathdiff",
"ron",
"rust-ini",
"rust-ini 0.17.0",
"serde",
"serde_json",
"toml",
@ -547,9 +547,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
[[package]]
name = "cpufeatures"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"
checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
dependencies = [
"libc",
]
@ -565,9 +565,9 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
version = "0.5.2"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa"
checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
dependencies = [
"cfg-if",
"crossbeam-utils",
@ -575,10 +575,11 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
version = "0.9.7"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9"
checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"lazy_static",
@ -588,9 +589,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.7"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6"
checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
dependencies = [
"cfg-if",
"lazy_static",
@ -618,9 +619,9 @@ dependencies = [
[[package]]
name = "dashmap"
version = "5.1.0"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0834a35a3fce649144119e18da2a4d8ed12ef3862f47183fd46f625d072d96c"
checksum = "4c8858831f7781322e539ea39e72449c46b059638250c14344fec8d0aa6e539c"
dependencies = [
"cfg-if",
"num_cpus",
@ -679,9 +680,9 @@ dependencies = [
[[package]]
name = "dirs-sys"
version = "0.3.6"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780"
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
dependencies = [
"libc",
"redox_users",
@ -697,6 +698,12 @@ dependencies = [
"rand",
]
[[package]]
name = "dlv-list"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
[[package]]
name = "either"
version = "1.6.1"
@ -901,14 +908,14 @@ checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
dependencies = [
"cfg-if",
"libc",
"wasi",
"wasi 0.10.2+wasi-snapshot-preview1",
]
[[package]]
name = "h2"
version = "0.3.11"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9f1f717ddc7b2ba36df7e871fd88db79326551d3d6f1fc406fbfd28b582ff8e"
checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b"
dependencies = [
"bytes",
"fnv",
@ -938,6 +945,15 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
[[package]]
name = "hashbrown"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758"
dependencies = [
"ahash 0.7.6",
]
[[package]]
name = "hdrhistogram"
version = "7.5.0"
@ -1169,9 +1185,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.119"
version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f"
[[package]]
name = "linked-hash-map"
@ -1292,14 +1308,15 @@ dependencies = [
[[package]]
name = "mio"
version = "0.8.0"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2"
checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9"
dependencies = [
"libc",
"log",
"miow",
"ntapi",
"wasi 0.11.0+wasi-snapshot-preview1",
"winapi",
]
@ -1320,13 +1337,12 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
[[package]]
name = "nom"
version = "7.1.0"
version = "7.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109"
checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
dependencies = [
"memchr",
"minimal-lexical",
"version_check",
]
[[package]]
@ -1359,9 +1375,9 @@ dependencies = [
[[package]]
name = "num_threads"
version = "0.1.3"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15"
checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0"
dependencies = [
"libc",
]
@ -1429,10 +1445,20 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485"
dependencies = [
"dlv-list",
"dlv-list 0.2.3",
"hashbrown 0.9.1",
]
[[package]]
name = "ordered-multimap"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
dependencies = [
"dlv-list 0.3.0",
"hashbrown 0.12.0",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
@ -1725,9 +1751,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.15"
version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145"
checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57"
dependencies = [
"proc-macro2",
]
@ -1773,12 +1799,13 @@ dependencies = [
[[package]]
name = "redox_users"
version = "0.4.0"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55"
dependencies = [
"getrandom",
"redox_syscall",
"thiserror",
]
[[package]]
@ -1818,9 +1845,9 @@ dependencies = [
[[package]]
name = "reqwest"
version = "0.11.9"
version = "0.11.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525"
checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb"
dependencies = [
"base64",
"bytes",
@ -1897,13 +1924,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63471c4aa97a1cf8332a5f97709a79a4234698de6a1f5087faf66f2dae810e22"
dependencies = [
"cfg-if",
"ordered-multimap",
"ordered-multimap 0.3.1",
]
[[package]]
name = "rust-ini"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
dependencies = [
"cfg-if",
"ordered-multimap 0.4.3",
]
[[package]]
name = "rust-s3"
version = "0.29.0"
source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#09e6d50799650490db5bd5b97e860e9222e11fa8"
source = "git+https://github.com/asonix/rust-s3?branch=asonix/generic-client#943aacf18c4e395ab3c162510cbb12844e2da6c5"
dependencies = [
"anyhow",
"async-trait",
@ -1952,9 +1989,9 @@ dependencies = [
[[package]]
name = "rustls-pemfile"
version = "0.2.1"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9"
checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360"
dependencies = [
"base64",
]
@ -2205,9 +2242,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "syn"
version = "1.0.86"
version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b"
checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54"
dependencies = [
"proc-macro2",
"quote",
@ -2344,9 +2381,9 @@ dependencies = [
[[package]]
name = "tokio-rustls"
version = "0.23.2"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b"
checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e"
dependencies = [
"rustls",
"tokio",
@ -2493,9 +2530,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
[[package]]
name = "tracing"
version = "0.1.31"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6c650a8ef0cd2dd93736f033d21cbd1224c5a967aa0c258d00fcf7dafef9b9f"
checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"
dependencies = [
"cfg-if",
"log",
@ -2749,6 +2786,12 @@ version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.79"
@ -2846,9 +2889,9 @@ dependencies = [
[[package]]
name = "which"
version = "4.2.4"
version = "4.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a5a7e487e921cf220206864a94a89b6c6905bfc19f1057fa26a4cb360e5c1d2"
checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae"
dependencies = [
"either",
"lazy_static",
@ -2922,9 +2965,9 @@ checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"
[[package]]
name = "winreg"
version = "0.7.0"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
dependencies = [
"winapi",
]

View file

@ -75,6 +75,10 @@ pub(crate) struct Overrides {
#[serde(skip_serializing_if = "Option::is_none")]
max_image_area: Option<usize>,
#[structopt(long, help = "Specify the maximum area in pixels allowed in an image")]
#[serde(skip_serializing_if = "Option::is_none")]
sled_cache_capacity: Option<u64>,
#[structopt(
long,
help = "An optional string to be checked on requests to privileged endpoints"
@ -106,6 +110,7 @@ impl Overrides {
&& self.max_image_width.is_none()
&& self.max_image_height.is_none()
&& self.max_image_area.is_none()
&& self.sled_cache_capacity.is_none()
&& self.api_key.is_none()
&& self.opentelemetry_url.is_none()
&& self.store.is_none()
@ -183,6 +188,7 @@ pub(crate) struct Config {
max_image_width: usize,
max_image_height: usize,
max_image_area: usize,
sled_cache_capacity: u64,
api_key: Option<String>,
opentelemetry_url: Option<Url>,
store: Store,
@ -197,6 +203,7 @@ pub(crate) struct Defaults {
max_image_width: usize,
max_image_height: usize,
max_image_area: usize,
sled_cache_capacity: u64,
store: Store,
}
@ -209,6 +216,7 @@ impl Defaults {
max_image_width: 10_000,
max_image_height: 10_000,
max_image_area: 40_000_000,
sled_cache_capacity: 1024 * 1024 * 64, // 16 times smaller than sled's default of 1GB
store: Store::FileStore { path: None },
}
}
@ -260,6 +268,10 @@ impl Config {
self.path.clone()
}
pub(crate) fn sled_cache_capacity(&self) -> u64 {
self.sled_cache_capacity
}
pub(crate) fn format(&self) -> Option<Format> {
self.image_format
}

View file

@ -884,7 +884,7 @@ where
async fn main() -> anyhow::Result<()> {
init_tracing("pict-rs", CONFIG.opentelemetry_url())?;
let db = LatestDb::exists(CONFIG.data_dir()).migrate()?;
let db = LatestDb::exists(CONFIG.data_dir(), CONFIG.sled_cache_capacity()).migrate()?;
let manager = UploadManager::new(db.clone(), CONFIG.format()).await?;

View file

@ -51,21 +51,30 @@ trait SledTree {
pub(crate) struct LatestDb {
root_dir: PathBuf,
version: DbVersion,
cache_capacity: u64,
}
impl LatestDb {
pub(crate) fn exists(root_dir: PathBuf) -> Self {
let version = DbVersion::exists(root_dir.clone());
pub(crate) fn exists(root_dir: PathBuf, cache_capacity: u64) -> Self {
let version = DbVersion::exists(root_dir.clone(), cache_capacity);
LatestDb { root_dir, version }
LatestDb {
root_dir,
version,
cache_capacity,
}
}
pub(crate) fn migrate(self) -> Result<sled::Db, UploadError> {
let LatestDb { root_dir, version } = self;
let LatestDb {
root_dir,
version,
cache_capacity,
} = self;
loop {
let root_dir2 = root_dir.clone();
let res = std::panic::catch_unwind(move || version.migrate(root_dir2));
let res = std::panic::catch_unwind(move || version.migrate(root_dir2, cache_capacity));
if let Ok(res) = res {
return res;
@ -81,17 +90,17 @@ enum DbVersion {
}
impl DbVersion {
fn exists(root: PathBuf) -> Self {
if s034::exists(root.clone()) && !s034::migrating(root) {
fn exists(root: PathBuf, cache_capacity: u64) -> Self {
if s034::exists(root.clone()) && !s034::migrating(root, cache_capacity) {
return DbVersion::Sled034;
}
DbVersion::Fresh
}
fn migrate(self, root: PathBuf) -> Result<sled::Db, UploadError> {
fn migrate(self, root: PathBuf, cache_capacity: u64) -> Result<sled::Db, UploadError> {
match self {
DbVersion::Sled034 | DbVersion::Fresh => s034::open(root),
DbVersion::Sled034 | DbVersion::Fresh => s034::open(root, cache_capacity),
}
}
}

View file

@ -14,8 +14,8 @@ pub(crate) fn exists(mut base: PathBuf) -> bool {
std::fs::metadata(base).is_ok()
}
pub(crate) fn migrating(base: PathBuf) -> bool {
if let Ok(db) = open(base) {
pub(crate) fn migrating(base: PathBuf, cache_capacity: u64) -> bool {
if let Ok(db) = open(base, cache_capacity) {
if let Ok(tree) = db.open_tree("migrate") {
if let Ok(Some(_)) = tree.get("done") {
return false;
@ -26,11 +26,16 @@ pub(crate) fn migrating(base: PathBuf) -> bool {
true
}
pub(crate) fn open(mut base: PathBuf) -> Result<sled034::Db, UploadError> {
pub(crate) fn open(mut base: PathBuf, cache_capacity: u64) -> Result<sled034::Db, UploadError> {
base.push("sled");
base.push(SLED_034);
Ok(sled034::open(base)?)
let db = sled034::Config::default()
.cache_capacity(cache_capacity)
.path(base)
.open()?;
Ok(db)
}
impl SledDb for sled034::Db {