Migrate from 0.32.0-rc1

Cargo.lock

@@ -1891,9 +1891,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
 
 [[package]]
 name = "sled"
-version = "0.32.0-rc1"
+version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49d0ada816c19135ab4644d908b896a76e43d9d7e45bed6b25c486ac692afd7e"
+checksum = "cdad3dc85d888056d3bd9954ffdf22d8a22701b6cd3aca4f6df4c436111898c4"
 dependencies = [
  "backtrace",
  "crc32fast",

Cargo.toml

@@ -27,7 +27,7 @@ rexiv2 = { version = "0.9.0", git = "https://git.asonix.dog/asonix/rexiv2" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 sha2 = "0.9.0"
-sled = "0.32.0-rc1"
+sled = "0.32.0"
 structopt = "0.3.14"
 thiserror = "1.0"
 tracing = "0.1.15"

src/main.rs

@@ -16,6 +16,7 @@ use tracing_subscriber::EnvFilter;
 mod config;
 mod error;
 mod middleware;
+mod migrate;
 mod processor;
 mod upload_manager;
 mod validate;

src/migrate.rs

@@ -0,0 +1,173 @@
+use crate::UploadError;
+use sled as sled032;
+use std::path::PathBuf;
+use tracing::{debug, info, warn};
+
+const SLED_032: &str = "db-0.32";
+const SLED_0320_RC1: &str = "db";
+
+pub(crate) struct LatestDb {
+    root_dir: PathBuf,
+    version: DbVersion,
+}
+
+impl LatestDb {
+    pub(crate) fn exists(root_dir: PathBuf) -> Self {
+        let version = DbVersion::exists(root_dir.clone());
+
+        LatestDb { root_dir, version }
+    }
+
+    pub(crate) fn migrate(self) -> Result<sled032::Db, UploadError> {
+        let LatestDb { root_dir, version } = self;
+
+        version.migrate(root_dir)
+    }
+}
+
+enum DbVersion {
+    Sled0320Rc1,
+    Sled032,
+    Fresh,
+}
+
+impl DbVersion {
+    fn exists(root: PathBuf) -> Self {
+        let mut sled_dir = root.clone();
+        sled_dir.push("sled");
+        sled_dir.push(SLED_032);
+        if std::fs::metadata(sled_dir).is_ok() {
+            return DbVersion::Sled032;
+        }
+
+        let mut sled_dir = root;
+        sled_dir.push(SLED_0320_RC1);
+        if std::fs::metadata(sled_dir).is_ok() {
+            return DbVersion::Sled0320Rc1;
+        }
+
+        DbVersion::Fresh
+    }
+
+    fn migrate(self, root: PathBuf) -> Result<sled032::Db, UploadError> {
+        match self {
+            DbVersion::Sled0320Rc1 => migrate_0_32_0_rc1(root),
+            DbVersion::Sled032 | DbVersion::Fresh => {
+                let mut sled_dir = root;
+                sled_dir.push("sled");
+                sled_dir.push(SLED_032);
+                Ok(sled032::open(sled_dir)?)
+            }
+        }
+    }
+}
+
+fn migrate_0_32_0_rc1(root: PathBuf) -> Result<sled032::Db, UploadError> {
+    info!("Migrating database from 0.32.0-rc1 to 0.32.0");
+    let mut sled_dir = root.clone();
+    sled_dir.push("db");
+
+    let mut new_sled_dir = root;
+    new_sled_dir.push("sled");
+    new_sled_dir.push(SLED_032);
+
+    let old_db = sled032::open(sled_dir)?;
+    let new_db = sled032::open(new_sled_dir)?;
+
+    let old_alias_tree = old_db.open_tree("alias")?;
+    let new_alias_tree = new_db.open_tree("alias")?;
+
+    let old_fname_tree = old_db.open_tree("filename")?;
+    let new_fname_tree = new_db.open_tree("filename")?;
+
+    for res in old_alias_tree.iter().keys() {
+        let k = res?;
+        if let Some(v) = old_alias_tree.get(&k)? {
+            if !k.contains(&b"/"[0]) {
+                // k is an alias
+                migrate_main_tree(&k, &v, &old_db, &new_db)?;
+                debug!(
+                    "Moving alias -> hash for alias {}",
+                    String::from_utf8_lossy(k.as_ref()),
+                );
+            } else {
+                debug!(
+                    "Moving {}, {}",
+                    String::from_utf8_lossy(k.as_ref()),
+                    String::from_utf8_lossy(v.as_ref())
+                );
+            }
+            new_alias_tree.insert(k, v)?;
+        } else {
+            warn!("MISSING {}", String::from_utf8_lossy(k.as_ref()));
+        }
+    }
+
+    for res in old_fname_tree.iter().keys() {
+        let k = res?;
+        if let Some(v) = old_fname_tree.get(&k)? {
+            debug!(
+                "Moving file -> hash for file {}",
+                String::from_utf8_lossy(k.as_ref()),
+            );
+            new_fname_tree.insert(&k, &v)?;
+        } else {
+            warn!("MISSING {}", String::from_utf8_lossy(k.as_ref()));
+        }
+    }
+
+    Ok(new_db) as Result<sled032::Db, UploadError>
+}
+
+fn migrate_main_tree(
+    alias: &sled032::IVec,
+    hash: &sled032::IVec,
+    old_db: &sled032::Db,
+    new_db: &sled032::Db,
+) -> Result<(), UploadError> {
+    debug!(
+        "Migrating files for {}",
+        String::from_utf8_lossy(alias.as_ref())
+    );
+    if let Some(v) = old_db.get(&hash)? {
+        new_db.insert(&hash, v)?;
+    } else {
+        warn!("Missing filename");
+    }
+
+    let (start, end) = alias_key_bounds(&hash);
+    for res in old_db.range(start..end) {
+        let (k, v) = res?;
+        debug!("Moving alias {}", String::from_utf8_lossy(v.as_ref()));
+        new_db.insert(k, v)?;
+    }
+
+    let (start, end) = variant_key_bounds(&hash);
+    for res in old_db.range(start..end) {
+        let (k, v) = res?;
+        debug!("Moving variant {}", String::from_utf8_lossy(v.as_ref()));
+        new_db.insert(k, v)?;
+    }
+
+    Ok(())
+}
+
+pub(crate) fn alias_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) {
+    let mut start = hash.to_vec();
+    start.extend(&[0]);
+
+    let mut end = hash.to_vec();
+    end.extend(&[1]);
+
+    (start, end)
+}
+
+pub(crate) fn variant_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) {
+    let mut start = hash.to_vec();
+    start.extend(&[2]);
+
+    let mut end = hash.to_vec();
+    end.extend(&[3]);
+
+    (start, end)
+}

src/upload_manager.rs

@@ -1,4 +1,10 @@
-use crate::{config::Format, error::UploadError, to_ext, validate::validate_image};
+use crate::{
+    config::Format,
+    error::UploadError,
+    migrate::{alias_key_bounds, variant_key_bounds, LatestDb},
+    to_ext,
+    validate::validate_image,
+};
 use actix_web::web;
 use futures::stream::{Stream, StreamExt, TryStreamExt};
 use sha2::Digest;
@@ -84,10 +90,9 @@ impl UploadManager {
         mut root_dir: PathBuf,
         format: Option<Format>,
     ) -> Result<Self, UploadError> {
-        let mut sled_dir = root_dir.clone();
-        sled_dir.push("db");
+        let root_clone = root_dir.clone();
         // sled automatically creates it's own directories
-        let db = web::block(move || sled::open(sled_dir)).await?;
+        let db = web::block(move || LatestDb::exists(root_clone).migrate()).await?;
 
         root_dir.push("files");
 
@@ -684,16 +689,6 @@ fn alias_key(hash: &[u8], id: &str) -> Vec<u8> {
     key
 }
 
-fn alias_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) {
-    let mut start = hash.to_vec();
-    start.extend(&[0]);
-
-    let mut end = hash.to_vec();
-    end.extend(&[1]);
-
-    (start, end)
-}
-
 fn alias_id_key(alias: &str) -> String {
     format!("{}/id", alias)
 }
@@ -708,13 +703,3 @@ fn variant_key(hash: &[u8], path: &str) -> Vec<u8> {
     key.extend(path.as_bytes());
     key
 }
-
-fn variant_key_bounds(hash: &[u8]) -> (Vec<u8>, Vec<u8>) {
-    let mut start = hash.to_vec();
-    start.extend(&[2]);
-
-    let mut end = hash.to_vec();
-    end.extend(&[3]);
-
-    (start, end)
-}