diff --git a/Cargo.lock b/Cargo.lock
index fb533d5..1d0816d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -292,9 +292,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.66"
+version = "1.0.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
+checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
 
 [[package]]
 name = "ap-relay"
@@ -401,9 +401,9 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.59"
+version = "0.1.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364"
+checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -663,9 +663,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.77"
+version = "1.0.78"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
+checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
 
 [[package]]
 name = "cfg-if"
@@ -1333,9 +1333,9 @@ dependencies = [
 
 [[package]]
 name = "http-signature-normalization-actix"
-version = "0.7.2"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eabdd590b7b712a168bf4b01a63dc3118f668c3c5a76d0b9f2abf45c20dff55a"
+checksum = "1dc95d9ca3b4e2f93a97e5ccf9f26992c69a272e0abad8807180f0a9e9b59e31"
 dependencies = [
  "actix-http",
  "actix-rt",
@@ -1521,9 +1521,9 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
+checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
 
 [[package]]
 name = "js-sys"
@@ -2153,9 +2153,9 @@ dependencies = [
 
 [[package]]
 name = "paste"
-version = "1.0.10"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf1c2c742266c2f1041c914ba65355a83ae8747b05f208319784083583494b4b"
+checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
 
 [[package]]
 name = "pathdiff"
@@ -2326,9 +2326,9 @@ dependencies = [
 
 [[package]]
 name = "portable-atomic"
-version = "0.3.17"
+version = "0.3.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ef946e2f765276038550e74abfbda40c84d73278417c071e0f19f8af6ba100b"
+checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92"
 
 [[package]]
 name = "ppv-lite86"
@@ -2344,9 +2344,9 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
 
 [[package]]
 name = "prettyplease"
-version = "0.1.21"
+version = "0.1.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c142c0e46b57171fe0c528bee8c5b7569e80f0c17e377cd0e30ea57dbc11bb51"
+checksum = "2c8992a85d8e93a28bdf76137db888d3874e3b230dee5ed8bebac4c9f7617773"
 dependencies = [
  "proc-macro2",
  "syn",
@@ -2378,9 +2378,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.47"
+version = "1.0.49"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
+checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5"
 dependencies = [
  "unicode-ident",
 ]
@@ -2458,9 +2458,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.21"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
 dependencies = [
  "proc-macro2",
 ]
@@ -2762,15 +2762,15 @@ dependencies = [
 
 [[package]]
 name = "rustversion"
-version = "1.0.9"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
+checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70"
 
 [[package]]
 name = "ryu"
-version = "1.0.11"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
 
 [[package]]
 name = "scopeguard"
@@ -2790,24 +2790,24 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.14"
+version = "1.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4"
+checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
 
 [[package]]
 name = "serde"
-version = "1.0.150"
+version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91"
+checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.150"
+version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e"
+checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2816,9 +2816,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.89"
+version = "1.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db"
+checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
 dependencies = [
  "itoa",
  "ryu",
@@ -3014,9 +3014,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.105"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908"
+checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3152,18 +3152,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
+checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
+checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3562,9 +3562,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.5"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
+checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
 
 [[package]]
 name = "unicode-normalization"
diff --git a/Cargo.toml b/Cargo.toml
index e3cdaac..a3c3272 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -86,7 +86,7 @@ default-features = false
 features = ["background-jobs-actix", "error-logging"]
 
 [dependencies.http-signature-normalization-actix]
-version = "0.7.1"
+version = "0.8.0"
 default-features = false
 features = ["client", "server", "sha-2"]
 
diff --git a/src/config.rs b/src/config.rs
index 957cbc8..3a46833 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,9 +1,6 @@
 use crate::{
-    data::{ActorCache, State},
     error::Error,
     extractors::{AdminConfig, XApiToken},
-    middleware::MyVerify,
-    requests::Requests,
 };
 use activitystreams::{
     iri,
@@ -14,7 +11,7 @@ use activitystreams::{
     },
 };
 use config::Environment;
-use http_signature_normalization_actix::prelude::{VerifyDigest, VerifySignature};
+use http_signature_normalization_actix::prelude::VerifyDigest;
 use rustls::{Certificate, PrivateKey};
 use sha2::{Digest, Sha256};
 use std::{io::BufReader, net::IpAddr, path::PathBuf};
@@ -277,19 +274,6 @@ impl Config {
         }
     }
 
-    pub(crate) fn signature_middleware(
-        &self,
-        requests: Requests,
-        actors: ActorCache,
-        state: State,
-    ) -> VerifySignature<MyVerify> {
-        if self.validate_signatures {
-            VerifySignature::new(MyVerify(requests, actors, state), Default::default())
-        } else {
-            VerifySignature::new(MyVerify(requests, actors, state), Default::default()).optional()
-        }
-    }
-
     pub(crate) fn x_api_token(&self) -> Option<XApiToken> {
         self.api_token.clone().map(XApiToken::new)
     }
diff --git a/src/error.rs b/src/error.rs
index 141b25e..ece3c06 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -126,7 +126,7 @@ pub(crate) enum ErrorKind {
     BadActor(String, String),
 
     #[error("Signature verification is required, but no signature was given")]
-    NoSignature(String),
+    NoSignature(Option<String>),
 
     #[error("Wrong ActivityPub kind, {0}")]
     Kind(String),
diff --git a/src/main.rs b/src/main.rs
index f31f63c..5605a48 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -7,6 +7,7 @@ use actix_web::{middleware::Compress, web, App, HttpServer};
 use collector::MemoryCollector;
 #[cfg(feature = "console")]
 use console_subscriber::ConsoleLayer;
+use http_signature_normalization_actix::middleware::VerifySignature;
 use opentelemetry::{sdk::Resource, KeyValue};
 use opentelemetry_otlp::WithExportConfig;
 use rustls::ServerConfig;
@@ -36,7 +37,7 @@ use self::{
     data::{ActorCache, MediaCache, State},
     db::Db,
     jobs::create_workers,
-    middleware::{DebugPayload, RelayResolver, Timings},
+    middleware::{DebugPayload, MyVerify, RelayResolver, Timings},
     routes::{actor, inbox, index, nodeinfo, nodeinfo_meta, statics},
 };
 
@@ -232,10 +233,9 @@ async fn do_server_main(
             .service(
                 web::resource("/inbox")
                     .wrap(config.digest_middleware())
-                    .wrap(config.signature_middleware(
-                        state.requests(&config),
-                        actors.clone(),
-                        state.clone(),
+                    .wrap(VerifySignature::new(
+                        MyVerify(state.requests(&config), actors.clone(), state.clone()),
+                        Default::default(),
                     ))
                     .wrap(DebugPayload(config.debug()))
                     .route(web::post().to(inbox)),
diff --git a/src/middleware/verifier.rs b/src/middleware/verifier.rs
index 6b201b0..a982563 100644
--- a/src/middleware/verifier.rs
+++ b/src/middleware/verifier.rs
@@ -73,8 +73,8 @@ impl MyVerify {
                 Ok(res) => res.actor_id().ok_or(ErrorKind::MissingId),
                 Err(e) => {
                     if e.is_gone() {
-                        tracing::warn!("Actor gone: {}, trusting it for now.", public_key_id);
-                        return Ok(true);
+                        tracing::warn!("Actor gone: {}", public_key_id);
+                        return Ok(false);
                     } else {
                         return Err(e);
                     }
diff --git a/src/routes/inbox.rs b/src/routes/inbox.rs
index 41c25f9..0729501 100644
--- a/src/routes/inbox.rs
+++ b/src/routes/inbox.rs
@@ -24,29 +24,28 @@ pub(crate) async fn route(
     client: web::Data<Requests>,
     jobs: web::Data<JobServer>,
     input: web::Json<AcceptedActivities>,
-    verified: Option<(SignatureVerified, DigestVerified)>,
+    digest_verified: Option<DigestVerified>,
+    signature_verified: Option<SignatureVerified>,
 ) -> Result<HttpResponse, Error> {
     let input = input.into_inner();
-    println!("ActivityActor: {:?}", input);
 
-    let actor = match actors
+    let kind = input.kind().ok_or(ErrorKind::MissingKind)?;
+
+    if digest_verified.is_some() && signature_verified.is_none() && *kind == ValidTypes::Delete {
+        return Ok(accepted(serde_json::json!({})));
+    } else if config.validate_signatures()
+        && (digest_verified.is_none() || signature_verified.is_none())
+    {
+        return Err(ErrorKind::NoSignature(None).into());
+    }
+
+    let actor = actors
         .get(
             input.actor()?.as_single_id().ok_or(ErrorKind::MissingId)?,
             &client,
         )
-        .await
-    {
-        Ok(actor) => actor.into_inner(),
-        Err(e) => {
-            // Eat up the message if actor is 410 and message is delete
-            let kind = input.kind().ok_or(ErrorKind::MissingKind)?;
-            if e.is_gone() && *kind == ValidTypes::Delete {
-                return Ok(accepted(serde_json::json!({})));
-            } else {
-                return Err(e);
-            }
-        }
-    };
+        .await?
+        .into_inner();
 
     let is_allowed = state.db.is_allowed(actor.id.clone()).await?;
     let is_connected = state.db.is_connected(actor.id.clone()).await?;
@@ -59,10 +58,8 @@ pub(crate) async fn route(
         return Err(ErrorKind::NotSubscribed(actor.id.to_string()).into());
     }
 
-    if config.validate_signatures() && verified.is_none() {
-        return Err(ErrorKind::NoSignature(actor.public_key_id.to_string()).into());
-    } else if config.validate_signatures() {
-        if let Some((verified, _)) = verified {
+    if config.validate_signatures() {
+        if let Some(verified) = signature_verified {
             if actor.public_key_id.as_str() != verified.key_id() {
                 tracing::error!("Actor signed with wrong key");
                 return Err(ErrorKind::BadActor(
@@ -71,10 +68,13 @@ pub(crate) async fn route(
                 )
                 .into());
             }
+        } else {
+            tracing::error!("This case should never be reachable, since I handle signature checks earlier in the flow. If you see this in a log it means I did it wrong");
+            return Err(ErrorKind::NoSignature(Some(actor.public_key_id.to_string())).into());
         }
     }
 
-    match input.kind().ok_or(ErrorKind::MissingKind)? {
+    match kind {
         ValidTypes::Accept => handle_accept(&config, input).await?,
         ValidTypes::Reject => handle_reject(&config, &jobs, input, actor).await?,
         ValidTypes::Announce | ValidTypes::Create => {