Never report a namespace whose entry was emptied as missing

libsql-server/src/error.rs

@@ -102,6 +102,8 @@ pub enum Error {
     UrlParseError(#[from] url::ParseError),
     #[error("Namespace store has shutdown")]
     NamespaceStoreShutdown,
+    #[error("Namespace `{0}` is temporarily unavailable")]
+    NamespaceUnavailable(String),
     #[error("Unable to update metastore: {0}")]
     MetaStoreUpdateFailure(Box<dyn std::error::Error + Send + Sync>),
     // This is for errors returned by moka
@@ -131,9 +133,10 @@ pub enum Error {
 }
 
 impl AsRef<Self> for Error {
+    /// The error behind any number of `Ref` wrappers, like `into_response`.
     fn as_ref(&self) -> &Self {
         match self {
-            Self::Ref(this) => this.as_ref(),
+            Self::Ref(this) => <Error as AsRef<Error>>::as_ref(this),
             _ => self,
         }
     }
@@ -212,6 +215,7 @@ impl IntoResponse for &Error {
             PrimaryStreamInterupted => self.format_err(StatusCode::INTERNAL_SERVER_ERROR),
             UrlParseError(_) => self.format_err(StatusCode::BAD_REQUEST),
             NamespaceStoreShutdown => self.format_err(StatusCode::SERVICE_UNAVAILABLE),
+            NamespaceUnavailable(_) => self.format_err(StatusCode::SERVICE_UNAVAILABLE),
             MetaStoreUpdateFailure(_) => self.format_err(StatusCode::INTERNAL_SERVER_ERROR),
             Ref(this) => this.as_ref().into_response(),
             ProstDecode(_) => self.format_err(StatusCode::INTERNAL_SERVER_ERROR),

libsql-server/src/metrics.rs

@@ -37,6 +37,22 @@ pub static STREAM_HANDLES_COUNT: Lazy<Gauge> = Lazy::new(|| {
     describe_gauge!(NAME, "amount of in-memory stream handles");
     register_gauge!(NAME)
 });
+pub static NAMESPACE_EMPTIED_ENTRY: Lazy<Counter> = Lazy::new(|| {
+    const NAME: &str = "libsql_server_namespace_emptied_entry";
+    describe_counter!(
+        NAME,
+        "number of times a request found its namespace cache entry emptied (by shutdown, eviction or destroy) and looked again"
+    );
+    register_counter!(NAME)
+});
+pub static NAMESPACE_EMPTIED_ENTRY_GAVE_UP: Lazy<Counter> = Lazy::new(|| {
+    const NAME: &str = "libsql_server_namespace_emptied_entry_gave_up";
+    describe_counter!(
+        NAME,
+        "number of requests answered `namespace unavailable` because every lookup of their namespace, which still exists, found its cache entry emptied"
+    );
+    register_counter!(NAME)
+});
 pub static NAMESPACE_LOAD_LATENCY: Lazy<Histogram> = Lazy::new(|| {
     const NAME: &str = "libsql_server_namespace_load_latency";
     describe_histogram!(NAME, "latency is us when loading a namespace");

libsql-server/src/namespace/mod.rs

@@ -16,6 +16,8 @@ use crate::stats::Stats;
 
 use self::meta_store::MetaStoreHandle;
 pub use self::name::NamespaceName;
+#[cfg(test)]
+pub(crate) use self::store::test::make_store as make_test_store;
 pub use self::store::NamespaceStore;
 
 pub mod broadcasters;