fix: relay sync failures with delayed bounded retries

Co-authored-by: userAdityaa <aditya.chaudhary1558@gmail.com>
Co-committed-by: userAdityaa <aditya.chaudhary1558@gmail.com>

backend/README.md

@@ -74,13 +74,20 @@ Public exceptions:
 - `GET /tenants` — list tenants (admin)
 - `POST /tenants` — idempotently ensure a tenant row exists for the current auth pubkey (creates Stripe customer + tenant on first call, returns existing tenant otherwise)
 - `GET /tenants/:pubkey` — get tenant (admin or same tenant)
-- `PUT /tenants/:pubkey/billing` — update tenant `nwc_url` (admin or same tenant)
+- `PUT /tenants/:pubkey` — update tenant `nwc_url` (admin or same tenant)
-- `GET /relays` — list relays (`?tenant=<pubkey>` allowed for admin only)
+- `GET /tenants/:pubkey/relays` — list tenant relays (admin or same tenant)
+- `GET /relays` — list relays (admin)
 - `POST /relays` — create relay (admin or relay tenant)
 - `GET /relays/:id` — get relay (admin or relay tenant)
+- `GET /relays/:id/members` — list relay members from zooid (admin or relay tenant)
 - `PUT /relays/:id` — update relay (admin or relay tenant)
+- `GET /relays/:id/activity` — list relay activity (admin or relay tenant)
 - `POST /relays/:id/deactivate` — deactivate relay (admin or relay tenant)
-- `GET /invoices` — list invoices (`?tenant=<pubkey>` allowed for admin only)
+- `POST /relays/:id/reactivate` — reactivate relay (admin or relay tenant)
+- `GET /tenants/:pubkey/invoices` — list tenant invoices (admin or same tenant)
+- `GET /invoices/:id` — get invoice (admin or same tenant)
+- `GET /invoices/:id/bolt11` — get invoice bolt11 (admin or same tenant)
+- `GET /tenants/:pubkey/stripe/session` — create Stripe customer portal session (admin or same tenant)
 
 ## API Auth Model
 

backend/migrations/0002_hot_path_indexes.sql

@@ -0,0 +1,11 @@
+CREATE INDEX IF NOT EXISTS idx_tenant_stripe_customer_id
+ON tenant (stripe_customer_id);
+
+CREATE INDEX IF NOT EXISTS idx_relay_tenant_id
+ON relay (tenant, id);
+
+CREATE INDEX IF NOT EXISTS idx_relay_tenant_status_plan
+ON relay (tenant, status, plan);
+
+CREATE INDEX IF NOT EXISTS idx_activity_resource_type_resource_id_created_at_id
+ON activity (resource_type, resource_id, created_at DESC, id DESC);

backend/spec/infra.md

@@ -15,12 +15,15 @@ Members:
 ## `pub async fn start(self)`
 
 - Subscribes to `command.notify`
+- On startup, schedules delayed sync retries for relays whose `sync_error` is non-empty.
 - Loops on `rx.recv()`, calling `handle_activity` for each received `Activity`.
 
 ## `async fn handle_activity(&self, activity: &Activity)`
 
-- For `create_relay`, `update_relay`, `activate_relay`, or `deactivate_relay` activity, calls `sync_and_report`.
+- For `create_relay`, `update_relay`, `activate_relay`, or `deactivate_relay` activity, calls `sync_and_report` immediately.
-- All other activity types are ignored (e.g. `fail_relay_sync`, `complete_relay_sync`).
+- For `fail_relay_sync`, schedules a delayed retry using exponential backoff based on consecutive failures for the relay.
+- Retry scheduling stops after the configured max attempts to avoid infinite retry loops.
+- Other activity types are ignored (e.g. `complete_relay_sync`).
 
 ## `async fn sync_and_report(&self, relay: &Relay, is_new: bool)`
 

backend/src/infra.rs

@@ -1,10 +1,15 @@
 use anyhow::Result;
 use nostr_sdk::prelude::*;
+use std::time::Duration;
 
 use crate::command::Command;
-use crate::models::{Activity, Relay, RELAY_STATUS_DELINQUENT, RELAY_STATUS_INACTIVE};
+use crate::models::{Activity, RELAY_STATUS_DELINQUENT, RELAY_STATUS_INACTIVE, Relay};
 use crate::query::Query;
 
+const RELAY_SYNC_RETRY_BASE_DELAY_SECS: u64 = 30;
+const RELAY_SYNC_RETRY_MAX_DELAY_SECS: u64 = 15 * 60;
+const RELAY_SYNC_RETRY_MAX_ATTEMPTS: usize = 6;
+
 #[derive(Clone)]
 pub struct Infra {
     api_url: String,
@@ -18,14 +23,22 @@ pub struct Infra {
 }
 
 impl Infra {
-    pub fn new(query: Query, command: Command) -> Self {
+    pub fn new(query: Query, command: Command) -> Result<Self> {
         let api_url = std::env::var("ZOOID_API_URL").unwrap_or_default();
         let relay_domain = std::env::var("RELAY_DOMAIN").unwrap_or_default();
         let livekit_url = std::env::var("LIVEKIT_URL").unwrap_or_default();
         let livekit_api_key = std::env::var("LIVEKIT_API_KEY").unwrap_or_default();
         let livekit_api_secret = std::env::var("LIVEKIT_API_SECRET").unwrap_or_default();
         let api_secret = std::env::var("ZOOID_API_SECRET").unwrap_or_default();
-        Self {
+
+        if api_url.trim().is_empty() {
+            anyhow::bail!("missing ZOOID_API_URL");
+        }
+        if api_secret.trim().is_empty() {
+            anyhow::bail!("missing ZOOID_API_SECRET");
+        }
+
+        Ok(Self {
             api_url,
             relay_domain,
             livekit_url,
@@ -34,12 +47,16 @@ impl Infra {
             api_secret,
             query,
             command,
-        }
+        })
     }
 
     pub async fn start(self) {
         let mut rx = self.command.notify.subscribe();
 
+        if let Err(e) = self.schedule_startup_retries().await {
+            tracing::error!(error = %e, "failed to schedule relay sync retries on startup");
+        }
+
         loop {
             match rx.recv().await {
                 Ok(activity) => {
@@ -58,15 +75,84 @@ impl Infra {
     async fn handle_activity(&self, activity: &Activity) -> Result<()> {
         let needs_sync = should_sync_relay_activity(activity.activity_type.as_str());
 
-        if needs_sync {
+        if !needs_sync || activity.resource_type != "relay" {
-            let Some(relay) = self.query.get_relay(&activity.resource_id).await? else {
+            return Ok(());
-                return Ok(());
-            };
-
-            let is_new = relay.synced == 0;
-            self.sync_and_report(&relay, is_new).await;
         }
 
+        if activity.activity_type == "fail_relay_sync" {
+            self.schedule_relay_sync_retry(&activity.resource_id, "activity")
+                .await?;
+            return Ok(());
+        }
+
+        let Some(relay) = self.query.get_relay(&activity.resource_id).await? else {
+            return Ok(());
+        };
+
+        let is_new = relay.synced == 0;
+        self.sync_and_report(&relay, is_new).await;
+
+        Ok(())
+    }
+
+    async fn schedule_startup_retries(&self) -> Result<()> {
+        let relays = self.query.list_relays_with_sync_error().await?;
+
+        for relay in relays {
+            self.schedule_relay_sync_retry(&relay.id, "startup").await?;
+        }
+
+        Ok(())
+    }
+
+    async fn schedule_relay_sync_retry(&self, relay_id: &str, source: &str) -> Result<()> {
+        let activities = self.query.list_activity_for_relay(relay_id).await?;
+        let consecutive_failures = consecutive_sync_failures(&activities);
+
+        let Some(delay) = relay_sync_retry_delay(consecutive_failures) else {
+            tracing::warn!(
+                relay = relay_id,
+                consecutive_failures,
+                max_attempts = RELAY_SYNC_RETRY_MAX_ATTEMPTS,
+                "relay sync retries exhausted; awaiting manual intervention"
+            );
+            return Ok(());
+        };
+
+        tracing::info!(
+            relay = relay_id,
+            source,
+            consecutive_failures,
+            delay_secs = delay.as_secs(),
+            "scheduled relay sync retry"
+        );
+
+        let relay_id = relay_id.to_string();
+        let infra = self.clone();
+
+        tokio::spawn(async move {
+            tokio::time::sleep(delay).await;
+
+            if let Err(e) = infra.retry_relay_sync(&relay_id).await {
+                tracing::error!(relay = %relay_id, error = %e, "relay sync retry task failed");
+            }
+        });
+
+        Ok(())
+    }
+
+    async fn retry_relay_sync(&self, relay_id: &str) -> Result<()> {
+        let Some(relay) = self.query.get_relay(relay_id).await? else {
+            return Ok(());
+        };
+
+        if relay.sync_error.trim().is_empty() {
+            tracing::debug!(relay = %relay.id, "skip relay sync retry; relay has no sync_error");
+            return Ok(());
+        }
+
+        let is_new = relay.synced == 0;
+        self.sync_and_report(&relay, is_new).await;
         Ok(())
     }
 
@@ -97,19 +183,16 @@ impl Infra {
     }
 
     pub async fn list_relay_members(&self, relay_id: &str) -> Result<Vec<String>> {
-        if self.api_url.trim().is_empty() {
-            anyhow::bail!("missing ZOOID_API_URL");
-        }
-        if self.api_secret.trim().is_empty() {
-            anyhow::bail!("missing ZOOID_API_SECRET");
-        }
-
         let client = reqwest::Client::new();
         let base = self.api_url.trim_end_matches('/');
         let url = format!("{base}/relay/{relay_id}/members");
         let auth = self.nip98_auth(&url, HttpMethod::GET).await?;
 
-        let response = client.get(&url).header("Authorization", auth).send().await?;
+        let response = client
+            .get(&url)
+            .header("Authorization", auth)
+            .send()
+            .await?;
 
         if !response.status().is_success() {
             let status = response.status();
@@ -150,7 +233,9 @@ impl Infra {
         );
 
         let url = format!("{}/relay/{}", base, relay.id);
-        let auth = self.nip98_auth(&url, zooid_sync_http_method(is_new)).await?;
+        let auth = self
+            .nip98_auth(&url, zooid_sync_http_method(is_new))
+            .await?;
 
         let request = if is_new {
             client.post(&url)
@@ -251,6 +336,28 @@ fn relay_sync_body(
 fn should_sync_relay_activity(activity_type: &str) -> bool {
     matches!(
         activity_type,
-        "create_relay" | "update_relay" | "activate_relay" | "deactivate_relay"
+        "create_relay" | "update_relay" | "activate_relay" | "deactivate_relay" | "fail_relay_sync"
     )
 }
+
+fn consecutive_sync_failures(activities: &[Activity]) -> usize {
+    activities
+        .iter()
+        .take_while(|activity| activity.activity_type == "fail_relay_sync")
+        .count()
+}
+
+fn relay_sync_retry_delay(consecutive_failures: usize) -> Option<Duration> {
+    let retry_attempt = consecutive_failures.max(1);
+    if retry_attempt > RELAY_SYNC_RETRY_MAX_ATTEMPTS {
+        return None;
+    }
+
+    let exponent = (retry_attempt - 1).min(31);
+    let multiplier = 1u64 << exponent;
+    let delay_secs = RELAY_SYNC_RETRY_BASE_DELAY_SECS
+        .saturating_mul(multiplier)
+        .min(RELAY_SYNC_RETRY_MAX_DELAY_SECS);
+
+    Some(Duration::from_secs(delay_secs))
+}

backend/src/main.rs

@@ -33,7 +33,7 @@ async fn main() -> Result<()> {
     let query = Query::new(pool.clone());
     let command = Command::new(pool);
     let billing = Billing::new(query.clone(), command.clone(), robot.clone());
-    let infra = Infra::new(query.clone(), command.clone());
+    let infra = Infra::new(query.clone(), command.clone())?;
     let api = Api::new(query, command, billing.clone(), infra.clone());
 
     let host = std::env::var("HOST").unwrap_or_else(|_| "127.0.0.1".to_string());

backend/src/query.rs

@@ -94,6 +94,23 @@ impl Query {
         Ok(rows)
     }
 
+    pub async fn list_relays_with_sync_error(&self) -> Result<Vec<Relay>> {
+        let rows = sqlx::query_as::<_, Relay>(
+            "SELECT id, tenant, schema, subdomain, plan, stripe_subscription_item_id,
+                     status, sync_error,
+                     info_name, info_icon, info_description,
+                     policy_public_join, policy_strip_signatures,
+                     groups_enabled, management_enabled, blossom_enabled,
+                     livekit_enabled, push_enabled, synced
+              FROM relay
+              WHERE TRIM(sync_error) != ''
+              ORDER BY id",
+        )
+        .fetch_all(&self.pool)
+        .await?;
+        Ok(rows)
+    }
+
     pub async fn list_relays_for_tenant(&self, tenant_id: &str) -> Result<Vec<Relay>> {
         let rows = sqlx::query_as::<_, Relay>(
             "SELECT id, tenant, schema, subdomain, plan, stripe_subscription_item_id,

frontend/src/pages/admin/AdminRelayDetail.tsx

@@ -1,11 +1,12 @@
 import { useParams } from "@solidjs/router"
-import { Show } from "solid-js"
+import { createResource, Show } from "solid-js"
 import BackLink from "@/components/BackLink"
 import PageContainer from "@/components/PageContainer"
 import RelayDetailCard from "@/components/RelayDetailCard"
 import ResourceState from "@/components/ResourceState"
 import useMinLoading from "@/components/useMinLoading"
 import ActivityFeed from "@/components/ActivityFeed"
+import { listRelayMembers } from "@/lib/api"
 import { useRelay, useRelayActivity } from "@/lib/hooks"
 import useRelayToggles from "@/lib/useRelayToggles"
 
@@ -13,6 +14,15 @@ export default function AdminRelayDetail() {
   const params = useParams()
   const relayId = () => params.id ?? ""
   const [relay, { refetch, mutate }] = useRelay(relayId)
+  const [members] = createResource(relayId, async (id) => {
+    if (!id) return []
+
+    try {
+      return (await listRelayMembers(id)).members
+    } catch {
+      return []
+    }
+  })
   const loading = useMinLoading(() => relay.loading && !relay())
   const [activity] = useRelayActivity(relayId)
   const { busy, handleDeactivate, handleReactivate, toggles } = useRelayToggles(relayId, relay, { refetch, mutate })
@@ -26,6 +36,7 @@ export default function AdminRelayDetail() {
           <div class="space-y-6 mb-6">
             <RelayDetailCard
               relay={r()}
+              currentMembers={members()?.length}
               showTenant
               editHref={`/admin/relays/${params.id}/edit`}
               onDeactivate={handleDeactivate}