fix: manual Lightning payment reconciliation with Stripe invoice state

Co-authored-by: userAdityaa <aditya.chaudhary1558@gmail.com>
Co-committed-by: userAdityaa <aditya.chaudhary1558@gmail.com>

backend/README.md

@@ -74,13 +74,20 @@ Public exceptions:
 - `GET /tenants` — list tenants (admin)
 - `POST /tenants` — idempotently ensure a tenant row exists for the current auth pubkey (creates Stripe customer + tenant on first call, returns existing tenant otherwise)
 - `GET /tenants/:pubkey` — get tenant (admin or same tenant)
-- `PUT /tenants/:pubkey/billing` — update tenant `nwc_url` (admin or same tenant)
-- `GET /relays` — list relays (`?tenant=<pubkey>` allowed for admin only)
+- `PUT /tenants/:pubkey` — update tenant `nwc_url` (admin or same tenant)
+- `GET /tenants/:pubkey/relays` — list tenant relays (admin or same tenant)
+- `GET /relays` — list relays (admin)
 - `POST /relays` — create relay (admin or relay tenant)
 - `GET /relays/:id` — get relay (admin or relay tenant)
+- `GET /relays/:id/members` — list relay members from zooid (admin or relay tenant)
 - `PUT /relays/:id` — update relay (admin or relay tenant)
+- `GET /relays/:id/activity` — list relay activity (admin or relay tenant)
 - `POST /relays/:id/deactivate` — deactivate relay (admin or relay tenant)
-- `GET /invoices` — list invoices (`?tenant=<pubkey>` allowed for admin only)
+- `POST /relays/:id/reactivate` — reactivate relay (admin or relay tenant)
+- `GET /tenants/:pubkey/invoices` — list tenant invoices (admin or same tenant)
+- `GET /invoices/:id` — get invoice (admin or same tenant)
+- `GET /invoices/:id/bolt11` — get invoice bolt11 (admin or same tenant)
+- `GET /tenants/:pubkey/stripe/session` — create Stripe customer portal session (admin or same tenant)
 
 ## API Auth Model
 

backend/migrations/0002_hot_path_indexes.sql

@@ -0,0 +1,11 @@
+CREATE INDEX IF NOT EXISTS idx_tenant_stripe_customer_id
+ON tenant (stripe_customer_id);
+
+CREATE INDEX IF NOT EXISTS idx_relay_tenant_id
+ON relay (tenant, id);
+
+CREATE INDEX IF NOT EXISTS idx_relay_tenant_status_plan
+ON relay (tenant, status, plan);
+
+CREATE INDEX IF NOT EXISTS idx_activity_resource_type_resource_id_created_at_id
+ON activity (resource_type, resource_id, created_at DESC, id DESC);

backend/migrations/0003_invoice_nwc_payment_guard.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS invoice_nwc_payment (
+  invoice_id TEXT PRIMARY KEY,
+  tenant_pubkey TEXT NOT NULL,
+  state TEXT NOT NULL CHECK (state IN ('pending', 'paid')),
+  created_at INTEGER NOT NULL,
+  updated_at INTEGER NOT NULL,
+  FOREIGN KEY (tenant_pubkey) REFERENCES tenant(pubkey)
+);
+
+CREATE INDEX IF NOT EXISTS idx_invoice_nwc_payment_tenant_pubkey
+ON invoice_nwc_payment (tenant_pubkey);

backend/migrations/0004_invoice_manual_lightning_payment.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS invoice_manual_lightning_payment (
+  invoice_id TEXT PRIMARY KEY,
+  tenant_pubkey TEXT NOT NULL,
+  bolt11 TEXT NOT NULL,
+  created_at INTEGER NOT NULL,
+  updated_at INTEGER NOT NULL,
+  FOREIGN KEY (tenant_pubkey) REFERENCES tenant(pubkey)
+);
+
+CREATE INDEX IF NOT EXISTS idx_invoice_manual_lightning_payment_tenant_pubkey
+ON invoice_manual_lightning_payment (tenant_pubkey);

backend/spec/api.md

@@ -9,6 +9,7 @@ Members:
 - `query: Query`
 - `command: Command`
 - `billing: Billing`
+- `infra: Infra`
 
 Notes:
 
@@ -103,6 +104,14 @@ Notes:
 - Authorizes admin or relay owner
 - Return `data` is a single relay struct from `query.get_relay`
 
+## `async fn list_relay_members(...) -> Response`
+
+- Serves `GET /relays/:id/members`
+- Authorizes admin or relay owner
+- For unsynced relays, returns an empty member list without calling zooid
+- For synced relays, proxies the member list from zooid via `infra`
+- Return `data` is `{ members }`
+
 ## `async fn create_relay(...) -> Response`
 
 - Serves `POST /relays`
@@ -117,6 +126,7 @@ Notes:
 - Serves `PUT /relays/:id`
 - Authorizes admin or relay owner
 - Validates/prepares the relay data to be saved using `prepare_relay`
+- If the requested plan changes to a plan with a finite member limit and the current member count exceeds that limit, return a `422` with `code=member-limit-exceeded`
 - Updates the given relay using `command.update_relay`
 - If relay is a duplicate by subdomain, return a `422` with `code=subdomain-exists`
 - Return `data` is a single relay struct.

backend/spec/infra.md

@@ -15,12 +15,15 @@ Members:
 ## `pub async fn start(self)`
 
 - Subscribes to `command.notify`
+- On startup, schedules delayed sync retries for relays whose `sync_error` is non-empty.
 - Loops on `rx.recv()`, calling `handle_activity` for each received `Activity`.
 
 ## `async fn handle_activity(&self, activity: &Activity)`
 
-- For `create_relay`, `update_relay`, `activate_relay`, or `deactivate_relay` activity, calls `sync_and_report`.
-- All other activity types are ignored (e.g. `fail_relay_sync`, `complete_relay_sync`).
+- For `create_relay`, `update_relay`, `activate_relay`, or `deactivate_relay` activity, calls `sync_and_report` immediately.
+- For `fail_relay_sync`, schedules a delayed retry using exponential backoff based on consecutive failures for the relay.
+- Retry scheduling stops after the configured max attempts to avoid infinite retry loops.
+- Other activity types are ignored (e.g. `complete_relay_sync`).
 
 ## `async fn sync_and_report(&self, relay: &Relay, is_new: bool)`
 

backend/src/api.rs

@@ -1,6 +1,6 @@
 use std::sync::Arc;
 
-use anyhow::anyhow;
+use anyhow::{Result, anyhow};
 use axum::{
     Json, Router,
     extract::{Path, State},
@@ -14,6 +14,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::billing::{Billing, InvoiceLookupError};
 use crate::command::Command;
+use crate::infra::Infra;
 use crate::models::{
     RELAY_STATUS_ACTIVE, RELAY_STATUS_DELINQUENT, RELAY_STATUS_INACTIVE, Relay, Tenant,
 };
@@ -27,6 +28,7 @@ pub struct Api {
     query: Query,
     command: Command,
     billing: Billing,
+    infra: Infra,
 }
 
 async fn stripe_webhook(
@@ -117,7 +119,7 @@ fn map_invoice_lookup_error(error: InvoiceLookupError) -> ApiError {
 }
 
 impl Api {
-    pub fn new(query: Query, command: Command, billing: Billing) -> Self {
+    pub fn new(query: Query, command: Command, billing: Billing, infra: Infra) -> Self {
         let host = std::env::var("HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
         let admins = std::env::var("ADMINS")
             .unwrap_or_default()
@@ -131,6 +133,7 @@ impl Api {
             query,
             command,
             billing,
+            infra,
         }
     }
 
@@ -148,6 +151,7 @@ impl Api {
             .route("/tenants/:pubkey/relays", get(list_tenant_relays))
             .route("/relays", get(list_relays).post(create_relay))
             .route("/relays/:id", get(get_relay).put(update_relay))
+            .route("/relays/:id/members", get(list_relay_members))
             .route("/relays/:id/activity", get(list_relay_activity))
             .route("/relays/:id/deactivate", post(deactivate_relay))
             .route("/relays/:id/reactivate", post(reactivate_relay))
@@ -251,6 +255,14 @@ impl Api {
         }
     }
 
+    async fn fetch_relay_members(&self, relay: &Relay) -> Result<Vec<String>> {
+        if relay.synced == 0 {
+            return Ok(Vec::new());
+        }
+
+        self.infra.list_relay_members(&relay.id).await
+    }
+
     fn prepare_relay(&self, mut relay: Relay) -> std::result::Result<Relay, RelayValidationError> {
         validate_subdomain_label(&relay.subdomain)?;
 
@@ -669,6 +681,40 @@ async fn list_relay_activity(
     }
 }
 
+async fn list_relay_members(
+    State(state): State<AppState>,
+    headers: HeaderMap,
+    Path(id): Path<String>,
+) -> std::result::Result<Response, ApiError> {
+    let auth = state.api.extract_auth_pubkey(&headers)?;
+
+    let relay = match state.api.query.get_relay(&id).await {
+        Ok(Some(r)) => r,
+        Ok(None) => return Ok(err(StatusCode::NOT_FOUND, "not-found", "relay not found")),
+        Err(e) => {
+            return Ok(err(
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "internal",
+                &e.to_string(),
+            ));
+        }
+    };
+
+    state.api.require_admin_or_tenant(&auth, &relay.tenant)?;
+
+    match state.api.fetch_relay_members(&relay).await {
+        Ok(members) => Ok(ok(
+            StatusCode::OK,
+            serde_json::json!({ "members": members }),
+        )),
+        Err(e) => Ok(err(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "internal",
+            &e.to_string(),
+        )),
+    }
+}
+
 async fn create_relay(
     State(state): State<AppState>,
     headers: HeaderMap,
@@ -748,10 +794,13 @@ async fn update_relay(
 
     state.api.require_admin_or_tenant(&auth, &relay.tenant)?;
 
+    let current_plan = relay.plan.clone();
+    let requested_plan = payload.plan.clone();
+
     if let Some(v) = payload.subdomain {
         relay.subdomain = v;
     }
-    if let Some(v) = payload.plan {
+    if let Some(v) = requested_plan.clone() {
         relay.plan = v;
     }
     if let Some(v) = payload.info_name {
@@ -792,6 +841,38 @@ async fn update_relay(
         }
     };
 
+    let plan_changed = requested_plan
+        .as_deref()
+        .is_some_and(|requested| requested != current_plan);
+
+    if plan_changed {
+        let selected_plan = Query::get_plan(&relay.plan).expect("validated plan must exist");
+        if let Some(limit) = selected_plan.members {
+            let current_members = match state.api.fetch_relay_members(&relay).await {
+                Ok(members) => members.len() as i64,
+                Err(e) => {
+                    return Ok(err(
+                        StatusCode::INTERNAL_SERVER_ERROR,
+                        "internal",
+                        &e.to_string(),
+                    ));
+                }
+            };
+
+            if current_members > limit {
+                let message = format!(
+                    "relay has {current_members} members, which exceeds the {} plan limit of {limit}",
+                    selected_plan.name.to_lowercase()
+                );
+                return Ok(err(
+                    StatusCode::UNPROCESSABLE_ENTITY,
+                    "member-limit-exceeded",
+                    &message,
+                ));
+            }
+        }
+    }
+
     match state.api.command.update_relay(&relay).await {
         Ok(()) => Ok(ok(StatusCode::OK, relay)),
         Err(e) => {
@@ -928,6 +1009,13 @@ async fn get_invoice(
         .map_err(map_invoice_lookup_error)?;
     state.api.require_admin_or_tenant(&auth, &tenant.pubkey)?;
 
+    let invoice = state
+        .api
+        .billing
+        .reconcile_manual_lightning_invoice(&id, &invoice)
+        .await
+        .map_err(map_invoice_lookup_error)?;
+
     Ok(ok(StatusCode::OK, invoice))
 }
 
@@ -945,6 +1033,13 @@ async fn get_invoice_bolt11(
         .map_err(map_invoice_lookup_error)?;
     state.api.require_admin_or_tenant(&auth, &tenant.pubkey)?;
 
+    let invoice = state
+        .api
+        .billing
+        .reconcile_manual_lightning_invoice(&id, &invoice)
+        .await
+        .map_err(map_invoice_lookup_error)?;
+
     let status = invoice["status"].as_str().unwrap_or_default();
     if status != "open" {
         return Ok(err(
@@ -957,7 +1052,12 @@ async fn get_invoice_bolt11(
     let amount_due = invoice["amount_due"].as_i64().unwrap_or(0);
     let currency = invoice["currency"].as_str().unwrap_or("usd");
 
-    match state.api.billing.create_bolt11(amount_due, currency).await {
+    match state
+        .api
+        .billing
+        .get_or_create_manual_lightning_bolt11(&id, &tenant.pubkey, amount_due, currency)
+        .await
+    {
         Ok(bolt11) => Ok(ok(StatusCode::OK, serde_json::json!({ "bolt11": bolt11 }))),
         Err(e) => Ok(err(
             StatusCode::INTERNAL_SERVER_ERROR,

backend/src/billing.rs

@@ -1,7 +1,8 @@
 use anyhow::{Result, anyhow};
 use hmac::{Hmac, Mac};
 use nwc::prelude::{
-    MakeInvoiceRequest, NWC, NostrWalletConnectURI, PayInvoiceRequest as NwcPayInvoiceRequest,
+    LookupInvoiceRequest, LookupInvoiceResponse, MakeInvoiceRequest, NWC, NostrWalletConnectURI,
+    PayInvoiceRequest as NwcPayInvoiceRequest, TransactionState,
 };
 use sha2::Sha256;
 
@@ -17,6 +18,9 @@ type HmacSha256 = Hmac<Sha256>;
 const STRIPE_API: &str = "https://api.stripe.com/v1";
 const COINBASE_SPOT_API: &str = "https://api.coinbase.com/v2/prices";
 const WEBHOOK_TOLERANCE_SECS: i64 = 300;
+const MANUAL_LIGHTNING_PAYMENT_DM: &str = "Payment is due for your relay subscription. Please visit the application to complete a manual Lightning payment.";
+const NWC_ERROR_DM_PREFIX: &str = "NWC auto-payment failed:";
+const NWC_ERROR_DM_MAX_CHARS: usize = 240;
 
 #[derive(Debug)]
 pub enum InvoiceLookupError {
@@ -75,12 +79,17 @@ struct CoinbaseSpotPriceData {
     amount: String,
 }
 
+enum NwcInvoicePaymentOutcome {
+    Paid,
+    Fallback(anyhow::Error),
+    Pending(anyhow::Error),
+}
+
 #[derive(Clone)]
 pub struct Billing {
     nwc_url: String,
     stripe_secret_key: String,
     stripe_webhook_secret: String,
-    btc_quote_api_base: String,
     http: reqwest::Client,
     query: Query,
     command: Command,
@@ -98,13 +107,10 @@ impl Billing {
         if stripe_webhook_secret.trim().is_empty() {
             panic!("missing STRIPE_WEBHOOK_SECRET environment variable");
         }
-        let btc_quote_api_base =
-            std::env::var("BTC_PRICE_API_BASE").unwrap_or_else(|_| COINBASE_SPOT_API.to_string());
         Self {
             nwc_url,
             stripe_secret_key,
             stripe_webhook_secret,
-            btc_quote_api_base,
             http: reqwest::Client::new(),
             query,
             command,
@@ -115,6 +121,10 @@ impl Billing {
     pub async fn start(self) {
         let mut rx = self.command.notify.subscribe();
 
+        if let Err(error) = self.reconcile_relay_subscriptions("startup").await {
+            tracing::error!(error = %error, "failed to reconcile relay billing state on startup");
+        }
+
         loop {
             match rx.recv().await {
                 Ok(activity) => {
@@ -124,12 +134,39 @@ impl Billing {
                 }
                 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
                     tracing::warn!(missed = n, "billing lagged");
+
+                    if let Err(error) = self.reconcile_relay_subscriptions("lagged").await {
+                        tracing::error!(error = %error, "failed to reconcile relay billing state after lag");
+                    }
                 }
                 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
             }
         }
     }
 
+    async fn reconcile_relay_subscriptions(&self, source: &str) -> Result<()> {
+        let relays = self.query.list_relays().await?;
+
+        if relays.is_empty() {
+            return Ok(());
+        }
+
+        tracing::info!(source, relay_count = relays.len(), "reconciling relay billing state");
+
+        for relay in relays {
+            if let Err(error) = self.sync_relay_subscription_for_relay(&relay).await {
+                tracing::error!(
+                    source,
+                    relay = %relay.id,
+                    error = %error,
+                    "failed to reconcile relay billing state"
+                );
+            }
+        }
+
+        Ok(())
+    }
+
     async fn handle_activity(&self, activity: &Activity) -> Result<()> {
         let needs_billing_sync = matches!(
             activity.activity_type.as_str(),
@@ -153,6 +190,10 @@ impl Billing {
             return Ok(());
         };
 
+        self.sync_relay_subscription_for_relay(&relay).await
+    }
+
+    async fn sync_relay_subscription_for_relay(&self, relay: &Relay) -> Result<()> {
         let Some(tenant) = self.query.get_tenant(&relay.tenant).await? else {
             return Ok(());
         };
@@ -261,6 +302,23 @@ impl Billing {
         Ok(())
     }
 
+    async fn existing_invoice_nwc_payment_outcome(
+        &self,
+        invoice_id: &str,
+    ) -> Result<Option<NwcInvoicePaymentOutcome>> {
+        let state = self.query.get_invoice_nwc_payment_state(invoice_id).await?;
+        match state.as_deref() {
+            Some("paid") => Ok(Some(NwcInvoicePaymentOutcome::Paid)),
+            Some("pending") => Ok(Some(NwcInvoicePaymentOutcome::Pending(anyhow!(
+                "invoice {invoice_id} has a pending NWC reconciliation; refusing to create a new Lightning charge"
+            )))),
+            Some(other) => Err(anyhow!(
+                "unknown invoice_nwc_payment state '{other}' for invoice {invoice_id}"
+            )),
+            None => Ok(None),
+        }
+    }
+
     pub async fn handle_webhook(&self, payload: &str, signature: &str) -> Result<()> {
         self.verify_webhook_signature(payload, signature)?;
 
@@ -360,24 +418,54 @@ impl Billing {
             return Ok(());
         };
 
+        let mut nwc_error_for_dm: Option<String> = None;
+
         // 1. NWC auto-pay: if the tenant has a nwc_url
         if !tenant.nwc_url.is_empty() {
             match self
-                .nwc_pay_invoice(amount_due, currency, &tenant.nwc_url)
-                .await
+                .nwc_pay_invoice(
+                    invoice_id,
+                    &tenant.pubkey,
+                    amount_due,
+                    currency,
+                    &tenant.nwc_url,
+                )
+                .await?
             {
-                Ok(()) => {
-                    self.stripe_pay_invoice_out_of_band(invoice_id).await?;
-                    self.command.clear_tenant_nwc_error(&tenant.pubkey).await?;
+                NwcInvoicePaymentOutcome::Paid => {
+                    self.mark_invoice_paid_out_of_band_after_nwc(invoice_id, &tenant.pubkey)
+                        .await?;
                     return Ok(());
                 }
-                Err(e) => {
+                NwcInvoicePaymentOutcome::Fallback(e) => {
                     let error_msg = format!("{e}");
                     self.command
                         .set_tenant_nwc_error(&tenant.pubkey, &error_msg)
                         .await?;
+                    tracing::warn!(
+                        error = %e,
+                        tenant_pubkey = %tenant.pubkey,
+                        stripe_customer_id,
+                        invoice_id,
+                        "nwc auto-payment failed for invoice.created"
+                    );
+                    nwc_error_for_dm = summarize_nwc_error_for_dm(&error_msg);
                     // Fall through to next option
                 }
+                NwcInvoicePaymentOutcome::Pending(e) => {
+                    let error_msg = format!("{e}");
+                    self.command
+                        .set_tenant_nwc_error(&tenant.pubkey, &error_msg)
+                        .await?;
+                    tracing::error!(
+                        error = %e,
+                        tenant_pubkey = %tenant.pubkey,
+                        stripe_customer_id,
+                        invoice_id,
+                        "nwc auto-payment requires reconciliation before retry"
+                    );
+                    return Err(e);
+                }
             }
         }
 
@@ -390,12 +478,8 @@ impl Billing {
         }
 
         // 3. Manual payment: send a DM
-        self.robot
-            .send_dm(
-                &tenant.pubkey,
-                "Payment is due for your relay subscription. Please visit the application to complete a manual Lightning payment.",
-            )
-            .await?;
+        let dm_message = manual_lightning_payment_dm(nwc_error_for_dm.as_deref());
+        self.robot.send_dm(&tenant.pubkey, &dm_message).await?;
 
         Ok(())
     }
@@ -635,14 +719,60 @@ impl Billing {
         Ok((invoice, tenant))
     }
 
+    pub async fn reconcile_manual_lightning_invoice(
+        &self,
+        invoice_id: &str,
+        invoice: &serde_json::Value,
+    ) -> std::result::Result<serde_json::Value, InvoiceLookupError> {
+        self.reconcile_manual_lightning_invoice_if_settled(invoice_id, invoice)
+            .await
+    }
+
+    pub async fn get_or_create_manual_lightning_bolt11(
+        &self,
+        invoice_id: &str,
+        tenant_pubkey: &str,
+        amount_due_minor: i64,
+        currency: &str,
+    ) -> Result<String> {
+        if let Some(existing_bolt11) = self
+            .query
+            .get_invoice_manual_lightning_bolt11(invoice_id)
+            .await?
+        {
+            return Ok(existing_bolt11);
+        }
+
+        let bolt11 = self.create_bolt11(amount_due_minor, currency).await?;
+
+        if self
+            .command
+            .insert_manual_lightning_invoice_payment(invoice_id, tenant_pubkey, &bolt11)
+            .await?
+        {
+            return Ok(bolt11);
+        }
+
+        self.query
+            .get_invoice_manual_lightning_bolt11(invoice_id)
+            .await?
+            .ok_or_else(|| {
+                anyhow!(
+                    "manual lightning payment row missing after insert race for invoice {invoice_id}"
+                )
+            })
+    }
+
     pub async fn stripe_create_customer(&self, tenant_pubkey: &str) -> Result<String> {
         let short_pubkey: String = tenant_pubkey.chars().take(12).collect();
         let display_name = format!("Caravel tenant {short_pubkey}");
+        let idempotency_key = self.idempotency_key(&["create_customer", tenant_pubkey]);
 
         let resp = self
             .http
             .post(format!("{STRIPE_API}/customers"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .form(&[
                 ("name", display_name.as_str()),
                 ("metadata[tenant_pubkey]", tenant_pubkey),
@@ -743,21 +873,28 @@ impl Billing {
             }
 
             match self
-                .nwc_pay_invoice(amount_due, currency, &tenant.nwc_url)
-                .await
+                .nwc_pay_invoice(
+                    invoice_id,
+                    &tenant.pubkey,
+                    amount_due,
+                    currency,
+                    &tenant.nwc_url,
+                )
+                .await?
             {
-                Ok(()) => {
-                    if let Err(e) = self.stripe_pay_invoice_out_of_band(invoice_id).await {
+                NwcInvoicePaymentOutcome::Paid => {
+                    if let Err(e) = self
+                        .mark_invoice_paid_out_of_band_after_nwc(invoice_id, &tenant.pubkey)
+                        .await
+                    {
                         tracing::error!(
                             error = %e,
                             invoice_id,
                             "failed to mark invoice paid out of band"
                         );
-                    } else {
-                        let _ = self.command.clear_tenant_nwc_error(&tenant.pubkey).await;
                     }
                 }
-                Err(e) => {
+                NwcInvoicePaymentOutcome::Fallback(e) => {
                     let error_msg = format!("{e}");
                     tracing::error!(
                         error = %e,
@@ -769,6 +906,18 @@ impl Billing {
                         .set_tenant_nwc_error(&tenant.pubkey, &error_msg)
                         .await;
                 }
+                NwcInvoicePaymentOutcome::Pending(e) => {
+                    let error_msg = format!("{e}");
+                    tracing::error!(
+                        error = %e,
+                        invoice_id,
+                        "outstanding invoice requires NWC reconciliation before retry"
+                    );
+                    let _ = self
+                        .command
+                        .set_tenant_nwc_error(&tenant.pubkey, &error_msg)
+                        .await;
+                }
             }
         }
 
@@ -829,15 +978,30 @@ impl Billing {
 
     // --- Stripe API helpers ---
 
+    fn idempotency_key(&self, parts: &[&str]) -> String {
+        let mut mac = HmacSha256::new_from_slice(self.stripe_secret_key.as_bytes())
+            .expect("HMAC accepts any key length");
+        for (i, part) in parts.iter().enumerate() {
+            if i > 0 {
+                mac.update(b":");
+            }
+            mac.update(part.as_bytes());
+        }
+        hex::encode(mac.finalize().into_bytes())
+    }
+
     async fn stripe_create_subscription(
         &self,
         customer_id: &str,
         price_id: &str,
     ) -> Result<(String, String)> {
+        let idempotency_key =
+            self.idempotency_key(&["create_subscription", customer_id, price_id]);
         let resp = self
             .http
             .post(format!("{STRIPE_API}/subscriptions"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .form(&[
                 ("customer", customer_id),
                 ("collection_method", "charge_automatically"),
@@ -864,10 +1028,13 @@ impl Billing {
         subscription_id: &str,
         price_id: &str,
     ) -> Result<String> {
+        let idempotency_key =
+            self.idempotency_key(&["create_subscription_item", subscription_id, price_id]);
         let resp = self
             .http
             .post(format!("{STRIPE_API}/subscription_items"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .form(&[("subscription", subscription_id), ("price", price_id)])
             .send()
             .await?;
@@ -886,10 +1053,13 @@ impl Billing {
         item_id: &str,
         price_id: &str,
     ) -> Result<String> {
+        let idempotency_key =
+            self.idempotency_key(&["update_subscription_item", item_id, price_id]);
         let resp = self
             .http
             .post(format!("{STRIPE_API}/subscription_items/{item_id}"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .form(&[("price", price_id)])
             .send()
             .await?;
@@ -926,9 +1096,11 @@ impl Billing {
     }
 
     async fn stripe_pay_invoice(&self, invoice_id: &str) -> Result<()> {
+        let idempotency_key = self.idempotency_key(&["pay_invoice", invoice_id]);
         self.http
             .post(format!("{STRIPE_API}/invoices/{invoice_id}/pay"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .send()
             .await?
             .error_for_status()?;
@@ -968,9 +1140,11 @@ impl Billing {
     }
 
     async fn stripe_pay_invoice_out_of_band(&self, invoice_id: &str) -> Result<()> {
+        let idempotency_key = self.idempotency_key(&["pay_invoice_oob", invoice_id]);
         self.http
             .post(format!("{STRIPE_API}/invoices/{invoice_id}/pay"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .form(&[("paid_out_of_band", "true")])
             .send()
             .await?
@@ -999,19 +1173,113 @@ impl Billing {
 
     // --- NWC helpers ---
 
+    async fn mark_invoice_paid_out_of_band_after_nwc(
+        &self,
+        invoice_id: &str,
+        tenant_pubkey: &str,
+    ) -> Result<()> {
+        self.stripe_pay_invoice_out_of_band(invoice_id).await?;
+        self.command.clear_tenant_nwc_error(tenant_pubkey).await?;
+        Ok(())
+    }
+
+    async fn reconcile_manual_lightning_invoice_if_settled(
+        &self,
+        invoice_id: &str,
+        invoice: &serde_json::Value,
+    ) -> std::result::Result<serde_json::Value, InvoiceLookupError> {
+        if invoice["status"].as_str().unwrap_or_default() != "open" {
+            return Ok(invoice.clone());
+        }
+
+        let Some(bolt11) = self
+            .query
+            .get_invoice_manual_lightning_bolt11(invoice_id)
+            .await?
+        else {
+            return Ok(invoice.clone());
+        };
+
+        let settled = match self.is_manual_lightning_invoice_settled(&bolt11).await {
+            Ok(settled) => settled,
+            Err(error) => {
+                tracing::warn!(
+                    error = %error,
+                    invoice_id,
+                    "failed to lookup manual lightning invoice settlement"
+                );
+                return Ok(invoice.clone());
+            }
+        };
+
+        if !settled {
+            return Ok(invoice.clone());
+        }
+
+        if let Err(error) = self.stripe_pay_invoice_out_of_band(invoice_id).await {
+            tracing::warn!(
+                error = %error,
+                invoice_id,
+                "failed to mark settled manual lightning invoice as paid_out_of_band"
+            );
+        }
+
+        self.stripe_get_invoice(invoice_id).await
+    }
+
+    async fn is_manual_lightning_invoice_settled(&self, bolt11: &str) -> Result<bool> {
+        let system_uri = Self::parse_nwc_uri(&self.nwc_url, "system")?;
+        let system_nwc = NWC::new(system_uri);
+
+        let lookup_req = LookupInvoiceRequest {
+            payment_hash: None,
+            invoice: Some(bolt11.to_string()),
+        };
+
+        let lookup_result = system_nwc.lookup_invoice(lookup_req).await;
+        system_nwc.shutdown().await;
+
+        let lookup_response =
+            lookup_result.map_err(|error| anyhow!("failed to lookup invoice: {error}"))?;
+
+        Ok(Self::lookup_invoice_response_is_settled(&lookup_response))
+    }
+
+    fn lookup_invoice_response_is_settled(response: &LookupInvoiceResponse) -> bool {
+        response.state == Some(TransactionState::Settled) || response.settled_at.is_some()
+    }
+
+    fn parse_nwc_uri(nwc_url: &str, role: &str) -> Result<NostrWalletConnectURI> {
+        nwc_url
+            .parse::<NostrWalletConnectURI>()
+            .map_err(|_| anyhow!("invalid {role} NWC URL"))
+    }
+
     async fn nwc_pay_invoice(
         &self,
+        invoice_id: &str,
+        tenant_pubkey: &str,
         amount_due_minor: i64,
         currency: &str,
         tenant_nwc_url: &str,
-    ) -> Result<()> {
-        let amount_msats = self.fiat_minor_to_msats(amount_due_minor, currency).await?;
+    ) -> Result<NwcInvoicePaymentOutcome> {
+        if let Some(existing_outcome) = self
+            .existing_invoice_nwc_payment_outcome(invoice_id)
+            .await?
+        {
+            return Ok(existing_outcome);
+        }
+
+        let amount_msats = match self.fiat_minor_to_msats(amount_due_minor, currency).await {
+            Ok(amount_msats) => amount_msats,
+            Err(error) => return Ok(NwcInvoicePaymentOutcome::Fallback(error)),
+        };
 
         // Create a bolt11 invoice using the system wallet (self.nwc_url)
-        let system_uri: NostrWalletConnectURI = self
-            .nwc_url
-            .parse()
-            .map_err(|_| anyhow!("invalid system NWC URL"))?;
+        let system_uri = match Self::parse_nwc_uri(&self.nwc_url, "system") {
+            Ok(system_uri) => system_uri,
+            Err(error) => return Ok(NwcInvoicePaymentOutcome::Fallback(error)),
+        };
         let system_nwc = NWC::new(system_uri);
 
         let make_req = MakeInvoiceRequest {
@@ -1021,29 +1289,61 @@ impl Billing {
             expiry: None,
         };
 
-        let invoice_response = system_nwc
-            .make_invoice(make_req)
-            .await
-            .map_err(|e| anyhow!("failed to create invoice: {e}"))?;
+        let invoice_response = system_nwc.make_invoice(make_req).await;
+
+        let invoice_response = match invoice_response {
+            Ok(invoice_response) => invoice_response,
+            Err(error) => {
+                system_nwc.shutdown().await;
+                return Ok(NwcInvoicePaymentOutcome::Fallback(anyhow!(
+                    "failed to create invoice: {error}"
+                )));
+            }
+        };
 
         system_nwc.shutdown().await;
 
         // Pay the bolt11 invoice using the tenant's wallet
-        let tenant_uri: NostrWalletConnectURI = tenant_nwc_url
-            .parse()
-            .map_err(|_| anyhow!("invalid tenant NWC URL"))?;
+        let tenant_uri = match Self::parse_nwc_uri(tenant_nwc_url, "tenant") {
+            Ok(tenant_uri) => tenant_uri,
+            Err(error) => return Ok(NwcInvoicePaymentOutcome::Fallback(error)),
+        };
+
+        if !self
+            .command
+            .insert_pending_invoice_nwc_payment(invoice_id, tenant_pubkey)
+            .await?
+        {
+            if let Some(existing_outcome) = self
+                .existing_invoice_nwc_payment_outcome(invoice_id)
+                .await?
+            {
+                return Ok(existing_outcome);
+            }
+            return Err(anyhow!(
+                "invoice_nwc_payment row missing after insert race for invoice {invoice_id}"
+            ));
+        }
+
         let tenant_nwc = NWC::new(tenant_uri);
 
         let pay_req = NwcPayInvoiceRequest::new(invoice_response.invoice);
 
-        tenant_nwc
-            .pay_invoice(pay_req)
-            .await
-            .map_err(|e| anyhow!("failed to pay invoice: {e}"))?;
+        let pay_result = tenant_nwc.pay_invoice(pay_req).await;
 
         tenant_nwc.shutdown().await;
 
-        Ok(())
+        match pay_result {
+            Ok(_) => match self.command.mark_invoice_nwc_payment_paid(invoice_id).await {
+                Ok(()) => Ok(NwcInvoicePaymentOutcome::Paid),
+                Err(error) => Ok(NwcInvoicePaymentOutcome::Pending(anyhow!(
+                    "invoice {invoice_id} was charged over NWC but failed to persist paid state: {error}"
+                ))),
+            },
+            Err(error) => Ok(NwcInvoicePaymentOutcome::Pending(anyhow!(
+                "invoice {invoice_id} NWC payment attempt requires reconciliation: {error}"
+            ))),
+        }
     }
 
     async fn fiat_minor_to_msats(&self, amount_due_minor: i64, currency: &str) -> Result<u64> {
@@ -1057,7 +1357,7 @@ impl Billing {
     }
 
     async fn fetch_btc_spot_price(&self, currency: &str) -> Result<f64> {
-        fetch_btc_spot_price_from_base(&self.http, &self.btc_quote_api_base, currency).await
+        fetch_btc_spot_price_from_base(&self.http, COINBASE_SPOT_API, currency).await
     }
 
     fn currency_minor_exponent(currency: &str) -> Result<u8> {
@@ -1101,6 +1401,31 @@ pub async fn fetch_btc_spot_price_from_base(
     Ok(amount)
 }
 
+fn summarize_nwc_error_for_dm(error: &str) -> Option<String> {
+    let normalized = error.split_whitespace().collect::<Vec<_>>().join(" ");
+    if normalized.is_empty() {
+        return None;
+    }
+
+    if normalized.chars().count() <= NWC_ERROR_DM_MAX_CHARS {
+        return Some(normalized);
+    }
+
+    let prefix_len = NWC_ERROR_DM_MAX_CHARS.saturating_sub(3);
+    let mut truncated = normalized.chars().take(prefix_len).collect::<String>();
+    truncated.push_str("...");
+    Some(truncated)
+}
+
+fn manual_lightning_payment_dm(nwc_error: Option<&str>) -> String {
+    match nwc_error {
+        Some(error) if !error.is_empty() => {
+            format!("{MANUAL_LIGHTNING_PAYMENT_DM}\n\n{NWC_ERROR_DM_PREFIX} {error}")
+        }
+        _ => MANUAL_LIGHTNING_PAYMENT_DM.to_string(),
+    }
+}
+
 pub fn fiat_minor_to_msats_from_quote(
     amount_due_minor: i64,
     currency: &str,
@@ -1398,4 +1723,5 @@ mod tests {
         assert_eq!(billing.stripe_secret_key, "sk_test_dummy");
         assert_eq!(billing.stripe_webhook_secret, "whsec_test_dummy");
     }
+
 }

backend/src/command.rs

@@ -113,12 +113,12 @@ impl Command {
 
         sqlx::query(
             "INSERT INTO relay (
-                id, tenant, schema, subdomain, plan, status, sync_error,
+                id, tenant, schema, subdomain, plan, status, synced, sync_error,
                 info_name, info_icon, info_description,
                 policy_public_join, policy_strip_signatures,
                 groups_enabled, management_enabled, blossom_enabled,
                 livekit_enabled, push_enabled
-            ) VALUES (?, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            ) VALUES (?, ?, ?, ?, ?, 'active', 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
         )
         .bind(&relay.id)
         .bind(&relay.tenant)
@@ -151,7 +151,7 @@ impl Command {
 
         sqlx::query(
             "UPDATE relay
-             SET tenant = ?, schema = ?, subdomain = ?, plan = ?, status = ?, sync_error = ?,
+             SET tenant = ?, schema = ?, subdomain = ?, plan = ?, status = ?, sync_error = ?, synced = 0,
                  info_name = ?, info_icon = ?, info_description = ?,
                  policy_public_join = ?, policy_strip_signatures = ?,
                  groups_enabled = ?, management_enabled = ?, blossom_enabled = ?,
@@ -203,7 +203,7 @@ impl Command {
     ) -> Result<()> {
         let mut tx = self.pool.begin().await?;
 
-        sqlx::query("UPDATE relay SET status = ? WHERE id = ?")
+        sqlx::query("UPDATE relay SET status = ?, synced = 0 WHERE id = ?")
             .bind(status)
             .bind(relay_id)
             .execute(&mut *tx)
@@ -224,7 +224,7 @@ impl Command {
     pub async fn fail_relay_sync(&self, relay: &Relay, sync_error: String) -> Result<()> {
         let mut tx = self.pool.begin().await?;
 
-        sqlx::query("UPDATE relay SET sync_error = ? WHERE id = ?")
+        sqlx::query("UPDATE relay SET synced = 0, sync_error = ? WHERE id = ?")
             .bind(&sync_error)
             .bind(&relay.id)
             .execute(&mut *tx)
@@ -313,6 +313,70 @@ impl Command {
         Ok(())
     }
 
+    pub async fn insert_pending_invoice_nwc_payment(
+        &self,
+        invoice_id: &str,
+        tenant_pubkey: &str,
+    ) -> Result<bool> {
+        let now = chrono::Utc::now().timestamp();
+        let result = sqlx::query(
+            "INSERT INTO invoice_nwc_payment (invoice_id, tenant_pubkey, state, created_at, updated_at)
+             VALUES (?, ?, 'pending', ?, ?)
+             ON CONFLICT(invoice_id) DO NOTHING",
+        )
+        .bind(invoice_id)
+        .bind(tenant_pubkey)
+        .bind(now)
+        .bind(now)
+        .execute(&self.pool)
+        .await?;
+
+        Ok(result.rows_affected() > 0)
+    }
+
+    pub async fn mark_invoice_nwc_payment_paid(&self, invoice_id: &str) -> Result<()> {
+        let now = chrono::Utc::now().timestamp();
+        let result = sqlx::query(
+            "UPDATE invoice_nwc_payment
+             SET state = 'paid', updated_at = ?
+             WHERE invoice_id = ?",
+        )
+        .bind(now)
+        .bind(invoice_id)
+        .execute(&self.pool)
+        .await?;
+
+        if result.rows_affected() == 0 {
+            anyhow::bail!("invoice_nwc_payment row missing for invoice_id: {invoice_id}");
+        }
+
+        Ok(())
+    }
+
+    pub async fn insert_manual_lightning_invoice_payment(
+        &self,
+        invoice_id: &str,
+        tenant_pubkey: &str,
+        bolt11: &str,
+    ) -> Result<bool> {
+        let now = chrono::Utc::now().timestamp();
+        let result = sqlx::query(
+            "INSERT INTO invoice_manual_lightning_payment
+             (invoice_id, tenant_pubkey, bolt11, created_at, updated_at)
+             VALUES (?, ?, ?, ?, ?)
+             ON CONFLICT(invoice_id) DO NOTHING",
+        )
+        .bind(invoice_id)
+        .bind(tenant_pubkey)
+        .bind(bolt11)
+        .bind(now)
+        .bind(now)
+        .execute(&self.pool)
+        .await?;
+
+        Ok(result.rows_affected() > 0)
+    }
+
     pub async fn set_tenant_past_due(&self, pubkey: &str) -> Result<()> {
         let now = chrono::Utc::now().timestamp();
         sqlx::query("UPDATE tenant SET past_due_at = ? WHERE pubkey = ?")

backend/src/infra.rs

@@ -1,10 +1,15 @@
 use anyhow::Result;
 use nostr_sdk::prelude::*;
+use std::time::Duration;
 
 use crate::command::Command;
-use crate::models::{Activity, Relay, RELAY_STATUS_DELINQUENT, RELAY_STATUS_INACTIVE};
+use crate::models::{Activity, RELAY_STATUS_DELINQUENT, RELAY_STATUS_INACTIVE, Relay};
 use crate::query::Query;
 
+const RELAY_SYNC_RETRY_BASE_DELAY_SECS: u64 = 30;
+const RELAY_SYNC_RETRY_MAX_DELAY_SECS: u64 = 15 * 60;
+const RELAY_SYNC_RETRY_MAX_ATTEMPTS: usize = 6;
+
 #[derive(Clone)]
 pub struct Infra {
     api_url: String,
@@ -18,14 +23,22 @@ pub struct Infra {
 }
 
 impl Infra {
-    pub fn new(query: Query, command: Command) -> Self {
+    pub fn new(query: Query, command: Command) -> Result<Self> {
         let api_url = std::env::var("ZOOID_API_URL").unwrap_or_default();
         let relay_domain = std::env::var("RELAY_DOMAIN").unwrap_or_default();
         let livekit_url = std::env::var("LIVEKIT_URL").unwrap_or_default();
         let livekit_api_key = std::env::var("LIVEKIT_API_KEY").unwrap_or_default();
         let livekit_api_secret = std::env::var("LIVEKIT_API_SECRET").unwrap_or_default();
         let api_secret = std::env::var("ZOOID_API_SECRET").unwrap_or_default();
-        Self {
+
+        if api_url.trim().is_empty() {
+            anyhow::bail!("missing ZOOID_API_URL");
+        }
+        if api_secret.trim().is_empty() {
+            anyhow::bail!("missing ZOOID_API_SECRET");
+        }
+
+        Ok(Self {
             api_url,
             relay_domain,
             livekit_url,
@@ -34,12 +47,16 @@ impl Infra {
             api_secret,
             query,
             command,
-        }
+        })
     }
 
     pub async fn start(self) {
         let mut rx = self.command.notify.subscribe();
 
+        if let Err(error) = self.reconcile_relay_state("startup").await {
+            tracing::error!(error = %error, "failed to reconcile relay state on startup");
+        }
+
         loop {
             match rx.recv().await {
                 Ok(activity) => {
@@ -49,6 +66,10 @@ impl Infra {
                 }
                 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
                     tracing::warn!(missed = n, "infra lagged");
+
+                    if let Err(error) = self.reconcile_relay_state("lagged").await {
+                        tracing::error!(error = %error, "failed to reconcile relay state after lag");
+                    }
                 }
                 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
             }
@@ -58,18 +79,107 @@ impl Infra {
     async fn handle_activity(&self, activity: &Activity) -> Result<()> {
         let needs_sync = should_sync_relay_activity(activity.activity_type.as_str());
 
-        if needs_sync {
-            let Some(relay) = self.query.get_relay(&activity.resource_id).await? else {
-                return Ok(());
-            };
+        if !needs_sync || activity.resource_type != "relay" {
+            return Ok(());
+        }
 
-            let is_new = relay.synced == 0;
-            self.sync_and_report(&relay, is_new).await;
+        if activity.activity_type == "fail_relay_sync" {
+            self.schedule_relay_sync_retry(&activity.resource_id, "activity")
+                .await?;
+            return Ok(());
+        }
+
+        let Some(relay) = self.query.get_relay(&activity.resource_id).await? else {
+            return Ok(());
+        };
+
+        let is_new = self.relay_sync_is_new(&relay).await?;
+        self.sync_and_report(&relay, is_new).await;
+
+        Ok(())
+    }
+
+    async fn reconcile_relay_state(&self, source: &str) -> Result<()> {
+        let relays = self.query.list_relays_pending_sync().await?;
+
+        if relays.is_empty() {
+            return Ok(());
+        }
+
+        tracing::info!(source, relay_count = relays.len(), "reconciling pending relay state");
+
+        for relay in relays {
+            if relay.sync_error.trim().is_empty() {
+                let is_new = self.relay_sync_is_new(&relay).await?;
+                self.sync_and_report(&relay, is_new).await;
+            } else {
+                self.schedule_relay_sync_retry(&relay.id, source).await?;
+            }
         }
 
         Ok(())
     }
 
+    async fn schedule_relay_sync_retry(&self, relay_id: &str, source: &str) -> Result<()> {
+        let activities = self.query.list_activity_for_relay(relay_id).await?;
+        let consecutive_failures = consecutive_sync_failures(&activities);
+
+        let Some(delay) = relay_sync_retry_delay(consecutive_failures) else {
+            tracing::warn!(
+                relay = relay_id,
+                consecutive_failures,
+                max_attempts = RELAY_SYNC_RETRY_MAX_ATTEMPTS,
+                "relay sync retries exhausted; awaiting manual intervention"
+            );
+            return Ok(());
+        };
+
+        tracing::info!(
+            relay = relay_id,
+            source,
+            consecutive_failures,
+            delay_secs = delay.as_secs(),
+            "scheduled relay sync retry"
+        );
+
+        let relay_id = relay_id.to_string();
+        let infra = self.clone();
+
+        tokio::spawn(async move {
+            tokio::time::sleep(delay).await;
+
+            if let Err(e) = infra.retry_relay_sync(&relay_id).await {
+                tracing::error!(relay = %relay_id, error = %e, "relay sync retry task failed");
+            }
+        });
+
+        Ok(())
+    }
+
+    async fn retry_relay_sync(&self, relay_id: &str) -> Result<()> {
+        let Some(relay) = self.query.get_relay(relay_id).await? else {
+            return Ok(());
+        };
+
+        if relay.sync_error.trim().is_empty() {
+            tracing::debug!(relay = %relay.id, "skip relay sync retry; relay has no sync_error");
+            return Ok(());
+        }
+
+        let is_new = self.relay_sync_is_new(&relay).await?;
+        self.sync_and_report(&relay, is_new).await;
+        Ok(())
+    }
+
+    async fn relay_sync_is_new(&self, relay: &Relay) -> Result<bool> {
+        if relay.synced == 1 {
+            return Ok(false);
+        }
+
+        let has_completed_sync = self.query.relay_has_completed_sync(&relay.id).await?;
+        Ok(!has_completed_sync)
+    }
+
     async fn sync_and_report(&self, relay: &Relay, is_new: bool) {
         match self.sync_relay(relay, is_new).await {
             Ok(()) => {
@@ -96,6 +206,28 @@ impl Infra {
         Ok(auth)
     }
 
+    pub async fn list_relay_members(&self, relay_id: &str) -> Result<Vec<String>> {
+        let client = reqwest::Client::new();
+        let base = self.api_url.trim_end_matches('/');
+        let url = format!("{base}/relay/{relay_id}/members");
+        let auth = self.nip98_auth(&url, HttpMethod::GET).await?;
+
+        let response = client
+            .get(&url)
+            .header("Authorization", auth)
+            .send()
+            .await?;
+
+        if !response.status().is_success() {
+            let status = response.status();
+            let body = response.text().await.unwrap_or_default();
+            anyhow::bail!("zooid members returned {status}: {body}");
+        }
+
+        let body = response.text().await?;
+        parse_relay_members_response(&body)
+    }
+
     async fn sync_relay(&self, relay: &Relay, is_new: bool) -> Result<()> {
         let client = reqwest::Client::new();
         let base = self.api_url.trim_end_matches('/');
@@ -125,7 +257,9 @@ impl Infra {
         );
 
         let url = format!("{}/relay/{}", base, relay.id);
-        let auth = self.nip98_auth(&url, zooid_sync_http_method(is_new)).await?;
+        let auth = self
+            .nip98_auth(&url, zooid_sync_http_method(is_new))
+            .await?;
 
         let request = if is_new {
             client.post(&url)
@@ -156,6 +290,34 @@ fn zooid_sync_http_method(is_new: bool) -> HttpMethod {
     }
 }
 
+fn parse_relay_members_response(body: &str) -> Result<Vec<String>> {
+    let value: serde_json::Value = serde_json::from_str(body)?;
+
+    if let Some(members) = members_from_value(&value) {
+        return Ok(members);
+    }
+    if let Some(members) = value.get("members").and_then(members_from_value) {
+        return Ok(members);
+    }
+    if let Some(members) = value
+        .get("data")
+        .and_then(|data| data.get("members"))
+        .and_then(members_from_value)
+    {
+        return Ok(members);
+    }
+
+    anyhow::bail!("zooid members response missing members array")
+}
+
+fn members_from_value(value: &serde_json::Value) -> Option<Vec<String>> {
+    let values = value.as_array()?;
+    values
+        .iter()
+        .map(|value| value.as_str().map(ToString::to_string))
+        .collect()
+}
+
 fn relay_sync_body(
     relay: &Relay,
     host: String,
@@ -198,6 +360,28 @@ fn relay_sync_body(
 fn should_sync_relay_activity(activity_type: &str) -> bool {
     matches!(
         activity_type,
-        "create_relay" | "update_relay" | "activate_relay" | "deactivate_relay"
+        "create_relay" | "update_relay" | "activate_relay" | "deactivate_relay" | "fail_relay_sync"
     )
 }
+
+fn consecutive_sync_failures(activities: &[Activity]) -> usize {
+    activities
+        .iter()
+        .take_while(|activity| activity.activity_type == "fail_relay_sync")
+        .count()
+}
+
+fn relay_sync_retry_delay(consecutive_failures: usize) -> Option<Duration> {
+    let retry_attempt = consecutive_failures.max(1);
+    if retry_attempt > RELAY_SYNC_RETRY_MAX_ATTEMPTS {
+        return None;
+    }
+
+    let exponent = (retry_attempt - 1).min(31);
+    let multiplier = 1u64 << exponent;
+    let delay_secs = RELAY_SYNC_RETRY_BASE_DELAY_SECS
+        .saturating_mul(multiplier)
+        .min(RELAY_SYNC_RETRY_MAX_DELAY_SECS);
+
+    Some(Duration::from_secs(delay_secs))
+}

backend/src/main.rs

@@ -33,8 +33,8 @@ async fn main() -> Result<()> {
     let query = Query::new(pool.clone());
     let command = Command::new(pool);
     let billing = Billing::new(query.clone(), command.clone(), robot.clone());
-    let infra = Infra::new(query.clone(), command.clone());
-    let api = Api::new(query, command, billing.clone());
+    let infra = Infra::new(query.clone(), command.clone())?;
+    let api = Api::new(query, command, billing.clone(), infra.clone());
 
     let host = std::env::var("HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
     let port: u16 = std::env::var("PORT")

backend/src/query.rs

@@ -94,6 +94,23 @@ impl Query {
         Ok(rows)
     }
 
+    pub async fn list_relays_pending_sync(&self) -> Result<Vec<Relay>> {
+        let rows = sqlx::query_as::<_, Relay>(
+            "SELECT id, tenant, schema, subdomain, plan, stripe_subscription_item_id,
+                     status, sync_error,
+                     info_name, info_icon, info_description,
+                     policy_public_join, policy_strip_signatures,
+                     groups_enabled, management_enabled, blossom_enabled,
+                     livekit_enabled, push_enabled, synced
+              FROM relay
+              WHERE synced = 0 OR TRIM(sync_error) != ''
+              ORDER BY id",
+        )
+        .fetch_all(&self.pool)
+        .await?;
+        Ok(rows)
+    }
+
     pub async fn list_relays_for_tenant(&self, tenant_id: &str) -> Result<Vec<Relay>> {
         let rows = sqlx::query_as::<_, Relay>(
             "SELECT id, tenant, schema, subdomain, plan, stripe_subscription_item_id,
@@ -144,6 +161,29 @@ impl Query {
         Ok(row)
     }
 
+    pub async fn get_invoice_nwc_payment_state(&self, invoice_id: &str) -> Result<Option<String>> {
+        let state = sqlx::query_scalar::<_, String>(
+            "SELECT state FROM invoice_nwc_payment WHERE invoice_id = ?",
+        )
+        .bind(invoice_id)
+        .fetch_optional(&self.pool)
+        .await?;
+        Ok(state)
+    }
+
+    pub async fn get_invoice_manual_lightning_bolt11(
+        &self,
+        invoice_id: &str,
+    ) -> Result<Option<String>> {
+        let bolt11 = sqlx::query_scalar::<_, String>(
+            "SELECT bolt11 FROM invoice_manual_lightning_payment WHERE invoice_id = ?",
+        )
+        .bind(invoice_id)
+        .fetch_optional(&self.pool)
+        .await?;
+        Ok(bolt11)
+    }
+
     pub async fn has_active_paid_relays(&self, tenant_id: &str) -> Result<bool> {
         let plans = sqlx::query_scalar::<_, String>(
             "SELECT plan FROM relay WHERE tenant = ? AND status = 'active'",
@@ -167,4 +207,20 @@ impl Query {
         .await?;
         Ok(rows)
     }
+
+    pub async fn relay_has_completed_sync(&self, relay_id: &str) -> Result<bool> {
+        let found = sqlx::query_scalar::<_, i64>(
+            "SELECT 1
+             FROM activity
+             WHERE resource_type = 'relay'
+               AND resource_id = ?
+               AND activity_type = 'complete_relay_sync'
+             LIMIT 1",
+        )
+        .bind(relay_id)
+        .fetch_optional(&self.pool)
+        .await?;
+
+        Ok(found.is_some())
+    }
 }

frontend/src/lib/api.ts

@@ -241,6 +241,10 @@ export function getRelay(id: string) {
   return callApi<undefined, Relay>("GET", `/relays/${id}`)
 }
 
+export function listRelayMembers(id: string) {
+  return callApi<undefined, { members: string[] }>("GET", `/relays/${id}/members`)
+}
+
 export function listRelayActivity(id: string) {
   return callApi<undefined, { activity: Activity[] }>("GET", `/relays/${id}/activity`)
 }

frontend/src/lib/hooks.ts

@@ -2,7 +2,6 @@ import { createEffect, createResource, createSignal, onCleanup } from "solid-js"
 import { getProfilePicture } from "applesauce-core/helpers/profile"
 import { createOutboxMap, selectOptimalRelays, setFallbackRelays } from "applesauce-core/helpers/relay-selection"
 import { includeMailboxes } from "applesauce-core/observable"
-import { Relay as NostrRelay, RelayManagement } from "applesauce-relay"
 import { map, of } from "rxjs"
 import {
   createRelay,
@@ -147,14 +146,4 @@ export async function getLatestOpenInvoice(): Promise<Invoice | null> {
   return open[0] ?? null
 }
 
-export async function getRelayMembers(url: string) {
-  const management = new RelayManagement(new NostrRelay(url), account()!.signer)
-
-  try {
-    return await management.listAllowedPubkeys()
-  } catch {
-    return []
-  }
-}
-
 export type { Activity, Invoice, Relay, Tenant }

frontend/src/pages/admin/AdminRelayDetail.tsx

@@ -1,11 +1,12 @@
 import { useParams } from "@solidjs/router"
-import { Show } from "solid-js"
+import { createResource, Show } from "solid-js"
 import BackLink from "@/components/BackLink"
 import PageContainer from "@/components/PageContainer"
 import RelayDetailCard from "@/components/RelayDetailCard"
 import ResourceState from "@/components/ResourceState"
 import useMinLoading from "@/components/useMinLoading"
 import ActivityFeed from "@/components/ActivityFeed"
+import { listRelayMembers } from "@/lib/api"
 import { useRelay, useRelayActivity } from "@/lib/hooks"
 import useRelayToggles from "@/lib/useRelayToggles"
 
@@ -13,6 +14,15 @@ export default function AdminRelayDetail() {
   const params = useParams()
   const relayId = () => params.id ?? ""
   const [relay, { refetch, mutate }] = useRelay(relayId)
+  const [members] = createResource(relayId, async (id) => {
+    if (!id) return []
+
+    try {
+      return (await listRelayMembers(id)).members
+    } catch {
+      return []
+    }
+  })
   const loading = useMinLoading(() => relay.loading && !relay())
   const [activity] = useRelayActivity(relayId)
   const { busy, handleDeactivate, handleReactivate, toggles } = useRelayToggles(relayId, relay, { refetch, mutate })
@@ -26,6 +36,7 @@ export default function AdminRelayDetail() {
           <div class="space-y-6 mb-6">
             <RelayDetailCard
               relay={r()}
+              currentMembers={members()?.length}
               showTenant
               editHref={`/admin/relays/${params.id}/edit`}
               onDeactivate={handleDeactivate}

frontend/src/pages/relays/RelayDetail.tsx

@@ -8,7 +8,8 @@ import RelayDetailCard from "@/components/RelayDetailCard"
 import ResourceState from "@/components/ResourceState"
 import useMinLoading from "@/components/useMinLoading"
 import ActivityFeed from "@/components/ActivityFeed"
-import { getLatestOpenInvoice, getRelayMembers, useRelay, useRelayActivity, useTenant } from "@/lib/hooks"
+import { listRelayMembers } from "@/lib/api"
+import { getLatestOpenInvoice, useRelay, useRelayActivity, useTenant } from "@/lib/hooks"
 import useRelayToggles from "@/lib/useRelayToggles"
 import { plans } from "@/lib/state"
 
@@ -16,11 +17,15 @@ export default function RelayDetail() {
   const params = useParams()
   const relayId = () => params.id ?? ""
   const [relay, { refetch, mutate }] = useRelay(relayId)
-  const relayUrl = createMemo(() => {
-    const subdomain = relay()?.subdomain
-    return subdomain ? `wss://${subdomain}.spaces.coracle.social` : undefined
+  const [members] = createResource(relayId, async (id) => {
+    if (!id) return []
+
+    try {
+      return (await listRelayMembers(id)).members
+    } catch {
+      return []
+    }
   })
-  const [members] = createResource(relayUrl, getRelayMembers)
   const loading = useMinLoading(() => relay.loading && !relay())
   const [activity] = useRelayActivity(relayId)
   const { busy, handleDeactivate, handleReactivate, handleUpdatePlan, pendingInvoice, clearPendingInvoice, toggles } = useRelayToggles(relayId, relay, { refetch, mutate })
@@ -97,7 +102,7 @@ export default function RelayDetail() {
             </Show>
             <RelayDetailCard
               relay={r()}
-              currentMembers={members.length}
+              currentMembers={members()?.length}
               editHref={`/relays/${params.id}/edit`}
               onDeactivate={handleDeactivate}
               onReactivate={handleReactivate}