fix: silent relay state drift when activity bus drops events

Reviewed-on: coracle/caravel#51
Co-authored-by: userAdityaa <aditya.chaudhary1558@gmail.com>
Co-committed-by: userAdityaa <aditya.chaudhary1558@gmail.com>

backend/src/billing.rs

@@ -120,6 +120,10 @@ impl Billing {
     pub async fn start(self) {
         let mut rx = self.command.notify.subscribe();
 
+        if let Err(error) = self.reconcile_relay_subscriptions("startup").await {
+            tracing::error!(error = %error, "failed to reconcile relay billing state on startup");
+        }
+
         loop {
             match rx.recv().await {
                 Ok(activity) => {
@@ -129,12 +133,39 @@ impl Billing {
                 }
                 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
                     tracing::warn!(missed = n, "billing lagged");
+
+                    if let Err(error) = self.reconcile_relay_subscriptions("lagged").await {
+                        tracing::error!(error = %error, "failed to reconcile relay billing state after lag");
+                    }
                 }
                 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
             }
         }
     }
 
+    async fn reconcile_relay_subscriptions(&self, source: &str) -> Result<()> {
+        let relays = self.query.list_relays().await?;
+
+        if relays.is_empty() {
+            return Ok(());
+        }
+
+        tracing::info!(source, relay_count = relays.len(), "reconciling relay billing state");
+
+        for relay in relays {
+            if let Err(error) = self.sync_relay_subscription_for_relay(&relay).await {
+                tracing::error!(
+                    source,
+                    relay = %relay.id,
+                    error = %error,
+                    "failed to reconcile relay billing state"
+                );
+            }
+        }
+
+        Ok(())
+    }
+
     async fn handle_activity(&self, activity: &Activity) -> Result<()> {
         let needs_billing_sync = matches!(
             activity.activity_type.as_str(),
@@ -158,6 +189,10 @@ impl Billing {
             return Ok(());
         };
 
+        self.sync_relay_subscription_for_relay(&relay).await
+    }
+
+    async fn sync_relay_subscription_for_relay(&self, relay: &Relay) -> Result<()> {
         let Some(tenant) = self.query.get_tenant(&relay.tenant).await? else {
             return Ok(());
         };
@@ -686,11 +721,13 @@ impl Billing {
     pub async fn stripe_create_customer(&self, tenant_pubkey: &str) -> Result<String> {
         let short_pubkey: String = tenant_pubkey.chars().take(12).collect();
         let display_name = format!("Caravel tenant {short_pubkey}");
+        let idempotency_key = self.idempotency_key(&["create_customer", tenant_pubkey]);
 
         let resp = self
             .http
             .post(format!("{STRIPE_API}/customers"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .form(&[
                 ("name", display_name.as_str()),
                 ("metadata[tenant_pubkey]", tenant_pubkey),
@@ -896,15 +933,30 @@ impl Billing {
 
     // --- Stripe API helpers ---
 
+    fn idempotency_key(&self, parts: &[&str]) -> String {
+        let mut mac = HmacSha256::new_from_slice(self.stripe_secret_key.as_bytes())
+            .expect("HMAC accepts any key length");
+        for (i, part) in parts.iter().enumerate() {
+            if i > 0 {
+                mac.update(b":");
+            }
+            mac.update(part.as_bytes());
+        }
+        hex::encode(mac.finalize().into_bytes())
+    }
+
     async fn stripe_create_subscription(
         &self,
         customer_id: &str,
         price_id: &str,
     ) -> Result<(String, String)> {
+        let idempotency_key =
+            self.idempotency_key(&["create_subscription", customer_id, price_id]);
         let resp = self
             .http
             .post(format!("{STRIPE_API}/subscriptions"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .form(&[
                 ("customer", customer_id),
                 ("collection_method", "charge_automatically"),
@@ -931,10 +983,13 @@ impl Billing {
         subscription_id: &str,
         price_id: &str,
     ) -> Result<String> {
+        let idempotency_key =
+            self.idempotency_key(&["create_subscription_item", subscription_id, price_id]);
         let resp = self
             .http
             .post(format!("{STRIPE_API}/subscription_items"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .form(&[("subscription", subscription_id), ("price", price_id)])
             .send()
             .await?;
@@ -953,10 +1008,13 @@ impl Billing {
         item_id: &str,
         price_id: &str,
     ) -> Result<String> {
+        let idempotency_key =
+            self.idempotency_key(&["update_subscription_item", item_id, price_id]);
         let resp = self
             .http
             .post(format!("{STRIPE_API}/subscription_items/{item_id}"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .form(&[("price", price_id)])
             .send()
             .await?;
@@ -993,9 +1051,11 @@ impl Billing {
     }
 
     async fn stripe_pay_invoice(&self, invoice_id: &str) -> Result<()> {
+        let idempotency_key = self.idempotency_key(&["pay_invoice", invoice_id]);
         self.http
             .post(format!("{STRIPE_API}/invoices/{invoice_id}/pay"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .send()
             .await?
             .error_for_status()?;
@@ -1035,9 +1095,11 @@ impl Billing {
     }
 
     async fn stripe_pay_invoice_out_of_band(&self, invoice_id: &str) -> Result<()> {
+        let idempotency_key = self.idempotency_key(&["pay_invoice_oob", invoice_id]);
         self.http
             .post(format!("{STRIPE_API}/invoices/{invoice_id}/pay"))
             .bearer_auth(&self.stripe_secret_key)
+            .header("Idempotency-Key", idempotency_key)
             .form(&[("paid_out_of_band", "true")])
             .send()
             .await?
@@ -1550,4 +1612,5 @@ mod tests {
         assert_eq!(billing.stripe_secret_key, "sk_test_dummy");
         assert_eq!(billing.stripe_webhook_secret, "whsec_test_dummy");
     }
+
 }

backend/src/command.rs

@@ -113,12 +113,12 @@ impl Command {
 
         sqlx::query(
             "INSERT INTO relay (
-                id, tenant, schema, subdomain, plan, status, sync_error,
+                id, tenant, schema, subdomain, plan, status, synced, sync_error,
                 info_name, info_icon, info_description,
                 policy_public_join, policy_strip_signatures,
                 groups_enabled, management_enabled, blossom_enabled,
                 livekit_enabled, push_enabled
-            ) VALUES (?, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            ) VALUES (?, ?, ?, ?, ?, 'active', 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
         )
         .bind(&relay.id)
         .bind(&relay.tenant)
@@ -151,7 +151,7 @@ impl Command {
 
         sqlx::query(
             "UPDATE relay
-             SET tenant = ?, schema = ?, subdomain = ?, plan = ?, status = ?, sync_error = ?,
+             SET tenant = ?, schema = ?, subdomain = ?, plan = ?, status = ?, sync_error = ?, synced = 0,
                  info_name = ?, info_icon = ?, info_description = ?,
                  policy_public_join = ?, policy_strip_signatures = ?,
                  groups_enabled = ?, management_enabled = ?, blossom_enabled = ?,
@@ -203,7 +203,7 @@ impl Command {
     ) -> Result<()> {
         let mut tx = self.pool.begin().await?;
 
-        sqlx::query("UPDATE relay SET status = ? WHERE id = ?")
+        sqlx::query("UPDATE relay SET status = ?, synced = 0 WHERE id = ?")
             .bind(status)
             .bind(relay_id)
             .execute(&mut *tx)
@@ -224,7 +224,7 @@ impl Command {
     pub async fn fail_relay_sync(&self, relay: &Relay, sync_error: String) -> Result<()> {
         let mut tx = self.pool.begin().await?;
 
-        sqlx::query("UPDATE relay SET sync_error = ? WHERE id = ?")
+        sqlx::query("UPDATE relay SET synced = 0, sync_error = ? WHERE id = ?")
             .bind(&sync_error)
             .bind(&relay.id)
             .execute(&mut *tx)

backend/src/infra.rs

@@ -53,8 +53,8 @@ impl Infra {
     pub async fn start(self) {
         let mut rx = self.command.notify.subscribe();
 
-        if let Err(e) = self.schedule_startup_retries().await {
+        if let Err(error) = self.reconcile_relay_state("startup").await {
-            tracing::error!(error = %e, "failed to schedule relay sync retries on startup");
+            tracing::error!(error = %error, "failed to reconcile relay state on startup");
         }
 
         loop {
@@ -66,6 +66,10 @@ impl Infra {
                 }
                 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
                     tracing::warn!(missed = n, "infra lagged");
+
+                    if let Err(error) = self.reconcile_relay_state("lagged").await {
+                        tracing::error!(error = %error, "failed to reconcile relay state after lag");
+                    }
                 }
                 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
             }
@@ -95,11 +99,22 @@ impl Infra {
         Ok(())
     }
 
-    async fn schedule_startup_retries(&self) -> Result<()> {
+    async fn reconcile_relay_state(&self, source: &str) -> Result<()> {
-        let relays = self.query.list_relays_with_sync_error().await?;
+        let relays = self.query.list_relays_pending_sync().await?;
+
+        if relays.is_empty() {
+            return Ok(());
+        }
+
+        tracing::info!(source, relay_count = relays.len(), "reconciling pending relay state");
 
         for relay in relays {
-            self.schedule_relay_sync_retry(&relay.id, "startup").await?;
+            if relay.sync_error.trim().is_empty() {
+                let is_new = relay.synced == 0;
+                self.sync_and_report(&relay, is_new).await;
+            } else {
+                self.schedule_relay_sync_retry(&relay.id, source).await?;
+            }
         }
 
         Ok(())

backend/src/query.rs

@@ -94,7 +94,7 @@ impl Query {
         Ok(rows)
     }
 
-    pub async fn list_relays_with_sync_error(&self) -> Result<Vec<Relay>> {
+    pub async fn list_relays_pending_sync(&self) -> Result<Vec<Relay>> {
         let rows = sqlx::query_as::<_, Relay>(
             "SELECT id, tenant, schema, subdomain, plan, stripe_subscription_item_id,
                      status, sync_error,
@@ -103,7 +103,7 @@ impl Query {
                      groups_enabled, management_enabled, blossom_enabled,
                      livekit_enabled, push_enabled, synced
               FROM relay
-              WHERE TRIM(sync_error) != ''
+              WHERE synced = 0 OR TRIM(sync_error) != ''
               ORDER BY id",
         )
         .fetch_all(&self.pool)