Permanently delete never-paid VMs + super-admin VM purge

ADMIN_API_ENDPOINTS.md

@@ -357,10 +357,19 @@ Body (optional):
 
 ```json
 {
-  "reason": "string"
+  "reason": "string",
+  "purge": false
 }
 ```
 
+- `reason` — optional admin reason recorded in the VM history audit trail.
+- `purge` — optional. When `true`, permanently deletes the VM and **all** related
+  records (history, firewall rules, IP assignments, and its subscription, line
+  items and payment history) instead of soft-deleting. Requires the
+  `super_admin` role — non-super-admins receive `403`. Intended for removing
+  test VMs. VMs that have never been paid are always purged regardless of this
+  flag.
+
 Response:
 
 ```json

API_CHANGELOG.md

@@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
+- **Permanently delete VMs (admin)** — `DELETE /api/admin/v1/vms/{id}` now accepts an optional `purge` flag in the request body (issue #168).
+  - VMs that have **never been paid** (their subscription was never set up) are now **hard-deleted** from the database instead of being soft-deleted, both when an admin deletes them and when the worker's hourly cleanup removes unpaid VMs. Nothing is left behind: the VM row and all related records (history, firewall rules, IP assignments, and the VM's own subscription + line items + payment history) are removed.
+  - `purge: true` lets a **super_admin** permanently delete *any* VM — including VMs with payment history — clearing up all related entities. This is intended for removing test VMs. The flag is rejected with `403` for non-super-admins (checked before the VM lookup); regular deletes and never-paid purges are unaffected.
+
 - **Import existing host VMs (admin)** — new admin endpoints to adopt VMs that exist on a host but aren't tracked in the database (issue #166).
   - `GET /api/admin/v1/hosts/{id}/vms/unmanaged` lists VMs present on the host with no matching database record (returns host vmid, mapped database id, name, CPU/memory/disk specs, storage, MAC, running state). The admin service dispatches a discovery job to the worker and reads the reply over a temporary Redis pub/sub channel.
   - `POST /api/admin/v1/hosts/{id}/vms/import` (`{ host_vm_id, user_id, reason? }`) imports one VM: it is assigned to `user_id` and billed via the region's **custom pricing** (required — import fails if the region has none), capturing the VM's current CPU/memory/disk specs into a custom template. Currently supports Proxmox hosts. Returns a `job_id`.

lnvps_api/src/provisioner/rollback_tests.rs

@@ -147,7 +147,7 @@ mod tests {
         );
 
         // Now clean up and verify cleanup works (this tests the delete_vm rollback)
-        provisioner.delete_vm(vm.id).await?;
+        provisioner.delete_vm(vm.id, false).await?;
 
         // Verify ARP entry for this VM was removed
         let cleanup_arp_entries = router.list_arp_entry().await?;
@@ -203,7 +203,7 @@ mod tests {
         );
 
         // Delete the VM to trigger cleanup (simulating rollback scenario)
-        provisioner.delete_vm(vm.id).await?;
+        provisioner.delete_vm(vm.id, false).await?;
 
         // Verify IP assignments are marked deleted
         let ips_after = db.list_vm_ip_assignments(vm.id).await?;
@@ -290,7 +290,7 @@ mod tests {
         );
 
         // Delete the VM
-        provisioner.delete_vm(vm.id).await?;
+        provisioner.delete_vm(vm.id, false).await?;
 
         // Verify DNS records are removed (count should be back to initial or less)
         {
@@ -358,7 +358,7 @@ mod tests {
         }
 
         // Delete VM
-        provisioner.delete_vm(vm.id).await?;
+        provisioner.delete_vm(vm.id, false).await?;
 
         // After delete - IPs should be soft-deleted (marked deleted=true)
         let ips_after_delete = db.list_vm_ip_assignments(vm.id).await?;
@@ -408,7 +408,7 @@ mod tests {
         assert_eq!(ips.len(), 2, "Should still have exactly 2 IP assignments");
 
         // Cleanup
-        provisioner.delete_vm(vm.id).await?;
+        provisioner.delete_vm(vm.id, false).await?;
 
         Ok(())
     }
@@ -448,7 +448,7 @@ mod tests {
         );
 
         // Delete and verify cleanup
-        provisioner.delete_vm(vm.id).await?;
+        provisioner.delete_vm(vm.id, false).await?;
 
         // The ARP entry with this MAC should be gone
         let router = MockRouter::new();
@@ -501,7 +501,7 @@ mod tests {
         assert!(arp_before.iter().any(|e| e.mac_address == mac_address));
 
         // Delete the VM
-        provisioner.delete_vm(vm_id).await?;
+        provisioner.delete_vm(vm_id, false).await?;
 
         // Verify complete cleanup:
         // VM should be soft-deleted (deleted = true), matching production MySQL behavior
@@ -666,7 +666,7 @@ mod tests {
         assert!(!ips_before.is_empty(), "Should have IP assignments");
 
         // Delete should work
-        let result = provisioner.delete_vm(vm_id).await;
+        let result = provisioner.delete_vm(vm_id, false).await;
         assert!(result.is_ok(), "Delete should succeed");
 
         // Verify VM is soft-deleted (matching production MySQL behavior)
@@ -682,6 +682,49 @@ mod tests {
         Ok(())
     }
 
+    /// A purging delete (`purge = true`, used for never-paid VMs and super-admin
+    /// forced deletions) removes the VM row entirely and hard-deletes its IP
+    /// assignments instead of soft-deleting them.
+    #[tokio::test]
+    async fn test_delete_vm_purge_removes_vm_row() -> Result<()> {
+        clear_mock_state().await;
+        let settings = mock_settings();
+        let db = Arc::new(MockDb::default());
+        let rates = Arc::new(MockExchangeRate::new());
+        const MOCK_RATE: f32 = 69_420.0;
+        rates.set_rate(Ticker::btc_rate("EUR")?, MOCK_RATE).await;
+
+        setup_db_with_static_arp(&db).await?;
+
+        let provisioner = VmProvisioner::new(settings, db.clone());
+
+        let (user, ssh_key) = add_user(&db).await?;
+        let vm = provisioner
+            .provision(user.id, 1, 1, ssh_key.id, None)
+            .await?;
+        let vm_id = vm.id;
+
+        let pipeline = provisioner.spawn_vm_pipeline(vm_id).await?;
+        pipeline.execute().await?;
+        assert!(!db.list_vm_ip_assignments(vm_id).await?.is_empty());
+
+        // Purge the VM.
+        provisioner.delete_vm(vm_id, true).await?;
+
+        // The VM row is gone entirely (not just soft-deleted).
+        assert!(
+            db.get_vm(vm_id).await.is_err(),
+            "VM row should be hard-deleted"
+        );
+        // IP assignments are hard-deleted too.
+        assert!(
+            db.list_vm_ip_assignments(vm_id).await?.is_empty(),
+            "IP assignments should be hard-deleted"
+        );
+
+        Ok(())
+    }
+
     /// Test that when the router's generate_mac returns an ArpEntry, the vm.mac_address
     /// is set from ArpEntry.mac_address (the actual MAC) and not ArpEntry.address (the IP).
     /// This covers the regression where mac and IP were mixed up in the OVH router path.

lnvps_api/src/provisioner/vm.rs

@@ -652,8 +652,17 @@ impl VmProvisioner {
             ))
     }
 
-    /// Delete a VM and its associated resources
-    pub async fn delete_vm(&self, vm_id: u64) -> OpResult<()> {
+    /// Delete a VM and its associated resources.
+    ///
+    /// When `purge` is set the VM is permanently removed from the database along
+    /// with all of its related records (history, firewall rules, IP
+    /// assignments, and its subscription + payment history) via
+    /// [`LNVpsDb::hard_delete_vm`]. Otherwise the VM is soft-deleted
+    /// (`deleted = 1`), matching production behaviour for paid VMs.
+    ///
+    /// Callers pass `purge = true` for never-paid (new) VMs and super-admin
+    /// forced deletions so that they leave nothing behind in the database.
+    pub async fn delete_vm(&self, vm_id: u64, purge: bool) -> OpResult<()> {
         let vm = self.db.get_vm(vm_id).await?;
         let host = self.db.get_host(vm.host_id).await?;
 
@@ -663,8 +672,14 @@ impl VmProvisioner {
             .step("delete_ips", |ctx| {
                 Box::pin(ctx.2.delete_all_ip_assignments(vm_id))
             })
-            .step("delete_vm_db", |ctx| {
-                Box::pin(async { Ok(ctx.0.delete_vm(vm_id).await?) })
+            .step("delete_vm_db", move |ctx| {
+                Box::pin(async move {
+                    if purge {
+                        Ok(ctx.0.hard_delete_vm(vm_id).await?)
+                    } else {
+                        Ok(ctx.0.delete_vm(vm_id).await?)
+                    }
+                })
             });
         pipeline.execute().await?;
         Ok(())
@@ -1191,7 +1206,7 @@ mod tests {
         }
 
         // now expire
-        provisioner.delete_vm(vm.id).await?;
+        provisioner.delete_vm(vm.id, false).await?;
 
         // test arp/dns is removed
         let arp = router.list_arp_entry().await?;

lnvps_api/src/subscription/vm.rs

@@ -249,7 +249,7 @@ impl SubscriptionLineItemHandler for VmLineItemHandler {
             return Ok(());
         }
 
-        if let Err(e) = self.provisioner.delete_vm(vm_id).await {
+        if let Err(e) = self.provisioner.delete_vm(vm_id, false).await {
             warn!("Failed to delete expired VM {}: {}", vm_id, e);
         } else {
             if let Err(e) = self

lnvps_api/src/worker.rs

@@ -1051,7 +1051,9 @@ impl Worker {
                 continue;
             }
             info!("Deleting unpaid VM {}", vm.id);
-            if let Err(e) = provisioner.delete_vm(vm.id).await {
+            // Never-paid (new) VMs carry no customer data, so purge them entirely
+            // rather than leaving a soft-deleted row and orphaned subscription.
+            if let Err(e) = provisioner.delete_vm(vm.id, true).await {
                 error!("Failed to delete unpaid VM {}: {}", vm.id, e);
                 self.queue_admin_notification(
                     format!("Failed to delete unpaid VM {}:\n{}", vm.id, e),
@@ -2039,31 +2041,49 @@ impl Worker {
                 vm_id,
                 reason,
                 admin_user_id,
+                purge,
             } => {
                 let vm = self.db.get_vm(*vm_id).await?;
                 if vm.deleted {
                     return Ok(None);
                 }
 
+                // A VM that has never had its first (purchase) payment confirmed
+                // carries no customer data, so purge it entirely. Super-admins
+                // can also force a purge of any VM (including paid ones) via the
+                // `purge` flag.
+                let ever_paid = self
+                    .db
+                    .get_subscription_by_line_item_id(vm.subscription_line_item_id)
+                    .await
+                    .map(|s| s.is_setup)
+                    .unwrap_or(false);
+                let hard_delete = *purge || !ever_paid;
+
                 // Delete the VM via provisioner
                 let provisioner = self.subscription_handler.vm_provisioner();
-                provisioner.delete_vm(*vm_id).await?;
-
-                // Log VM deletion
-                let metadata = if let Some(admin_id) = admin_user_id {
-                    Some(serde_json::json!({
-                        "admin_user_id": admin_id,
-                        "admin_action": true
-                    }))
-                } else {
-                    Some(serde_json::json!({
-                        "admin_action": true
-                    }))
-                };
-
-                self.vm_history_logger
-                    .log_vm_deleted(*vm_id, *admin_user_id, reason.as_deref(), metadata)
-                    .await?;
+                provisioner.delete_vm(*vm_id, hard_delete).await?;
+
+                // A hard delete removes the VM row (and its history), so logging
+                // a vm_history entry afterwards would fail the foreign key.
+                // Only record deletion history for soft-deleted VMs.
+                if !hard_delete {
+                    // Log VM deletion
+                    let metadata = if let Some(admin_id) = admin_user_id {
+                        Some(serde_json::json!({
+                            "admin_user_id": admin_id,
+                            "admin_action": true
+                        }))
+                    } else {
+                        Some(serde_json::json!({
+                            "admin_action": true
+                        }))
+                    };
+
+                    self.vm_history_logger
+                        .log_vm_deleted(*vm_id, *admin_user_id, reason.as_deref(), metadata)
+                        .await?;
+                }
 
                 // Send notifications
                 let reason_text = reason.as_deref().unwrap_or("Admin requested deletion");
@@ -3218,10 +3238,12 @@ mod tests {
         let worker = setup_worker(db.clone()).await?;
         worker.check_vms().await?;
 
-        // VM should be soft-deleted
+        // Never-paid VMs are purged entirely, not just soft-deleted.
         let vms = db.vms.lock().await;
-        let deleted = vms.get(&vm_id).map(|v| v.deleted).unwrap_or(false);
-        assert!(deleted, "Unpaid VM older than 1 hour should be deleted");
+        assert!(
+            !vms.contains_key(&vm_id),
+            "Unpaid VM older than 1 hour should be purged"
+        );
         Ok(())
     }
 
@@ -3327,12 +3349,11 @@ mod tests {
         let worker = setup_worker(db.clone()).await?;
         worker.check_vms().await?;
 
-        // VM should be soft-deleted because the only payment is expired.
+        // VM should be purged because the only payment is expired.
         let vms = db.vms.lock().await;
-        let deleted = vms.get(&vm_id).map(|v| v.deleted).unwrap_or(false);
         assert!(
-            deleted,
-            "Unpaid VM with only an expired payment should still be deleted"
+            !vms.contains_key(&vm_id),
+            "Unpaid VM with only an expired payment should still be purged"
         );
         Ok(())
     }

lnvps_api_admin/src/admin/auth.rs

@@ -44,6 +44,21 @@ impl AdminAuth {
         })
     }
 
+    /// Check whether the authenticated admin holds the `super_admin` role.
+    ///
+    /// Permissions alone can't express "super admin only" actions (a custom role
+    /// could be granted the same permission tuples), so destructive operations
+    /// like permanently purging a paid VM are gated on the role by name.
+    pub async fn is_super_admin(&self, db: &Arc<dyn LNVpsDb>) -> Result<bool> {
+        let role_ids = db.get_user_roles(self.user_id).await?;
+        for role_id in role_ids {
+            if db.get_role(role_id).await?.name == "super_admin" {
+                return Ok(true);
+            }
+        }
+        Ok(false)
+    }
+
     /// Check if the authenticated admin has a specific permission
     pub fn has_permission(&self, resource: AdminResource, action: AdminAction) -> bool {
         self.permissions.contains(&Permission { resource, action })

lnvps_api_admin/src/admin/vms.rs

@@ -389,6 +389,11 @@ async fn admin_stop_vm(
 #[serde(default)]
 struct AdminDeleteVmRequest {
     reason: Option<String>,
+    /// Permanently purge the VM and all related records (history, payments,
+    /// subscription) from the database, even if it has payment history.
+    /// Requires the `super_admin` role. Never-paid VMs are always purged
+    /// regardless of this flag.
+    purge: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -415,6 +420,16 @@ async fn admin_delete_vm(
     // Check permission
     auth.require_permission(AdminResource::VirtualMachines, AdminAction::Delete)?;
 
+    // Purging a VM with payment history is destructive and irreversible, so it
+    // is restricted to super-admins. Never-paid VMs are purged automatically by
+    // the worker regardless of this flag. Authorize before looking up the VM.
+    let purge = req.purge.unwrap_or(false);
+    if purge && !auth.is_super_admin(&this.db).await? {
+        return Err(ApiError::forbidden(
+            "Only super admins can permanently purge a VM",
+        ));
+    }
+
     // Verify VM exists
     let vm = this.db.get_vm(id).await?;
 
@@ -427,6 +442,7 @@ async fn admin_delete_vm(
         vm_id: id,
         reason: req.reason.clone(),
         admin_user_id: Some(auth.user_id),
+        purge,
     };
 
     match this.work_commander.send(delete_job).await {