[sled-agent] Destroy orphaned datasets (PR 3/2)

common/src/api/internal/shared.rs

@@ -937,6 +937,7 @@ pub enum DatasetKind {
     // Zone filesystems
     TransientZoneRoot,
     TransientZone {
+        #[cfg_attr(any(test, feature = "testing"), strategy("[^/]+"))]
         name: String,
     },
 

dev-tools/omdb/src/bin/omdb/sled_agent.rs

@@ -5,11 +5,13 @@
 //! omdb commands that query or update specific Sleds
 
 use crate::Omdb;
+use crate::check_allow_destructive::DestructiveOperationToken;
 use crate::helpers::CONNECTION_OPTIONS_HEADING;
 use anyhow::Context;
 use anyhow::bail;
 use clap::Args;
 use clap::Subcommand;
+use sled_agent_client::types::ChickenSwitchDestroyOrphanedDatasets;
 
 /// Arguments to the "omdb sled-agent" subcommand
 #[derive(Debug, Args)]
@@ -37,6 +39,11 @@ enum SledAgentCommands {
     /// print information about the local bootstore node
     #[clap(subcommand)]
     Bootstore(BootstoreCommands),
+
+    /// control "chicken switches" (potentially-destructive sled-agent behavior
+    /// that can be toggled on or off via `omdb`)
+    #[clap(subcommand)]
+    ChickenSwitch(ChickenSwitchCommands),
 }
 
 #[derive(Debug, Subcommand)]
@@ -46,32 +53,38 @@ enum ZoneCommands {
 }
 
 #[derive(Debug, Subcommand)]
-enum ZpoolCommands {
-    /// Print list of all zpools managed by the sled agent
-    List,
+enum BootstoreCommands {
+    /// Show the internal state of the local bootstore node
+    Status,
 }
 
 #[derive(Debug, Subcommand)]
-enum DatasetCommands {
-    /// Print list of all datasets the sled agent is configured to manage
-    ///
-    /// Note that the set of actual datasets on the sled may be distinct,
-    /// use the `omdb db inventory collections show` command to see the latest
-    /// set of datasets collected from sleds.
-    List,
+enum ChickenSwitchCommands {
+    /// interact with the "destroy orphaned datasets" chicken switch
+    DestroyOrphans(DestroyOrphansArgs),
+}
+
+#[derive(Debug, Args)]
+struct DestroyOrphansArgs {
+    #[command(subcommand)]
+    command: DestroyOrphansCommands,
 }
 
 #[derive(Debug, Subcommand)]
-enum BootstoreCommands {
-    /// Show the internal state of the local bootstore node
-    Status,
+enum DestroyOrphansCommands {
+    /// Get the current chicken switch setting
+    Get,
+    /// Enable the current chicken switch setting
+    Enable,
+    /// Disable the current chicken switch setting
+    Disable,
 }
 
 impl SledAgentArgs {
     /// Run a `omdb sled-agent` subcommand.
     pub(crate) async fn run_cmd(
         &self,
-        _omdb: &Omdb,
+        omdb: &Omdb,
         log: &slog::Logger,
     ) -> Result<(), anyhow::Error> {
         // This is a little goofy. The sled URL is required, but can come
@@ -92,6 +105,29 @@ impl SledAgentArgs {
             SledAgentCommands::Bootstore(BootstoreCommands::Status) => {
                 cmd_bootstore_status(&client).await
             }
+            SledAgentCommands::ChickenSwitch(
+                ChickenSwitchCommands::DestroyOrphans(DestroyOrphansArgs {
+                    command: DestroyOrphansCommands::Get,
+                }),
+            ) => cmd_chicken_switch_destroy_orphans_get(&client).await,
+            SledAgentCommands::ChickenSwitch(
+                ChickenSwitchCommands::DestroyOrphans(DestroyOrphansArgs {
+                    command: DestroyOrphansCommands::Enable,
+                }),
+            ) => {
+                let token = omdb.check_allow_destructive()?;
+                cmd_chicken_switch_destroy_orphans_set(&client, true, token)
+                    .await
+            }
+            SledAgentCommands::ChickenSwitch(
+                ChickenSwitchCommands::DestroyOrphans(DestroyOrphansArgs {
+                    command: DestroyOrphansCommands::Disable,
+                }),
+            ) => {
+                let token = omdb.check_allow_destructive()?;
+                cmd_chicken_switch_destroy_orphans_set(&client, false, token)
+                    .await
+            }
         }
     }
 }
@@ -157,3 +193,33 @@ async fn cmd_bootstore_status(
 
     Ok(())
 }
+
+/// Runs `omdb sled-agent chicken-switch destroy-orphans get`
+async fn cmd_chicken_switch_destroy_orphans_get(
+    client: &sled_agent_client::Client,
+) -> Result<(), anyhow::Error> {
+    let ChickenSwitchDestroyOrphanedDatasets { destroy_orphans } = client
+        .chicken_switch_destroy_orphaned_datasets_get()
+        .await
+        .context("get chicken switch value")?
+        .into_inner();
+    let status = if destroy_orphans { "enabled" } else { "disabled" };
+    println!("destroy orphaned datasets {status}");
+    Ok(())
+}
+
+/// Runs `omdb sled-agent chicken-switch destroy-orphans {enable,disable}`
+async fn cmd_chicken_switch_destroy_orphans_set(
+    client: &sled_agent_client::Client,
+    destroy_orphans: bool,
+    _token: DestructiveOperationToken,
+) -> Result<(), anyhow::Error> {
+    let options = ChickenSwitchDestroyOrphanedDatasets { destroy_orphans };
+    client
+        .chicken_switch_destroy_orphaned_datasets_put(&options)
+        .await
+        .context("put chicken switch value")?;
+    let status = if destroy_orphans { "enabled" } else { "disabled" };
+    println!("destroy orphaned datasets {status}");
+    Ok(())
+}

dev-tools/omdb/tests/usage_errors.out

@@ -909,9 +909,11 @@ Debug a specific Sled
 Usage: omdb sled-agent [OPTIONS] <COMMAND>
 
 Commands:
-  zones      print information about zones
-  bootstore  print information about the local bootstore node
-  help       Print this message or the help of the given subcommand(s)
+  zones           print information about zones
+  bootstore       print information about the local bootstore node
+  chicken-switch  control "chicken switches" (potentially-destructive sled-agent behavior that can
+                  be toggled on or off via `omdb`)
+  help            Print this message or the help of the given subcommand(s)
 
 Options:
       --log-level <LOG_LEVEL>  log level filter [env: LOG_LEVEL=] [default: warn]

openapi/sled-agent.json

@@ -334,6 +334,57 @@
         }
       }
     },
+    "/chicken-switch/destroy-orphaned-datasets": {
+      "get": {
+        "summary": "This endpoint reports the status of the `destroy_orphaned_datasets`",
+        "description": "chicken switch. It will be removed with omicron#6177.",
+        "operationId": "chicken_switch_destroy_orphaned_datasets_get",
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ChickenSwitchDestroyOrphanedDatasets"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      },
+      "put": {
+        "summary": "This endpoint sets the `destroy_orphaned_datasets` chicken switch",
+        "description": "(allowing sled-agent to delete datasets it believes are orphaned). It will be removed with omicron#6177.",
+        "operationId": "chicken_switch_destroy_orphaned_datasets_put",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/ChickenSwitchDestroyOrphanedDatasets"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "204": {
+            "description": "resource updated"
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/disks/{disk_id}": {
       "put": {
         "operationId": "disk_put",
@@ -2926,6 +2977,18 @@
         "format": "uint64",
         "minimum": 0
       },
+      "ChickenSwitchDestroyOrphanedDatasets": {
+        "type": "object",
+        "properties": {
+          "destroy_orphans": {
+            "description": "If true, sled-agent will attempt to destroy durable ZFS datasets that it believes were associated with now-expunged Omicron zones.",
+            "type": "boolean"
+          }
+        },
+        "required": [
+          "destroy_orphans"
+        ]
+      },
       "Chipset": {
         "description": "A kind of virtual chipset.",
         "oneOf": [

sled-agent/api/src/lib.rs

@@ -682,6 +682,35 @@ pub trait SledAgentApi {
         path_params: Path<SledDiagnosticsLogsDownloadPathParm>,
         query_params: Query<SledDiagnosticsLogsDownloadQueryParam>,
     ) -> Result<http::Response<Body>, HttpError>;
+
+    /// This endpoint reports the status of the `destroy_orphaned_datasets`
+    /// chicken switch. It will be removed with omicron#6177.
+    #[endpoint {
+        method = GET,
+        path = "/chicken-switch/destroy-orphaned-datasets",
+    }]
+    async fn chicken_switch_destroy_orphaned_datasets_get(
+        request_context: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<ChickenSwitchDestroyOrphanedDatasets>, HttpError>;
+
+    /// This endpoint sets the `destroy_orphaned_datasets` chicken switch
+    /// (allowing sled-agent to delete datasets it believes are orphaned). It
+    /// will be removed with omicron#6177.
+    #[endpoint {
+        method = PUT,
+        path = "/chicken-switch/destroy-orphaned-datasets",
+    }]
+    async fn chicken_switch_destroy_orphaned_datasets_put(
+        request_context: RequestContext<Self::Context>,
+        body: TypedBody<ChickenSwitchDestroyOrphanedDatasets>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+}
+
+#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
+pub struct ChickenSwitchDestroyOrphanedDatasets {
+    /// If true, sled-agent will attempt to destroy durable ZFS datasets that it
+    /// believes were associated with now-expunged Omicron zones.
+    pub destroy_orphans: bool,
 }
 
 #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]