From 901c900d929879315410fbd97bbada1a09179147 Mon Sep 17 00:00:00 2001 From: Somanath Date: Mon, 15 Jun 2026 11:18:17 +0530 Subject: [PATCH 1/4] feat: add Go workflow scripts for ONTAP automation Add Go implementations of the existing Python, Ansible, and Terraform workflows. --- .gitignore | 4 + go/cluster_setup_basic/main.go | 272 ++++++++ go/go.mod | 3 + go/ontapclient/ontap_client.go | 301 +++++++++ go/snapmirror_cleanup_test_failover/main.go | 284 +++++++++ go/snapmirror_provision_dest_managed/main.go | 635 +++++++++++++++++++ go/snapmirror_provision_src_managed/main.go | 334 ++++++++++ go/snapmirror_test_failover/main.go | 303 +++++++++ 8 files changed, 2136 insertions(+) create mode 100644 go/cluster_setup_basic/main.go create mode 100644 go/go.mod create mode 100644 go/ontapclient/ontap_client.go create mode 100644 go/snapmirror_cleanup_test_failover/main.go create mode 100644 go/snapmirror_provision_dest_managed/main.go create mode 100644 go/snapmirror_provision_src_managed/main.go create mode 100644 go/snapmirror_test_failover/main.go diff --git a/.gitignore b/.gitignore index 0eee18f..9c650b3 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,7 @@ terraform.tfvars # POC poc/ + +# Go build outputs +go/**/*.exe +go/**/*.test diff --git a/go/cluster_setup_basic/main.go b/go/cluster_setup_basic/main.go new file mode 100644 index 0000000..46779b5 --- /dev/null +++ b/go/cluster_setup_basic/main.go @@ -0,0 +1,272 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// Cluster Setup — create a storage cluster from two pre-cluster nodes (ONTAP 9 unified). +// +// Steps: +// +// 1 discoverNodes — GET /cluster/nodes (membership=available, retry 3x/30s) +// 2 discoverLocal — isolate the local node (management_interfaces != null) +// 3 discoverPartner — isolate the partner node (exclude local node UUID) +// 4 createCluster — POST /cluster +// 5 trackJob — switch to cluster credentials, poll job until complete +// +// Prerequisites: +// 1. Two ONTAP 9 nodes in pre-cluster state (factory default or freshly wiped) +// 2. Both nodes reachable at their management IPs +// 3. Node 1 (ONTAP_HOST) must have at least one cluster interface already configured +// +// Usage: +// +// export ONTAP_HOST=10.x.x.x ONTAP_USER=admin ONTAP_PASS= +// export CLUSTER_NAME=mycluster CLUSTER_PASS=secret +// export CLUSTER_MGMT_IP=10.x.x.x CLUSTER_NETMASK=255.255.192.0 CLUSTER_GATEWAY=10.x.x.1 +// export PARTNER_MGMT_IP=10.x.x.y +// go run . +package main + +import ( + "fmt" + "log" + "os" + "strings" + "time" + + ontapclient "github.com/netapp/pace/go/ontapclient" +) + +// --------------------------------------------------------------------------- + +const nodeFields = "name,uuid,model,state,ha,version,serial_number,membership," + + "cluster_interfaces,management_interfaces,metrocluster" + +const clusterNodesPath = "/cluster/nodes" + +func main() { + log.SetFlags(log.LstdFlags) + loadDotEnv() + + host := mustEnv("ONTAP_HOST") + user := envOrDefault("ONTAP_USER", "admin") + pass := envOrDefault("ONTAP_PASS", "") // empty on pre-cluster nodes + + log.Printf("Cluster setup starting — connecting to %s", host) + + client := ontapclient.New(host, user, pass, false) + defer client.Close() + + // Step 1: Discover available nodes (retry 3x) + log.Println("=== Step 1: Discover nodes ===") + discoverNodes(client, 3, 30) + + // Step 2: Find local node + log.Println("=== Step 2: Discover local node ===") + localNode := discoverLocal(client) + localUUID := ontapclient.NestedStr(localNode, "uuid") + + // Step 3: Find partner node + log.Println("=== Step 3: Discover partner node ===") + partnerNode := discoverPartner(client, localUUID) + + // Step 4: Create cluster + log.Println("=== Step 4: Create cluster ===") + jobUUID := createCluster(client, localNode, partnerNode) + + // Step 5: Track job — switch to cluster credentials first + log.Println("=== Step 5: Track cluster creation job ===") + clusterPass := mustEnv("CLUSTER_PASS") + clusterMgmtIP := mustEnv("CLUSTER_MGMT_IP") + trackJob(host, user, clusterPass, jobUUID) + + log.Printf("=== CLUSTER CREATED ===\n"+ + " Name : %s\n"+ + " UI : https://%s\n"+ + " Login : %s / %s", + mustEnv("CLUSTER_NAME"), clusterMgmtIP, user, clusterPass) +} + +// discoverNodes GETs /cluster/nodes with membership=available, retrying up to maxAttempts times. +func discoverNodes(client *ontapclient.Client, maxAttempts, delaySecs int) { + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + resp, err := client.Get(clusterNodesPath, map[string]string{ + "fields": nodeFields, + "membership": "available", + }) + if err == nil { + log.Printf("discover_nodes — %d node(s) found", ontapclient.NumRecords(resp)) + return + } + lastErr = err + if attempt < maxAttempts { + log.Printf("discover_nodes failed (attempt %d/%d), retrying in %ds — %v", + attempt, maxAttempts, delaySecs, err) + time.Sleep(time.Duration(delaySecs) * time.Second) + } + } + log.Fatalf("discover_nodes failed after %d attempts: %v", maxAttempts, lastErr) +} + +// discoverLocal finds the local node (the one with management_interfaces set). +// Returns the first matching node record. +func discoverLocal(client *ontapclient.Client) map[string]interface{} { + resp, err := client.Get(clusterNodesPath, map[string]string{ + "fields": nodeFields, + "membership": "available", + "management_interfaces": "!null", + }) + dieOnErr("discover_local", err) + nodes := ontapclient.Records(resp) + if len(nodes) == 0 { + log.Fatal("discover_local: no local node returned") + } + log.Printf("discover_local — %s", ontapclient.NestedStr(nodes[0], "name")) + return nodes[0] +} + +// discoverPartner finds the partner node by excluding the local node UUID. +// Returns the first matching node record. +func discoverPartner(client *ontapclient.Client, localUUID string) map[string]interface{} { + resp, err := client.Get(clusterNodesPath, map[string]string{ + "fields": nodeFields, + "membership": "available", + "uuid": "!" + localUUID, + }) + dieOnErr("discover_partner", err) + nodes := ontapclient.Records(resp) + if len(nodes) == 0 { + log.Fatal("discover_partner: no partner node returned") + } + log.Printf("discover_partner — %s", ontapclient.NestedStr(nodes[0], "name")) + return nodes[0] +} + +// createCluster POSTs /cluster to create the cluster; returns the job UUID. +func createCluster(client *ontapclient.Client, localNode, partnerNode map[string]interface{}) string { + clusterName := mustEnv("CLUSTER_NAME") + clusterPass := mustEnv("CLUSTER_PASS") + clusterMgmtIP := mustEnv("CLUSTER_MGMT_IP") + clusterNetmask := mustEnv("CLUSTER_NETMASK") + clusterGateway := mustEnv("CLUSTER_GATEWAY") + ontapHost := mustEnv("ONTAP_HOST") + partnerMgmtIP := mustEnv("PARTNER_MGMT_IP") + + localClusterIP := clusterIfaceIP(localNode) + partnerClusterIP := clusterIfaceIP(partnerNode) + + if localClusterIP == "" { + log.Fatal("ABORTED — local node has no cluster interface IP") + } + if partnerClusterIP == "" { + log.Fatal("ABORTED — partner node has no cluster interface IP") + } + + body := map[string]interface{}{ + "name": clusterName, + "password": clusterPass, + "management_interface": map[string]interface{}{ + "ip": map[string]string{ + "address": clusterMgmtIP, + "netmask": clusterNetmask, + "gateway": clusterGateway, + }, + }, + "nodes": []map[string]interface{}{ + { + "name": fmt.Sprintf("%s-01", clusterName), + "management_interface": map[string]interface{}{ + "ip": map[string]string{"address": ontapHost}, + }, + "cluster_interface": map[string]interface{}{ + "ip": map[string]string{"address": localClusterIP}, + }, + }, + { + "name": fmt.Sprintf("%s-02", clusterName), + "management_interface": map[string]interface{}{ + "ip": map[string]string{"address": partnerMgmtIP}, + }, + "cluster_interface": map[string]interface{}{ + "ip": map[string]string{"address": partnerClusterIP}, + }, + }, + }, + "name_servers": map[string]interface{}{}, + "ntp_servers": map[string]interface{}{}, + "dns_domains": map[string]interface{}{}, + "configuration_backup": map[string]interface{}{}, + } + + resp, err := client.Post("/cluster?keep_precluster_config=true", body) + dieOnErr("create_cluster", err) + + jobUUID := ontapclient.JobUUID(resp) + log.Printf("create_cluster — job %s", jobUUID) + return jobUUID +} + +// trackJob switches to cluster credentials then polls the job until complete. +// After POST /cluster the node switches to full cluster mode and requires CLUSTER_PASS. +func trackJob(host, user, clusterPass, jobUUID string) { + clusterClient := ontapclient.New(host, user, clusterPass, false) + defer clusterClient.Close() + + if _, err := clusterClient.PollJob(jobUUID, 10); err != nil { + log.Fatalf("track_job: %v", err) + } +} + +// clusterIfaceIP extracts the IP address of the first cluster interface from a node record. +func clusterIfaceIP(node map[string]interface{}) string { + ifaces, _ := node["cluster_interfaces"].([]interface{}) + if len(ifaces) == 0 { + return "" + } + iface, _ := ifaces[0].(map[string]interface{}) + return ontapclient.NestedStr(iface, "ip", "address") +} + +func mustEnv(key string) string { + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) + return "" +} + +func envOrDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + +func dieOnErr(context string, err error) { + if err != nil { + log.Fatalf("%s: %v", context, err) + } +} + +// loadDotEnv reads a .env file from the current directory and exports each +// KEY=VALUE pair as an environment variable (only if not already set). +// The file is gitignored — safe to store credentials there for local testing. +func loadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +} diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 0000000..f8778af --- /dev/null +++ b/go/go.mod @@ -0,0 +1,3 @@ +module github.com/netapp/pace/go + +go 1.22 diff --git a/go/ontapclient/ontap_client.go b/go/ontapclient/ontap_client.go new file mode 100644 index 0000000..9897e61 --- /dev/null +++ b/go/ontapclient/ontap_client.go @@ -0,0 +1,301 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// Package ontapclient provides a lightweight ONTAP REST API client. +// +// Usage: +// +// client := ontapclient.New("10.x.x.x", "admin", "secret", false) +// defer client.Close() +// cluster, err := client.Get("/cluster", map[string]string{"fields": "name,version"}) +package ontapclient + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "time" +) + +const ( + defaultTimeout = 30 * time.Second + clientAppHdr = "pace-example" + maxJobWait = 10 * time.Minute +) + +// OntapApiError is returned when the ONTAP REST API responds with a non-2xx status. +type OntapApiError struct { + StatusCode int + Detail interface{} +} + +func (e *OntapApiError) Error() string { + return fmt.Sprintf("HTTP %d: %v", e.StatusCode, e.Detail) +} + +// Client is a thin HTTP client for the ONTAP REST API. +type Client struct { + baseURL string + username string + password string + httpClient *http.Client +} + +// New creates a new Client. +// Set verifySSL=false to allow self-signed certificates (common in lab environments). +func New(host, username, password string, verifySSL bool) *Client { + transport := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: !verifySSL}, // #nosec G402 — intentional for lab certs + } + return &Client{ + baseURL: fmt.Sprintf("https://%s/api", host), + username: username, + password: password, + httpClient: &http.Client{ + Timeout: defaultTimeout, + Transport: transport, + }, + } +} + +// FromEnv creates a Client from standard ONTAP_* environment variables. +// Required: ONTAP_HOST, ONTAP_PASS. Optional: ONTAP_USER (default "admin"). +func FromEnv() *Client { + host := os.Getenv("ONTAP_HOST") + if host == "" { + log.Fatal("ONTAP_HOST environment variable is required") + } + password := os.Getenv("ONTAP_PASS") + if password == "" { + log.Fatal("ONTAP_PASS environment variable is required") + } + user := os.Getenv("ONTAP_USER") + if user == "" { + user = "admin" + } + return New(host, user, password, false) +} + +// Close is a no-op provided for symmetry with connection-pooling patterns. +func (c *Client) Close() { + c.httpClient.CloseIdleConnections() +} + +// buildURL constructs a full API URL with query parameters. +func (c *Client) buildURL(path string, params map[string]string) string { + u := c.baseURL + path + if len(params) == 0 { + return u + } + q := url.Values{} + for k, v := range params { + q.Set(k, v) + } + return u + "?" + q.Encode() +} + +// do executes an HTTP request and decodes the JSON response body. +func (c *Client) do(method, rawURL string, body interface{}) (map[string]interface{}, error) { + var bodyReader io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request body: %w", err) + } + bodyReader = bytes.NewReader(b) + } + + req, err := http.NewRequest(method, rawURL, bodyReader) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.SetBasicAuth(c.username, c.password) + req.Header.Set("Accept", "application/hal+json") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Dot-Client-App", clientAppHdr) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("execute request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + var result map[string]interface{} + if len(respBytes) > 0 { + if err := json.Unmarshal(respBytes, &result); err != nil { + result = map[string]interface{}{"_raw": string(respBytes)} + } + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return result, &OntapApiError{StatusCode: resp.StatusCode, Detail: result} + } + return result, nil +} + +// Get sends a GET request to the given API path with optional query params. +func (c *Client) Get(path string, params map[string]string) (map[string]interface{}, error) { + return c.do(http.MethodGet, c.buildURL(path, params), nil) +} + +// Post sends a POST request with a JSON body. +func (c *Client) Post(path string, body interface{}) (map[string]interface{}, error) { + return c.do(http.MethodPost, c.baseURL+path, body) +} + +// Patch sends a PATCH request with a JSON body. +func (c *Client) Patch(path string, body interface{}) (map[string]interface{}, error) { + return c.do(http.MethodPatch, c.baseURL+path, body) +} + +// Delete sends a DELETE request. +func (c *Client) Delete(path string) (map[string]interface{}, error) { + return c.do(http.MethodDelete, c.baseURL+path, nil) +} + +// PollJob polls /cluster/jobs/{uuid} until the job reaches a terminal state. +// Returns an error if the job ends in any state other than "success". +func (c *Client) PollJob(jobUUID string, intervalSecs int) (map[string]interface{}, error) { + if intervalSecs <= 0 { + intervalSecs = 10 + } + deadline := time.Now().Add(maxJobWait) + for { + if time.Now().After(deadline) { + return nil, fmt.Errorf("poll job %s: timed out after %s", jobUUID, maxJobWait) + } + result, err := c.Get(fmt.Sprintf("/cluster/jobs/%s", jobUUID), + map[string]string{"fields": "state,message,error,code"}) + if err != nil { + return nil, fmt.Errorf("poll job %s: %w", jobUUID, err) + } + state, _ := result["state"].(string) + log.Printf(" job %s — state=%s", jobUUID, state) + switch state { + case "running", "queued", "paused": + time.Sleep(time.Duration(intervalSecs) * time.Second) + case "success": + return result, nil + default: + msg, _ := result["message"].(string) + return nil, fmt.Errorf("job %s ended with state=%s: %s", jobUUID, state, msg) + } + } +} + +// WaitSnapmirrored polls a SnapMirror relationship until state == "snapmirrored". +// maxWaitSecs defaults to 1800 if <= 0. +func (c *Client) WaitSnapmirrored(relUUID string, intervalSecs, maxWaitSecs int) (map[string]interface{}, error) { + if intervalSecs <= 0 { + intervalSecs = 15 + } + if maxWaitSecs <= 0 { + maxWaitSecs = 1800 + } + elapsed := 0 + for elapsed < maxWaitSecs { + result, err := c.Get(fmt.Sprintf("/snapmirror/relationships/%s", relUUID), + map[string]string{"fields": "state,lag_time,healthy"}) + if err != nil { + return nil, fmt.Errorf("poll relationship %s: %w", relUUID, err) + } + state, _ := result["state"].(string) + log.Printf(" relationship %s — state=%s", relUUID, state) + if state == "snapmirrored" { + return result, nil + } + time.Sleep(time.Duration(intervalSecs) * time.Second) + elapsed += intervalSecs + } + return nil, fmt.Errorf("timed out waiting for relationship %s to reach snapmirrored", relUUID) +} + +// NestedStr safely extracts a nested string value from a map[string]interface{}. +// Keys are applied in order: NestedStr(m, "a", "b") => m["a"].(map)["b"].(string). +func NestedStr(m map[string]interface{}, keys ...string) string { + cur := m + for i, k := range keys { + v, ok := cur[k] + if !ok { + return "" + } + if i == len(keys)-1 { + s, _ := v.(string) + return s + } + cur, ok = v.(map[string]interface{}) + if !ok { + return "" + } + } + return "" +} + +// NestedFloat safely extracts a float64 from a nested map. +func NestedFloat(m map[string]interface{}, keys ...string) float64 { + cur := m + for i, k := range keys { + v, ok := cur[k] + if !ok { + return 0 + } + if i == len(keys)-1 { + f, _ := v.(float64) + return f + } + cur, ok = v.(map[string]interface{}) + if !ok { + return 0 + } + } + return 0 +} + +// Records returns the "records" slice from a collection response. +func Records(resp map[string]interface{}) []map[string]interface{} { + raw, ok := resp["records"] + if !ok { + return nil + } + slice, ok := raw.([]interface{}) + if !ok { + return nil + } + out := make([]map[string]interface{}, 0, len(slice)) + for _, item := range slice { + if m, ok := item.(map[string]interface{}); ok { + out = append(out, m) + } + } + return out +} + +// NumRecords returns the num_records integer from a collection response. +func NumRecords(resp map[string]interface{}) int { + v, ok := resp["num_records"] + if !ok { + return 0 + } + f, ok := v.(float64) + if !ok { + return 0 + } + return int(f) +} + +// JobUUID extracts job.uuid from a response. +func JobUUID(resp map[string]interface{}) string { + return NestedStr(resp, "job", "uuid") +} diff --git a/go/snapmirror_cleanup_test_failover/main.go b/go/snapmirror_cleanup_test_failover/main.go new file mode 100644 index 0000000..8c8cde2 --- /dev/null +++ b/go/snapmirror_cleanup_test_failover/main.go @@ -0,0 +1,284 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// SnapMirror Test Failover Cleanup — deletes the FlexClone created by test_failover. +// +// Finds the clone via SnapMirror relationship UUID tag (":test"). +// Only clones tagged by the snapmirror_test_failover workflow are touched — manually +// created volumes are never matched or deleted. +// +// Phases: +// +// 0 Relationship-pick — find SM relationship on correct cluster +// A Tag-based find — locate clone tagged with ":test" +// B SMAS removal — delete any SMAS relationship on the clone (releases lock) +// C Unmount — remove NAS junction path (with retry) +// D Offline — set volume state to offline +// E Delete — delete the clone and confirm removal +// +// Prerequisites: +// 1. ONTAP 9.8+ on both clusters +// 2. snapmirror_test_failover.go must have been run first +// 3. The SnapMirror relationship must still be accessible on one of the clusters +// 4. Admin credentials for both clusters +// +// Usage: +// +// export CLUSTER_A=10.x.x.x CLUSTER_B=10.y.y.y +// export DEST_USER=admin DEST_PASS=secret +// export SOURCE_VOLUME=vol_rw_01 +// export SOURCE_SVM=vs0 +// go run . +package main + +import ( + "fmt" + "log" + "os" + "strings" + "time" + + ontapclient "github.com/netapp/pace/go/ontapclient" +) + +const volumePatchPath = "/storage/volumes/%s?return_timeout=120" + +// --------------------------------------------------------------------------- + +func main() { + log.SetFlags(log.LstdFlags) + loadDotEnv() + + clusterA := mustEnv("CLUSTER_A") + clusterB := mustEnv("CLUSTER_B") + destUser := envOrDefault("DEST_USER", "admin") + destPass := mustEnv("DEST_PASS") + sourceVolume := mustEnv("SOURCE_VOLUME") + sourceSVM := mustEnv("SOURCE_SVM") + + // === Phase 0: Find SnapMirror relationship === + log.Println("=== Phase 0: Find SnapMirror relationship ===") + destHost, rel := pickClusterByRelationship(clusterA, clusterB, destUser, destPass, sourceSVM, sourceVolume) + relUUID := ontapclient.NestedStr(rel, "uuid") + log.Printf("RELATIONSHIP FOUND | cluster=%s | uuid=%s | source=%s | dest=%s | state=%s | healthy=%v", + destHost, + relUUID, + ontapclient.NestedStr(rel, "source", "path"), + ontapclient.NestedStr(rel, "destination", "path"), + ontapclient.NestedStr(rel, "state"), + rel["healthy"]) + + if ontapclient.NestedStr(rel, "state") != "snapmirrored" { + log.Printf("Relationship state=%s healthy=%v — proceeding with cleanup anyway", + ontapclient.NestedStr(rel, "state"), rel["healthy"]) + } + + client := ontapclient.New(destHost, destUser, destPass, false) + defer client.Close() + + // === Phase A: Find tagged clone === + log.Println("=== Phase A: Find tagged clone ===") + clone := findTaggedClone(client, relUUID) + if clone == nil { + log.Printf("NO TAGGED CLONE FOUND for %s:%s on %s — nothing to clean up", + sourceSVM, sourceVolume, destHost) + return + } + log.Printf("CLONE FOUND | name=%s | uuid=%s | svm=%s | cluster=%s", + clone["name"], clone["uuid"], clone["svm"], destHost) + + cloneUUID, _ := clone["uuid"].(string) + cloneSVM, _ := clone["svm"].(string) + cloneName, _ := clone["name"].(string) + + removeSMASAndBringOnline(client, cloneUUID, cloneSVM, cloneName) + unmountClone(client, cloneUUID) + offlineClone(client, cloneUUID) + deleteAndConfirmClone(client, cloneUUID, cloneName, destHost) +} + +// pickClusterByRelationship returns (clusterIP, relationshipRecord) for the cluster owning this SM rel. +func pickClusterByRelationship(clusterA, clusterB, user, passwd, sourceSVM, sourceVolume string) (string, map[string]interface{}) { + sourcePath := sourceSVM + ":" + sourceVolume + for _, host := range []string{clusterA, clusterB} { + client := ontapclient.New(host, user, passwd, false) + resp, err := client.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,source.path,destination.path,state,healthy", + "source.path": sourcePath, + "max_records": "1", + }) + client.Close() + if err != nil { + log.Printf(" cluster %s — %v", host, err) + continue + } + if ontapclient.NumRecords(resp) >= 1 { + return host, ontapclient.Records(resp)[0] + } + } + log.Fatalf("No SM relationship found for %s on either cluster (%s, %s)", sourcePath, clusterA, clusterB) + return "", nil +} + +// findTaggedClone returns the clone tagged ':test', or nil if not found. +func findTaggedClone(client *ontapclient.Client, relUUID string) map[string]interface{} { + resp, err := client.Get("/storage/volumes", map[string]string{ + "fields": "name,uuid,svm.name,state,nas.path", + "_tags": relUUID + ":test", + "max_records": "1", + }) + if err != nil || ontapclient.NumRecords(resp) == 0 { + return nil + } + rec := ontapclient.Records(resp)[0] + return map[string]interface{}{ + "uuid": ontapclient.NestedStr(rec, "uuid"), + "name": ontapclient.NestedStr(rec, "name"), + "svm": ontapclient.NestedStr(rec, "svm", "name"), + } +} + +// removeSMASAndBringOnline deletes any SMAS relationship on the clone, then ensures it is online. +func removeSMASAndBringOnline(client *ontapclient.Client, cloneUUID, cloneSVM, cloneName string) { + log.Println("=== Phase B: Remove SMAS relationship on clone (if any) ===") + smasResp, err := client.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,state", + "destination.path": cloneSVM + ":" + cloneName, + "max_records": "10", + }) + if err != nil { + log.Printf("list smas relationships: %v (continuing)", err) + } + smasRels := ontapclient.Records(smasResp) + for _, r := range smasRels { + smasUUID := ontapclient.NestedStr(r, "uuid") + log.Printf(" Deleting SMAS relationship %s on clone", smasUUID) + resp, err := client.Delete(fmt.Sprintf("/snapmirror/relationships/%s?return_timeout=120&force=true", smasUUID)) + if err != nil { + log.Printf("delete_smas_rel %s — %v (continuing)", smasUUID, err) + continue + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll delete smas job — %v", err) + } + } + } + if len(smasRels) == 0 { + log.Println(" No SMAS relationships found on clone — continuing") + } + + resp, err := client.Patch(fmt.Sprintf(volumePatchPath, cloneUUID), + map[string]interface{}{"state": "online"}) + if err != nil { + log.Printf("bring_online — %v (continuing)", err) + return + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll bring-online job — %v", err) + } + } +} + +// unmountClone removes the NAS junction path; retries up to 6 times before aborting. +func unmountClone(client *ontapclient.Client, cloneUUID string) { + log.Println("=== Phase C: Unmount clone ===") + for attempt := 1; attempt <= 6; attempt++ { + resp, err := client.Patch(fmt.Sprintf(volumePatchPath, cloneUUID), + map[string]interface{}{"nas": map[string]string{"path": ""}}) + if err != nil { + log.Printf("unmount_clone attempt %d/6 — %v", attempt, err) + if attempt < 6 { + time.Sleep(10 * time.Second) + } + continue + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll unmount job — %v", err) + } + } + return + } + log.Fatal("Failed to unmount clone after 6 attempts — aborting") +} + +// offlineClone sets the volume state to offline (required before delete). +func offlineClone(client *ontapclient.Client, cloneUUID string) { + log.Println("=== Phase D: Offline clone ===") + resp, err := client.Patch(fmt.Sprintf(volumePatchPath, cloneUUID), + map[string]interface{}{"state": "offline"}) + if err != nil { + log.Printf("offline_clone — %v", err) + return + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll offline job — %v", err) + } + } +} + +// deleteAndConfirmClone deletes the clone volume and confirms it is gone. +func deleteAndConfirmClone(client *ontapclient.Client, cloneUUID, cloneName, destHost string) { + log.Println("=== Phase E: Delete clone ===") + resp, err := client.Delete(fmt.Sprintf(volumePatchPath, cloneUUID)) + if err != nil { + log.Printf("delete_clone — %v", err) + } else if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll delete job — %v", err) + } + } + + confirm, err := client.Get("/storage/volumes", map[string]string{ + "fields": "name,uuid", + "uuid": cloneUUID, + "max_records": "1", + }) + if err != nil || ontapclient.NumRecords(confirm) == 0 { + log.Printf("=== CLEANUP COMPLETE — clone '%s' deleted from cluster %s ===", cloneName, destHost) + } else { + log.Fatalf("Clone '%s' still exists after delete attempt", cloneName) + } +} + +// loadDotEnv reads a .env file from the current directory and exports each +// KEY=VALUE pair as an environment variable (only if not already set). +// The file is gitignored — safe to store credentials there for local testing. +func loadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +} + +func mustEnv(key string) string { + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) + return "" +} + +func envOrDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} diff --git a/go/snapmirror_provision_dest_managed/main.go b/go/snapmirror_provision_dest_managed/main.go new file mode 100644 index 0000000..68007bc --- /dev/null +++ b/go/snapmirror_provision_dest_managed/main.go @@ -0,0 +1,635 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// SnapMirror Provision — Destination-Managed view. +// +// All SnapMirror API calls driven from the DESTINATION cluster. +// Source RW volume must already exist; dest DP volume is auto-created. +// +// Steps: +// 1. Verify source cluster connectivity +// 2. Verify dest cluster connectivity +// 3. Setup cluster peer (auto-create if missing) +// 4. Validate source volume exists and is RW +// 5. Get dest aggregate +// 6. Setup SVM peer (auto-create if missing) +// 7. Auto-create dest DP volume (skip if already exists) +// 8. Validate dest DP volume exists +// 9. Check if relationship already exists +// 10. Create + initialize SnapMirror relationship +// 11. Poll create/init job +// 12. Fetch relationship UUID +// 13. Initialize relationship (trigger baseline transfer) +// 14. Wait for state = snapmirrored +// 15. Validate health + print final report +// +// Prerequisites: +// 1. ONTAP 9.8+ on both clusters +// 2. SnapMirror licence installed on both clusters +// 3. At least one intercluster LIF on each cluster +// 4. Admin credentials for both clusters +// +// Usage: +// +// export SOURCE_HOST=10.x.x.x SOURCE_USER=admin SOURCE_PASS=secret +// export SOURCE_SVM=vs0 SOURCE_VOLUME=vol_rw_01 +// export DEST_HOST=10.y.y.y DEST_USER=admin DEST_PASS=secret +// export DEST_SVM=vs1 +// export SM_POLICY=Asynchronous +// go run . +package main + +import ( + "fmt" + "log" + "os" + "strings" + "time" + + ontapclient "github.com/netapp/pace/go/ontapclient" +) + +const ( + pathStorageVolumes = "/storage/volumes" // NOSONAR + pathClusterPeers = "/cluster/peers" + pathSVMPeers = "/svm/peers" + keySVMName = "svm.name" + peerFields = "name,uuid,status.state" +) + +// smRelConfig groups SnapMirror relationship parameters to keep function signatures compact. +type smRelConfig struct { + destSVM, destVolume, sourceSVMAlias, sourceVolume, peerName, smPolicy string +} + +// --------------------------------------------------------------------------- +// USER INPUTS — fill in your values here before running +// --------------------------------------------------------------------------- +var inputs = map[string]string{ + "SOURCE_HOST": "", // set via SOURCE_HOST in go/.env or env var + "SOURCE_USER": "admin", + "SOURCE_PASS": "", // set via SOURCE_PASS in go/.env or env var + "SOURCE_SVM": "", // set via SOURCE_SVM in go/.env or env var + "SOURCE_VOLUME": "", // set via SOURCE_VOLUME in go/.env or env var + "DEST_HOST": "", // set via DEST_HOST in go/.env or env var + "DEST_USER": "admin", + "DEST_PASS": "", // set via DEST_PASS in go/.env or env var + "DEST_SVM": "", // set via DEST_SVM in go/.env or env var + "SM_POLICY": "Asynchronous", +} + +// --------------------------------------------------------------------------- + +func main() { + log.SetFlags(log.LstdFlags) + loadDotEnv() + + sourceHost := mustEnv("SOURCE_HOST") + sourceUser := envOrDefault("SOURCE_USER", "admin") + sourcePass := mustEnv("SOURCE_PASS") + sourceSVM := mustEnv("SOURCE_SVM") + sourceVolume := mustEnv("SOURCE_VOLUME") + + destHost := mustEnv("DEST_HOST") + destUser := envOrDefault("DEST_USER", "admin") + destPass := mustEnv("DEST_PASS") + destSVM := mustEnv("DEST_SVM") + smPolicy := envOrDefault("SM_POLICY", "Asynchronous") + + destVolume := sourceVolume + "_dest" + + src := ontapclient.New(sourceHost, sourceUser, sourcePass, false) + defer src.Close() + dst := ontapclient.New(destHost, destUser, destPass, false) + defer dst.Close() + + // === Phase A: Source pre-flight === + log.Println("=== Phase A: Source pre-flight ===") + srcVol := phaseASourcePreflight(src, sourceSVM, sourceVolume, sourceHost) + srcVolSize := fmt.Sprintf("%.0f", ontapclient.NestedFloat(srcVol, "space", "size")) + + // === Phase B: Dest pre-flight === + log.Println("=== Phase B: Dest pre-flight ===") + dstCluster, err := dst.Get("/cluster", map[string]string{"fields": "name,version"}) + dieOnErr("get dest cluster", err) + log.Printf("DEST CLUSTER | name=%s | ontap=%s", + ontapclient.NestedStr(dstCluster, "name"), + ontapclient.NestedStr(dstCluster, "version", "full")) + + // === Phase B0: Cluster peer setup === + log.Println("=== Phase B0: Cluster peer setup ===") + srcPeerName, peerName, dstPeerUUID := setupClusterPeer(src, dst, sourceSVM, destSVM) + + aggrResp, err := dst.Get("/storage/aggregates", map[string]string{ + "fields": "name,state", + "max_records": "1", + }) + dieOnErr("get dest aggregate", err) + aggrName := "" + aggrs := ontapclient.Records(aggrResp) + if len(aggrs) > 0 { + aggrName = ontapclient.NestedStr(aggrs[0], "name") + } + if aggrName == "" { + log.Fatal("ABORTED — no aggregates found on destination cluster") + } + log.Printf("DEST AGGREGATE | name=%s", aggrName) + + // === Phase B1: SVM peer setup === + log.Println("=== Phase B1: SVM peer setup ===") + sourceSVMAlias := setupSVMPeer(src, dst, sourceSVM, destSVM, srcPeerName, peerName, dstPeerUUID) + + // === Phase C: Dest volume setup === + log.Println("=== Phase C: Dest volume setup ===") + _, err = dst.Post("/storage/volumes?return_timeout=120", map[string]interface{}{ + "name": destVolume, + "type": "dp", + "svm": map[string]string{"name": destSVM}, + "aggregates": []map[string]string{ + {"name": aggrName}, + }, + "space": map[string]string{"size": srcVolSize}, + }) + if err != nil { + log.Printf("create_dest_volume — %v (skipped — may already exist)", err) + } else { + log.Printf("DEST VOLUME | created '%s' on aggregate '%s'", destVolume, aggrName) + } + + dstVolResp, err := dst.Get("/storage/volumes", map[string]string{ + "fields": "name,uuid,state,type", + "max_records": "1", + "name": destVolume, + keySVMName: destSVM, + }) + dieOnErr("verify dest volume", err) + dstVols := ontapclient.Records(dstVolResp) + if len(dstVols) == 0 { + log.Fatalf("ABORTED — dest volume '%s' not found on SVM '%s' after create", destVolume, destSVM) + } + dstVol := dstVols[0] + log.Printf("DEST VOLUME | svm=%s | name=%s | uuid=%s | state=%s | type=%s", + destSVM, + ontapclient.NestedStr(dstVol, "name"), + ontapclient.NestedStr(dstVol, "uuid"), + ontapclient.NestedStr(dstVol, "state"), + ontapclient.NestedStr(dstVol, "type")) + + // === Phase D: Relationship setup === + log.Println("=== Phase D: Relationship setup ===") + relUUID := phaseDSetupRelationship(src, dst, smRelConfig{ + destSVM: destSVM, destVolume: destVolume, + sourceSVMAlias: sourceSVMAlias, sourceVolume: sourceVolume, + peerName: peerName, smPolicy: smPolicy, + }) + + // === Phase E: Convergence polling === + log.Println("=== Phase E: Convergence polling ===") + if _, err := dst.WaitSnapmirrored(relUUID, 15, 1800); err != nil { + log.Fatalf("wait snapmirrored: %v", err) + } + + // === Phase F: Final validation === + log.Println("=== Phase F: Final validation ===") + final, err := dst.Get(fmt.Sprintf("/snapmirror/relationships/%s", relUUID), + map[string]string{"fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name"}) + dieOnErr("final validation", err) + log.Printf("=== SNAPMIRROR PROVISION COMPLETE ===\n"+ + " source : %s:%s\n"+ + " destination : %s:%s\n"+ + " state : %s\n"+ + " healthy : %v\n"+ + " policy : %s\n"+ + " lag_time : %v", + sourceSVM, sourceVolume, + destSVM, destVolume, + ontapclient.NestedStr(final, "state"), + final["healthy"], + ontapclient.NestedStr(final, "policy", "name"), + final["lag_time"]) +} + +// phaseASourcePreflight verifies source cluster connectivity and validates the source volume. +func phaseASourcePreflight(src *ontapclient.Client, sourceSVM, sourceVolume, sourceHost string) map[string]interface{} { + srcCluster, err := src.Get("/cluster", map[string]string{"fields": "name,version"}) + dieOnErr("get source cluster", err) + log.Printf("SOURCE CLUSTER | name=%s | ontap=%s", + ontapclient.NestedStr(srcCluster, "name"), + ontapclient.NestedStr(srcCluster, "version", "full")) + + srcVolResp, err := src.Get("/storage/volumes", map[string]string{ + "fields": "name,uuid,state,type,space.size", + "max_records": "1", + "name": sourceVolume, + keySVMName: sourceSVM, + }) + dieOnErr("get source volume", err) + if ontapclient.NumRecords(srcVolResp) == 0 { + log.Fatalf("ABORTED — source volume '%s' not found on %s", sourceVolume, sourceHost) + } + srcVol := ontapclient.Records(srcVolResp)[0] + if ontapclient.NestedStr(srcVol, "type") == "dp" { + log.Fatal("ABORTED — source volume is type=dp; specify the RW volume") + } + log.Printf("SOURCE VOLUME | svm=%s | name=%s | uuid=%s | state=%s | type=%s | size=%.0f", + sourceSVM, + ontapclient.NestedStr(srcVol, "name"), + ontapclient.NestedStr(srcVol, "uuid"), + ontapclient.NestedStr(srcVol, "state"), + ontapclient.NestedStr(srcVol, "type"), + ontapclient.NestedFloat(srcVol, "space", "size")) + return srcVol +} + +// getICLIFIPs returns intercluster LIF IP addresses from a cluster. +func getICLIFIPs(client *ontapclient.Client) []string { + resp, err := client.Get("/network/ip/interfaces", map[string]string{ + "fields": "name,ip.address,services", + "max_records": "50", + }) + if err != nil { + return nil + } + var ips []string + for _, r := range ontapclient.Records(resp) { + services, _ := r["services"].([]interface{}) + for _, s := range services { + if strings.Contains(fmt.Sprintf("%v", s), "intercluster") { + if ip := ontapclient.NestedStr(r, "ip", "address"); ip != "" { + ips = append(ips, ip) + break + } + } + } + } + return ips +} + +// checkICLIFPreconditions validates intercluster LIFs exist on both clusters. +func checkICLIFPreconditions(srcIPs, dstIPs []string) { + if len(srcIPs) == 0 { + log.Fatal("PRE-CONDITION FAILED | Source cluster has no intercluster LIFs.\n" + + " SnapMirror requires at least one IC LIF on each cluster.") + } + if len(dstIPs) == 0 { + log.Fatal("PRE-CONDITION FAILED | Dest cluster has no intercluster LIFs.\n" + + " SnapMirror requires at least one IC LIF on each cluster.") + } + subnet24 := func(ip string) string { + parts := strings.SplitN(ip, ".", 4) + if len(parts) >= 3 { + return parts[0] + "." + parts[1] + "." + parts[2] + } + return ip + } + srcSubnets := map[string]bool{} + for _, ip := range srcIPs { + srcSubnets[subnet24(ip)] = true + } + dstSubnets := map[string]bool{} + for _, ip := range dstIPs { + dstSubnets[subnet24(ip)] = true + } + shared := false + for s := range srcSubnets { + if dstSubnets[s] { + shared = true + break + } + } + if !shared { + log.Printf("PRE-CONDITION WARNING | IC LIFs are on different subnets.\n"+ + " src IPs : %v\n dst IPs : %v\n"+ + " SnapMirror data transfers require TCP 11104 and 11105 to be open between these subnets.", + srcIPs, dstIPs) + } else { + log.Printf("PRE-CONDITION OK | IC LIFs share a common subnet — transfers should work") + } +} + +// setupClusterPeer ensures a cluster peer exists; auto-creates if missing. +// Returns (srcPeerName, dstPeerName, dstPeerUUID). +func setupClusterPeer(src, dst *ontapclient.Client, sourceSVM, destSVM string) (string, string, string) { + okStates := map[string]bool{"available": true, "partial": true, "pending": true} + + dstCP, err := dst.Get(pathClusterPeers, map[string]string{ + "fields": peerFields, + "max_records": "10", + }) + dieOnErr("get dest cluster peers", err) + + for _, p := range ontapclient.Records(dstCP) { + state := ontapclient.NestedStr(p, "status", "state") + if !okStates[state] { + continue + } + // Peer already exists + srcCP, err2 := src.Get(pathClusterPeers, map[string]string{ + "fields": peerFields, + "max_records": "10", + }) + if err2 != nil { + log.Printf("get src cluster peers: %v", err2) + } + srcPeerName := "" + for _, q := range ontapclient.Records(srcCP) { + if okStates[ontapclient.NestedStr(q, "status", "state")] { + srcPeerName = ontapclient.NestedStr(q, "name") + break + } + } + srcIPs := getICLIFIPs(src) + dstIPs := getICLIFIPs(dst) + log.Printf("CLUSTER PEER | already peered — dst sees src as '%s' (state=%s) — skipping", + ontapclient.NestedStr(p, "name"), state) + log.Printf("IC LIFs | src=%v dst=%v", srcIPs, dstIPs) + checkICLIFPreconditions(srcIPs, dstIPs) + return srcPeerName, ontapclient.NestedStr(p, "name"), ontapclient.NestedStr(p, "uuid") + } + + // No existing peer — auto-create + log.Println("CLUSTER PEER | no existing peer found — auto-creating") + srcIPs := getICLIFIPs(src) + dstIPs := getICLIFIPs(dst) + log.Printf("CLUSTER PEER | src IC LIFs=%v dst IC LIFs=%v", srcIPs, dstIPs) + checkICLIFPreconditions(srcIPs, dstIPs) + return createNewClusterPeer(src, dst, srcIPs, dstIPs, sourceSVM, destSVM) +} + +// createNewClusterPeer posts a new cluster peer on both sides. +// Returns (srcPeerName, dstPeerName, dstPeerUUID). +func createNewClusterPeer(src, dst *ontapclient.Client, srcIPs, dstIPs []string, sourceSVM, destSVM string) (string, string, string) { + if len(srcIPs) == 0 { + log.Fatal("ABORTED — no intercluster LIFs found on source cluster.") + } + if len(dstIPs) == 0 { + log.Fatal("ABORTED — no intercluster LIFs found on dest cluster.") + } + + peerAddrs := make([]string, len(dstIPs)) + copy(peerAddrs, dstIPs) + srcResp, err := src.Post(pathClusterPeers, map[string]interface{}{ + "peer_addresses": peerAddrs, + "generate_passphrase": true, + "encryption": map[string]string{"proposed": "tls-psk"}, + "initial_allowed_svms": []map[string]string{ + {"name": sourceSVM}, + }, + }) + dieOnErr("create cluster peer on source", err) + passphrase, _ := srcResp["passphrase"].(string) + log.Println("CLUSTER PEER | created on source") + + dstPeerAddrs := make([]string, len(srcIPs)) + copy(dstPeerAddrs, srcIPs) + _, err = dst.Post(pathClusterPeers, map[string]interface{}{ + "peer_addresses": dstPeerAddrs, + "passphrase": passphrase, + "initial_allowed_svms": []map[string]string{ + {"name": destSVM}, + }, + }) + dieOnErr("accept cluster peer on dest", err) + log.Println("CLUSTER PEER | accepted on dest") + + time.Sleep(5 * time.Second) + return fetchCreatedPeerNames(src, dst) +} + +// fetchCreatedPeerNames retrieves peer names from both clusters after creation. +func fetchCreatedPeerNames(src, dst *ontapclient.Client) (string, string, string) { + okStates := map[string]bool{"available": true, "partial": true, "pending": true} + dstCP, err := dst.Get(pathClusterPeers, map[string]string{"fields": peerFields, "max_records": "10"}) + if err != nil { + log.Fatalf("ABORTED — could not query cluster peers on destination: %v", err) + } + dstPeer := map[string]interface{}{} + for _, p := range ontapclient.Records(dstCP) { + if okStates[ontapclient.NestedStr(p, "status", "state")] { + dstPeer = p + break + } + } + if len(dstPeer) == 0 { + log.Fatal("ABORTED — no usable cluster peer found on destination after creation") + } + srcCP, err := src.Get(pathClusterPeers, map[string]string{"fields": peerFields, "max_records": "10"}) + if err != nil { + log.Fatalf("ABORTED — could not query cluster peers on source: %v", err) + } + srcPeer := map[string]interface{}{} + for _, p := range ontapclient.Records(srcCP) { + if okStates[ontapclient.NestedStr(p, "status", "state")] { + srcPeer = p + break + } + } + if len(srcPeer) == 0 { + log.Fatal("ABORTED — no usable cluster peer found on source after creation") + } + log.Printf("CLUSTER PEER | dst sees src as '%s'", ontapclient.NestedStr(dstPeer, "name")) + return ontapclient.NestedStr(srcPeer, "name"), + ontapclient.NestedStr(dstPeer, "name"), + ontapclient.NestedStr(dstPeer, "uuid") +} + +// grantSVMPeerPermission grants SnapMirror peer-permission on the source SVM. +func grantSVMPeerPermission(src *ontapclient.Client, sourceSVM, srcPeerName string) { + _, err := src.Post("/svm/peer-permissions", map[string]interface{}{ + "svm": map[string]string{"name": sourceSVM}, + "cluster_peer": map[string]string{"name": srcPeerName}, + "applications": []string{"snapmirror"}, + }) + if err != nil { + s := err.Error() + if strings.Contains(s, "already exists") || strings.Contains(strings.ToLower(s), "duplicate") || strings.Contains(s, "13001") { + log.Println("SVM PEER | peer-permission already exists — skipping") + return + } + log.Fatalf("SVM PEER | peer-permission failed: %v", err) + } + log.Println("SVM PEER | peer-permission granted on source") +} + +// createSVMPeerRelationship creates the SVM peer relationship on the destination. +func createSVMPeerRelationship(dst *ontapclient.Client, destSVM, sourceSVM, dstPeerName string) { + resp, err := dst.Post(pathSVMPeers, map[string]interface{}{ + "svm": map[string]string{"name": destSVM}, + "peer": map[string]interface{}{ + "svm": map[string]string{"name": sourceSVM}, + "cluster": map[string]string{"name": dstPeerName}, + }, + "applications": []string{"snapmirror"}, + }) + if err != nil { + s := err.Error() + if strings.Contains(s, "already exists") || strings.Contains(strings.ToLower(s), "duplicate") || strings.Contains(s, "13001") { + log.Println("SVM PEER | already exists — skipping") + return + } + log.Fatalf("SVM PEER | create failed: %v", err) + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := dst.PollJob(jobUUID, 10); err != nil { + log.Printf("poll svm peer job: %v", err) + } + } + log.Printf("SVM PEER | created '%s' <-> '%s'", destSVM, sourceSVM) +} + +// setupSVMPeer ensures SVM peer exists; returns the source SVM alias used in SnapMirror paths. +func setupSVMPeer(src, dst *ontapclient.Client, sourceSVM, destSVM, srcPeerName, dstPeerName, srcClusterPeerUUID string) string { + svmResp, err := dst.Get(pathSVMPeers, map[string]string{ + "fields": "uuid,name,state,peer", + keySVMName: destSVM, + }) + dieOnErr("get svm peers", err) + + for _, p := range ontapclient.Records(svmResp) { + state := ontapclient.NestedStr(p, "state") + if state != "peered" && state != "initiated" { + continue + } + if ontapclient.NestedStr(p, "peer", "cluster", "uuid") != srcClusterPeerUUID { + continue + } + alias := ontapclient.NestedStr(p, "peer", "svm", "name") + if alias == "" { + alias = sourceSVM + } + log.Printf("SVM PEER | already peered '%s' <-> '%s' (alias='%s', state=%s) — skipping", + destSVM, sourceSVM, alias, state) + return alias + } + + grantSVMPeerPermission(src, sourceSVM, srcPeerName) + createSVMPeerRelationship(dst, destSVM, sourceSVM, dstPeerName) + + svmResp2, err := dst.Get(pathSVMPeers, map[string]string{ + "fields": "uuid,name,state,peer", + keySVMName: destSVM, + }) + if err != nil { + return sourceSVM + } + for _, p := range ontapclient.Records(svmResp2) { + if ontapclient.NestedStr(p, "peer", "cluster", "uuid") == srcClusterPeerUUID { + alias := ontapclient.NestedStr(p, "peer", "svm", "name") + if alias == "" { + alias = sourceSVM + } + return alias + } + } + return sourceSVM +} + +// phaseDSetupRelationship creates and initializes the SnapMirror relationship; returns its UUID. +func phaseDSetupRelationship(src, dst *ontapclient.Client, cfg smRelConfig) string { + existing, err := dst.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,state,healthy", + "destination.path": cfg.destSVM + ":" + cfg.destVolume, + "max_records": "1", + }) + dieOnErr("check existing relationship", err) + log.Printf("RELATIONSHIP CHECK | existing=%d", ontapclient.NumRecords(existing)) + + createResp, err := dst.Post("/snapmirror/relationships?return_timeout=120", map[string]interface{}{ + "source": map[string]interface{}{ + "path": cfg.sourceSVMAlias + ":" + cfg.sourceVolume, + "cluster": map[string]string{"name": cfg.peerName}, + }, + "destination": map[string]string{"path": cfg.destSVM + ":" + cfg.destVolume}, + "policy": map[string]string{"name": cfg.smPolicy}, + }) + if err != nil { + log.Printf("create_and_initialize_relationship — %v (may already exist)", err) + } else if jobUUID := ontapclient.JobUUID(createResp); jobUUID != "" { + if _, err := dst.PollJob(jobUUID, 10); err != nil { + log.Printf("poll create job — %v", err) + } + } + + relResp, err := dst.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name", + "destination.path": cfg.destSVM + ":" + cfg.destVolume, + "max_records": "1", + }) + dieOnErr("get relationship", err) + relRecords := ontapclient.Records(relResp) + if len(relRecords) == 0 { + log.Fatalf("ABORTED — SnapMirror relationship not found for '%s:%s'", cfg.destSVM, cfg.destVolume) + } + rel := relRecords[0] + relUUID := ontapclient.NestedStr(rel, "uuid") + log.Printf("RELATIONSHIP | uuid=%s | state=%s | healthy=%v | policy=%s", + relUUID, + ontapclient.NestedStr(rel, "state"), + rel["healthy"], + ontapclient.NestedStr(rel, "policy", "name")) + + _, err = dst.Post(fmt.Sprintf("/snapmirror/relationships/%s/transfers?return_timeout=120", relUUID), map[string]interface{}{}) + if err != nil { + s := err.Error() + if strings.Contains(s, "13303812") { + srcIPs := getICLIFIPs(src) + dstIPs := getICLIFIPs(dst) + log.Fatalf("ABORTED — SnapMirror initialize failed: intercluster LIF connectivity issue.\n"+ + " Error : %s\n src IC : %v\n dst IC : %v\n"+ + " Cause : TCP ports 11104/11105 are likely blocked between these IPs.", + s, srcIPs, dstIPs) + } + log.Printf("initialize_relationship — %v (may already be initialized)", err) + } + return relUUID +} + +func mustEnv(key string) string { + if v := inputs[key]; v != "" { + return v + } + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in the INPUTS block at the top of this file", key) + return "" +} + +func envOrDefault(key, defaultVal string) string { + if v := inputs[key]; v != "" { + return v + } + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + +func dieOnErr(context string, err error) { + if err != nil { + log.Fatalf("%s: %v", context, err) + } +} + +// loadDotEnv reads go/.env and sets each KEY=VALUE as an env var (if not already set). +// Equivalent to Python's os.environ — credentials stay out of source code. +func loadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +} diff --git a/go/snapmirror_provision_src_managed/main.go b/go/snapmirror_provision_src_managed/main.go new file mode 100644 index 0000000..cc8d013 --- /dev/null +++ b/go/snapmirror_provision_src_managed/main.go @@ -0,0 +1,334 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// SnapMirror Provision — Source-Managed view. +// +// Connects to BOTH clusters for pre-flight verification, then drives all +// relationship/volume API calls from the DESTINATION cluster (ONTAP requirement). +// +// Phases: +// +// A Source pre-flight — verify source cluster + volume +// B Dest pre-flight — verify dest cluster + aggregate +// C Dest volume — auto-create DP volume if missing +// D Relationship — create + initialize SnapMirror +// E Convergence — poll until state=snapmirrored +// F Validation — health check + final report +// +// Prerequisites: +// 1. ONTAP 9.8+ on both clusters +// 2. SnapMirror licence installed on both clusters +// 3. At least one intercluster LIF on each cluster +// 4. Cluster peer relationship already exists between source and dest clusters +// 5. SVM peer relationship already exists (source SVM <-> dest SVM) +// 6. Source RW volume (SOURCE_VOLUME) already exists on SOURCE_SVM +// 7. At least one online aggregate on the destination cluster +// 8. Admin credentials for both clusters +// +// Usage: +// +// export SOURCE_HOST=10.x.x.x SOURCE_USER=admin SOURCE_PASS=secret +// export SOURCE_SVM=vs0 SOURCE_VOLUME=vol_rw_01 +// export DEST_HOST=10.y.y.y DEST_USER=admin DEST_PASS=secret +// export DEST_SVM=vs1 +// export SM_POLICY=Asynchronous +// go run . +package main + +import ( + "fmt" + "log" + "os" + "strings" + + ontapclient "github.com/netapp/pace/go/ontapclient" +) + +const ( + pathStorageVolumes = "/storage/volumes" // NOSONAR + keySVMName = "svm.name" +) + +// --------------------------------------------------------------------------- + +func main() { + log.SetFlags(log.LstdFlags) + loadDotEnv() + + sourceHost := mustEnv("SOURCE_HOST") + sourceUser := envOrDefault("SOURCE_USER", "admin") + sourcePass := mustEnv("SOURCE_PASS") + sourceSVM := mustEnv("SOURCE_SVM") + sourceVolume := mustEnv("SOURCE_VOLUME") + + destHost := mustEnv("DEST_HOST") + destUser := envOrDefault("DEST_USER", "admin") + destPass := mustEnv("DEST_PASS") + destSVM := mustEnv("DEST_SVM") + smPolicy := envOrDefault("SM_POLICY", "Asynchronous") + + destVolume := sourceVolume + "_dest" + + src := ontapclient.New(sourceHost, sourceUser, sourcePass, false) + defer src.Close() + dst := ontapclient.New(destHost, destUser, destPass, false) + defer dst.Close() + + log.Println("=== Phase A: Source pre-flight ===") + srcVolSize, srcVol := smSrcPhaseA(src, sourceSVM, sourceVolume, sourceHost) + + log.Println("=== Phase B: Dest pre-flight ===") + peerName, aggrName := smSrcPhaseB(dst) + + log.Println("=== Phase C: Dest volume setup ===") + smSrcPhaseC(dst, destSVM, destVolume, aggrName, srcVolSize) + + log.Println("=== Phase D: Relationship setup ===") + relUUID := smSrcPhaseD(dst, sourceSVM, sourceVolume, destSVM, destVolume, peerName, smPolicy) + + log.Println("=== Phase E: Convergence polling ===") + if _, err := dst.WaitSnapmirrored(relUUID, 15, 1800); err != nil { + log.Fatalf("wait snapmirrored: %v", err) + } + + log.Println("=== Phase F: Final validation ===") + smSrcPhaseF(dst, relUUID, sourceSVM, sourceVolume, destSVM, destVolume) + + _ = srcVol // used via srcVolSize +} + +// smSrcPhaseA verifies the source cluster and validates the source volume. +// Returns (srcVolSize string, srcVol record). +func smSrcPhaseA(src *ontapclient.Client, sourceSVM, sourceVolume, sourceHost string) (string, map[string]interface{}) { + srcCluster, err := src.Get("/cluster", map[string]string{"fields": "name,version"}) + dieOnErr("get source cluster", err) + log.Printf("SOURCE CLUSTER | name=%s | ontap=%s", + ontapclient.NestedStr(srcCluster, "name"), + ontapclient.NestedStr(srcCluster, "version", "full")) + + srcVolResp, err := src.Get(pathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,type,space.size", + "max_records": "1", + "name": sourceVolume, + keySVMName: sourceSVM, + }) + dieOnErr("get source volume", err) + if ontapclient.NumRecords(srcVolResp) == 0 { + log.Fatalf("ABORTED — source volume '%s' not found on %s", sourceVolume, sourceHost) + } + srcVol := ontapclient.Records(srcVolResp)[0] + if ontapclient.NestedStr(srcVol, "type") == "dp" { + log.Fatal("ABORTED — source volume is type=dp; specify the RW volume") + } + srcVolSize := fmt.Sprintf("%.0f", ontapclient.NestedFloat(srcVol, "space", "size")) + log.Printf("SOURCE VOLUME | name=%s | uuid=%s | state=%s | type=%s | size=%s", + ontapclient.NestedStr(srcVol, "name"), + ontapclient.NestedStr(srcVol, "uuid"), + ontapclient.NestedStr(srcVol, "state"), + ontapclient.NestedStr(srcVol, "type"), + srcVolSize) + return srcVolSize, srcVol +} + +// smSrcPhaseB verifies the dest cluster, fetches peer name and best aggregate. +// Returns (peerName, aggrName). +func smSrcPhaseB(dst *ontapclient.Client) (string, string) { + dstCluster, err := dst.Get("/cluster", map[string]string{"fields": "name,version"}) + dieOnErr("get dest cluster", err) + log.Printf("DEST CLUSTER | name=%s | ontap=%s", + ontapclient.NestedStr(dstCluster, "name"), + ontapclient.NestedStr(dstCluster, "version", "full")) + + peerResp, err := dst.Get("/cluster/peers", map[string]string{ + "fields": "name,status.state", + "max_records": "1", + }) + dieOnErr("get cluster peers", err) + peerName := "" + if peers := ontapclient.Records(peerResp); len(peers) > 0 { + peerName = ontapclient.NestedStr(peers[0], "name") + } + if peerName == "" { + log.Fatal("ABORTED — no cluster peer found on destination cluster; run snapmirror_peer_setup first") + } + log.Printf("CLUSTER PEER | name=%s", peerName) + + aggrResp, err := dst.Get("/storage/aggregates", map[string]string{ + "fields": "name,space.block_storage.available", + "state": "online", + "max_records": "1", + "order_by": "space.block_storage.available desc", + }) + dieOnErr("get dest aggregates", err) + aggrName := "" + if aggrs := ontapclient.Records(aggrResp); len(aggrs) > 0 { + aggrName = ontapclient.NestedStr(aggrs[0], "name") + } + if aggrName == "" { + log.Fatal("ABORTED — no online aggregates found on destination cluster") + } + log.Printf("DEST AGGREGATE | name=%s", aggrName) + return peerName, aggrName +} + +// smSrcPhaseC ensures the dest DP volume exists, creating it if needed. +func smSrcPhaseC(dst *ontapclient.Client, destSVM, destVolume, aggrName, srcVolSize string) { + checkDest, err := dst.Get(pathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,type", + "max_records": "1", + "name": destVolume, + keySVMName: destSVM, + }) + dieOnErr("check dest volume", err) + if ontapclient.NumRecords(checkDest) == 0 { + log.Printf("Creating dest DP volume '%s' on '%s'…", destVolume, aggrName) + _, err = dst.Post(pathStorageVolumes+"?return_timeout=120", map[string]interface{}{ + "name": destVolume, + "type": "dp", + "svm": map[string]string{"name": destSVM}, + "aggregates": []map[string]string{ + {"name": aggrName}, + }, + "size": srcVolSize, + }) + if err != nil { + log.Printf("create_dest_volume — %v (may already exist)", err) + } + } else { + log.Printf("Dest volume '%s' already exists — skipping create", destVolume) + } + + dstVolResp, err := dst.Get(pathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,type", + "max_records": "1", + "name": destVolume, + keySVMName: destSVM, + }) + dieOnErr("verify dest volume", err) + vols := ontapclient.Records(dstVolResp) + if len(vols) == 0 { + log.Fatalf("ABORTED — dest volume '%s' not found on SVM '%s' after create", destVolume, destSVM) + } + dstVol := vols[0] + log.Printf("DEST VOLUME | name=%s | uuid=%s | state=%s | type=%s", + ontapclient.NestedStr(dstVol, "name"), + ontapclient.NestedStr(dstVol, "uuid"), + ontapclient.NestedStr(dstVol, "state"), + ontapclient.NestedStr(dstVol, "type")) +} + +// smSrcPhaseD creates and initializes the SnapMirror relationship; returns the relationship UUID. +func smSrcPhaseD(dst *ontapclient.Client, sourceSVM, sourceVolume, destSVM, destVolume, peerName, smPolicy string) string { + existing, err := dst.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,state,healthy", + "destination.path": destSVM + ":" + destVolume, + "max_records": "1", + }) + dieOnErr("check existing relationship", err) + log.Printf("RELATIONSHIP CHECK | existing=%d", ontapclient.NumRecords(existing)) + + createResp, err := dst.Post("/snapmirror/relationships?return_timeout=120", map[string]interface{}{ + "source": map[string]interface{}{ + "path": sourceSVM + ":" + sourceVolume, + "cluster": map[string]string{"name": peerName}, + }, + "destination": map[string]string{"path": destSVM + ":" + destVolume}, + "policy": map[string]string{"name": smPolicy}, + }) + if err != nil { + log.Printf("create_and_initialize_relationship — %v (may already exist)", err) + } else if jobUUID := ontapclient.JobUUID(createResp); jobUUID != "" { + if _, err := dst.PollJob(jobUUID, 10); err != nil { + log.Printf("poll create job — %v", err) + } + } + + relResp, err := dst.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name", + "destination.path": destSVM + ":" + destVolume, + "max_records": "1", + }) + dieOnErr("get relationship", err) + rels := ontapclient.Records(relResp) + if len(rels) == 0 { + log.Fatalf("ABORTED — SnapMirror relationship not found for '%s:%s'", destSVM, destVolume) + } + rel := rels[0] + relUUID := ontapclient.NestedStr(rel, "uuid") + log.Printf("RELATIONSHIP FOUND | uuid=%s | state=%s | healthy=%v", + relUUID, ontapclient.NestedStr(rel, "state"), rel["healthy"]) + + _, err = dst.Post(fmt.Sprintf("/snapmirror/relationships/%s/transfers?return_timeout=120", relUUID), map[string]interface{}{}) + if err != nil { + log.Printf("initialize_relationship — %v (may already be initialized)", err) + } + return relUUID +} + +// smSrcPhaseF prints the final validation report. +func smSrcPhaseF(dst *ontapclient.Client, relUUID, sourceSVM, sourceVolume, destSVM, destVolume string) { + final, err := dst.Get(fmt.Sprintf("/snapmirror/relationships/%s", relUUID), + map[string]string{"fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name"}) + dieOnErr("final validation", err) + log.Printf("=== SNAPMIRROR PROVISION COMPLETE ===\n"+ + " source : %s:%s\n"+ + " destination : %s:%s\n"+ + " state : %s\n"+ + " healthy : %v\n"+ + " policy : %s\n"+ + " lag_time : %v", + sourceSVM, sourceVolume, + destSVM, destVolume, + ontapclient.NestedStr(final, "state"), + final["healthy"], + ontapclient.NestedStr(final, "policy", "name"), + final["lag_time"]) +} + +// mustEnv reads an environment variable and exits if it is not set. +func mustEnv(key string) string { + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) + return "" +} + +// envOrDefault reads an environment variable, returning defaultVal if unset. +func envOrDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + +// dieOnErr logs a fatal error if err is non-nil. +func dieOnErr(context string, err error) { + if err != nil { + log.Fatalf("%s: %v", context, err) + } +} + +// loadDotEnv reads a .env file from the current directory and exports each +// KEY=VALUE pair as an environment variable (only if not already set). +// The file is gitignored — safe to store credentials there for local testing. +func loadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +} diff --git a/go/snapmirror_test_failover/main.go b/go/snapmirror_test_failover/main.go new file mode 100644 index 0000000..316a536 --- /dev/null +++ b/go/snapmirror_test_failover/main.go @@ -0,0 +1,303 @@ +// © 2026 NetApp, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// See the NOTICE file in the repo root for trademark and attribution details. + +// SnapMirror Test Failover — creates a writable FlexClone of a SnapMirror dest volume. +// +// AUTO mode (SOURCE_VOLUME=* or unset): +// +// Queries clusters A then B and selects the first cluster that has a matching DP volume +// (the newest matching DP volume within that cluster). +// +// TARGETED mode (SOURCE_VOLUME=vol_rw_01): +// +// Finds vol_rw_01_dest on either cluster. +// +// Phases: +// +// 0 Auto-detect which cluster has the target DP volume +// A Pre-flight — verify cluster + relationship health +// B Snapshot — get latest SnapMirror snapshot on dest volume +// C Clone — create writable FlexClone +// D Verify — confirm clone online + tag with SM relationship UUID +// E Resync — resync SnapMirror + validate healthy state +// +// Prerequisites: +// 1. ONTAP 9.8+ on both clusters +// 2. A healthy SnapMirror relationship must already exist +// 3. Relationship state must be 'snapmirrored' (baseline transfer complete) +// 4. At least one SnapMirror snapshot on the destination volume +// 5. Admin credentials for both clusters +// +// Usage: +// +// export CLUSTER_A=10.x.x.x CLUSTER_B=10.y.y.y +// export DEST_USER=admin DEST_PASS=secret +// export SOURCE_VOLUME=* # or a specific volume name e.g. "vol_rw_01" +// go run . +package main + +import ( + "fmt" + "log" + "os" + "strings" + + ontapclient "github.com/netapp/pace/go/ontapclient" +) + +const pathStorageVolumes = "/storage/volumes" // NOSONAR + +// --------------------------------------------------------------------------- + +func main() { + log.SetFlags(log.LstdFlags) + loadDotEnv() + + clusterA := mustEnv("CLUSTER_A") + clusterB := mustEnv("CLUSTER_B") + destUser := envOrDefault("DEST_USER", "admin") + destPass := mustEnv("DEST_PASS") + sourceVolume := envOrDefault("SOURCE_VOLUME", "*") + + log.Println("=== Phase 0: Auto-detect target cluster ===") + destHost, dpVol := pickCluster(clusterA, clusterB, destUser, destPass, sourceVolume) + dpVolName := ontapclient.NestedStr(dpVol, "name") + dpSVMName := ontapclient.NestedStr(dpVol, "svm", "name") + dpVolUUID := ontapclient.NestedStr(dpVol, "uuid") + log.Printf("SELECTED | cluster=%s | volume=%s | svm=%s | uuid=%s | state=%s | size=%.0f", + destHost, dpVolName, dpSVMName, dpVolUUID, + ontapclient.NestedStr(dpVol, "state"), + ontapclient.NestedFloat(dpVol, "space", "size")) + + client := ontapclient.New(destHost, destUser, destPass, false) + defer client.Close() + + log.Println("=== Phase A: Pre-flight ===") + relUUID := tfPhaseA(client, dpSVMName, dpVolName) + + log.Println("=== Phase B: Get latest SnapMirror snapshot ===") + snapshotName := tfPhaseB(client, dpVolUUID, dpVolName) + + log.Println("=== Phase C: Create FlexClone ===") + cloneName, cloneUUID := tfPhaseC(client, dpVolName, dpSVMName, snapshotName) + + log.Println("=== Phase D: Verify clone + tag ===") + tfPhaseD(client, cloneName, cloneUUID, relUUID, dpSVMName, snapshotName) + + log.Println("=== Phase E: Resync SnapMirror ===") + tfPhaseE(client, relUUID) +} + +// tfPhaseA verifies cluster connectivity and fetches the SnapMirror relationship UUID. +func tfPhaseA(client *ontapclient.Client, dpSVMName, dpVolName string) string { + cluster, err := client.Get("/cluster", map[string]string{"fields": "name,version"}) + dieOnErr("get cluster", err) + log.Printf("DEST CLUSTER | name=%s | ontap=%s", + ontapclient.NestedStr(cluster, "name"), + ontapclient.NestedStr(cluster, "version", "full")) + + relResp, err := client.Get("/snapmirror/relationships", map[string]string{ + "fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name", + "destination.path": dpSVMName + ":" + dpVolName, + "max_records": "1", + }) + dieOnErr("get snapmirror relationship", err) + rels := ontapclient.Records(relResp) + if len(rels) == 0 { + log.Fatalf("No SnapMirror relationship found for %s:%s", dpSVMName, dpVolName) + } + rel := rels[0] + relUUID := ontapclient.NestedStr(rel, "uuid") + log.Printf("RELATIONSHIP | uuid=%s | source=%s | dest=%s | state=%s | healthy=%v | lag=%v", + relUUID, + ontapclient.NestedStr(rel, "source", "path"), + ontapclient.NestedStr(rel, "destination", "path"), + ontapclient.NestedStr(rel, "state"), + rel["healthy"], rel["lag_time"]) + return relUUID +} + +// tfPhaseB fetches the latest SnapMirror snapshot name from the DP volume. +func tfPhaseB(client *ontapclient.Client, dpVolUUID, dpVolName string) string { + snapResp, err := client.Get(fmt.Sprintf("/storage/volumes/%s/snapshots", dpVolUUID), map[string]string{ + "fields": "name,create_time", + "max_records": "1", + "order_by": "create_time desc", + }) + dieOnErr("get snapshots", err) + if ontapclient.NumRecords(snapResp) == 0 { + log.Fatalf("No SnapMirror snapshots on %s — run provision workflow first", dpVolName) + } + snap := ontapclient.Records(snapResp)[0] + snapshotName := ontapclient.NestedStr(snap, "name") + log.Printf("LATEST SM SNAPSHOT | name=%s | created=%v", snapshotName, snap["create_time"]) + return snapshotName +} + +// tfPhaseC creates the writable FlexClone; returns (cloneName, cloneUUID). +func tfPhaseC(client *ontapclient.Client, dpVolName, dpSVMName, snapshotName string) (string, string) { + cloneName := dpVolName + "_clone" + cloneResp, err := client.Post("/storage/volumes?return_timeout=120", map[string]interface{}{ + "name": cloneName, + "svm": map[string]string{"name": dpSVMName}, + "nas": map[string]string{"path": "/" + cloneName}, + "clone": map[string]interface{}{ + "is_flexclone": true, + "parent_volume": map[string]string{"name": dpVolName}, + "parent_snapshot": map[string]string{"name": snapshotName}, + }, + }) + if err != nil { + log.Printf("create_test_clone — %v (may already exist)", err) + } else if jobUUID := ontapclient.JobUUID(cloneResp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll clone job — %v", err) + } + } + + cloneVolResp, err := client.Get(pathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,nas.path,space.size", + "max_records": "1", + "name": cloneName, + "svm.name": dpSVMName, + }) + dieOnErr("get clone volume", err) + cloneVol := map[string]interface{}{} + if vols := ontapclient.Records(cloneVolResp); len(vols) > 0 { + cloneVol = vols[0] + } + cloneUUID := ontapclient.NestedStr(cloneVol, "uuid") + if cloneUUID == "" { + log.Fatalf("ABORTED — FlexClone '%s' not found after create (create may have failed)", cloneName) + } + log.Printf("CLONE | name=%s | uuid=%s | state=%s | junction=%s", + ontapclient.NestedStr(cloneVol, "name"), cloneUUID, + ontapclient.NestedStr(cloneVol, "state"), + ontapclient.NestedStr(cloneVol, "nas", "path")) + return cloneName, cloneUUID +} + +// tfPhaseD tags the clone and prints the test-failover-ready message. +func tfPhaseD(client *ontapclient.Client, cloneName, cloneUUID, relUUID, dpSVMName, snapshotName string) { + _, err := client.Patch(fmt.Sprintf("/storage/volumes/%s?return_timeout=120", cloneUUID), + map[string]interface{}{"_tags": []string{relUUID + ":test"}}) + if err != nil { + log.Printf("tag_clone_volume — %v", err) + } else { + log.Printf("TAG APPLIED | clone=%s | tag=%s:test", cloneName, relUUID) + } + + cloneVolResp, err := client.Get(pathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,nas.path", + "max_records": "1", + "name": cloneName, + "svm.name": dpSVMName, + }) + if err != nil { + log.Printf("re-fetch clone — %v", err) + return + } + cloneVol := map[string]interface{}{} + if vols := ontapclient.Records(cloneVolResp); len(vols) > 0 { + cloneVol = vols[0] + } + junctionPath := ontapclient.NestedStr(cloneVol, "nas", "path") + log.Printf("=== TEST FAILOVER READY ===\n"+ + " Clone : %s\n UUID : %s\n State : %s\n"+ + " Junction : %s\n SVM : %s\n Snapshot : %s\n\n"+ + " ACTION: Mount %s from SVM %s on a test client.", + ontapclient.NestedStr(cloneVol, "name"), cloneUUID, + ontapclient.NestedStr(cloneVol, "state"), + junctionPath, dpSVMName, snapshotName, + junctionPath, dpSVMName) +} + +// tfPhaseE resyncs the SnapMirror relationship and waits for snapmirrored state. +func tfPhaseE(client *ontapclient.Client, relUUID string) { + resyncResp, err := client.Patch(fmt.Sprintf("/snapmirror/relationships/%s?return_timeout=120", relUUID), + map[string]interface{}{"state": "snapmirrored"}) + if err != nil { + log.Printf("resync_sm_relationship — %v", err) + } else if jobUUID := ontapclient.JobUUID(resyncResp); jobUUID != "" { + if _, err := client.PollJob(jobUUID, 10); err != nil { + log.Printf("poll resync job — %v", err) + } + } + if _, err := client.WaitSnapmirrored(relUUID, 15, 1800); err != nil { + log.Fatalf("wait snapmirrored: %v", err) + } + log.Println("=== TEST FAILOVER COMPLETE — SnapMirror resynced ===") +} + +// pickCluster finds which cluster has the target DP volume; returns (clusterIP, volRecord). +func pickCluster(clusterA, clusterB, user, passwd, volNameFilter string) (string, map[string]interface{}) { + destFilter := volNameFilter + "_dest" + if volNameFilter == "*" { + destFilter = "*_dest" + } + for _, host := range []string{clusterA, clusterB} { + client := ontapclient.New(host, user, passwd, false) + resp, err := client.Get(pathStorageVolumes, map[string]string{ + "fields": "name,create_time,uuid,svm.name,state,space.size", + "type": "dp", + "name": destFilter, + "order_by": "create_time desc", + "max_records": "1", + }) + client.Close() + if err != nil { + log.Printf(" cluster %s — %v", host, err) + continue + } + if ontapclient.NumRecords(resp) >= 1 { + return host, ontapclient.Records(resp)[0] + } + } + log.Fatalf("No DP volumes found on either cluster (%s, %s)", clusterA, clusterB) + return "", nil +} + +func mustEnv(key string) string { + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) + return "" +} + +func envOrDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + +func dieOnErr(context string, err error) { + if err != nil { + log.Fatalf("%s: %v", context, err) + } +} + +// loadDotEnv reads a .env file from the current directory and exports each +// KEY=VALUE pair as an environment variable (only if not already set). +// The file is gitignored — safe to store credentials there for local testing. +func loadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +} From 62ac34107e4bec80f447da4f12fe0162424fd726 Mon Sep 17 00:00:00 2001 From: Somanath Date: Mon, 15 Jun 2026 18:24:56 +0530 Subject: [PATCH 2/4] fix: handle node reboot during cluster creation in trackJob After POST /cluster ONTAP restarts its management stack, forcibly closing the TCP connection. The previous code treated any poll error as fatal; this change retries on network-level errors (expected reboot drop) and only fails on HTTP-level errors (4xx/5xx). Also adds 'errors' import required by errors.As(). --- go/cluster_setup_basic/main.go | 39 +++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/go/cluster_setup_basic/main.go b/go/cluster_setup_basic/main.go index 46779b5..2241bf8 100644 --- a/go/cluster_setup_basic/main.go +++ b/go/cluster_setup_basic/main.go @@ -27,6 +27,7 @@ package main import ( + "errors" "fmt" "log" "os" @@ -207,13 +208,45 @@ func createCluster(client *ontapclient.Client, localNode, partnerNode map[string } // trackJob switches to cluster credentials then polls the job until complete. -// After POST /cluster the node switches to full cluster mode and requires CLUSTER_PASS. +// After POST /cluster the node reboots its management stack — network errors +// are expected and retried until the deadline. HTTP-level errors (4xx/5xx) are fatal. func trackJob(host, user, clusterPass, jobUUID string) { clusterClient := ontapclient.New(host, user, clusterPass, false) defer clusterClient.Close() - if _, err := clusterClient.PollJob(jobUUID, 10); err != nil { - log.Fatalf("track_job: %v", err) + deadline := time.Now().Add(10 * time.Minute) + jobPath := fmt.Sprintf("/cluster/jobs/%s", jobUUID) + + for { + if time.Now().After(deadline) { + log.Fatal("track_job: timed out waiting for cluster creation") + } + + result, err := clusterClient.Get(jobPath, + map[string]string{"fields": "state,message,error,code"}) + if err != nil { + var apiErr *ontapclient.OntapApiError + if errors.As(err, &apiErr) { + // HTTP-level error (e.g. 401, 500) — something is wrong. + log.Fatalf("track_job: %v", err) + } + // Network error: node is rebooting as part of cluster creation. + log.Printf(" node rebooting (network error), retrying in 15s — %v", err) + time.Sleep(15 * time.Second) + continue + } + + state, _ := result["state"].(string) + log.Printf(" job %s — state=%s", jobUUID, state) + switch state { + case "running", "queued", "paused": + time.Sleep(10 * time.Second) + case "success": + return + default: + msg, _ := result["message"].(string) + log.Fatalf("job %s ended with state=%s: %s", jobUUID, state, msg) + } } } From 41075034fc4880d6959e7d9762d6b7e674dffeab Mon Sep 17 00:00:00 2001 From: Somanath Date: Thu, 18 Jun 2026 16:29:38 +0530 Subject: [PATCH 3/4] fix(go): address critical and major code review issues - Remove password from log output in cluster_setup_basic - Remove inputs map override bug in snapmirror_provision_dest_managed; env vars read directly - Fix volume size field from string to int64 in both provisioning scripts - Fix FromEnv() to return error instead of calling log.Fatal - Fix deleteAndConfirmClone false-success: confirmation GET is now single source of truth --- go/cluster_setup_basic/main.go | 142 +++-------- go/ontapclient/ontap_client.go | 189 ++++++++++++--- go/snapmirror_cleanup_test_failover/main.go | 157 ++++++------ go/snapmirror_provision_dest_managed/main.go | 241 ++++++++----------- go/snapmirror_provision_src_managed/main.go | 195 +++++++-------- go/snapmirror_test_failover/main.go | 128 ++++------ 6 files changed, 499 insertions(+), 553 deletions(-) diff --git a/go/cluster_setup_basic/main.go b/go/cluster_setup_basic/main.go index 2241bf8..b66e65f 100644 --- a/go/cluster_setup_basic/main.go +++ b/go/cluster_setup_basic/main.go @@ -6,7 +6,7 @@ // // Steps: // -// 1 discoverNodes — GET /cluster/nodes (membership=available, retry 3x/30s) +// 1 waitForNodes — GET /cluster/nodes (membership=available, retry 3x/30s) // 2 discoverLocal — isolate the local node (management_interfaces != null) // 3 discoverPartner — isolate the partner node (exclude local node UUID) // 4 createCluster — POST /cluster @@ -27,11 +27,9 @@ package main import ( - "errors" + "context" "fmt" "log" - "os" - "strings" "time" ontapclient "github.com/netapp/pace/go/ontapclient" @@ -47,6 +45,7 @@ const clusterNodesPath = "/cluster/nodes" func main() { log.SetFlags(log.LstdFlags) loadDotEnv() + ctx := context.Background() host := mustEnv("ONTAP_HOST") user := envOrDefault("ONTAP_USER", "admin") @@ -59,60 +58,61 @@ func main() { // Step 1: Discover available nodes (retry 3x) log.Println("=== Step 1: Discover nodes ===") - discoverNodes(client, 3, 30) + waitForNodes(ctx, client, 3, 30*time.Second) // Step 2: Find local node log.Println("=== Step 2: Discover local node ===") - localNode := discoverLocal(client) + localNode := discoverLocal(ctx, client) localUUID := ontapclient.NestedStr(localNode, "uuid") // Step 3: Find partner node log.Println("=== Step 3: Discover partner node ===") - partnerNode := discoverPartner(client, localUUID) + partnerNode := discoverPartner(ctx, client, localUUID) // Step 4: Create cluster log.Println("=== Step 4: Create cluster ===") - jobUUID := createCluster(client, localNode, partnerNode) + jobUUID := createCluster(ctx, client, localNode, partnerNode) // Step 5: Track job — switch to cluster credentials first log.Println("=== Step 5: Track cluster creation job ===") clusterPass := mustEnv("CLUSTER_PASS") clusterMgmtIP := mustEnv("CLUSTER_MGMT_IP") - trackJob(host, user, clusterPass, jobUUID) + trackJob(ctx, host, user, clusterPass, jobUUID) log.Printf("=== CLUSTER CREATED ===\n"+ " Name : %s\n"+ " UI : https://%s\n"+ - " Login : %s / %s", - mustEnv("CLUSTER_NAME"), clusterMgmtIP, user, clusterPass) + " User : %s", + mustEnv("CLUSTER_NAME"), clusterMgmtIP, user) } -// discoverNodes GETs /cluster/nodes with membership=available, retrying up to maxAttempts times. -func discoverNodes(client *ontapclient.Client, maxAttempts, delaySecs int) { +// waitForNodes GETs /cluster/nodes with membership=available, retrying up to maxAttempts times. +// Acts as a readiness guard — the caller proceeds only when nodes are reachable. +func waitForNodes(ctx context.Context, client *ontapclient.Client, maxAttempts int, delay time.Duration) { var lastErr error for attempt := 1; attempt <= maxAttempts; attempt++ { - resp, err := client.Get(clusterNodesPath, map[string]string{ + resp, err := client.Get(ctx, clusterNodesPath, map[string]string{ "fields": nodeFields, "membership": "available", }) if err == nil { - log.Printf("discover_nodes — %d node(s) found", ontapclient.NumRecords(resp)) + log.Printf("wait_for_nodes — %d node(s) found", ontapclient.NumRecords(resp)) return } lastErr = err if attempt < maxAttempts { - log.Printf("discover_nodes failed (attempt %d/%d), retrying in %ds — %v", - attempt, maxAttempts, delaySecs, err) - time.Sleep(time.Duration(delaySecs) * time.Second) + log.Printf("wait_for_nodes failed (attempt %d/%d), retrying in %s — %v", + attempt, maxAttempts, delay, err) + time.Sleep(delay) } } - log.Fatalf("discover_nodes failed after %d attempts: %v", maxAttempts, lastErr) + log.Fatalf("wait_for_nodes failed after %d attempts: %v", maxAttempts, lastErr) } // discoverLocal finds the local node (the one with management_interfaces set). // Returns the first matching node record. -func discoverLocal(client *ontapclient.Client) map[string]interface{} { - resp, err := client.Get(clusterNodesPath, map[string]string{ +func discoverLocal(ctx context.Context, client *ontapclient.Client) map[string]interface{} { + resp, err := client.Get(ctx, clusterNodesPath, map[string]string{ "fields": nodeFields, "membership": "available", "management_interfaces": "!null", @@ -128,8 +128,8 @@ func discoverLocal(client *ontapclient.Client) map[string]interface{} { // discoverPartner finds the partner node by excluding the local node UUID. // Returns the first matching node record. -func discoverPartner(client *ontapclient.Client, localUUID string) map[string]interface{} { - resp, err := client.Get(clusterNodesPath, map[string]string{ +func discoverPartner(ctx context.Context, client *ontapclient.Client, localUUID string) map[string]interface{} { + resp, err := client.Get(ctx, clusterNodesPath, map[string]string{ "fields": nodeFields, "membership": "available", "uuid": "!" + localUUID, @@ -144,7 +144,7 @@ func discoverPartner(client *ontapclient.Client, localUUID string) map[string]in } // createCluster POSTs /cluster to create the cluster; returns the job UUID. -func createCluster(client *ontapclient.Client, localNode, partnerNode map[string]interface{}) string { +func createCluster(ctx context.Context, client *ontapclient.Client, localNode, partnerNode map[string]interface{}) string { clusterName := mustEnv("CLUSTER_NAME") clusterPass := mustEnv("CLUSTER_PASS") clusterMgmtIP := mustEnv("CLUSTER_MGMT_IP") @@ -193,13 +193,9 @@ func createCluster(client *ontapclient.Client, localNode, partnerNode map[string }, }, }, - "name_servers": map[string]interface{}{}, - "ntp_servers": map[string]interface{}{}, - "dns_domains": map[string]interface{}{}, - "configuration_backup": map[string]interface{}{}, } - resp, err := client.Post("/cluster?keep_precluster_config=true", body) + resp, err := client.Post(ctx, "/cluster", map[string]string{"keep_precluster_config": "true"}, body) dieOnErr("create_cluster", err) jobUUID := ontapclient.JobUUID(resp) @@ -209,44 +205,13 @@ func createCluster(client *ontapclient.Client, localNode, partnerNode map[string // trackJob switches to cluster credentials then polls the job until complete. // After POST /cluster the node reboots its management stack — network errors -// are expected and retried until the deadline. HTTP-level errors (4xx/5xx) are fatal. -func trackJob(host, user, clusterPass, jobUUID string) { +// are expected and retried. HTTP-level errors (4xx/5xx) are fatal. +// Delegates to PollJobTolerant which encapsulates the network-retry logic. +func trackJob(ctx context.Context, host, user, clusterPass, jobUUID string) { clusterClient := ontapclient.New(host, user, clusterPass, false) defer clusterClient.Close() - - deadline := time.Now().Add(10 * time.Minute) - jobPath := fmt.Sprintf("/cluster/jobs/%s", jobUUID) - - for { - if time.Now().After(deadline) { - log.Fatal("track_job: timed out waiting for cluster creation") - } - - result, err := clusterClient.Get(jobPath, - map[string]string{"fields": "state,message,error,code"}) - if err != nil { - var apiErr *ontapclient.OntapApiError - if errors.As(err, &apiErr) { - // HTTP-level error (e.g. 401, 500) — something is wrong. - log.Fatalf("track_job: %v", err) - } - // Network error: node is rebooting as part of cluster creation. - log.Printf(" node rebooting (network error), retrying in 15s — %v", err) - time.Sleep(15 * time.Second) - continue - } - - state, _ := result["state"].(string) - log.Printf(" job %s — state=%s", jobUUID, state) - switch state { - case "running", "queued", "paused": - time.Sleep(10 * time.Second) - case "success": - return - default: - msg, _ := result["message"].(string) - log.Fatalf("job %s ended with state=%s: %s", jobUUID, state, msg) - } + if _, err := clusterClient.PollJobTolerant(ctx, jobUUID, 15*time.Second); err != nil { + log.Fatalf("track_job: %v", err) } } @@ -260,46 +225,7 @@ func clusterIfaceIP(node map[string]interface{}) string { return ontapclient.NestedStr(iface, "ip", "address") } -func mustEnv(key string) string { - if v := os.Getenv(key); v != "" { - return v - } - log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) - return "" -} - -func envOrDefault(key, defaultVal string) string { - if v := os.Getenv(key); v != "" { - return v - } - return defaultVal -} - -func dieOnErr(context string, err error) { - if err != nil { - log.Fatalf("%s: %v", context, err) - } -} - -// loadDotEnv reads a .env file from the current directory and exports each -// KEY=VALUE pair as an environment variable (only if not already set). -// The file is gitignored — safe to store credentials there for local testing. -func loadDotEnv() { - data, err := os.ReadFile(".env") - if err != nil { - return - } - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - k, v, ok := strings.Cut(line, "=") - if !ok { - continue - } - if os.Getenv(strings.TrimSpace(k)) == "" { - _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) - } - } -} +func mustEnv(key string) string { return ontapclient.MustEnv(key) } +func envOrDefault(k, def string) string { return ontapclient.EnvOrDefault(k, def) } +func dieOnErr(op string, err error) { ontapclient.DieOnErr(op, err) } +func loadDotEnv() { ontapclient.LoadDotEnv() } diff --git a/go/ontapclient/ontap_client.go b/go/ontapclient/ontap_client.go index 9897e61..387fb76 100644 --- a/go/ontapclient/ontap_client.go +++ b/go/ontapclient/ontap_client.go @@ -13,14 +13,17 @@ package ontapclient import ( "bytes" + "context" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "log" "net/http" "net/url" "os" + "strings" "time" ) @@ -28,6 +31,13 @@ const ( defaultTimeout = 30 * time.Second clientAppHdr = "pace-example" maxJobWait = 10 * time.Minute + maxRespBytes = 32 << 20 // 32 MiB safety cap on response body size + + // PathStorageVolumes is the ONTAP REST API path for storage volume operations. + PathStorageVolumes = "/storage/volumes" + + // KeySVMName is the query parameter key for filtering resources by SVM name. + KeySVMName = "svm.name" ) // OntapApiError is returned when the ONTAP REST API responds with a non-2xx status. @@ -40,6 +50,22 @@ func (e *OntapApiError) Error() string { return fmt.Sprintf("HTTP %d: %v", e.StatusCode, e.Detail) } +// ErrorCode extracts the ONTAP API error code string from the parsed response body. +// Returns an empty string if the code field is absent or unparseable. +// Example ONTAP error body: {"error": {"code": "917927", "message": "entry already exists"}} +func (e *OntapApiError) ErrorCode() string { + m, ok := e.Detail.(map[string]interface{}) + if !ok { + return "" + } + errMap, ok := m["error"].(map[string]interface{}) + if !ok { + return "" + } + code, _ := errMap["code"].(string) + return code +} + // Client is a thin HTTP client for the ONTAP REST API. type Client struct { baseURL string @@ -67,20 +93,22 @@ func New(host, username, password string, verifySSL bool) *Client { // FromEnv creates a Client from standard ONTAP_* environment variables. // Required: ONTAP_HOST, ONTAP_PASS. Optional: ONTAP_USER (default "admin"). -func FromEnv() *Client { +// Returns an error if a required variable is unset so callers can handle it +// without os.Exit side-effects (important for testability). +func FromEnv() (*Client, error) { host := os.Getenv("ONTAP_HOST") if host == "" { - log.Fatal("ONTAP_HOST environment variable is required") + return nil, fmt.Errorf("ONTAP_HOST environment variable is required") } password := os.Getenv("ONTAP_PASS") if password == "" { - log.Fatal("ONTAP_PASS environment variable is required") + return nil, fmt.Errorf("ONTAP_PASS environment variable is required") } user := os.Getenv("ONTAP_USER") if user == "" { user = "admin" } - return New(host, user, password, false) + return New(host, user, password, false), nil } // Close is a no-op provided for symmetry with connection-pooling patterns. @@ -102,7 +130,7 @@ func (c *Client) buildURL(path string, params map[string]string) string { } // do executes an HTTP request and decodes the JSON response body. -func (c *Client) do(method, rawURL string, body interface{}) (map[string]interface{}, error) { +func (c *Client) do(ctx context.Context, method, rawURL string, body interface{}) (map[string]interface{}, error) { var bodyReader io.Reader if body != nil { b, err := json.Marshal(body) @@ -112,7 +140,7 @@ func (c *Client) do(method, rawURL string, body interface{}) (map[string]interfa bodyReader = bytes.NewReader(b) } - req, err := http.NewRequest(method, rawURL, bodyReader) + req, err := http.NewRequestWithContext(ctx, method, rawURL, bodyReader) if err != nil { return nil, fmt.Errorf("create request: %w", err) } @@ -127,10 +155,14 @@ func (c *Client) do(method, rawURL string, body interface{}) (map[string]interfa } defer func() { _ = resp.Body.Close() }() - respBytes, err := io.ReadAll(resp.Body) + limited := io.LimitReader(resp.Body, maxRespBytes+1) + respBytes, err := io.ReadAll(limited) if err != nil { return nil, fmt.Errorf("read response body: %w", err) } + if int64(len(respBytes)) > maxRespBytes { + return nil, fmt.Errorf("response body exceeds 32 MiB — add max_records to reduce response size") + } var result map[string]interface{} if len(respBytes) > 0 { @@ -146,37 +178,37 @@ func (c *Client) do(method, rawURL string, body interface{}) (map[string]interfa } // Get sends a GET request to the given API path with optional query params. -func (c *Client) Get(path string, params map[string]string) (map[string]interface{}, error) { - return c.do(http.MethodGet, c.buildURL(path, params), nil) +func (c *Client) Get(ctx context.Context, path string, params map[string]string) (map[string]interface{}, error) { + return c.do(ctx, http.MethodGet, c.buildURL(path, params), nil) } -// Post sends a POST request with a JSON body. -func (c *Client) Post(path string, body interface{}) (map[string]interface{}, error) { - return c.do(http.MethodPost, c.baseURL+path, body) +// Post sends a POST request with a JSON body and optional query params. +func (c *Client) Post(ctx context.Context, path string, params map[string]string, body interface{}) (map[string]interface{}, error) { + return c.do(ctx, http.MethodPost, c.buildURL(path, params), body) } -// Patch sends a PATCH request with a JSON body. -func (c *Client) Patch(path string, body interface{}) (map[string]interface{}, error) { - return c.do(http.MethodPatch, c.baseURL+path, body) +// Patch sends a PATCH request with a JSON body and optional query params. +func (c *Client) Patch(ctx context.Context, path string, params map[string]string, body interface{}) (map[string]interface{}, error) { + return c.do(ctx, http.MethodPatch, c.buildURL(path, params), body) } -// Delete sends a DELETE request. -func (c *Client) Delete(path string) (map[string]interface{}, error) { - return c.do(http.MethodDelete, c.baseURL+path, nil) +// Delete sends a DELETE request with optional query params. +func (c *Client) Delete(ctx context.Context, path string, params map[string]string) (map[string]interface{}, error) { + return c.do(ctx, http.MethodDelete, c.buildURL(path, params), nil) } // PollJob polls /cluster/jobs/{uuid} until the job reaches a terminal state. // Returns an error if the job ends in any state other than "success". -func (c *Client) PollJob(jobUUID string, intervalSecs int) (map[string]interface{}, error) { - if intervalSecs <= 0 { - intervalSecs = 10 +func (c *Client) PollJob(ctx context.Context, jobUUID string, interval time.Duration) (map[string]interface{}, error) { + if interval <= 0 { + interval = 10 * time.Second } deadline := time.Now().Add(maxJobWait) for { if time.Now().After(deadline) { return nil, fmt.Errorf("poll job %s: timed out after %s", jobUUID, maxJobWait) } - result, err := c.Get(fmt.Sprintf("/cluster/jobs/%s", jobUUID), + result, err := c.Get(ctx, fmt.Sprintf("/cluster/jobs/%s", jobUUID), map[string]string{"fields": "state,message,error,code"}) if err != nil { return nil, fmt.Errorf("poll job %s: %w", jobUUID, err) @@ -185,7 +217,7 @@ func (c *Client) PollJob(jobUUID string, intervalSecs int) (map[string]interface log.Printf(" job %s — state=%s", jobUUID, state) switch state { case "running", "queued", "paused": - time.Sleep(time.Duration(intervalSecs) * time.Second) + time.Sleep(interval) case "success": return result, nil default: @@ -196,17 +228,20 @@ func (c *Client) PollJob(jobUUID string, intervalSecs int) (map[string]interface } // WaitSnapmirrored polls a SnapMirror relationship until state == "snapmirrored". -// maxWaitSecs defaults to 1800 if <= 0. -func (c *Client) WaitSnapmirrored(relUUID string, intervalSecs, maxWaitSecs int) (map[string]interface{}, error) { - if intervalSecs <= 0 { - intervalSecs = 15 +// Defaults: interval=15s, maxWait=30m when zero or negative values are provided. +func (c *Client) WaitSnapmirrored(ctx context.Context, relUUID string, interval, maxWait time.Duration) (map[string]interface{}, error) { + if interval <= 0 { + interval = 15 * time.Second } - if maxWaitSecs <= 0 { - maxWaitSecs = 1800 + if maxWait <= 0 { + maxWait = 30 * time.Minute } - elapsed := 0 - for elapsed < maxWaitSecs { - result, err := c.Get(fmt.Sprintf("/snapmirror/relationships/%s", relUUID), + deadline := time.Now().Add(maxWait) + for { + if time.Now().After(deadline) { + return nil, fmt.Errorf("timed out waiting for relationship %s to reach snapmirrored", relUUID) + } + result, err := c.Get(ctx, fmt.Sprintf("/snapmirror/relationships/%s", relUUID), map[string]string{"fields": "state,lag_time,healthy"}) if err != nil { return nil, fmt.Errorf("poll relationship %s: %w", relUUID, err) @@ -216,10 +251,8 @@ func (c *Client) WaitSnapmirrored(relUUID string, intervalSecs, maxWaitSecs int) if state == "snapmirrored" { return result, nil } - time.Sleep(time.Duration(intervalSecs) * time.Second) - elapsed += intervalSecs + time.Sleep(interval) } - return nil, fmt.Errorf("timed out waiting for relationship %s to reach snapmirrored", relUUID) } // NestedStr safely extracts a nested string value from a map[string]interface{}. @@ -299,3 +332,89 @@ func NumRecords(resp map[string]interface{}) int { func JobUUID(resp map[string]interface{}) string { return NestedStr(resp, "job", "uuid") } + +// PollJobTolerant is like PollJob but retries on transient network errors. +// Use when the management stack may restart during the operation (e.g. POST /cluster). +// HTTP-level API errors (4xx/5xx) are returned immediately without retrying. +func (c *Client) PollJobTolerant(ctx context.Context, jobUUID string, interval time.Duration) (map[string]interface{}, error) { + if interval <= 0 { + interval = 10 * time.Second + } + deadline := time.Now().Add(maxJobWait) + jobPath := fmt.Sprintf("/cluster/jobs/%s", jobUUID) + for { + if time.Now().After(deadline) { + return nil, fmt.Errorf("poll job %s: timed out after %s", jobUUID, maxJobWait) + } + result, err := c.Get(ctx, jobPath, map[string]string{"fields": "state,message,error,code"}) + if err != nil { + var apiErr *OntapApiError + if errors.As(err, &apiErr) { + // HTTP-level error — not a transient reboot; return immediately. + return nil, fmt.Errorf("poll job %s: %w", jobUUID, err) + } + // Network error — management stack may be restarting; retry. + log.Printf(" job %s — network error, retrying in %s — %v", jobUUID, interval, err) + time.Sleep(interval) + continue + } + state, _ := result["state"].(string) + log.Printf(" job %s — state=%s", jobUUID, state) + switch state { + case "running", "queued", "paused": + time.Sleep(interval) + case "success": + return result, nil + default: + msg, _ := result["message"].(string) + return nil, fmt.Errorf("job %s ended with state=%s: %s", jobUUID, state, msg) + } + } +} + +// MustEnv reads an environment variable and calls log.Fatal if it is not set or empty. +func MustEnv(key string) string { + if v := os.Getenv(key); v != "" { + return v + } + log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) + return "" +} + +// EnvOrDefault reads an environment variable, returning defaultVal if unset or empty. +func EnvOrDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + +// DieOnErr calls log.Fatal if err is non-nil. For use in main packages only. +func DieOnErr(op string, err error) { + if err != nil { + log.Fatalf("%s: %v", op, err) + } +} + +// LoadDotEnv reads a .env file from the current directory and sets each KEY=VALUE +// pair as an environment variable (only if not already set). The file is gitignored — +// safe to store credentials there for local testing. +func LoadDotEnv() { + data, err := os.ReadFile(".env") + if err != nil { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok { + continue + } + if os.Getenv(strings.TrimSpace(k)) == "" { + _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) + } + } +} diff --git a/go/snapmirror_cleanup_test_failover/main.go b/go/snapmirror_cleanup_test_failover/main.go index 8c8cde2..ee59298 100644 --- a/go/snapmirror_cleanup_test_failover/main.go +++ b/go/snapmirror_cleanup_test_failover/main.go @@ -33,22 +33,22 @@ package main import ( + "context" "fmt" "log" - "os" - "strings" "time" ontapclient "github.com/netapp/pace/go/ontapclient" ) -const volumePatchPath = "/storage/volumes/%s?return_timeout=120" +const volumeOpPathFmt = "/storage/volumes/%s" // --------------------------------------------------------------------------- func main() { log.SetFlags(log.LstdFlags) loadDotEnv() + ctx := context.Background() clusterA := mustEnv("CLUSTER_A") clusterB := mustEnv("CLUSTER_B") @@ -59,7 +59,7 @@ func main() { // === Phase 0: Find SnapMirror relationship === log.Println("=== Phase 0: Find SnapMirror relationship ===") - destHost, rel := pickClusterByRelationship(clusterA, clusterB, destUser, destPass, sourceSVM, sourceVolume) + destHost, rel := pickClusterByRelationship(ctx, clusterA, clusterB, destUser, destPass, sourceSVM, sourceVolume) relUUID := ontapclient.NestedStr(rel, "uuid") log.Printf("RELATIONSHIP FOUND | cluster=%s | uuid=%s | source=%s | dest=%s | state=%s | healthy=%v", destHost, @@ -79,7 +79,10 @@ func main() { // === Phase A: Find tagged clone === log.Println("=== Phase A: Find tagged clone ===") - clone := findTaggedClone(client, relUUID) + clone, findErr := findTaggedClone(ctx, client, relUUID) + if findErr != nil { + log.Fatalf("find tagged clone: %v", findErr) + } if clone == nil { log.Printf("NO TAGGED CLONE FOUND for %s:%s on %s — nothing to clean up", sourceSVM, sourceVolume, destHost) @@ -92,57 +95,69 @@ func main() { cloneSVM, _ := clone["svm"].(string) cloneName, _ := clone["name"].(string) - removeSMASAndBringOnline(client, cloneUUID, cloneSVM, cloneName) - unmountClone(client, cloneUUID) - offlineClone(client, cloneUUID) - deleteAndConfirmClone(client, cloneUUID, cloneName, destHost) + removeSMASRelationships(ctx, client, cloneSVM, cloneName) + bringCloneOnline(ctx, client, cloneUUID) + unmountClone(ctx, client, cloneUUID) + offlineClone(ctx, client, cloneUUID) + deleteAndConfirmClone(ctx, client, cloneUUID, cloneName, destHost) } // pickClusterByRelationship returns (clusterIP, relationshipRecord) for the cluster owning this SM rel. -func pickClusterByRelationship(clusterA, clusterB, user, passwd, sourceSVM, sourceVolume string) (string, map[string]interface{}) { +func pickClusterByRelationship(ctx context.Context, clusterA, clusterB, user, passwd, sourceSVM, sourceVolume string) (string, map[string]interface{}) { sourcePath := sourceSVM + ":" + sourceVolume - for _, host := range []string{clusterA, clusterB} { - client := ontapclient.New(host, user, passwd, false) - resp, err := client.Get("/snapmirror/relationships", map[string]string{ + tryHost := func(host string) (map[string]interface{}, bool) { + c := ontapclient.New(host, user, passwd, false) + defer c.Close() + resp, err := c.Get(ctx, "/snapmirror/relationships", map[string]string{ "fields": "uuid,source.path,destination.path,state,healthy", "source.path": sourcePath, "max_records": "1", }) - client.Close() if err != nil { log.Printf(" cluster %s — %v", host, err) - continue + return nil, false } if ontapclient.NumRecords(resp) >= 1 { - return host, ontapclient.Records(resp)[0] + return ontapclient.Records(resp)[0], true + } + return nil, false + } + for _, host := range []string{clusterA, clusterB} { + if rel, ok := tryHost(host); ok { + return host, rel } } log.Fatalf("No SM relationship found for %s on either cluster (%s, %s)", sourcePath, clusterA, clusterB) return "", nil } -// findTaggedClone returns the clone tagged ':test', or nil if not found. -func findTaggedClone(client *ontapclient.Client, relUUID string) map[string]interface{} { - resp, err := client.Get("/storage/volumes", map[string]string{ +// findTaggedClone returns the clone tagged ':test', or (nil, nil) if not found. +// Returns a non-nil error if the API call itself fails (e.g. auth error, network failure), +// distinguishing a genuine "nothing to clean up" from a broken connection. +func findTaggedClone(ctx context.Context, client *ontapclient.Client, relUUID string) (map[string]interface{}, error) { + resp, err := client.Get(ctx, "/storage/volumes", map[string]string{ "fields": "name,uuid,svm.name,state,nas.path", "_tags": relUUID + ":test", "max_records": "1", }) - if err != nil || ontapclient.NumRecords(resp) == 0 { - return nil + if err != nil { + return nil, fmt.Errorf("find tagged clone: %w", err) + } + if ontapclient.NumRecords(resp) == 0 { + return nil, nil } rec := ontapclient.Records(resp)[0] return map[string]interface{}{ "uuid": ontapclient.NestedStr(rec, "uuid"), "name": ontapclient.NestedStr(rec, "name"), "svm": ontapclient.NestedStr(rec, "svm", "name"), - } + }, nil } -// removeSMASAndBringOnline deletes any SMAS relationship on the clone, then ensures it is online. -func removeSMASAndBringOnline(client *ontapclient.Client, cloneUUID, cloneSVM, cloneName string) { +// removeSMASRelationships deletes any SMAS SnapMirror relationships on the clone volume. +func removeSMASRelationships(ctx context.Context, client *ontapclient.Client, cloneSVM, cloneName string) { log.Println("=== Phase B: Remove SMAS relationship on clone (if any) ===") - smasResp, err := client.Get("/snapmirror/relationships", map[string]string{ + smasResp, err := client.Get(ctx, "/snapmirror/relationships", map[string]string{ "fields": "uuid,state", "destination.path": cloneSVM + ":" + cloneName, "max_records": "10", @@ -154,13 +169,14 @@ func removeSMASAndBringOnline(client *ontapclient.Client, cloneUUID, cloneSVM, c for _, r := range smasRels { smasUUID := ontapclient.NestedStr(r, "uuid") log.Printf(" Deleting SMAS relationship %s on clone", smasUUID) - resp, err := client.Delete(fmt.Sprintf("/snapmirror/relationships/%s?return_timeout=120&force=true", smasUUID)) + resp, err := client.Delete(ctx, fmt.Sprintf("/snapmirror/relationships/%s", smasUUID), + map[string]string{"return_timeout": "120", "force": "true"}) if err != nil { log.Printf("delete_smas_rel %s — %v (continuing)", smasUUID, err) continue } if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { - if _, err := client.PollJob(jobUUID, 10); err != nil { + if _, err := client.PollJob(ctx, jobUUID, 10*time.Second); err != nil { log.Printf("poll delete smas job — %v", err) } } @@ -168,25 +184,30 @@ func removeSMASAndBringOnline(client *ontapclient.Client, cloneUUID, cloneSVM, c if len(smasRels) == 0 { log.Println(" No SMAS relationships found on clone — continuing") } +} - resp, err := client.Patch(fmt.Sprintf(volumePatchPath, cloneUUID), +// bringCloneOnline sets the clone volume state to online. +func bringCloneOnline(ctx context.Context, client *ontapclient.Client, cloneUUID string) { + resp, err := client.Patch(ctx, fmt.Sprintf(volumeOpPathFmt, cloneUUID), + map[string]string{"return_timeout": "120"}, map[string]interface{}{"state": "online"}) if err != nil { log.Printf("bring_online — %v (continuing)", err) return } if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { - if _, err := client.PollJob(jobUUID, 10); err != nil { + if _, err := client.PollJob(ctx, jobUUID, 10*time.Second); err != nil { log.Printf("poll bring-online job — %v", err) } } } // unmountClone removes the NAS junction path; retries up to 6 times before aborting. -func unmountClone(client *ontapclient.Client, cloneUUID string) { +func unmountClone(ctx context.Context, client *ontapclient.Client, cloneUUID string) { log.Println("=== Phase C: Unmount clone ===") for attempt := 1; attempt <= 6; attempt++ { - resp, err := client.Patch(fmt.Sprintf(volumePatchPath, cloneUUID), + resp, err := client.Patch(ctx, fmt.Sprintf(volumeOpPathFmt, cloneUUID), + map[string]string{"return_timeout": "120"}, map[string]interface{}{"nas": map[string]string{"path": ""}}) if err != nil { log.Printf("unmount_clone attempt %d/6 — %v", attempt, err) @@ -196,7 +217,7 @@ func unmountClone(client *ontapclient.Client, cloneUUID string) { continue } if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { - if _, err := client.PollJob(jobUUID, 10); err != nil { + if _, err := client.PollJob(ctx, jobUUID, 10*time.Second); err != nil { log.Printf("poll unmount job — %v", err) } } @@ -206,34 +227,48 @@ func unmountClone(client *ontapclient.Client, cloneUUID string) { } // offlineClone sets the volume state to offline (required before delete). -func offlineClone(client *ontapclient.Client, cloneUUID string) { +func offlineClone(ctx context.Context, client *ontapclient.Client, cloneUUID string) { log.Println("=== Phase D: Offline clone ===") - resp, err := client.Patch(fmt.Sprintf(volumePatchPath, cloneUUID), + resp, err := client.Patch(ctx, fmt.Sprintf(volumeOpPathFmt, cloneUUID), + map[string]string{"return_timeout": "120"}, map[string]interface{}{"state": "offline"}) if err != nil { log.Printf("offline_clone — %v", err) return } if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { - if _, err := client.PollJob(jobUUID, 10); err != nil { + if _, err := client.PollJob(ctx, jobUUID, 10*time.Second); err != nil { log.Printf("poll offline job — %v", err) } } } // deleteAndConfirmClone deletes the clone volume and confirms it is gone. -func deleteAndConfirmClone(client *ontapclient.Client, cloneUUID, cloneName, destHost string) { +// The confirmation GET is the single source of truth: if the volume is already +// absent when the delete call errors, the function reports success rather than fataling. +func deleteAndConfirmClone(ctx context.Context, client *ontapclient.Client, cloneUUID, cloneName, destHost string) { log.Println("=== Phase E: Delete clone ===") - resp, err := client.Delete(fmt.Sprintf(volumePatchPath, cloneUUID)) + resp, err := client.Delete(ctx, fmt.Sprintf(volumeOpPathFmt, cloneUUID), map[string]string{"return_timeout": "120"}) if err != nil { - log.Printf("delete_clone — %v", err) - } else if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { - if _, err := client.PollJob(jobUUID, 10); err != nil { + // Volume may already be gone — confirm before declaring failure. + confirm, cErr := client.Get(ctx, "/storage/volumes", map[string]string{ + "fields": "name,uuid", + "uuid": cloneUUID, + "max_records": "1", + }) + if cErr == nil && ontapclient.NumRecords(confirm) == 0 { + log.Printf("=== CLEANUP COMPLETE — clone '%s' already removed from cluster %s ===", cloneName, destHost) + return + } + log.Fatalf("delete_clone: %v", err) + } + if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { + if _, err := client.PollJob(ctx, jobUUID, 10*time.Second); err != nil { log.Printf("poll delete job — %v", err) } } - confirm, err := client.Get("/storage/volumes", map[string]string{ + confirm, err := client.Get(ctx, "/storage/volumes", map[string]string{ "fields": "name,uuid", "uuid": cloneUUID, "max_records": "1", @@ -245,40 +280,6 @@ func deleteAndConfirmClone(client *ontapclient.Client, cloneUUID, cloneName, des } } -// loadDotEnv reads a .env file from the current directory and exports each -// KEY=VALUE pair as an environment variable (only if not already set). -// The file is gitignored — safe to store credentials there for local testing. -func loadDotEnv() { - data, err := os.ReadFile(".env") - if err != nil { - return - } - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - k, v, ok := strings.Cut(line, "=") - if !ok { - continue - } - if os.Getenv(strings.TrimSpace(k)) == "" { - _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) - } - } -} - -func mustEnv(key string) string { - if v := os.Getenv(key); v != "" { - return v - } - log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) - return "" -} - -func envOrDefault(key, defaultVal string) string { - if v := os.Getenv(key); v != "" { - return v - } - return defaultVal -} +func mustEnv(key string) string { return ontapclient.MustEnv(key) } +func envOrDefault(k, def string) string { return ontapclient.EnvOrDefault(k, def) } +func loadDotEnv() { ontapclient.LoadDotEnv() } diff --git a/go/snapmirror_provision_dest_managed/main.go b/go/snapmirror_provision_dest_managed/main.go index 68007bc..35932e9 100644 --- a/go/snapmirror_provision_dest_managed/main.go +++ b/go/snapmirror_provision_dest_managed/main.go @@ -41,9 +41,10 @@ package main import ( + "context" + "errors" "fmt" "log" - "os" "strings" "time" @@ -51,11 +52,9 @@ import ( ) const ( - pathStorageVolumes = "/storage/volumes" // NOSONAR - pathClusterPeers = "/cluster/peers" - pathSVMPeers = "/svm/peers" - keySVMName = "svm.name" - peerFields = "name,uuid,status.state" + pathClusterPeers = "/cluster/peers" + pathSVMPeers = "/svm/peers" + peerFields = "name,uuid,status.state" ) // smRelConfig groups SnapMirror relationship parameters to keep function signatures compact. @@ -63,27 +62,12 @@ type smRelConfig struct { destSVM, destVolume, sourceSVMAlias, sourceVolume, peerName, smPolicy string } -// --------------------------------------------------------------------------- -// USER INPUTS — fill in your values here before running -// --------------------------------------------------------------------------- -var inputs = map[string]string{ - "SOURCE_HOST": "", // set via SOURCE_HOST in go/.env or env var - "SOURCE_USER": "admin", - "SOURCE_PASS": "", // set via SOURCE_PASS in go/.env or env var - "SOURCE_SVM": "", // set via SOURCE_SVM in go/.env or env var - "SOURCE_VOLUME": "", // set via SOURCE_VOLUME in go/.env or env var - "DEST_HOST": "", // set via DEST_HOST in go/.env or env var - "DEST_USER": "admin", - "DEST_PASS": "", // set via DEST_PASS in go/.env or env var - "DEST_SVM": "", // set via DEST_SVM in go/.env or env var - "SM_POLICY": "Asynchronous", -} - // --------------------------------------------------------------------------- func main() { log.SetFlags(log.LstdFlags) loadDotEnv() + ctx := context.Background() sourceHost := mustEnv("SOURCE_HOST") sourceUser := envOrDefault("SOURCE_USER", "admin") @@ -106,12 +90,12 @@ func main() { // === Phase A: Source pre-flight === log.Println("=== Phase A: Source pre-flight ===") - srcVol := phaseASourcePreflight(src, sourceSVM, sourceVolume, sourceHost) - srcVolSize := fmt.Sprintf("%.0f", ontapclient.NestedFloat(srcVol, "space", "size")) + srcVol := phaseASourcePreflight(ctx, src, sourceSVM, sourceVolume, sourceHost) + srcVolSize := int64(ontapclient.NestedFloat(srcVol, "space", "size")) // === Phase B: Dest pre-flight === log.Println("=== Phase B: Dest pre-flight ===") - dstCluster, err := dst.Get("/cluster", map[string]string{"fields": "name,version"}) + dstCluster, err := dst.Get(ctx, "/cluster", map[string]string{"fields": "name,version"}) dieOnErr("get dest cluster", err) log.Printf("DEST CLUSTER | name=%s | ontap=%s", ontapclient.NestedStr(dstCluster, "name"), @@ -119,11 +103,13 @@ func main() { // === Phase B0: Cluster peer setup === log.Println("=== Phase B0: Cluster peer setup ===") - srcPeerName, peerName, dstPeerUUID := setupClusterPeer(src, dst, sourceSVM, destSVM) + srcPeerName, peerName, dstPeerUUID := setupClusterPeer(ctx, src, dst, sourceSVM, destSVM) - aggrResp, err := dst.Get("/storage/aggregates", map[string]string{ - "fields": "name,state", + aggrResp, err := dst.Get(ctx, "/storage/aggregates", map[string]string{ + "fields": "name,state,space.block_storage.available", + "state": "online", "max_records": "1", + "order_by": "space.block_storage.available desc", }) dieOnErr("get dest aggregate", err) aggrName := "" @@ -138,30 +124,35 @@ func main() { // === Phase B1: SVM peer setup === log.Println("=== Phase B1: SVM peer setup ===") - sourceSVMAlias := setupSVMPeer(src, dst, sourceSVM, destSVM, srcPeerName, peerName, dstPeerUUID) + sourceSVMAlias := setupSVMPeer(ctx, src, dst, sourceSVM, destSVM, srcPeerName, peerName, dstPeerUUID) // === Phase C: Dest volume setup === log.Println("=== Phase C: Dest volume setup ===") - _, err = dst.Post("/storage/volumes?return_timeout=120", map[string]interface{}{ + _, err = dst.Post(ctx, "/storage/volumes", map[string]string{"return_timeout": "120"}, map[string]interface{}{ "name": destVolume, "type": "dp", "svm": map[string]string{"name": destSVM}, "aggregates": []map[string]string{ {"name": aggrName}, }, - "space": map[string]string{"size": srcVolSize}, + "size": srcVolSize, }) if err != nil { - log.Printf("create_dest_volume — %v (skipped — may already exist)", err) + var apiErr *ontapclient.OntapApiError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "917927" { + log.Printf("create_dest_volume — volume already exists, skipping") + } else { + log.Fatalf("create_dest_volume: %v", err) + } } else { log.Printf("DEST VOLUME | created '%s' on aggregate '%s'", destVolume, aggrName) } - dstVolResp, err := dst.Get("/storage/volumes", map[string]string{ - "fields": "name,uuid,state,type", - "max_records": "1", - "name": destVolume, - keySVMName: destSVM, + dstVolResp, err := dst.Get(ctx, "/storage/volumes", map[string]string{ + "fields": "name,uuid,state,type", + "max_records": "1", + "name": destVolume, + ontapclient.KeySVMName: destSVM, }) dieOnErr("verify dest volume", err) dstVols := ontapclient.Records(dstVolResp) @@ -178,7 +169,7 @@ func main() { // === Phase D: Relationship setup === log.Println("=== Phase D: Relationship setup ===") - relUUID := phaseDSetupRelationship(src, dst, smRelConfig{ + relUUID := phaseDSetupRelationship(ctx, src, dst, smRelConfig{ destSVM: destSVM, destVolume: destVolume, sourceSVMAlias: sourceSVMAlias, sourceVolume: sourceVolume, peerName: peerName, smPolicy: smPolicy, @@ -186,13 +177,13 @@ func main() { // === Phase E: Convergence polling === log.Println("=== Phase E: Convergence polling ===") - if _, err := dst.WaitSnapmirrored(relUUID, 15, 1800); err != nil { + if _, err := dst.WaitSnapmirrored(ctx, relUUID, 15*time.Second, 30*time.Minute); err != nil { log.Fatalf("wait snapmirrored: %v", err) } // === Phase F: Final validation === log.Println("=== Phase F: Final validation ===") - final, err := dst.Get(fmt.Sprintf("/snapmirror/relationships/%s", relUUID), + final, err := dst.Get(ctx, fmt.Sprintf("/snapmirror/relationships/%s", relUUID), map[string]string{"fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name"}) dieOnErr("final validation", err) log.Printf("=== SNAPMIRROR PROVISION COMPLETE ===\n"+ @@ -211,18 +202,18 @@ func main() { } // phaseASourcePreflight verifies source cluster connectivity and validates the source volume. -func phaseASourcePreflight(src *ontapclient.Client, sourceSVM, sourceVolume, sourceHost string) map[string]interface{} { - srcCluster, err := src.Get("/cluster", map[string]string{"fields": "name,version"}) +func phaseASourcePreflight(ctx context.Context, src *ontapclient.Client, sourceSVM, sourceVolume, sourceHost string) map[string]interface{} { + srcCluster, err := src.Get(ctx, "/cluster", map[string]string{"fields": "name,version"}) dieOnErr("get source cluster", err) log.Printf("SOURCE CLUSTER | name=%s | ontap=%s", ontapclient.NestedStr(srcCluster, "name"), ontapclient.NestedStr(srcCluster, "version", "full")) - srcVolResp, err := src.Get("/storage/volumes", map[string]string{ - "fields": "name,uuid,state,type,space.size", - "max_records": "1", - "name": sourceVolume, - keySVMName: sourceSVM, + srcVolResp, err := src.Get(ctx, "/storage/volumes", map[string]string{ + "fields": "name,uuid,state,type,space.size", + "max_records": "1", + "name": sourceVolume, + ontapclient.KeySVMName: sourceSVM, }) dieOnErr("get source volume", err) if ontapclient.NumRecords(srcVolResp) == 0 { @@ -243,8 +234,8 @@ func phaseASourcePreflight(src *ontapclient.Client, sourceSVM, sourceVolume, sou } // getICLIFIPs returns intercluster LIF IP addresses from a cluster. -func getICLIFIPs(client *ontapclient.Client) []string { - resp, err := client.Get("/network/ip/interfaces", map[string]string{ +func getICLIFIPs(ctx context.Context, client *ontapclient.Client) []string { + resp, err := client.Get(ctx, "/network/ip/interfaces", map[string]string{ "fields": "name,ip.address,services", "max_records": "50", }) @@ -310,10 +301,10 @@ func checkICLIFPreconditions(srcIPs, dstIPs []string) { // setupClusterPeer ensures a cluster peer exists; auto-creates if missing. // Returns (srcPeerName, dstPeerName, dstPeerUUID). -func setupClusterPeer(src, dst *ontapclient.Client, sourceSVM, destSVM string) (string, string, string) { +func setupClusterPeer(ctx context.Context, src, dst *ontapclient.Client, sourceSVM, destSVM string) (string, string, string) { okStates := map[string]bool{"available": true, "partial": true, "pending": true} - dstCP, err := dst.Get(pathClusterPeers, map[string]string{ + dstCP, err := dst.Get(ctx, pathClusterPeers, map[string]string{ "fields": peerFields, "max_records": "10", }) @@ -325,7 +316,7 @@ func setupClusterPeer(src, dst *ontapclient.Client, sourceSVM, destSVM string) ( continue } // Peer already exists - srcCP, err2 := src.Get(pathClusterPeers, map[string]string{ + srcCP, err2 := src.Get(ctx, pathClusterPeers, map[string]string{ "fields": peerFields, "max_records": "10", }) @@ -339,8 +330,8 @@ func setupClusterPeer(src, dst *ontapclient.Client, sourceSVM, destSVM string) ( break } } - srcIPs := getICLIFIPs(src) - dstIPs := getICLIFIPs(dst) + srcIPs := getICLIFIPs(ctx, src) + dstIPs := getICLIFIPs(ctx, dst) log.Printf("CLUSTER PEER | already peered — dst sees src as '%s' (state=%s) — skipping", ontapclient.NestedStr(p, "name"), state) log.Printf("IC LIFs | src=%v dst=%v", srcIPs, dstIPs) @@ -350,16 +341,16 @@ func setupClusterPeer(src, dst *ontapclient.Client, sourceSVM, destSVM string) ( // No existing peer — auto-create log.Println("CLUSTER PEER | no existing peer found — auto-creating") - srcIPs := getICLIFIPs(src) - dstIPs := getICLIFIPs(dst) + srcIPs := getICLIFIPs(ctx, src) + dstIPs := getICLIFIPs(ctx, dst) log.Printf("CLUSTER PEER | src IC LIFs=%v dst IC LIFs=%v", srcIPs, dstIPs) checkICLIFPreconditions(srcIPs, dstIPs) - return createNewClusterPeer(src, dst, srcIPs, dstIPs, sourceSVM, destSVM) + return createNewClusterPeer(ctx, src, dst, srcIPs, dstIPs, sourceSVM, destSVM) } // createNewClusterPeer posts a new cluster peer on both sides. // Returns (srcPeerName, dstPeerName, dstPeerUUID). -func createNewClusterPeer(src, dst *ontapclient.Client, srcIPs, dstIPs []string, sourceSVM, destSVM string) (string, string, string) { +func createNewClusterPeer(ctx context.Context, src, dst *ontapclient.Client, srcIPs, dstIPs []string, sourceSVM, destSVM string) (string, string, string) { if len(srcIPs) == 0 { log.Fatal("ABORTED — no intercluster LIFs found on source cluster.") } @@ -369,7 +360,7 @@ func createNewClusterPeer(src, dst *ontapclient.Client, srcIPs, dstIPs []string, peerAddrs := make([]string, len(dstIPs)) copy(peerAddrs, dstIPs) - srcResp, err := src.Post(pathClusterPeers, map[string]interface{}{ + srcResp, err := src.Post(ctx, pathClusterPeers, nil, map[string]interface{}{ "peer_addresses": peerAddrs, "generate_passphrase": true, "encryption": map[string]string{"proposed": "tls-psk"}, @@ -379,11 +370,14 @@ func createNewClusterPeer(src, dst *ontapclient.Client, srcIPs, dstIPs []string, }) dieOnErr("create cluster peer on source", err) passphrase, _ := srcResp["passphrase"].(string) + if passphrase == "" { + log.Fatal("ABORTED — cluster peer response missing passphrase field") + } log.Println("CLUSTER PEER | created on source") dstPeerAddrs := make([]string, len(srcIPs)) copy(dstPeerAddrs, srcIPs) - _, err = dst.Post(pathClusterPeers, map[string]interface{}{ + _, err = dst.Post(ctx, pathClusterPeers, nil, map[string]interface{}{ "peer_addresses": dstPeerAddrs, "passphrase": passphrase, "initial_allowed_svms": []map[string]string{ @@ -394,13 +388,13 @@ func createNewClusterPeer(src, dst *ontapclient.Client, srcIPs, dstIPs []string, log.Println("CLUSTER PEER | accepted on dest") time.Sleep(5 * time.Second) - return fetchCreatedPeerNames(src, dst) + return fetchCreatedPeerNames(ctx, src, dst) } // fetchCreatedPeerNames retrieves peer names from both clusters after creation. -func fetchCreatedPeerNames(src, dst *ontapclient.Client) (string, string, string) { +func fetchCreatedPeerNames(ctx context.Context, src, dst *ontapclient.Client) (string, string, string) { okStates := map[string]bool{"available": true, "partial": true, "pending": true} - dstCP, err := dst.Get(pathClusterPeers, map[string]string{"fields": peerFields, "max_records": "10"}) + dstCP, err := dst.Get(ctx, pathClusterPeers, map[string]string{"fields": peerFields, "max_records": "10"}) if err != nil { log.Fatalf("ABORTED — could not query cluster peers on destination: %v", err) } @@ -414,7 +408,7 @@ func fetchCreatedPeerNames(src, dst *ontapclient.Client) (string, string, string if len(dstPeer) == 0 { log.Fatal("ABORTED — no usable cluster peer found on destination after creation") } - srcCP, err := src.Get(pathClusterPeers, map[string]string{"fields": peerFields, "max_records": "10"}) + srcCP, err := src.Get(ctx, pathClusterPeers, map[string]string{"fields": peerFields, "max_records": "10"}) if err != nil { log.Fatalf("ABORTED — could not query cluster peers on source: %v", err) } @@ -435,15 +429,15 @@ func fetchCreatedPeerNames(src, dst *ontapclient.Client) (string, string, string } // grantSVMPeerPermission grants SnapMirror peer-permission on the source SVM. -func grantSVMPeerPermission(src *ontapclient.Client, sourceSVM, srcPeerName string) { - _, err := src.Post("/svm/peer-permissions", map[string]interface{}{ +func grantSVMPeerPermission(ctx context.Context, src *ontapclient.Client, sourceSVM, srcPeerName string) { + _, err := src.Post(ctx, "/svm/peer-permissions", nil, map[string]interface{}{ "svm": map[string]string{"name": sourceSVM}, "cluster_peer": map[string]string{"name": srcPeerName}, "applications": []string{"snapmirror"}, }) if err != nil { - s := err.Error() - if strings.Contains(s, "already exists") || strings.Contains(strings.ToLower(s), "duplicate") || strings.Contains(s, "13001") { + var apiErr *ontapclient.OntapApiError + if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "13001" || apiErr.ErrorCode() == "917927") { log.Println("SVM PEER | peer-permission already exists — skipping") return } @@ -453,8 +447,8 @@ func grantSVMPeerPermission(src *ontapclient.Client, sourceSVM, srcPeerName stri } // createSVMPeerRelationship creates the SVM peer relationship on the destination. -func createSVMPeerRelationship(dst *ontapclient.Client, destSVM, sourceSVM, dstPeerName string) { - resp, err := dst.Post(pathSVMPeers, map[string]interface{}{ +func createSVMPeerRelationship(ctx context.Context, dst *ontapclient.Client, destSVM, sourceSVM, dstPeerName string) { + resp, err := dst.Post(ctx, pathSVMPeers, nil, map[string]interface{}{ "svm": map[string]string{"name": destSVM}, "peer": map[string]interface{}{ "svm": map[string]string{"name": sourceSVM}, @@ -463,15 +457,15 @@ func createSVMPeerRelationship(dst *ontapclient.Client, destSVM, sourceSVM, dstP "applications": []string{"snapmirror"}, }) if err != nil { - s := err.Error() - if strings.Contains(s, "already exists") || strings.Contains(strings.ToLower(s), "duplicate") || strings.Contains(s, "13001") { + var apiErr *ontapclient.OntapApiError + if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "13001" || apiErr.ErrorCode() == "917927") { log.Println("SVM PEER | already exists — skipping") return } log.Fatalf("SVM PEER | create failed: %v", err) } if jobUUID := ontapclient.JobUUID(resp); jobUUID != "" { - if _, err := dst.PollJob(jobUUID, 10); err != nil { + if _, err := dst.PollJob(ctx, jobUUID, 10*time.Second); err != nil { log.Printf("poll svm peer job: %v", err) } } @@ -479,10 +473,10 @@ func createSVMPeerRelationship(dst *ontapclient.Client, destSVM, sourceSVM, dstP } // setupSVMPeer ensures SVM peer exists; returns the source SVM alias used in SnapMirror paths. -func setupSVMPeer(src, dst *ontapclient.Client, sourceSVM, destSVM, srcPeerName, dstPeerName, srcClusterPeerUUID string) string { - svmResp, err := dst.Get(pathSVMPeers, map[string]string{ - "fields": "uuid,name,state,peer", - keySVMName: destSVM, +func setupSVMPeer(ctx context.Context, src, dst *ontapclient.Client, sourceSVM, destSVM, srcPeerName, dstPeerName, srcClusterPeerUUID string) string { + svmResp, err := dst.Get(ctx, pathSVMPeers, map[string]string{ + "fields": "uuid,name,state,peer", + ontapclient.KeySVMName: destSVM, }) dieOnErr("get svm peers", err) @@ -503,12 +497,12 @@ func setupSVMPeer(src, dst *ontapclient.Client, sourceSVM, destSVM, srcPeerName, return alias } - grantSVMPeerPermission(src, sourceSVM, srcPeerName) - createSVMPeerRelationship(dst, destSVM, sourceSVM, dstPeerName) + grantSVMPeerPermission(ctx, src, sourceSVM, srcPeerName) + createSVMPeerRelationship(ctx, dst, destSVM, sourceSVM, dstPeerName) - svmResp2, err := dst.Get(pathSVMPeers, map[string]string{ - "fields": "uuid,name,state,peer", - keySVMName: destSVM, + svmResp2, err := dst.Get(ctx, pathSVMPeers, map[string]string{ + "fields": "uuid,name,state,peer", + ontapclient.KeySVMName: destSVM, }) if err != nil { return sourceSVM @@ -526,8 +520,8 @@ func setupSVMPeer(src, dst *ontapclient.Client, sourceSVM, destSVM, srcPeerName, } // phaseDSetupRelationship creates and initializes the SnapMirror relationship; returns its UUID. -func phaseDSetupRelationship(src, dst *ontapclient.Client, cfg smRelConfig) string { - existing, err := dst.Get("/snapmirror/relationships", map[string]string{ +func phaseDSetupRelationship(ctx context.Context, src, dst *ontapclient.Client, cfg smRelConfig) string { + existing, err := dst.Get(ctx, "/snapmirror/relationships", map[string]string{ "fields": "uuid,state,healthy", "destination.path": cfg.destSVM + ":" + cfg.destVolume, "max_records": "1", @@ -535,7 +529,7 @@ func phaseDSetupRelationship(src, dst *ontapclient.Client, cfg smRelConfig) stri dieOnErr("check existing relationship", err) log.Printf("RELATIONSHIP CHECK | existing=%d", ontapclient.NumRecords(existing)) - createResp, err := dst.Post("/snapmirror/relationships?return_timeout=120", map[string]interface{}{ + createResp, err := dst.Post(ctx, "/snapmirror/relationships", map[string]string{"return_timeout": "120"}, map[string]interface{}{ "source": map[string]interface{}{ "path": cfg.sourceSVMAlias + ":" + cfg.sourceVolume, "cluster": map[string]string{"name": cfg.peerName}, @@ -544,14 +538,19 @@ func phaseDSetupRelationship(src, dst *ontapclient.Client, cfg smRelConfig) stri "policy": map[string]string{"name": cfg.smPolicy}, }) if err != nil { - log.Printf("create_and_initialize_relationship — %v (may already exist)", err) + var apiErr *ontapclient.OntapApiError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "917927" { + log.Printf("create_and_initialize_relationship — relationship already exists, skipping create") + } else { + log.Fatalf("create_and_initialize_relationship: %v", err) + } } else if jobUUID := ontapclient.JobUUID(createResp); jobUUID != "" { - if _, err := dst.PollJob(jobUUID, 10); err != nil { + if _, err := dst.PollJob(ctx, jobUUID, 10*time.Second); err != nil { log.Printf("poll create job — %v", err) } } - relResp, err := dst.Get("/snapmirror/relationships", map[string]string{ + relResp, err := dst.Get(ctx, "/snapmirror/relationships", map[string]string{ "fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name", "destination.path": cfg.destSVM + ":" + cfg.destVolume, "max_records": "1", @@ -569,67 +568,23 @@ func phaseDSetupRelationship(src, dst *ontapclient.Client, cfg smRelConfig) stri rel["healthy"], ontapclient.NestedStr(rel, "policy", "name")) - _, err = dst.Post(fmt.Sprintf("/snapmirror/relationships/%s/transfers?return_timeout=120", relUUID), map[string]interface{}{}) + _, err = dst.Post(ctx, fmt.Sprintf("/snapmirror/relationships/%s/transfers", relUUID), map[string]string{"return_timeout": "120"}, map[string]interface{}{}) if err != nil { - s := err.Error() - if strings.Contains(s, "13303812") { - srcIPs := getICLIFIPs(src) - dstIPs := getICLIFIPs(dst) + var apiErr *ontapclient.OntapApiError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "13303812" { + srcIPs := getICLIFIPs(ctx, src) + dstIPs := getICLIFIPs(ctx, dst) log.Fatalf("ABORTED — SnapMirror initialize failed: intercluster LIF connectivity issue.\n"+ - " Error : %s\n src IC : %v\n dst IC : %v\n"+ + " Error : %v\n src IC : %v\n dst IC : %v\n"+ " Cause : TCP ports 11104/11105 are likely blocked between these IPs.", - s, srcIPs, dstIPs) + err, srcIPs, dstIPs) } log.Printf("initialize_relationship — %v (may already be initialized)", err) } return relUUID } -func mustEnv(key string) string { - if v := inputs[key]; v != "" { - return v - } - if v := os.Getenv(key); v != "" { - return v - } - log.Fatalf("'%s' is required — set it in the INPUTS block at the top of this file", key) - return "" -} - -func envOrDefault(key, defaultVal string) string { - if v := inputs[key]; v != "" { - return v - } - if v := os.Getenv(key); v != "" { - return v - } - return defaultVal -} - -func dieOnErr(context string, err error) { - if err != nil { - log.Fatalf("%s: %v", context, err) - } -} - -// loadDotEnv reads go/.env and sets each KEY=VALUE as an env var (if not already set). -// Equivalent to Python's os.environ — credentials stay out of source code. -func loadDotEnv() { - data, err := os.ReadFile(".env") - if err != nil { - return - } - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - k, v, ok := strings.Cut(line, "=") - if !ok { - continue - } - if os.Getenv(strings.TrimSpace(k)) == "" { - _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) - } - } -} +func mustEnv(key string) string { return ontapclient.MustEnv(key) } +func envOrDefault(k, def string) string { return ontapclient.EnvOrDefault(k, def) } +func dieOnErr(op string, err error) { ontapclient.DieOnErr(op, err) } +func loadDotEnv() { ontapclient.LoadDotEnv() } diff --git a/go/snapmirror_provision_src_managed/main.go b/go/snapmirror_provision_src_managed/main.go index cc8d013..b3803fd 100644 --- a/go/snapmirror_provision_src_managed/main.go +++ b/go/snapmirror_provision_src_managed/main.go @@ -37,24 +37,21 @@ package main import ( + "context" + "errors" "fmt" "log" - "os" - "strings" + "time" ontapclient "github.com/netapp/pace/go/ontapclient" ) -const ( - pathStorageVolumes = "/storage/volumes" // NOSONAR - keySVMName = "svm.name" -) - // --------------------------------------------------------------------------- func main() { log.SetFlags(log.LstdFlags) loadDotEnv() + ctx := context.Background() sourceHost := mustEnv("SOURCE_HOST") sourceUser := envOrDefault("SOURCE_USER", "admin") @@ -76,42 +73,40 @@ func main() { defer dst.Close() log.Println("=== Phase A: Source pre-flight ===") - srcVolSize, srcVol := smSrcPhaseA(src, sourceSVM, sourceVolume, sourceHost) + srcVolSize := smSrcPhaseA(ctx, src, sourceSVM, sourceVolume, sourceHost) log.Println("=== Phase B: Dest pre-flight ===") - peerName, aggrName := smSrcPhaseB(dst) + peerName, aggrName := smSrcPhaseB(ctx, dst) log.Println("=== Phase C: Dest volume setup ===") - smSrcPhaseC(dst, destSVM, destVolume, aggrName, srcVolSize) + smSrcPhaseC(ctx, dst, destSVM, destVolume, aggrName, srcVolSize) log.Println("=== Phase D: Relationship setup ===") - relUUID := smSrcPhaseD(dst, sourceSVM, sourceVolume, destSVM, destVolume, peerName, smPolicy) + relUUID := smSrcPhaseD(ctx, dst, sourceSVM, sourceVolume, destSVM, destVolume, peerName, smPolicy) log.Println("=== Phase E: Convergence polling ===") - if _, err := dst.WaitSnapmirrored(relUUID, 15, 1800); err != nil { + if _, err := dst.WaitSnapmirrored(ctx, relUUID, 15*time.Second, 30*time.Minute); err != nil { log.Fatalf("wait snapmirrored: %v", err) } log.Println("=== Phase F: Final validation ===") - smSrcPhaseF(dst, relUUID, sourceSVM, sourceVolume, destSVM, destVolume) - - _ = srcVol // used via srcVolSize + smSrcPhaseF(ctx, dst, relUUID, sourceSVM, sourceVolume, destSVM, destVolume) } // smSrcPhaseA verifies the source cluster and validates the source volume. -// Returns (srcVolSize string, srcVol record). -func smSrcPhaseA(src *ontapclient.Client, sourceSVM, sourceVolume, sourceHost string) (string, map[string]interface{}) { - srcCluster, err := src.Get("/cluster", map[string]string{"fields": "name,version"}) +// Returns srcVolSize as int64 bytes, ready to pass directly to the ONTAP size field. +func smSrcPhaseA(ctx context.Context, src *ontapclient.Client, sourceSVM, sourceVolume, sourceHost string) int64 { + srcCluster, err := src.Get(ctx, "/cluster", map[string]string{"fields": "name,version"}) dieOnErr("get source cluster", err) log.Printf("SOURCE CLUSTER | name=%s | ontap=%s", ontapclient.NestedStr(srcCluster, "name"), ontapclient.NestedStr(srcCluster, "version", "full")) - srcVolResp, err := src.Get(pathStorageVolumes, map[string]string{ - "fields": "name,uuid,state,type,space.size", - "max_records": "1", - "name": sourceVolume, - keySVMName: sourceSVM, + srcVolResp, err := src.Get(ctx, ontapclient.PathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,type,space.size", + "max_records": "1", + "name": sourceVolume, + ontapclient.KeySVMName: sourceSVM, }) dieOnErr("get source volume", err) if ontapclient.NumRecords(srcVolResp) == 0 { @@ -121,28 +116,29 @@ func smSrcPhaseA(src *ontapclient.Client, sourceSVM, sourceVolume, sourceHost st if ontapclient.NestedStr(srcVol, "type") == "dp" { log.Fatal("ABORTED — source volume is type=dp; specify the RW volume") } - srcVolSize := fmt.Sprintf("%.0f", ontapclient.NestedFloat(srcVol, "space", "size")) - log.Printf("SOURCE VOLUME | name=%s | uuid=%s | state=%s | type=%s | size=%s", + srcVolSize := int64(ontapclient.NestedFloat(srcVol, "space", "size")) + log.Printf("SOURCE VOLUME | name=%s | uuid=%s | state=%s | type=%s | size=%d", ontapclient.NestedStr(srcVol, "name"), ontapclient.NestedStr(srcVol, "uuid"), ontapclient.NestedStr(srcVol, "state"), ontapclient.NestedStr(srcVol, "type"), srcVolSize) - return srcVolSize, srcVol + return srcVolSize } // smSrcPhaseB verifies the dest cluster, fetches peer name and best aggregate. // Returns (peerName, aggrName). -func smSrcPhaseB(dst *ontapclient.Client) (string, string) { - dstCluster, err := dst.Get("/cluster", map[string]string{"fields": "name,version"}) +func smSrcPhaseB(ctx context.Context, dst *ontapclient.Client) (string, string) { + dstCluster, err := dst.Get(ctx, "/cluster", map[string]string{"fields": "name,version"}) dieOnErr("get dest cluster", err) log.Printf("DEST CLUSTER | name=%s | ontap=%s", ontapclient.NestedStr(dstCluster, "name"), ontapclient.NestedStr(dstCluster, "version", "full")) - peerResp, err := dst.Get("/cluster/peers", map[string]string{ - "fields": "name,status.state", - "max_records": "1", + peerResp, err := dst.Get(ctx, "/cluster/peers", map[string]string{ + "fields": "name,status.state", + "status.state": "available", + "max_records": "1", }) dieOnErr("get cluster peers", err) peerName := "" @@ -150,11 +146,11 @@ func smSrcPhaseB(dst *ontapclient.Client) (string, string) { peerName = ontapclient.NestedStr(peers[0], "name") } if peerName == "" { - log.Fatal("ABORTED — no cluster peer found on destination cluster; run snapmirror_peer_setup first") + log.Fatal("ABORTED — no available cluster peer found on destination cluster; run snapmirror_peer_setup first") } log.Printf("CLUSTER PEER | name=%s", peerName) - aggrResp, err := dst.Get("/storage/aggregates", map[string]string{ + aggrResp, err := dst.Get(ctx, "/storage/aggregates", map[string]string{ "fields": "name,space.block_storage.available", "state": "online", "max_records": "1", @@ -173,17 +169,17 @@ func smSrcPhaseB(dst *ontapclient.Client) (string, string) { } // smSrcPhaseC ensures the dest DP volume exists, creating it if needed. -func smSrcPhaseC(dst *ontapclient.Client, destSVM, destVolume, aggrName, srcVolSize string) { - checkDest, err := dst.Get(pathStorageVolumes, map[string]string{ - "fields": "name,uuid,state,type", - "max_records": "1", - "name": destVolume, - keySVMName: destSVM, +func smSrcPhaseC(ctx context.Context, dst *ontapclient.Client, destSVM, destVolume, aggrName string, srcVolSize int64) { + checkDest, err := dst.Get(ctx, ontapclient.PathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,type", + "max_records": "1", + "name": destVolume, + ontapclient.KeySVMName: destSVM, }) dieOnErr("check dest volume", err) if ontapclient.NumRecords(checkDest) == 0 { log.Printf("Creating dest DP volume '%s' on '%s'…", destVolume, aggrName) - _, err = dst.Post(pathStorageVolumes+"?return_timeout=120", map[string]interface{}{ + _, err = dst.Post(ctx, ontapclient.PathStorageVolumes, map[string]string{"return_timeout": "120"}, map[string]interface{}{ "name": destVolume, "type": "dp", "svm": map[string]string{"name": destSVM}, @@ -193,17 +189,22 @@ func smSrcPhaseC(dst *ontapclient.Client, destSVM, destVolume, aggrName, srcVolS "size": srcVolSize, }) if err != nil { - log.Printf("create_dest_volume — %v (may already exist)", err) + var apiErr *ontapclient.OntapApiError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "917927" { + log.Printf("create_dest_volume — volume already exists, skipping") + } else { + log.Fatalf("create_dest_volume: %v", err) + } } } else { log.Printf("Dest volume '%s' already exists — skipping create", destVolume) } - dstVolResp, err := dst.Get(pathStorageVolumes, map[string]string{ - "fields": "name,uuid,state,type", - "max_records": "1", - "name": destVolume, - keySVMName: destSVM, + dstVolResp, err := dst.Get(ctx, ontapclient.PathStorageVolumes, map[string]string{ + "fields": "name,uuid,state,type", + "max_records": "1", + "name": destVolume, + ontapclient.KeySVMName: destSVM, }) dieOnErr("verify dest volume", err) vols := ontapclient.Records(dstVolResp) @@ -219,32 +220,37 @@ func smSrcPhaseC(dst *ontapclient.Client, destSVM, destVolume, aggrName, srcVolS } // smSrcPhaseD creates and initializes the SnapMirror relationship; returns the relationship UUID. -func smSrcPhaseD(dst *ontapclient.Client, sourceSVM, sourceVolume, destSVM, destVolume, peerName, smPolicy string) string { - existing, err := dst.Get("/snapmirror/relationships", map[string]string{ +func smSrcPhaseD(ctx context.Context, dst *ontapclient.Client, sourceSVM, sourceVolume, destSVM, destVolume, peerName, smPolicy string) string { + existing, err := dst.Get(ctx, "/snapmirror/relationships", map[string]string{ "fields": "uuid,state,healthy", "destination.path": destSVM + ":" + destVolume, "max_records": "1", }) dieOnErr("check existing relationship", err) - log.Printf("RELATIONSHIP CHECK | existing=%d", ontapclient.NumRecords(existing)) - createResp, err := dst.Post("/snapmirror/relationships?return_timeout=120", map[string]interface{}{ - "source": map[string]interface{}{ - "path": sourceSVM + ":" + sourceVolume, - "cluster": map[string]string{"name": peerName}, - }, - "destination": map[string]string{"path": destSVM + ":" + destVolume}, - "policy": map[string]string{"name": smPolicy}, - }) - if err != nil { - log.Printf("create_and_initialize_relationship — %v (may already exist)", err) - } else if jobUUID := ontapclient.JobUUID(createResp); jobUUID != "" { - if _, err := dst.PollJob(jobUUID, 10); err != nil { - log.Printf("poll create job — %v", err) + if ontapclient.NumRecords(existing) == 0 { + createResp, err := dst.Post(ctx, "/snapmirror/relationships", map[string]string{"return_timeout": "120"}, map[string]interface{}{ + "source": map[string]interface{}{ + "path": sourceSVM + ":" + sourceVolume, + "cluster": map[string]string{"name": peerName}, + }, + "destination": map[string]string{"path": destSVM + ":" + destVolume}, + "policy": map[string]string{"name": smPolicy}, + }) + if err != nil { + log.Fatalf("create_and_initialize_relationship: %v", err) } + if jobUUID := ontapclient.JobUUID(createResp); jobUUID != "" { + if _, err := dst.PollJob(ctx, jobUUID, 10*time.Second); err != nil { + log.Printf("poll create job — %v", err) + } + } + log.Println("RELATIONSHIP | created") + } else { + log.Println("RELATIONSHIP | already exists — skipping create") } - relResp, err := dst.Get("/snapmirror/relationships", map[string]string{ + relResp, err := dst.Get(ctx, "/snapmirror/relationships", map[string]string{ "fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name", "destination.path": destSVM + ":" + destVolume, "max_records": "1", @@ -259,16 +265,23 @@ func smSrcPhaseD(dst *ontapclient.Client, sourceSVM, sourceVolume, destSVM, dest log.Printf("RELATIONSHIP FOUND | uuid=%s | state=%s | healthy=%v", relUUID, ontapclient.NestedStr(rel, "state"), rel["healthy"]) - _, err = dst.Post(fmt.Sprintf("/snapmirror/relationships/%s/transfers?return_timeout=120", relUUID), map[string]interface{}{}) + _, err = dst.Post(ctx, fmt.Sprintf("/snapmirror/relationships/%s/transfers", relUUID), map[string]string{"return_timeout": "120"}, map[string]interface{}{}) if err != nil { + var apiErr *ontapclient.OntapApiError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "13303812" { + log.Fatalf("ABORTED — SnapMirror initialize failed: intercluster LIF connectivity issue.\n"+ + " Error : %v\n"+ + " Cause : TCP ports 11104/11105 are likely blocked between the source and dest IC LIFs.", + err) + } log.Printf("initialize_relationship — %v (may already be initialized)", err) } return relUUID } // smSrcPhaseF prints the final validation report. -func smSrcPhaseF(dst *ontapclient.Client, relUUID, sourceSVM, sourceVolume, destSVM, destVolume string) { - final, err := dst.Get(fmt.Sprintf("/snapmirror/relationships/%s", relUUID), +func smSrcPhaseF(ctx context.Context, dst *ontapclient.Client, relUUID, sourceSVM, sourceVolume, destSVM, destVolume string) { + final, err := dst.Get(ctx, fmt.Sprintf("/snapmirror/relationships/%s", relUUID), map[string]string{"fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name"}) dieOnErr("final validation", err) log.Printf("=== SNAPMIRROR PROVISION COMPLETE ===\n"+ @@ -286,49 +299,7 @@ func smSrcPhaseF(dst *ontapclient.Client, relUUID, sourceSVM, sourceVolume, dest final["lag_time"]) } -// mustEnv reads an environment variable and exits if it is not set. -func mustEnv(key string) string { - if v := os.Getenv(key); v != "" { - return v - } - log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) - return "" -} - -// envOrDefault reads an environment variable, returning defaultVal if unset. -func envOrDefault(key, defaultVal string) string { - if v := os.Getenv(key); v != "" { - return v - } - return defaultVal -} - -// dieOnErr logs a fatal error if err is non-nil. -func dieOnErr(context string, err error) { - if err != nil { - log.Fatalf("%s: %v", context, err) - } -} - -// loadDotEnv reads a .env file from the current directory and exports each -// KEY=VALUE pair as an environment variable (only if not already set). -// The file is gitignored — safe to store credentials there for local testing. -func loadDotEnv() { - data, err := os.ReadFile(".env") - if err != nil { - return - } - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - k, v, ok := strings.Cut(line, "=") - if !ok { - continue - } - if os.Getenv(strings.TrimSpace(k)) == "" { - _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) - } - } -} +func mustEnv(key string) string { return ontapclient.MustEnv(key) } +func envOrDefault(k, def string) string { return ontapclient.EnvOrDefault(k, def) } +func dieOnErr(op string, err error) { ontapclient.DieOnErr(op, err) } +func loadDotEnv() { ontapclient.LoadDotEnv() } diff --git a/go/snapmirror_test_failover/main.go b/go/snapmirror_test_failover/main.go index 316a536..d529b68 100644 --- a/go/snapmirror_test_failover/main.go +++ b/go/snapmirror_test_failover/main.go @@ -38,21 +38,21 @@ package main import ( + "context" + "errors" "fmt" "log" - "os" - "strings" + "time" ontapclient "github.com/netapp/pace/go/ontapclient" ) -const pathStorageVolumes = "/storage/volumes" // NOSONAR - // --------------------------------------------------------------------------- func main() { log.SetFlags(log.LstdFlags) loadDotEnv() + ctx := context.Background() clusterA := mustEnv("CLUSTER_A") clusterB := mustEnv("CLUSTER_B") @@ -61,7 +61,7 @@ func main() { sourceVolume := envOrDefault("SOURCE_VOLUME", "*") log.Println("=== Phase 0: Auto-detect target cluster ===") - destHost, dpVol := pickCluster(clusterA, clusterB, destUser, destPass, sourceVolume) + destHost, dpVol := pickCluster(ctx, clusterA, clusterB, destUser, destPass, sourceVolume) dpVolName := ontapclient.NestedStr(dpVol, "name") dpSVMName := ontapclient.NestedStr(dpVol, "svm", "name") dpVolUUID := ontapclient.NestedStr(dpVol, "uuid") @@ -74,30 +74,30 @@ func main() { defer client.Close() log.Println("=== Phase A: Pre-flight ===") - relUUID := tfPhaseA(client, dpSVMName, dpVolName) + relUUID := tfPhaseA(ctx, client, dpSVMName, dpVolName) log.Println("=== Phase B: Get latest SnapMirror snapshot ===") - snapshotName := tfPhaseB(client, dpVolUUID, dpVolName) + snapshotName := tfPhaseB(ctx, client, dpVolUUID, dpVolName) log.Println("=== Phase C: Create FlexClone ===") - cloneName, cloneUUID := tfPhaseC(client, dpVolName, dpSVMName, snapshotName) + cloneName, cloneUUID := tfPhaseC(ctx, client, dpVolName, dpSVMName, snapshotName) log.Println("=== Phase D: Verify clone + tag ===") - tfPhaseD(client, cloneName, cloneUUID, relUUID, dpSVMName, snapshotName) + tfPhaseD(ctx, client, cloneName, cloneUUID, relUUID, dpSVMName, snapshotName) log.Println("=== Phase E: Resync SnapMirror ===") - tfPhaseE(client, relUUID) + tfPhaseE(ctx, client, relUUID) } // tfPhaseA verifies cluster connectivity and fetches the SnapMirror relationship UUID. -func tfPhaseA(client *ontapclient.Client, dpSVMName, dpVolName string) string { - cluster, err := client.Get("/cluster", map[string]string{"fields": "name,version"}) +func tfPhaseA(ctx context.Context, client *ontapclient.Client, dpSVMName, dpVolName string) string { + cluster, err := client.Get(ctx, "/cluster", map[string]string{"fields": "name,version"}) dieOnErr("get cluster", err) log.Printf("DEST CLUSTER | name=%s | ontap=%s", ontapclient.NestedStr(cluster, "name"), ontapclient.NestedStr(cluster, "version", "full")) - relResp, err := client.Get("/snapmirror/relationships", map[string]string{ + relResp, err := client.Get(ctx, "/snapmirror/relationships", map[string]string{ "fields": "uuid,source.path,destination.path,state,lag_time,healthy,policy.name", "destination.path": dpSVMName + ":" + dpVolName, "max_records": "1", @@ -119,8 +119,8 @@ func tfPhaseA(client *ontapclient.Client, dpSVMName, dpVolName string) string { } // tfPhaseB fetches the latest SnapMirror snapshot name from the DP volume. -func tfPhaseB(client *ontapclient.Client, dpVolUUID, dpVolName string) string { - snapResp, err := client.Get(fmt.Sprintf("/storage/volumes/%s/snapshots", dpVolUUID), map[string]string{ +func tfPhaseB(ctx context.Context, client *ontapclient.Client, dpVolUUID, dpVolName string) string { + snapResp, err := client.Get(ctx, fmt.Sprintf("/storage/volumes/%s/snapshots", dpVolUUID), map[string]string{ "fields": "name,create_time", "max_records": "1", "order_by": "create_time desc", @@ -136,9 +136,9 @@ func tfPhaseB(client *ontapclient.Client, dpVolUUID, dpVolName string) string { } // tfPhaseC creates the writable FlexClone; returns (cloneName, cloneUUID). -func tfPhaseC(client *ontapclient.Client, dpVolName, dpSVMName, snapshotName string) (string, string) { +func tfPhaseC(ctx context.Context, client *ontapclient.Client, dpVolName, dpSVMName, snapshotName string) (string, string) { cloneName := dpVolName + "_clone" - cloneResp, err := client.Post("/storage/volumes?return_timeout=120", map[string]interface{}{ + cloneResp, err := client.Post(ctx, "/storage/volumes", map[string]string{"return_timeout": "120"}, map[string]interface{}{ "name": cloneName, "svm": map[string]string{"name": dpSVMName}, "nas": map[string]string{"path": "/" + cloneName}, @@ -149,14 +149,19 @@ func tfPhaseC(client *ontapclient.Client, dpVolName, dpSVMName, snapshotName str }, }) if err != nil { - log.Printf("create_test_clone — %v (may already exist)", err) + var apiErr *ontapclient.OntapApiError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "917927" { + log.Printf("create_test_clone — clone already exists, skipping create") + } else { + log.Fatalf("create_test_clone: %v", err) + } } else if jobUUID := ontapclient.JobUUID(cloneResp); jobUUID != "" { - if _, err := client.PollJob(jobUUID, 10); err != nil { + if _, err := client.PollJob(ctx, jobUUID, 10*time.Second); err != nil { log.Printf("poll clone job — %v", err) } } - cloneVolResp, err := client.Get(pathStorageVolumes, map[string]string{ + cloneVolResp, err := client.Get(ctx, ontapclient.PathStorageVolumes, map[string]string{ "fields": "name,uuid,state,nas.path,space.size", "max_records": "1", "name": cloneName, @@ -179,8 +184,9 @@ func tfPhaseC(client *ontapclient.Client, dpVolName, dpSVMName, snapshotName str } // tfPhaseD tags the clone and prints the test-failover-ready message. -func tfPhaseD(client *ontapclient.Client, cloneName, cloneUUID, relUUID, dpSVMName, snapshotName string) { - _, err := client.Patch(fmt.Sprintf("/storage/volumes/%s?return_timeout=120", cloneUUID), +func tfPhaseD(ctx context.Context, client *ontapclient.Client, cloneName, cloneUUID, relUUID, dpSVMName, snapshotName string) { + _, err := client.Patch(ctx, fmt.Sprintf("/storage/volumes/%s", cloneUUID), + map[string]string{"return_timeout": "120"}, map[string]interface{}{"_tags": []string{relUUID + ":test"}}) if err != nil { log.Printf("tag_clone_volume — %v", err) @@ -188,7 +194,7 @@ func tfPhaseD(client *ontapclient.Client, cloneName, cloneUUID, relUUID, dpSVMNa log.Printf("TAG APPLIED | clone=%s | tag=%s:test", cloneName, relUUID) } - cloneVolResp, err := client.Get(pathStorageVolumes, map[string]string{ + cloneVolResp, err := client.Get(ctx, ontapclient.PathStorageVolumes, map[string]string{ "fields": "name,uuid,state,nas.path", "max_records": "1", "name": cloneName, @@ -214,90 +220,58 @@ func tfPhaseD(client *ontapclient.Client, cloneName, cloneUUID, relUUID, dpSVMNa } // tfPhaseE resyncs the SnapMirror relationship and waits for snapmirrored state. -func tfPhaseE(client *ontapclient.Client, relUUID string) { - resyncResp, err := client.Patch(fmt.Sprintf("/snapmirror/relationships/%s?return_timeout=120", relUUID), +func tfPhaseE(ctx context.Context, client *ontapclient.Client, relUUID string) { + resyncResp, err := client.Patch(ctx, fmt.Sprintf("/snapmirror/relationships/%s", relUUID), + map[string]string{"return_timeout": "120"}, map[string]interface{}{"state": "snapmirrored"}) if err != nil { log.Printf("resync_sm_relationship — %v", err) } else if jobUUID := ontapclient.JobUUID(resyncResp); jobUUID != "" { - if _, err := client.PollJob(jobUUID, 10); err != nil { + if _, err := client.PollJob(ctx, jobUUID, 10*time.Second); err != nil { log.Printf("poll resync job — %v", err) } } - if _, err := client.WaitSnapmirrored(relUUID, 15, 1800); err != nil { + if _, err := client.WaitSnapmirrored(ctx, relUUID, 15*time.Second, 30*time.Minute); err != nil { log.Fatalf("wait snapmirrored: %v", err) } log.Println("=== TEST FAILOVER COMPLETE — SnapMirror resynced ===") } // pickCluster finds which cluster has the target DP volume; returns (clusterIP, volRecord). -func pickCluster(clusterA, clusterB, user, passwd, volNameFilter string) (string, map[string]interface{}) { +func pickCluster(ctx context.Context, clusterA, clusterB, user, passwd, volNameFilter string) (string, map[string]interface{}) { destFilter := volNameFilter + "_dest" if volNameFilter == "*" { destFilter = "*_dest" } - for _, host := range []string{clusterA, clusterB} { - client := ontapclient.New(host, user, passwd, false) - resp, err := client.Get(pathStorageVolumes, map[string]string{ + tryHost := func(host string) (map[string]interface{}, bool) { + c := ontapclient.New(host, user, passwd, false) + defer c.Close() + resp, err := c.Get(ctx, ontapclient.PathStorageVolumes, map[string]string{ "fields": "name,create_time,uuid,svm.name,state,space.size", "type": "dp", "name": destFilter, "order_by": "create_time desc", "max_records": "1", }) - client.Close() if err != nil { log.Printf(" cluster %s — %v", host, err) - continue + return nil, false } if ontapclient.NumRecords(resp) >= 1 { - return host, ontapclient.Records(resp)[0] + return ontapclient.Records(resp)[0], true + } + return nil, false + } + for _, host := range []string{clusterA, clusterB} { + if vol, ok := tryHost(host); ok { + return host, vol } } log.Fatalf("No DP volumes found on either cluster (%s, %s)", clusterA, clusterB) return "", nil } -func mustEnv(key string) string { - if v := os.Getenv(key); v != "" { - return v - } - log.Fatalf("'%s' is required — set it in go/.env or as an environment variable", key) - return "" -} - -func envOrDefault(key, defaultVal string) string { - if v := os.Getenv(key); v != "" { - return v - } - return defaultVal -} - -func dieOnErr(context string, err error) { - if err != nil { - log.Fatalf("%s: %v", context, err) - } -} - -// loadDotEnv reads a .env file from the current directory and exports each -// KEY=VALUE pair as an environment variable (only if not already set). -// The file is gitignored — safe to store credentials there for local testing. -func loadDotEnv() { - data, err := os.ReadFile(".env") - if err != nil { - return - } - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - k, v, ok := strings.Cut(line, "=") - if !ok { - continue - } - if os.Getenv(strings.TrimSpace(k)) == "" { - _ = os.Setenv(strings.TrimSpace(k), strings.TrimSpace(v)) - } - } -} +func mustEnv(key string) string { return ontapclient.MustEnv(key) } +func envOrDefault(k, def string) string { return ontapclient.EnvOrDefault(k, def) } +func dieOnErr(op string, err error) { ontapclient.DieOnErr(op, err) } +func loadDotEnv() { ontapclient.LoadDotEnv() } From c7c1d26a2f1d25e0ebd30c1b96ce788115ebd080 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Jun 2026 11:05:24 +0000 Subject: [PATCH 4/4] ci: add go to commitlint scope-enum --- commitlint.config.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/commitlint.config.mjs b/commitlint.config.mjs index 694b461..7dea85b 100644 --- a/commitlint.config.mjs +++ b/commitlint.config.mjs @@ -12,6 +12,7 @@ export default { 'ci', 'deps', 'docs', + 'go', 'python', 'terraform', ],