diff --git a/go.mod b/go.mod index cac379c6..79720426 100644 --- a/go.mod +++ b/go.mod @@ -66,3 +66,5 @@ replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.21.1 replace k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.21.1 replace k8s.io/sample-controller => k8s.io/sample-controller v0.21.1 + +replace github.com/kubernetes-csi/csi-lib-iscsi => ./pkg/lib/iscsi diff --git a/go.sum b/go.sum index c4211a16..9745ba6c 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,6 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubernetes-csi/csi-lib-iscsi v0.0.0-20211110090527-5c802c48a124 h1:cajnZqFO7WAGwKHNzS4RXKJOLiB6RbN5VMVB38FYxcc= -github.com/kubernetes-csi/csi-lib-iscsi v0.0.0-20211110090527-5c802c48a124/go.mod h1:c/keGS6bErOzLrFyNgafdDWT6h72v2XQiA/p2R7yghU= github.com/kubernetes-csi/csi-lib-utils v0.10.0 h1:Aqm8X81eCzzfH/bvIEqSWtcbK9HF9NbFk4d+le1snVA= github.com/kubernetes-csi/csi-lib-utils v0.10.0/go.mod h1:BmGZZB16L18+9+Lgg9YWwBKfNEHIDdgGfAyuW6p2NV0= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= diff --git a/pkg/lib/iscsi/.gitignore b/pkg/lib/iscsi/.gitignore new file mode 100644 index 00000000..a7d4f9a5 --- /dev/null +++ b/pkg/lib/iscsi/.gitignore @@ -0,0 +1,3 @@ +*.out +_output +example/example diff --git a/pkg/lib/iscsi/Makefile b/pkg/lib/iscsi/Makefile new file mode 100644 index 00000000..9e3fde2d --- /dev/null +++ b/pkg/lib/iscsi/Makefile @@ -0,0 +1,21 @@ +.PHONY: all build clean install test coverage + +all: clean build install + +clean: + go clean -r -x + -rm -rf _output + +build: + go build ./iscsi/ + go build -o _output/example ./example/main.go + +install: + go install ./iscsi/ + +test: + go test ./iscsi/ + +coverage: + go test ./iscsi -coverprofile=coverage.out + go tool cover -html=coverage.out diff --git a/pkg/lib/iscsi/OWNERS b/pkg/lib/iscsi/OWNERS new file mode 100644 index 00000000..c7f62cbf --- /dev/null +++ b/pkg/lib/iscsi/OWNERS @@ -0,0 +1,8 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +approvers: +- saad-ali +- j-griffith +reviews: +- saad-ali +- j-griffith diff --git a/pkg/lib/iscsi/README.md b/pkg/lib/iscsi/README.md new file mode 100644 index 00000000..b4ae426d --- /dev/null +++ b/pkg/lib/iscsi/README.md @@ -0,0 +1,55 @@ +# csi lib-iscsi + +A simple go package intended to assist CSI plugin authors by providing a tool set to manage iscsi connections. + +## Goals + +Provide a basic, lightweight library for CSI Plugin Authors to leverage some of the common tasks like connecting +and disconnecting iscsi devices to a node. This library includes a high level abstraction for iscsi that exposes +simple Connect and Disconnect functions. These are built on top of exported iscsiadm calls, so if you need more +control you can access the iscsiadm calls directly. + +## Design Philosophy + +The idea is to keep this as lightweight and generic as possible. We intentionally avoid the use of any third party +libraries or packages in this project. We don't have a vendor directory, because we attempt to rely only on the std +golang libs. This may prove to not be ideal, and may be changed over time, but initially it's a worthwhile goal. + +## Logging and Debug + +By default the library does not provide any logging, but provides an error message that includes any messages from +iscsiadm as well as exit-codes. In the event that you need to debug the library, we provide a function: + +``` +func EnableDebugLogging(writer io.Writer) +``` + +This will turn on verbose logging directed to the provided io.Writer and include the response of every iscsiadm command +issued. + +## Intended Usage + +Currently, the intended usage of this library is simply to provide a golang +package to standardize how plugins are implementing +iscsi connect and disconnect. It's not intended to be a "service", +although that's a possible next step. It's currently been +used for plugins where iscsid is installed in containers only, as well as designs where it uses the nodes iscsid. Each of these +approaches has their own pros and cons. Currently, it's up to the plugin author to determine which model suits them best +and to deploy their node plugin appropriately. + +## Community, discussion, contribution, and support + +Learn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/). + +You can reach the maintainers of this project at: + +- [Slack](http://slack.k8s.io/) + * sig-storage +- [Mailing List](https://groups.google.com/forum/#!forum/kubernetes-dev) + +### Code of conduct + +Participation in the Kubernetes community is governed by the [Kubernetes Code of Conduct](code-of-conduct.md). + +[owners]: https://git.k8s.io/community/contributors/guide/owners.md +[Creative Commons 4.0]: https://git.k8s.io/website/LICENSE diff --git a/pkg/lib/iscsi/RELEASE.md b/pkg/lib/iscsi/RELEASE.md new file mode 100644 index 00000000..7274b344 --- /dev/null +++ b/pkg/lib/iscsi/RELEASE.md @@ -0,0 +1,9 @@ +# Release Process + +The Kubernetes Template Project is released on an as-needed basis. The process is as follows: + +1. An issue is proposing a new release with a changelog since the last release +1. All [OWNERS](OWNERS) must LGTM this release +1. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION` +1. The release issue is closed +1. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] kubernetes-template-project $VERSION is released` diff --git a/pkg/lib/iscsi/SECURITY_CONTACTS b/pkg/lib/iscsi/SECURITY_CONTACTS new file mode 100644 index 00000000..5da66e7d --- /dev/null +++ b/pkg/lib/iscsi/SECURITY_CONTACTS @@ -0,0 +1,14 @@ +# Defined below are the security contacts for this repo. +# +# They are the contact point for the Product Security Team to reach out +# to for triaging and handling of incoming issues. +# +# The below names agree to abide by the +# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) +# and will be removed and replaced if they violate that agreement. +# +# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE +# INSTRUCTIONS AT https://kubernetes.io/security/ + +childsb +saad-ali diff --git a/pkg/lib/iscsi/code-of-conduct.md b/pkg/lib/iscsi/code-of-conduct.md new file mode 100644 index 00000000..0d15c00c --- /dev/null +++ b/pkg/lib/iscsi/code-of-conduct.md @@ -0,0 +1,3 @@ +# Kubernetes Community Code of Conduct + +Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) diff --git a/pkg/lib/iscsi/example/main.go b/pkg/lib/iscsi/example/main.go new file mode 100644 index 00000000..c8b821d5 --- /dev/null +++ b/pkg/lib/iscsi/example/main.go @@ -0,0 +1,71 @@ +package main + +import ( + "flag" + "log" + "os" + "strings" + "time" + + "github.com/kubernetes-csi/csi-lib-iscsi/iscsi" +) + +var ( + portals = flag.String("portals", "192.168.1.112:3260", "Comma delimited. Eg: 1.1.1.1,2.2.2.2") + iqn = flag.String("iqn", "iqn.2010-10.org.openstack:volume-95739000-1557-44f8-9f40-e9d29fe6ec47", "") + username = flag.String("username", "3aX7EEf3CEgvESQG75qh", "") + password = flag.String("password", "eJBDC7Bt7WE3XFDq", "") + lun = flag.Int("lun", 1, "") + debug = flag.Bool("debug", false, "enable logging") +) + +func main() { + flag.Parse() + tgtps := strings.Split(*portals, ",") + if *debug { + iscsi.EnableDebugLogging(os.Stdout) + } + + // You can utilize the iscsiadm calls directly if you wish, but by creating a Connector + // you can simplify interactions to simple calls like "Connect" and "Disconnect" + c := &iscsi.Connector{ + // Our example uses chap + AuthType: "chap", + // List of targets must be >= 1 (>1 signals multipath/mpio) + TargetIqn: *iqn, + TargetPortals: tgtps, + // CHAP can be setup up for discovery as well as sessions, our example + // device only uses CHAP security for sessions, for those that use Discovery + // as well, we'd add a DiscoverySecrets entry the same way + SessionSecrets: iscsi.Secrets{ + UserName: *username, + Password: *password, + SecretsType: "chap"}, + // Lun is the lun number the devices uses for exports + Lun: int32(*lun), + // Number of times we check for device path, waiting for CheckInterval seconds in between each check (defaults to 10 if omitted) + RetryCount: 11, + // CheckInterval is the time in seconds to wait in between device path checks when logging in to a target + CheckInterval: 1, + } + + // Now we can just issue a connection request using our Connector + // A successful connection will include the device path to access our iscsi volume + path, err := c.Connect() + if err != nil { + log.Printf("Error returned from c.Connect: %s", err.Error()) + os.Exit(1) + } + + log.Printf("Connected device at path: %s\n", path) + time.Sleep(3 * time.Second) + + // This will disconnect the volume + if err := c.DisconnectVolume(); err != nil { + log.Printf("Error returned from c.DisconnectVolume: %s", err.Error()) + os.Exit(1) + } + + // This will disconnect the session as well as clear out the iscsi DB entries associated with it + c.Disconnect() +} diff --git a/pkg/lib/iscsi/go.mod b/pkg/lib/iscsi/go.mod new file mode 100644 index 00000000..c3584003 --- /dev/null +++ b/pkg/lib/iscsi/go.mod @@ -0,0 +1,8 @@ +module github.com/kubernetes-csi/csi-lib-iscsi + +go 1.15 + +require ( + github.com/prashantv/gostub v1.0.0 + github.com/stretchr/testify v1.7.0 +) diff --git a/pkg/lib/iscsi/go.sum b/pkg/lib/iscsi/go.sum new file mode 100644 index 00000000..f0297e0e --- /dev/null +++ b/pkg/lib/iscsi/go.sum @@ -0,0 +1,13 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prashantv/gostub v1.0.0 h1:wTzvgO04xSS3gHuz6Vhuo0/kvWelyJxwNS0IRBPAwGY= +github.com/prashantv/gostub v1.0.0/go.mod h1:dP1v6T1QzyGJJKFocwAU0lSZKpfjstjH8TlhkEU0on0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/lib/iscsi/iscsi/iscsi.go b/pkg/lib/iscsi/iscsi/iscsi.go new file mode 100644 index 00000000..465ca0bd --- /dev/null +++ b/pkg/lib/iscsi/iscsi/iscsi.go @@ -0,0 +1,801 @@ +package iscsi + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "syscall" + "time" +) + +const defaultPort = "3260" + +var ( + debug *log.Logger + execCommand = exec.Command + execCommandContext = exec.CommandContext + execWithTimeout = ExecWithTimeout + osStat = os.Stat + filepathGlob = filepath.Glob + osOpenFile = os.OpenFile + sleep = time.Sleep +) + +// iscsiSession contains information about an iSCSI session +type iscsiSession struct { + Protocol string + ID int32 + Portal string + IQN string + Name string +} + +type deviceInfo []Device + +// Device contains informations about a device +type Device struct { + Name string `json:"name"` + Hctl string `json:"hctl"` + Children []Device `json:"children"` + Type string `json:"type"` + Transport string `json:"tran"` + Size string `json:"size,omitempty"` +} + +type HCTL struct { + HBA int + Channel int + Target int + LUN int +} + +// Connector provides a struct to hold all of the needed parameters to make our iSCSI connection +type Connector struct { + VolumeName string `json:"volume_name"` + TargetIqn string `json:"target_iqn"` + TargetPortals []string `json:"target_portal"` + Lun int32 `json:"lun"` + AuthType string `json:"auth_type"` + DiscoverySecrets Secrets `json:"discovery_secrets"` + SessionSecrets Secrets `json:"session_secrets"` + Interface string `json:"interface"` + + MountTargetDevice *Device `json:"mount_target_device"` + Devices []Device `json:"devices"` + + RetryCount uint `json:"retry_count"` + CheckInterval uint `json:"check_interval"` + DoDiscovery bool `json:"do_discovery"` + DoCHAPDiscovery bool `json:"do_chap_discovery"` +} + +func init() { + // by default we don't log anything, EnableDebugLogging() can turn on some tracing + debug = log.New(ioutil.Discard, "", 0) +} + +// EnableDebugLogging provides a mechanism to turn on debug logging for this package +// output is written to the provided io.Writer +func EnableDebugLogging(writer io.Writer) { + debug = log.New(writer, "DEBUG: ", log.Ldate|log.Ltime|log.Lshortfile) +} + +// parseSession takes the raw stdout from the iscsiadm -m session command and encodes it into an iSCSI session type +func parseSessions(lines string) []iscsiSession { + entries := strings.Split(strings.TrimSpace(lines), "\n") + r := strings.NewReplacer("[", "", + "]", "") + + var sessions []iscsiSession + for _, entry := range entries { + e := strings.Fields(entry) + if len(e) < 4 { + continue + } + protocol := strings.Split(e[0], ":")[0] + id := r.Replace(e[1]) + id64, _ := strconv.ParseInt(id, 10, 32) + portal := strings.Split(e[2], ",")[0] + + s := iscsiSession{ + Protocol: protocol, + ID: int32(id64), + Portal: portal, + IQN: e[3], + Name: strings.Split(e[3], ":")[1], + } + sessions = append(sessions, s) + } + return sessions +} + +// sessionExists checks if an iSCSI session exists +func sessionExists(tgtPortal, tgtIQN string) (bool, error) { + sessions, err := getCurrentSessions() + if err != nil { + return false, err + } + for _, s := range sessions { + if tgtIQN == s.IQN && tgtPortal == s.Portal { + return true, nil + } + } + return false, nil +} + +// extractTransportName returns a transport_name from getCurrentSessions output +func extractTransportName(output string) string { + res := regexp.MustCompile(`iface.transport_name = (.*)\n`).FindStringSubmatch(output) + if res == nil { + return "" + } + if res[1] == "" { + return "tcp" + } + return res[1] +} + +// getCurrentSessions list current iSCSI sessions +func getCurrentSessions() ([]iscsiSession, error) { + out, err := GetSessions() + if err != nil { + exitErr, ok := err.(*exec.ExitError) + if ok && exitErr.ProcessState.Sys().(syscall.WaitStatus).ExitStatus() == 21 { + return []iscsiSession{}, nil + } + return nil, err + } + sessions := parseSessions(out) + return sessions, err +} + +// waitForPathToExist wait for a file at a path to exists on disk +func waitForPathToExist(devicePath *string, maxRetries, intervalSeconds uint, deviceTransport string) error { + if devicePath == nil || *devicePath == "" { + return fmt.Errorf("unable to check unspecified devicePath") + } + + for i := uint(0); i <= maxRetries; i++ { + if i != 0 { + debug.Printf("Device path %q doesn't exists yet, retrying in %d seconds (%d/%d)", *devicePath, intervalSeconds, i, maxRetries) + sleep(time.Second * time.Duration(intervalSeconds)) + } + + if err := pathExists(devicePath, deviceTransport); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + } + + return os.ErrNotExist +} + +// pathExists checks if a file at a path exists on disk +func pathExists(devicePath *string, deviceTransport string) error { + if deviceTransport == "tcp" { + _, err := osStat(*devicePath) + if err != nil { + if !os.IsNotExist(err) { + debug.Printf("Error attempting to stat device: %s", err.Error()) + return err + } + debug.Printf("Device not found for: %s", *devicePath) + return err + } + } else { + fpath, err := filepathGlob(*devicePath) + + if err != nil { + return err + } + if fpath == nil { + return os.ErrNotExist + } + // There might be a case that fpath contains multiple device paths if + // multiple PCI devices connect to same iscsi target. We handle this + // case at subsequent logic. Pick up only first path here. + *devicePath = fpath[0] + } + + return nil +} + +// getMultipathDevice returns a multipath device for the configured targets if it exists +func getMultipathDevice(devices []Device) (*Device, error) { + var multipathDevice *Device + + for _, device := range devices { + if len(device.Children) != 1 { + multipathdNotRunning := "" + if len(device.Children) == 0 { + multipathdNotRunning = " (is multipathd running?)" + } + return nil, fmt.Errorf("device is not mapped to exactly one multipath device%s: %v", multipathdNotRunning, device.Children) + } + if multipathDevice != nil && device.Children[0].Name != multipathDevice.Name { + return nil, fmt.Errorf("devices don't share a common multipath device: %v", devices) + } + multipathDevice = &device.Children[0] + } + + if multipathDevice == nil { + return nil, fmt.Errorf("multipath device not found") + } + + if multipathDevice.Type != "mpath" { + return nil, fmt.Errorf("device is not of mpath type: %v", multipathDevice) + } + + return multipathDevice, nil +} + +// Connect is for backward-compatibility with c.Connect() +func Connect(c Connector) (string, error) { + return c.Connect() +} + +// Connect attempts to connect a volume to this node using the provided Connector info +func (c *Connector) Connect() (string, error) { + if c.RetryCount == 0 { + c.RetryCount = 10 + } + if c.CheckInterval == 0 { + c.CheckInterval = 1 + } + + iFace := "default" + if c.Interface != "" { + iFace = c.Interface + } + + // make sure our iface exists and extract the transport type + out, err := ShowInterface(iFace) + if err != nil { + return "", err + } + iscsiTransport := extractTransportName(out) + + var lastErr error + var devicePaths []string + for _, target := range c.TargetPortals { + devicePath, err := c.connectTarget(c.TargetIqn, target, iFace, iscsiTransport) + if err != nil { + lastErr = err + } else { + debug.Printf("Appending device path: %s", devicePath) + devicePaths = append(devicePaths, devicePath) + } + } + + // GetISCSIDevices returns all devices if no paths are given + if len(devicePaths) < 1 { + c.Devices = []Device{} + } else if c.Devices, err = GetISCSIDevices(devicePaths, true); err != nil { + return "", err + } + + if len(c.Devices) < 1 { + iscsiCmd([]string{"-m", "iface", "-I", iFace, "-o", "delete"}...) + return "", fmt.Errorf("failed to find device path: %s, last error seen: %v", devicePaths, lastErr) + } + + mountTargetDevice, err := c.getMountTargetDevice() + c.MountTargetDevice = mountTargetDevice + if err != nil { + debug.Printf("Connect failed: %v", err) + RemoveSCSIDevices(c.Devices...) + c.MountTargetDevice = nil + c.Devices = []Device{} + return "", err + } + + if c.IsMultipathEnabled() { + if err := c.IsMultipathConsistent(); err != nil { + return "", fmt.Errorf("multipath is inconsistent: %v", err) + } + } + + return c.MountTargetDevice.GetPath(), nil +} + +func (c *Connector) connectTarget(targetIqn string, target string, iFace string, iscsiTransport string) (string, error) { + debug.Printf("Process targetIqn: %s, portal: %s\n", targetIqn, target) + targetParts := strings.Split(target, ":") + targetPortal := targetParts[0] + targetPort := defaultPort + if len(targetParts) > 1 { + targetPort = targetParts[1] + } + baseArgs := []string{"-m", "node", "-T", targetIqn, "-p", targetPortal} + // Rescan sessions to discover newly mapped LUNs. Do not specify the interface when rescanning + // to avoid establishing additional sessions to the same target. + if _, err := iscsiCmd(append(baseArgs, []string{"-R"}...)...); err != nil { + debug.Printf("Failed to rescan session, err: %v", err) + if os.IsTimeout(err) { + debug.Printf("iscsiadm timeouted, logging out") + cmd := execCommand("iscsiadm", append(baseArgs, []string{"-u"}...)...) + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("could not logout from target: %s", out) + } + } + } + + // create our devicePath that we'll be looking for based on the transport being used + // portal with port + portal := strings.Join([]string{targetPortal, targetPort}, ":") + devicePath := strings.Join([]string{"/dev/disk/by-path/ip", portal, "iscsi", targetIqn, "lun", fmt.Sprint(c.Lun)}, "-") + if iscsiTransport != "tcp" { + devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", portal, "iscsi", targetIqn, "lun", fmt.Sprint(c.Lun)}, "-") + } + + exists, _ := sessionExists(portal, targetIqn) + if exists { + debug.Printf("Session already exists, checking if device path %q exists", devicePath) + if err := waitForPathToExist(&devicePath, c.RetryCount, c.CheckInterval, iscsiTransport); err != nil { + return "", err + } + return devicePath, nil + } + + if err := c.discoverTarget(targetIqn, iFace, portal); err != nil { + return "", err + } + + // perform the login + err := Login(targetIqn, portal) + if err != nil { + debug.Printf("Failed to login: %v", err) + return "", err + } + + debug.Printf("Waiting for device path %q to exist", devicePath) + if err := waitForPathToExist(&devicePath, c.RetryCount, c.CheckInterval, iscsiTransport); err != nil { + return "", err + } + + return devicePath, nil +} + +func (c *Connector) discoverTarget(targetIqn string, iFace string, portal string) error { + if c.DoDiscovery { + // build discoverydb and discover iscsi target + if err := Discoverydb(portal, iFace, c.DiscoverySecrets, c.DoCHAPDiscovery); err != nil { + debug.Printf("Error in discovery of the target: %s\n", err.Error()) + return err + } + } + + if c.DoCHAPDiscovery { + // Make sure we don't log the secrets + err := CreateDBEntry(targetIqn, portal, iFace, c.DiscoverySecrets, c.SessionSecrets) + if err != nil { + debug.Printf("Error creating db entry: %s\n", err.Error()) + return err + } + } + + return nil +} + +// Disconnect is for backward-compatibility with c.Disconnect() +func Disconnect(targetIqn string, targets []string) { + for _, target := range targets { + targetPortal := strings.Split(target, ":")[0] + Logout(targetIqn, targetPortal) + } + + deleted := map[string]bool{} + if _, ok := deleted[targetIqn]; ok { + return + } + deleted[targetIqn] = true + DeleteDBEntry(targetIqn) +} + +// Disconnect performs a disconnect operation from an appliance. +// Be sure to disconnect all deivces properly before doing this as it can result in data loss. +func (c *Connector) Disconnect() { + Disconnect(c.TargetIqn, c.TargetPortals) +} + +// DisconnectVolume removes a volume from a Linux host. +func (c *Connector) DisconnectVolume() error { + // Steps to safely remove an iSCSI storage volume from a Linux host are as following: + // 1. Unmount the disk from a filesystem on the system. + // 2. Flush the multipath map for the disk we’re removing (if multipath is enabled). + // 3. Remove the physical disk entities that Linux maintains. + // 4. Take the storage volume (disk) offline on the storage subsystem. + // 5. Rescan the iSCSI sessions (after unmapping only). + // + // DisconnectVolume focuses on step 2 and 3. + // Note: make sure the volume is already unmounted before calling this method. + + if c.IsMultipathEnabled() { + if err := c.IsMultipathConsistent(); err != nil { + return fmt.Errorf("multipath is inconsistent: %v", err) + } + + debug.Printf("Removing multipath device in path %s.\n", c.MountTargetDevice.GetPath()) + err := FlushMultipathDevice(c.MountTargetDevice) + if err != nil { + return err + } + if err := RemoveSCSIDevices(c.Devices...); err != nil { + return err + } + } else { + devicePath := c.MountTargetDevice.GetPath() + debug.Printf("Removing normal device in path %s.\n", devicePath) + if err := RemoveSCSIDevices(*c.MountTargetDevice); err != nil { + return err + } + } + + debug.Printf("Finished disconnecting volume.\n") + return nil +} + +// getMountTargetDevice returns the device to be mounted among the configured devices +func (c *Connector) getMountTargetDevice() (*Device, error) { + if len(c.Devices) > 1 { + multipathDevice, err := getMultipathDevice(c.Devices) + if err != nil { + debug.Printf("mount target is not a multipath device: %v", err) + return nil, err + } + debug.Printf("mount target is a multipath device") + return multipathDevice, nil + } + + if len(c.Devices) == 0 { + return nil, fmt.Errorf("could not find mount target device: connector does not contain any device") + } + + return &c.Devices[0], nil +} + +// IsMultipathEnabled check if multipath is enabled on devices handled by this connector +func (c *Connector) IsMultipathEnabled() bool { + return c.MountTargetDevice.Type == "mpath" +} + +// GetSCSIDevices get SCSI devices from device paths +// It will returns all SCSI devices if no paths are given +func GetSCSIDevices(devicePaths []string, strict bool) ([]Device, error) { + debug.Printf("Getting info about SCSI devices %s.\n", devicePaths) + + deviceInfo, err := lsblk(devicePaths, strict) + if err != nil { + debug.Printf("An error occurred while looking info about SCSI devices: %v", err) + return nil, err + } + + return deviceInfo, nil +} + +// GetISCSIDevices get iSCSI devices from device paths +// It will returns all iSCSI devices if no paths are given +func GetISCSIDevices(devicePaths []string, strict bool) (devices []Device, err error) { + scsiDevices, err := GetSCSIDevices(devicePaths, strict) + if err != nil { + return + } + + for i := range scsiDevices { + device := &scsiDevices[i] + if device.Transport == "iscsi" { + devices = append(devices, *device) + } + } + + return +} + +// lsblk execute the lsblk commands +func lsblk(devicePaths []string, strict bool) (deviceInfo, error) { + flags := []string{"-rn", "-o", "NAME,KNAME,PKNAME,HCTL,TYPE,TRAN,SIZE"} + command := execCommand("lsblk", append(flags, devicePaths...)...) + debug.Println(command.String()) + out, err := command.Output() + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + err = fmt.Errorf("%s, (%w)", strings.Trim(string(ee.Stderr), "\n"), ee) + if strict || ee.ExitCode() != 64 { // ignore the error if some devices have been found when not strict + return nil, err + } + debug.Printf("Could find only some devices: %v", err) + } else { + return nil, err + } + } + + var devices []*Device + devicesMap := make(map[string]*Device) + pkNames := []string{} + + // Parse devices + lines := strings.Split(strings.Trim(string(out), "\n"), "\n") + for _, line := range lines { + columns := strings.Split(line, " ") + if len(columns) < 5 { + return nil, fmt.Errorf("invalid output from lsblk: %s", line) + } + device := &Device{ + Name: columns[0], + Hctl: columns[3], + Type: columns[4], + Transport: columns[5], + Size: columns[6], + } + devices = append(devices, device) + pkNames = append(pkNames, columns[2]) + devicesMap[columns[1]] = device + } + + // Reconstruct devices tree + for i, pkName := range pkNames { + if pkName == "" { + continue + } + device := devices[i] + parent, ok := devicesMap[pkName] + if !ok { + return nil, fmt.Errorf("invalid output from lsblk: parent device %q not found", pkName) + } + if parent.Children == nil { + parent.Children = []Device{} + } + parent.Children = append(devicesMap[pkName].Children, *device) + } + + // Filter devices to keep only the roots of the tree + var deviceInfo deviceInfo + for i, device := range devices { + if pkNames[i] == "" { + deviceInfo = append(deviceInfo, *device) + } + } + + return deviceInfo, nil +} + +// writeInSCSIDeviceFile write into special devices files to change devices state +func writeInSCSIDeviceFile(hctl string, file string, content string) error { + filename := filepath.Join("/sys/class/scsi_device", hctl, "device", file) + debug.Printf("Write %q in %q.\n", content, filename) + + f, err := osOpenFile(filename, os.O_TRUNC|os.O_WRONLY, 0200) + if err != nil { + debug.Printf("Error while opening file %v: %v\n", filename, err) + return err + } + + defer f.Close() + if _, err := f.WriteString(content); err != nil { + debug.Printf("Error while writing to file %v: %v", filename, err) + return err + } + + return nil +} + +// RemoveSCSIDevices removes SCSI device(s) from a Linux host. +func RemoveSCSIDevices(devices ...Device) error { + debug.Printf("Removing SCSI devices %v.\n", devices) + + var errs []error + for _, device := range devices { + debug.Printf("Flush SCSI device %v.\n", device.Name) + if err := device.Exists(); err == nil { + out, err := execCommand("blockdev", "--flushbufs", device.GetPath()).CombinedOutput() + if err != nil { + debug.Printf("Command 'blockdev --flushbufs %s' did not succeed to flush the device: %v\n", device.GetPath(), err) + return errors.New(string(out)) + } + } else if !os.IsNotExist(err) { + return err + } + + debug.Printf("Put SCSI device %q offline.\n", device.Name) + err := device.Shutdown() + if err != nil { + if !os.IsNotExist(err) { // Ignore device already removed + errs = append(errs, err) + } + continue + } + + debug.Printf("Delete SCSI device %q.\n", device.Name) + err = device.Delete() + if err != nil { + if !os.IsNotExist(err) { // Ignore device already removed + errs = append(errs, err) + } + continue + } + } + + if len(errs) > 0 { + return errs[0] + } + debug.Println("Finished removing SCSI devices.") + return nil +} + +// PersistConnector is for backward-compatibility with c.Persist() +func PersistConnector(c *Connector, filePath string) error { + return c.Persist(filePath) +} + +// Persist persists the Connector to the specified file (ie /var/lib/pfile/myConnector.json) +func (c *Connector) Persist(filePath string) error { + //file := path.Join("mnt", c.VolumeName+".json") + f, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("error creating iSCSI persistence file %s: %s", filePath, err) + } + defer f.Close() + encoder := json.NewEncoder(f) + if err = encoder.Encode(c); err != nil { + return fmt.Errorf("error encoding connector: %v", err) + } + return nil +} + +// GetConnectorFromFile attempts to create a Connector using the specified json file (ie /var/lib/pfile/myConnector.json) +func GetConnectorFromFile(filePath string) (*Connector, error) { + f, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + c := Connector{} + err = json.Unmarshal([]byte(f), &c) + if err != nil { + return nil, err + } + + devicePaths := []string{} + for _, device := range c.Devices { + devicePaths = append(devicePaths, device.GetPath()) + } + + if devices, err := GetSCSIDevices([]string{c.MountTargetDevice.GetPath()}, false); err != nil { + return nil, err + } else { + c.MountTargetDevice = &devices[0] + } + + if c.Devices, err = GetSCSIDevices(devicePaths, false); err != nil { + return nil, err + } + + return &c, nil +} + +// IsMultipathConsistent check if the currently used device is using a consistent multipath mapping +func (c *Connector) IsMultipathConsistent() error { + devices := append([]Device{*c.MountTargetDevice}, c.Devices...) + + referenceLUN := struct { + LUN int + Name string + }{LUN: -1, Name: ""} + HBA := map[int]string{} + referenceDevice := devices[0] + for _, device := range devices { + if device.Size != referenceDevice.Size { + return fmt.Errorf("devices size differ: %s (%s) != %s (%s)", device.Name, device.Size, referenceDevice.Name, referenceDevice.Size) + } + + if device.Type != "mpath" { + hctl, err := device.HCTL() + if err != nil { + return err + } + if referenceLUN.LUN == -1 { + referenceLUN.LUN = hctl.LUN + referenceLUN.Name = device.Name + } else if hctl.LUN != referenceLUN.LUN { + return fmt.Errorf("devices LUNs differ: %s (%d) != %s (%d)", device.Name, hctl.LUN, referenceLUN.Name, referenceLUN.LUN) + } + + if name, ok := HBA[hctl.HBA]; !ok { + HBA[hctl.HBA] = device.Name + } else { + return fmt.Errorf("two devices are using the same controller (%d): %s and %s", hctl.HBA, device.Name, name) + } + } + + wwid, err := device.WWID() + if err != nil { + return fmt.Errorf("could not find WWID for device %s: %v", device.Name, err) + } + if wwid != referenceDevice.Name { + return fmt.Errorf("devices WWIDs differ: %s (wwid:%s) != %s (wwid:%s)", device.Name, wwid, referenceDevice.Name, referenceDevice.Name) + } + } + + return nil +} + +// Exists check if the device exists at its path and returns an error otherwise +func (d *Device) Exists() error { + _, err := osStat(d.GetPath()) + return err +} + +// GetPath returns the path of a device +func (d *Device) GetPath() string { + if d.Type == "mpath" { + return filepath.Join("/dev/mapper", d.Name) + } + + return filepath.Join("/dev", d.Name) +} + +// WWID returns the WWID of a device +func (d *Device) WWID() (string, error) { + timeout := 1 * time.Second + out, err := execWithTimeout("scsi_id", []string{"-g", "-u", d.GetPath()}, timeout) + if err != nil { + return "", err + } + + return string(out[:len(out)-1]), nil +} + +// HCTL returns the HCTL of a device +func (d *Device) HCTL() (*HCTL, error) { + var hctl []int + + for _, idstr := range strings.Split(d.Hctl, ":") { + id, err := strconv.Atoi(idstr) + if err != nil { + hctl = []int{} + break + } + hctl = append(hctl, id) + } + + if len(hctl) != 4 { + return nil, fmt.Errorf("invalid HCTL (%s) for device %q", d.Hctl, d.Name) + } + + return &HCTL{ + HBA: hctl[0], + Channel: hctl[1], + Target: hctl[2], + LUN: hctl[3], + }, nil +} + +// WriteDeviceFile write in a device file +func (d *Device) WriteDeviceFile(name string, content string) error { + return writeInSCSIDeviceFile(d.Hctl, name, content) +} + +// Shutdown turn off an SCSI device by writing offline\n in /sys/class/scsi_device/h:c:t:l/device/state +func (d *Device) Shutdown() error { + return d.WriteDeviceFile("state", "offline\n") +} + +// Delete detach an SCSI device by writing 1 in /sys/class/scsi_device/h:c:t:l/device/delete +func (d *Device) Delete() error { + return d.WriteDeviceFile("delete", "1") +} + +// Rescan rescan an SCSI device by writing 1 in /sys/class/scsi_device/h:c:t:l/device/rescan +func (d *Device) Rescan() error { + return d.WriteDeviceFile("rescan", "1") +} diff --git a/pkg/lib/iscsi/iscsi/iscsi_test.go b/pkg/lib/iscsi/iscsi/iscsi_test.go new file mode 100644 index 00000000..3ef70d17 --- /dev/null +++ b/pkg/lib/iscsi/iscsi/iscsi_test.go @@ -0,0 +1,855 @@ +package iscsi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/prashantv/gostub" + "github.com/stretchr/testify/assert" +) + +type testWriter struct { + data *[]byte +} + +func (w testWriter) Write(data []byte) (n int, err error) { + *w.data = append(*w.data, data...) + return len(data), nil +} + +const nodeDB = ` +# BEGIN RECORD 6.2.0.874 +node.name = iqn.2010-10.org.openstack:volume-eb393993-73d0-4e39-9ef4-b5841e244ced +node.tpgt = -1 +node.startup = automatic +node.leading_login = No +iface.iscsi_ifacename = default +iface.transport_name = tcp +iface.vlan_id = 0 +iface.vlan_priority = 0 +iface.iface_num = 0 +iface.mtu = 0 +iface.port = 0 +iface.tos = 0 +iface.ttl = 0 +iface.tcp_wsf = 0 +iface.tcp_timer_scale = 0 +iface.def_task_mgmt_timeout = 0 +iface.erl = 0 +iface.max_receive_data_len = 0 +iface.first_burst_len = 0 +iface.max_outstanding_r2t = 0 +iface.max_burst_len = 0 +node.discovery_port = 0 +node.discovery_type = static +node.session.initial_cmdsn = 0 +node.session.initial_login_retry_max = 8 +node.session.xmit_thread_priority = -20 +node.session.cmds_max = 128 +node.session.queue_depth = 32 +node.session.nr_sessions = 1 +node.session.auth.authmethod = CHAP +node.session.auth.username = 86Jx6hXYqDYpKamtgx4d +node.session.auth.password = Qj3MuzmHu8cJBpkv +node.session.timeo.replacement_timeout = 120 +node.session.err_timeo.abort_timeout = 15 +node.session.err_timeo.lu_reset_timeout = 30 +node.session.err_timeo.tgt_reset_timeout = 30 +node.session.err_timeo.host_reset_timeout = 60 +node.session.iscsi.FastAbort = Yes +node.session.iscsi.InitialR2T = No +node.session.iscsi.ImmediateData = Yes +node.session.iscsi.FirstBurstLength = 262144 +node.session.iscsi.MaxBurstLength = 16776192 +node.session.iscsi.DefaultTime2Retain = 0 +node.session.iscsi.DefaultTime2Wait = 2 +node.session.iscsi.MaxConnections = 1 +node.session.iscsi.MaxOutstandingR2T = 1 +node.session.iscsi.ERL = 0 +node.conn[0].address = 192.168.1.107 +node.conn[0].port = 3260 +node.conn[0].startup = manual +node.conn[0].tcp.window_size = 524288 +node.conn[0].tcp.type_of_service = 0 +node.conn[0].timeo.logout_timeout = 15 +node.conn[0].timeo.login_timeout = 15 +node.conn[0].timeo.auth_timeout = 45 +node.conn[0].timeo.noop_out_interval = 5 +node.conn[0].timeo.noop_out_timeout = 5 +node.conn[0].iscsi.MaxXmitDataSegmentLength = 0 +node.conn[0].iscsi.MaxRecvDataSegmentLength = 262144 +node.conn[0].iscsi.HeaderDigest = None +node.conn[0].iscsi.IFMarker = No +node.conn[0].iscsi.OFMarker = No +# END RECORD +` + +const emptyTransportName = "iface.transport_name = \n" +const emptyDbRecord = "\n\n\n" +const testRootFS = "/tmp/iscsi-tests" + +func makeFakeExecCommand(exitStatus int, stdout string) func(string, ...string) *exec.Cmd { + return func(command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestExecCommandHelper", "--", command} + cs = append(cs, args...) + cmd := exec.Command(os.Args[0], cs...) + es := strconv.Itoa(exitStatus) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1", + "STDOUT=" + stdout, + "EXIT_STATUS=" + es} + return cmd + } +} + +func makeFakeExecCommandContext(exitStatus int, stdout string) func(context.Context, string, ...string) *exec.Cmd { + return func(ctx context.Context, command string, args ...string) *exec.Cmd { + return makeFakeExecCommand(exitStatus, stdout)(command, args...) + } +} + +func makeFakeExecWithTimeout(withTimeout bool, output []byte, err error) func(string, []string, time.Duration) ([]byte, error) { + return func(command string, args []string, timeout time.Duration) ([]byte, error) { + if withTimeout { + return nil, context.DeadlineExceeded + } + return output, err + } +} + +func marshalDeviceInfo(d *deviceInfo) string { + var output string + pkNames := map[string]string{} + for _, device := range *d { + for _, child := range device.Children { + pkNames[child.Name] = device.Name + } + } + for _, device := range *d { + output += fmt.Sprintf("%s %s %s %s %s %s %s\n", device.Name, device.Name, pkNames[device.Name], device.Hctl, device.Type, device.Transport, device.Size) + } + return output +} + +func TestExecCommandHelper(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + + fmt.Fprintf(os.Stdout, os.Getenv("STDOUT")) + i, _ := strconv.Atoi(os.Getenv("EXIT_STATUS")) + os.Exit(i) +} + +func getDevicePath(device *Device) string { + sysDevicePath := "/tmp/iscsi-tests/sys/class/scsi_device/" + return filepath.Join(sysDevicePath, device.Hctl, "device") +} + +func preparePaths(devices []Device) error { + for _, d := range devices { + devicePath := getDevicePath(&d) + + if err := os.MkdirAll(devicePath, os.ModePerm); err != nil { + return err + } + + for _, filename := range []string{"delete", "state"} { + if err := ioutil.WriteFile(filepath.Join(devicePath, filename), []byte(""), 0600); err != nil { + return err + } + } + } + + return nil +} + +func checkFileContents(t *testing.T, path string, contents string) { + if out, err := ioutil.ReadFile(path); err != nil { + t.Errorf("could not read file: %v", err) + return + } else if string(out) != contents { + t.Errorf("file content mismatch, got = %q, want = %q", string(out), contents) + return + } +} + +func Test_parseSessions(t *testing.T) { + var sessions []iscsiSession + output := "tcp: [2] 192.168.1.107:3260,1 iqn.2010-10.org.openstack:volume-eb393993-73d0-4e39-9ef4-b5841e244ced (non-flash)\n" + + "tcp: [2] 192.168.1.200:3260,1 iqn.2010-10.org.openstack:volume-eb393993-73d0-4e39-9ef4-b5841e244ced (non-flash)\n" + + sessions = append(sessions, + iscsiSession{ + Protocol: "tcp", + ID: 2, + Portal: "192.168.1.107:3260", + IQN: "iqn.2010-10.org.openstack:volume-eb393993-73d0-4e39-9ef4-b5841e244ced", + Name: "volume-eb393993-73d0-4e39-9ef4-b5841e244ced", + }) + sessions = append(sessions, + iscsiSession{ + Protocol: "tcp", + ID: 2, + Portal: "192.168.1.200:3260", + IQN: "iqn.2010-10.org.openstack:volume-eb393993-73d0-4e39-9ef4-b5841e244ced", + Name: "volume-eb393993-73d0-4e39-9ef4-b5841e244ced", + }) + + type args struct { + lines string + } + validSession := args{ + lines: output, + } + tests := []struct { + name string + args args + want []iscsiSession + }{ + {"ValidParseSession", validSession, sessions}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseSessions(tt.args.lines) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseSessions() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_extractTransportName(t *testing.T) { + type args struct { + output string + } + validRecord := args{ + output: nodeDB, + } + emptyRecord := args{ + output: emptyDbRecord, + } + emptyTransportRecord := args{ + output: emptyTransportName, + } + tests := []struct { + name string + args args + want string + }{ + {"tcp-check", validRecord, "tcp"}, + {"tcp-check", emptyRecord, ""}, + {"tcp-check", emptyTransportRecord, "tcp"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := extractTransportName(tt.args.output); got != tt.want { + t.Errorf("extractTransportName() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_sessionExists(t *testing.T) { + fakeOutput := "tcp: [4] 192.168.1.107:3260,1 iqn.2010-10.org.openstack:volume-eb393993-73d0-4e39-9ef4-b5841e244ced (non-flash)\n" + defer gostub.Stub(&execWithTimeout, makeFakeExecWithTimeout(false, []byte(fakeOutput), nil)).Reset() + + type args struct { + tgtPortal string + tgtIQN string + } + testExistsArgs := args{ + tgtPortal: "192.168.1.107:3260", + tgtIQN: "iqn.2010-10.org.openstack:volume-eb393993-73d0-4e39-9ef4-b5841e244ced", + } + testWrongPortalArgs := args{ + tgtPortal: "10.0.0.1:3260", + tgtIQN: "iqn.2010-10.org.openstack:volume-eb393993-73d0-4e39-9ef4-b5841e244ced", + } + + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + {"TestSessionExists", testExistsArgs, true, false}, + {"TestSessionDoesNotExist", testWrongPortalArgs, false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := sessionExists(tt.args.tgtPortal, tt.args.tgtIQN) + if (err != nil) != tt.wantErr { + t.Errorf("sessionExists() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("sessionExists() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_DisconnectNormalVolume(t *testing.T) { + deleteDeviceFile := "/tmp/deleteDevice" + defer gostub.Stub(&osOpenFile, func(name string, flag int, perm os.FileMode) (*os.File, error) { + return os.OpenFile(deleteDeviceFile, flag, perm) + }).Reset() + + tests := []struct { + name string + withDeviceFile bool + wantErr bool + }{ + {"DisconnectNormalVolume", true, false}, + {"DisconnectNonexistentNormalVolume", false, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.withDeviceFile { + os.Create(deleteDeviceFile) + } else { + os.RemoveAll(testRootFS) + } + + device := Device{Name: "test"} + c := Connector{Devices: []Device{device}, MountTargetDevice: &device} + err := c.DisconnectVolume() + if (err != nil) != tt.wantErr { + t.Errorf("DisconnectVolume() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if tt.withDeviceFile { + out, err := ioutil.ReadFile(deleteDeviceFile) + if err != nil { + t.Errorf("can not read file %v: %v", deleteDeviceFile, err) + return + } + if string(out) != "1" { + t.Errorf("file content mismatch, got = %s, want = 1", string(out)) + return + } + } + }) + } +} + +func Test_DisconnectMultipathVolume(t *testing.T) { + defer gostub.Stub(&osStat, func(name string) (os.FileInfo, error) { + return nil, nil + }).Reset() + + tests := []struct { + name string + timeout bool + withDeviceFile bool + wantErr bool + }{ + {"DisconnectMultipathVolume", false, true, false}, + {"DisconnectMultipathVolumeFlushTimeout", true, true, true}, + {"DisconnectNonexistentMultipathVolume", false, false, false}, + } + + wwid := "3600c0ff0000000000000000000000000" + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer gostub.Stub(&execWithTimeout, func(cmd string, args []string, timeout time.Duration) ([]byte, error) { + mockedOutput := []byte("") + if cmd == "scsi_id" { + mockedOutput = []byte(wwid + "\n") + } + return makeFakeExecWithTimeout(tt.timeout, mockedOutput, nil)(cmd, args, timeout) + }).Reset() + c := Connector{ + Devices: []Device{{Hctl: "0:0:0:0"}, {Hctl: "1:0:0:0"}}, + MountTargetDevice: &Device{Name: wwid, Type: "mpath"}, + } + + defer gostub.Stub(&osOpenFile, func(name string, flag int, perm os.FileMode) (*os.File, error) { + return os.OpenFile(testRootFS+name, flag, perm) + }).Reset() + + defer gostub.Stub(&execCommand, makeFakeExecCommand(0, wwid)).Reset() + + if tt.withDeviceFile { + if err := preparePaths(c.Devices); err != nil { + t.Errorf("could not prepare paths: %v", err) + return + } + } else { + os.Remove(testRootFS) + } + + err := c.DisconnectVolume() + if (err != nil) != tt.wantErr { + t.Errorf("DisconnectVolume() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if tt.timeout { + assert.New(t).Contains(err.Error(), "context deadline exceeded") + } + + if tt.withDeviceFile && !tt.wantErr { + for _, device := range c.Devices { + checkFileContents(t, getDevicePath(&device)+"/delete", "1") + checkFileContents(t, getDevicePath(&device)+"/state", "offline\n") + } + } + }) + } +} + +func Test_EnableDebugLogging(t *testing.T) { + assert := assert.New(t) + data := []byte{} + writer := testWriter{data: &data} + EnableDebugLogging(writer) + + assert.Equal("", string(data)) + assert.Len(strings.Split(string(data), "\n"), 1) + + debug.Print("testing debug logs") + assert.Contains(string(data), "testing debug logs") + assert.Len(strings.Split(string(data), "\n"), 2) +} + +func Test_waitForPathToExist(t *testing.T) { + tests := map[string]struct { + attempts int + fileNotFound bool + withErr bool + transport string + }{ + "Basic": { + attempts: 1, + }, + "WithRetry": { + attempts: 2, + }, + "WithRetryFail": { + attempts: 3, + fileNotFound: true, + }, + "WithError": { + withErr: true, + }, + } + + for name, tt := range tests { + tt.transport = "tcp" + tests[name+"OverTCP"] = tt + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + attempts := 0 + maxRetries := tt.attempts - 1 + if tt.fileNotFound { + maxRetries-- + } + if maxRetries < 0 { + maxRetries = 0 + } + doAttempt := func(err error) error { + attempts++ + if tt.withErr { + return err + } + if attempts < tt.attempts { + return os.ErrNotExist + } + return nil + } + defer gostub.Stub(&osStat, func(name string) (os.FileInfo, error) { + if err := doAttempt(os.ErrPermission); err != nil { + return nil, err + } + return nil, nil + }).Reset() + defer gostub.Stub(&filepathGlob, func(name string) ([]string, error) { + if err := doAttempt(filepath.ErrBadPattern); err != nil { + return nil, err + } + return []string{"/somefilewithalongname"}, nil + }).Reset() + defer gostub.Stub(&sleep, func(_ time.Duration) {}).Reset() + path := "/somefile" + err := waitForPathToExist(&path, uint(maxRetries), 1, tt.transport) + + if tt.withErr { + if tt.transport == "tcp" { + assert.Equal(os.ErrPermission, err) + } else { + assert.Equal(filepath.ErrBadPattern, err) + } + return + } + if tt.fileNotFound { + assert.Equal(os.ErrNotExist, err) + assert.Equal(maxRetries, attempts-1) + } else { + assert.Nil(err) + assert.Equal(tt.attempts, attempts) + if tt.transport == "tcp" { + assert.Equal("/somefile", path) + } else { + assert.Equal("/somefilewithalongname", path) + } + } + }) + } + + t.Run("PathEmptyOrNil", func(t *testing.T) { + assert := assert.New(t) + path := "" + + err := waitForPathToExist(&path, 0, 0, "tcp") + assert.NotNil(err) + + err = waitForPathToExist(&path, 0, 0, "") + assert.NotNil(err) + + err = waitForPathToExist(nil, 0, 0, "tcp") + assert.NotNil(err) + + err = waitForPathToExist(nil, 0, 0, "") + assert.NotNil(err) + }) + + t.Run("PathNotFound", func(t *testing.T) { + assert := assert.New(t) + defer gostub.Stub(&filepathGlob, func(name string) ([]string, error) { + return nil, nil + }).Reset() + + path := "/test" + err := waitForPathToExist(&path, 0, 0, "") + assert.NotNil(err) + assert.Equal(os.ErrNotExist, err) + }) +} + +func Test_getMultipathDevice(t *testing.T) { + mpath1 := Device{Name: "3600c0ff0000000000000000000000000", Type: "mpath"} + mpath2 := Device{Name: "3600c0ff1111111111111111111111111", Type: "mpath"} + sda := Device{Name: "sda", Children: []Device{{Name: "sda1"}}} + sdb := Device{Name: "sdb", Children: []Device{mpath1}} + sdc := Device{Name: "sdc", Children: []Device{mpath1}} + sdd := Device{Name: "sdc", Children: []Device{mpath2}} + sde := Device{Name: "sdc", Children: []Device{mpath1, mpath2}} + + tests := map[string]struct { + mockedDevices []Device + multipathDevice *Device + wantErr bool + }{ + "Basic": { + mockedDevices: []Device{sdb, sdc}, + multipathDevice: &mpath1, + }, + "NotSharingTheSameMultipathDevice": { + mockedDevices: []Device{sdb, sdd}, + wantErr: true, + }, + "MoreThanOneMultipathDevice": { + mockedDevices: []Device{sde}, + wantErr: true, + }, + "NotAMultipathDevice": { + mockedDevices: []Device{sda}, + wantErr: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + multipathDevice, err := getMultipathDevice(tt.mockedDevices) + + if tt.wantErr { + assert.Nil(multipathDevice) + assert.NotNil(err) + } else { + assert.Equal(tt.multipathDevice, multipathDevice) + assert.Nil(err) + } + }) + } +} + +func Test_lsblk(t *testing.T) { + sda1 := Device{Name: "sda1"} + sda := Device{Name: "sda", Children: []Device{sda1}} + sdaOutput := marshalDeviceInfo(&deviceInfo{sda, sda1}) + + tests := map[string]struct { + devicePaths []string + strict bool + mockedStdout string + mockedDevices deviceInfo + mockedExitStatus int + wantErr bool + }{ + "Basic": { + devicePaths: []string{"/dev/sda"}, + mockedDevices: []Device{sda}, + mockedStdout: string(sdaOutput), + }, + "NotABlockDevice": { + devicePaths: []string{"/dev/sdzz"}, + mockedStdout: "lsblk: sdzz: not a block device", + mockedExitStatus: 32, + wantErr: true, + }, + "InvalidOutput": { + mockedStdout: "{", + mockedExitStatus: 0, + wantErr: true, + }, + "StrictWithMissingDevices": { + devicePaths: []string{"/dev/sda", "/dev/sdb"}, + strict: true, + mockedDevices: []Device{sda}, + mockedStdout: string(sdaOutput), + mockedExitStatus: 64, + wantErr: true, + }, + "NotStrictWithMissingDevices": { + devicePaths: []string{"/dev/sda", "/dev/sdb"}, + mockedDevices: []Device{sda}, + mockedStdout: string(sdaOutput), + mockedExitStatus: 64, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + defer gostub.Stub(&execCommand, makeFakeExecCommand(tt.mockedExitStatus, tt.mockedStdout)).Reset() + deviceInfo, err := lsblk(tt.devicePaths, tt.strict) + + if tt.wantErr { + assert.Nil(deviceInfo) + assert.NotNil(err) + } else { + assert.NotNil(deviceInfo) + assert.Equal(tt.mockedDevices, deviceInfo) + assert.Nil(err) + } + }) + } +} + +func TestConnectorPersistance(t *testing.T) { + assert := assert.New(t) + + secret := Secrets{ + SecretsType: "fake secret type", + UserName: "fake username", + Password: "fake password", + UserNameIn: "fake username in", + PasswordIn: "fake password in", + } + childDevice := Device{ + Name: "child-name", + Hctl: "child-hctl", + Type: "child-type", + Transport: "child-transport", + } + device := Device{ + Name: "device-name", + Hctl: "device-hctl", + Children: []Device{childDevice}, + Type: "device-type", + Transport: "device-transport", + } + c := Connector{ + VolumeName: "fake volume name", + TargetIqn: "fake target iqn", + TargetPortals: []string{}, + Lun: 42, + AuthType: "fake auth type", + DiscoverySecrets: secret, + SessionSecrets: secret, + Interface: "fake interface", + MountTargetDevice: &device, + Devices: []Device{childDevice}, + RetryCount: 24, + CheckInterval: 13, + DoDiscovery: true, + DoCHAPDiscovery: true, + } + devicesByPath := map[string]*Device{} + devicesByPath[childDevice.GetPath()] = &childDevice + devicesByPath[device.GetPath()] = &device + + defer gostub.Stub(&execCommand, func(name string, arg ...string) *exec.Cmd { + blockDevices := deviceInfo{} + for _, path := range arg[3:] { + blockDevices = append(blockDevices, *devicesByPath[path]) + } + + out := marshalDeviceInfo(&blockDevices) + return makeFakeExecCommand(0, string(out))(name, arg...) + }).Reset() + + defer gostub.Stub(&execCommand, func(cmd string, args ...string) *exec.Cmd { + devInfo := &deviceInfo{device, childDevice} + if args[3] == "/dev/child-name" { + devInfo = &deviceInfo{childDevice} + } + + mockedOutput := marshalDeviceInfo(devInfo) + return makeFakeExecCommand(0, string(mockedOutput))(cmd, args...) + }).Reset() + + c.Persist("/tmp/connector.json") + c2, err := GetConnectorFromFile("/tmp/connector.json") + assert.Nil(err) + assert.NotNil(c2) + if c2 != nil { + assert.Equal(c, *c2) + } + + err = c.Persist("/tmp") + assert.NotNil(err) + + os.Remove("/tmp/shouldNotExists.json") + _, err = GetConnectorFromFile("/tmp/shouldNotExists.json") + assert.NotNil(err) + assert.IsType(&os.PathError{}, err) + + ioutil.WriteFile("/tmp/connector.json", []byte("not a connector"), 0600) + _, err = GetConnectorFromFile("/tmp/connector.json") + assert.NotNil(err) + assert.IsType(&json.SyntaxError{}, err) +} + +func Test_IsMultipathConsistent(t *testing.T) { + mpath1 := Device{Name: "3600c0ff0000000000000000000000000", Type: "mpath", Size: "10G", Hctl: "0:0:0:1"} + mpath2 := Device{Name: "3600c0ff0000000000000000000000042", Type: "mpath", Size: "5G", Hctl: "0:0:0:2"} + sda := Device{Name: "sda", Size: "10G", Hctl: "1:0:0:1"} + sdb := Device{Name: "sdb", Size: "10G", Hctl: "2:0:0:1"} + sdc := Device{Name: "sdc", Size: "5G", Hctl: "1:0:0:2"} + sdd := Device{Name: "sdd", Size: "5G", Hctl: "2:0:0:2"} + invalidHCTL := Device{Name: "sde", Size: "5G", Hctl: "2:b"} + sdf := Device{Name: "sdf", Size: "10G", Hctl: "2:0:0:3"} + sdg := Device{Name: "sdg", Size: "10G", Hctl: "1:0:0:1"} + devicesWWIDs := map[string]string{} + devicesWWIDs[mpath1.GetPath()] = "3600c0ff0000000000000000000000000" + devicesWWIDs[sda.GetPath()] = "3600c0ff0000000000000000000000000" + devicesWWIDs[sdb.GetPath()] = "3600c0ff0000000000000000000000000" + devicesWWIDs[sdg.GetPath()] = "3600c0ff0000000000000000000000024" + + tests := map[string]struct { + connector *Connector + wantErr bool + errContains string + }{ + "Basic": { + connector: &Connector{ + MountTargetDevice: &mpath1, + Devices: []Device{sda, sdb}, + }, + }, + "Different sizes 1": { + connector: &Connector{ + MountTargetDevice: &mpath1, + Devices: []Device{sda, sdc}, + }, + wantErr: true, + errContains: "size differ", + }, + "Different sizes 2": { + connector: &Connector{ + MountTargetDevice: &mpath1, + Devices: []Device{sdc, sdd}, + }, + wantErr: true, + errContains: "size differ", + }, + "Invalid HCTL": { + connector: &Connector{ + MountTargetDevice: &invalidHCTL, + Devices: []Device{}, + }, + wantErr: true, + errContains: "invalid HCTL", + }, + "LUNs differs": { + connector: &Connector{ + MountTargetDevice: &mpath1, + Devices: []Device{sda, sdf}, + }, + wantErr: true, + errContains: "LUNs differ", + }, + "Same controller": { + connector: &Connector{ + MountTargetDevice: &mpath1, + Devices: []Device{sda, sdg}, + }, + wantErr: true, + errContains: "same controller", + }, + "Missing WWID": { + connector: &Connector{ + MountTargetDevice: &mpath2, + Devices: []Device{sdc, sdd}, + }, + wantErr: true, + errContains: "could not find WWID", + }, + "WWIDs differ": { + connector: &Connector{ + MountTargetDevice: &mpath1, + Devices: []Device{sdb, sdg}, + }, + wantErr: true, + errContains: "WWIDs differ", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + c := tt.connector + + defer gostub.Stub(&execWithTimeout, func(_ string, args []string, _ time.Duration) ([]byte, error) { + devicePath := args[len(args)-1] + wwid, ok := devicesWWIDs[devicePath] + if !ok { + return []byte(""), errors.New("") + } + return []byte(wwid + "\n"), nil + }).Reset() + + err := c.IsMultipathConsistent() + + if tt.wantErr { + assert.Error(err) + if tt.errContains != "" { + assert.Contains(err.Error(), tt.errContains) + } + } else { + assert.Nil(err) + } + }) + } +} diff --git a/pkg/lib/iscsi/iscsi/iscsiadm.go b/pkg/lib/iscsi/iscsi/iscsiadm.go new file mode 100644 index 00000000..b87b5c66 --- /dev/null +++ b/pkg/lib/iscsi/iscsi/iscsiadm.go @@ -0,0 +1,181 @@ +package iscsi + +import ( + "fmt" + "strings" + "time" +) + +// Secrets provides optional iscsi security credentials (CHAP settings) +type Secrets struct { + // SecretsType is the type of Secrets being utilized (currently we only impleemnent "chap" + SecretsType string `json:"secretsType,omitempty"` + // UserName is the configured iscsi user login + UserName string `json:"userName"` + // Password is the configured iscsi password + Password string `json:"password"` + // UserNameIn provides a specific input login for directional CHAP configurations + UserNameIn string `json:"userNameIn,omitempty"` + // PasswordIn provides a specific input password for directional CHAP configurations + PasswordIn string `json:"passwordIn,omitempty"` +} + +func iscsiCmd(args ...string) (string, error) { + stdout, err := execWithTimeout("iscsiadm", args, time.Second*3) + + debug.Printf("Run iscsiadm command: %s", strings.Join(append([]string{"iscsiadm"}, args...), " ")) + iscsiadmDebug(string(stdout), err) + + return string(stdout), err +} + +func iscsiadmDebug(output string, cmdError error) { + debugOutput := strings.Replace(output, "\n", "\\n", -1) + debug.Printf("Output of iscsiadm command: {output: %s}", debugOutput) + if cmdError != nil { + debug.Printf("Error message returned from iscsiadm command: %s", cmdError.Error()) + } +} + +// ListInterfaces returns a list of all iscsi interfaces configured on the node +/// along with the raw output in Response.StdOut we add the convenience of +// returning a list of entries found +func ListInterfaces() ([]string, error) { + debug.Println("Begin ListInterface...") + out, err := iscsiCmd("-m", "iface", "-o", "show") + return strings.Split(out, "\n"), err +} + +// ShowInterface retrieves the details for the specified iscsi interface +// caller should inspect r.Err and use r.StdOut for interface details +func ShowInterface(iface string) (string, error) { + debug.Println("Begin ShowInterface...") + out, err := iscsiCmd("-m", "iface", "-o", "show", "-I", iface) + return out, err +} + +// CreateDBEntry sets up a node entry for the specified tgt in the nodes iscsi nodes db +func CreateDBEntry(tgtIQN, portal, iFace string, discoverySecrets, sessionSecrets Secrets) error { + debug.Println("Begin CreateDBEntry...") + baseArgs := []string{"-m", "node", "-T", tgtIQN, "-p", portal} + _, err := iscsiCmd(append(baseArgs, "-I", iFace, "-o", "new")...) + if err != nil { + return err + } + + if discoverySecrets.SecretsType == "chap" { + debug.Printf("Setting CHAP Discovery...") + createCHAPEntries(baseArgs, discoverySecrets, true) + } + + if sessionSecrets.SecretsType == "chap" { + debug.Printf("Setting CHAP Session...") + createCHAPEntries(baseArgs, sessionSecrets, false) + } + + return err + +} + +// Discoverydb discovers the iscsi target +func Discoverydb(tp, iface string, discoverySecrets Secrets, chapDiscovery bool) error { + debug.Println("Begin Discoverydb...") + baseArgs := []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", iface} + out, err := iscsiCmd(append(baseArgs, []string{"-o", "new"}...)...) + if err != nil { + return fmt.Errorf("failed to create new entry of target in discoverydb, output: %v, err: %v", out, err) + } + + if chapDiscovery { + if err := createCHAPEntries(baseArgs, discoverySecrets, true); err != nil { + return err + } + } + + _, err = iscsiCmd(append(baseArgs, []string{"--discover"}...)...) + if err != nil { + //delete the discoverydb record + iscsiCmd(append(baseArgs, []string{"-o", "delete"}...)...) + return fmt.Errorf("failed to sendtargets to portal %s, err: %v", tp, err) + } + return nil +} + +func createCHAPEntries(baseArgs []string, secrets Secrets, discovery bool) error { + args := []string{} + debug.Printf("Begin createCHAPEntries (discovery=%t)...", discovery) + if discovery { + args = append(baseArgs, []string{"-o", "update", + "-n", "discovery.sendtargets.auth.authmethod", "-v", "CHAP", + "-n", "discovery.sendtargets.auth.username", "-v", secrets.UserName, + "-n", "discovery.sendtargets.auth.password", "-v", secrets.Password}...) + if secrets.UserNameIn != "" { + args = append(args, []string{"-n", "discovery.sendtargets.auth.username_in", "-v", secrets.UserNameIn}...) + } + if secrets.PasswordIn != "" { + args = append(args, []string{"-n", "discovery.sendtargets.auth.password_in", "-v", secrets.PasswordIn}...) + } + + } else { + + args = append(baseArgs, []string{"-o", "update", + "-n", "node.session.auth.authmethod", "-v", "CHAP", + "-n", "node.session.auth.username", "-v", secrets.UserName, + "-n", "node.session.auth.password", "-v", secrets.Password}...) + if secrets.UserNameIn != "" { + args = append(args, []string{"-n", "node.session.auth.username_in", "-v", secrets.UserNameIn}...) + } + if secrets.PasswordIn != "" { + args = append(args, []string{"-n", "node.session.auth.password_in", "-v", secrets.PasswordIn}...) + } + } + + _, err := iscsiCmd(args...) + if err != nil { + return fmt.Errorf("failed to update discoverydb with CHAP, err: %v", err) + } + + return nil +} + +// GetSessions retrieves a list of current iscsi sessions on the node +func GetSessions() (string, error) { + debug.Println("Begin GetSessions...") + out, err := iscsiCmd("-m", "session") + return out, err +} + +// Login performs an iscsi login for the specified target +func Login(tgtIQN, portal string) error { + debug.Println("Begin Login...") + baseArgs := []string{"-m", "node", "-T", tgtIQN, "-p", portal} + if _, err := iscsiCmd(append(baseArgs, []string{"-l"}...)...); err != nil { + //delete the node record from database + iscsiCmd(append(baseArgs, []string{"-o", "delete"}...)...) + return fmt.Errorf("failed to sendtargets to portal %s, err: %v", portal, err) + } + return nil +} + +// Logout logs out the specified target +func Logout(tgtIQN, portal string) error { + debug.Println("Begin Logout...") + args := []string{"-m", "node", "-T", tgtIQN, "-p", portal, "-u"} + iscsiCmd(args...) + return nil +} + +// DeleteDBEntry deletes the iscsi db entry for the specified target +func DeleteDBEntry(tgtIQN string) error { + debug.Println("Begin DeleteDBEntry...") + args := []string{"-m", "node", "-T", tgtIQN, "-o", "delete"} + iscsiCmd(args...) + return nil +} + +// DeleteIFace delete the iface +func DeleteIFace(iface string) error { + debug.Println("Begin DeleteIFace...") + iscsiCmd([]string{"-m", "iface", "-I", iface, "-o", "delete"}...) + return nil +} diff --git a/pkg/lib/iscsi/iscsi/iscsiadm_test.go b/pkg/lib/iscsi/iscsi/iscsiadm_test.go new file mode 100644 index 00000000..1d84d800 --- /dev/null +++ b/pkg/lib/iscsi/iscsi/iscsiadm_test.go @@ -0,0 +1,275 @@ +package iscsi + +import ( + "os/exec" + "testing" + + "github.com/prashantv/gostub" + "github.com/stretchr/testify/assert" +) + +const defaultInterface = ` +# BEGIN RECORD 2.0-874 +iface.iscsi_ifacename = default +iface.net_ifacename = +iface.ipaddress = +iface.hwaddress = +iface.transport_name = tcp +iface.initiatorname = +iface.state = +iface.vlan_id = 0 +iface.vlan_priority = 0 +iface.vlan_state = +iface.iface_num = 0 +iface.mtu = 0 +iface.port = 0 +iface.bootproto = +iface.subnet_mask = +iface.gateway = +iface.dhcp_alt_client_id_state = +iface.dhcp_alt_client_id = +iface.dhcp_dns = +iface.dhcp_learn_iqn = +iface.dhcp_req_vendor_id_state = +iface.dhcp_vendor_id_state = +iface.dhcp_vendor_id = +iface.dhcp_slp_da = +iface.fragmentation = +iface.gratuitous_arp = +iface.incoming_forwarding = +iface.tos_state = +iface.tos = 0 +iface.ttl = 0 +iface.delayed_ack = +iface.tcp_nagle = +iface.tcp_wsf_state = +iface.tcp_wsf = 0 +iface.tcp_timer_scale = 0 +iface.tcp_timestamp = +iface.redirect = +iface.def_task_mgmt_timeout = 0 +iface.header_digest = +iface.data_digest = +iface.immediate_data = +iface.initial_r2t = +iface.data_seq_inorder = +iface.data_pdu_inorder = +iface.erl = 0 +iface.max_receive_data_len = 0 +iface.first_burst_len = 0 +iface.max_outstanding_r2t = 0 +iface.max_burst_len = 0 +iface.chap_auth = +iface.bidi_chap = +iface.strict_login_compliance = +iface.discovery_auth = +iface.discovery_logout = +# END RECORD +` + +func TestDiscovery(t *testing.T) { + tests := map[string]struct { + tgtPortal string + iface string + discoverySecret Secrets + chapDiscovery bool + wantErr bool + mockedStdout string + mockedCmdError error + }{ + "DiscoverySuccess": { + tgtPortal: "172.18.0.2:3260", + iface: "default", + chapDiscovery: false, + mockedStdout: "172.18.0.2:3260,1 iqn.2016-09.com.openebs.jiva:store1\n", + mockedCmdError: nil, + }, + + "ConnectionFailure": { + tgtPortal: "172.18.0.2:3262", + iface: "default", + chapDiscovery: false, + mockedStdout: `iscsiadm: cannot make connection to 172.18.0.2: Connection refused +iscsiadm: cannot make connection to 172.18.0.2: Connection refused +iscsiadm: connection login retries (reopen_max) 5 exceeded +iscsiadm: Could not perform SendTargets discovery: encountered connection failure\n`, + mockedCmdError: exec.Command("exit", "4").Run(), + wantErr: true, + }, + + "ChapEntrySuccess": { + tgtPortal: "172.18.0.2:3260", + iface: "default", + chapDiscovery: true, + discoverySecret: Secrets{ + UserNameIn: "dummyuser", + PasswordIn: "dummypass", + }, + mockedStdout: "172.18.0.2:3260,1 iqn.2016-09.com.openebs.jiva:store1\n", + mockedCmdError: nil, + }, + + "ChapEntryFailure": { + tgtPortal: "172.18.0.2:3260", + iface: "default", + discoverySecret: Secrets{ + UserNameIn: "dummyuser", + PasswordIn: "dummypass", + }, + chapDiscovery: true, + mockedStdout: `iscsiadm: Login failed to authenticate with target +iscsiadm: discovery login to 172.18.0.2 rejected: initiator error (02/01), non-retryable, giving up +iscsiadm: Could not perform SendTargets discovery.\n`, + mockedCmdError: exec.Command("exit", "4").Run(), + wantErr: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + defer gostub.Stub(&execWithTimeout, makeFakeExecWithTimeout(false, []byte(tt.mockedStdout), tt.mockedCmdError)).Reset() + err := Discoverydb(tt.tgtPortal, tt.iface, tt.discoverySecret, tt.chapDiscovery) + if (err != nil) != tt.wantErr { + t.Errorf("Discoverydb() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestCreateDBEntry(t *testing.T) { + tests := map[string]struct { + tgtPortal string + tgtIQN string + iface string + discoverySecret Secrets + sessionSecret Secrets + wantErr bool + mockedStdout string + mockedCmdError error + }{ + "CreateDBEntryWithChapDiscoverySuccess": { + tgtPortal: "192.168.1.107:3260", + tgtIQN: "iqn.2010-10.org.openstack:volume-eb393993-73d0-4e39-9ef4-b5841e244ced", + iface: "default", + discoverySecret: Secrets{ + UserNameIn: "dummyuser", + PasswordIn: "dummypass", + SecretsType: "chap", + }, + sessionSecret: Secrets{ + UserNameIn: "dummyuser", + PasswordIn: "dummypass", + SecretsType: "chap", + }, + mockedStdout: nodeDB, + mockedCmdError: nil, + }, + "CreateDBEntryWithChapDiscoveryFailure": { + tgtPortal: "172.18.0.2:3260", + tgtIQN: "iqn.2016-09.com.openebs.jiva:store1", + iface: "default", + mockedStdout: "iscsiadm: No records found\n", + mockedCmdError: exec.Command("exit", "21").Run(), + wantErr: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + defer gostub.Stub(&execWithTimeout, makeFakeExecWithTimeout(false, []byte(tt.mockedStdout), tt.mockedCmdError)).Reset() + err := CreateDBEntry(tt.tgtIQN, tt.tgtPortal, tt.iface, tt.discoverySecret, tt.sessionSecret) + if (err != nil) != tt.wantErr { + t.Errorf("CreateDBEntry() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } + +} + +func TestListInterfaces(t *testing.T) { + tests := map[string]struct { + mockedStdout string + mockedCmdError error + interfaces []string + wantErr bool + }{ + "EmptyOutput": { + mockedStdout: "", + mockedCmdError: nil, + interfaces: []string{""}, + wantErr: false, + }, + "DefaultInterface": { + mockedStdout: "default", + mockedCmdError: nil, + interfaces: []string{"default"}, + wantErr: false, + }, + "TwoInterface": { + mockedStdout: "default\ntest", + mockedCmdError: nil, + interfaces: []string{"default", "test"}, + wantErr: false, + }, + "HasError": { + mockedStdout: "", + mockedCmdError: exec.Command("exit", "1").Run(), + interfaces: []string{}, + wantErr: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + defer gostub.Stub(&execWithTimeout, makeFakeExecWithTimeout(false, []byte(tt.mockedStdout), tt.mockedCmdError)).Reset() + interfaces, err := ListInterfaces() + + if tt.wantErr { + assert.NotNil(err) + } else { + assert.Nil(err) + assert.Equal(interfaces, tt.interfaces) + } + }) + } +} + +func TestShowInterface(t *testing.T) { + tests := map[string]struct { + mockedStdout string + mockedCmdError error + iFace string + wantErr bool + }{ + "DefaultInterface": { + mockedStdout: defaultInterface, + mockedCmdError: nil, + iFace: defaultInterface, + wantErr: false, + }, + "HasError": { + mockedStdout: "", + mockedCmdError: exec.Command("exit", "1").Run(), + iFace: "", + wantErr: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + defer gostub.Stub(&execWithTimeout, makeFakeExecWithTimeout(false, []byte(tt.mockedStdout), tt.mockedCmdError)).Reset() + interfaces, err := ShowInterface("default") + + if tt.wantErr { + assert.NotNil(err) + } else { + assert.Nil(err) + assert.Equal(interfaces, tt.iFace) + } + }) + } +} diff --git a/pkg/lib/iscsi/iscsi/multipath.go b/pkg/lib/iscsi/iscsi/multipath.go new file mode 100644 index 00000000..347ae33b --- /dev/null +++ b/pkg/lib/iscsi/iscsi/multipath.go @@ -0,0 +1,87 @@ +package iscsi + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +type pathGroup struct { + Paths []path `json:"paths"` +} + +type path struct { + Device string `json:"dev"` +} + +// ExecWithTimeout execute a command with a timeout and returns an error if timeout is excedeed +func ExecWithTimeout(command string, args []string, timeout time.Duration) ([]byte, error) { + debug.Printf("Executing command '%v' with args: '%v'.\n", command, args) + + // Create a new context and add a timeout to it + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + // Create command with context + cmd := execCommandContext(ctx, command, args...) + + // This time we can simply use Output() to get the result. + out, err := cmd.Output() + debug.Println(err) + + // We want to check the context error to see if the timeout was executed. + // The error returned by cmd.Output() will be OS specific based on what + // happens when a process is killed. + if ctx.Err() == context.DeadlineExceeded { + debug.Printf("Command '%s' timeout reached.\n", command) + return nil, ctx.Err() + } + + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + debug.Printf("Non-zero exit code: %s\n", err) + err = fmt.Errorf("%s", ee.Stderr) + } + } + + debug.Println("Finished executing command.") + return out, err +} + +// FlushMultipathDevice flushes a multipath device dm-x with command multipath -f /dev/dm-x +func FlushMultipathDevice(device *Device) error { + devicePath := device.GetPath() + debug.Printf("Flushing multipath device '%v'.\n", devicePath) + + timeout := 5 * time.Second + _, err := execWithTimeout("multipath", []string{"-f", devicePath}, timeout) + + if err != nil { + if _, e := osStat(devicePath); os.IsNotExist(e) { + debug.Printf("Multipath device %v has been removed.\n", devicePath) + } else { + if strings.Contains(err.Error(), "map in use") { + err = fmt.Errorf("device is probably still in use somewhere else: %v", err) + } + debug.Printf("Command 'multipath -f %v' did not succeed to delete the device: %v\n", devicePath, err) + return err + } + } + + debug.Printf("Finshed flushing multipath device %v.\n", devicePath) + return nil +} + +// ResizeMultipathDevice resize a multipath device based on its underlying devices +func ResizeMultipathDevice(device *Device) error { + debug.Printf("Resizing multipath device %s\n", device.GetPath()) + + if output, err := execCommand("multipathd", "resize", "map", device.Name).CombinedOutput(); err != nil { + return fmt.Errorf("could not resize multipath device: %s (%v)", output, err) + } + + return nil +} diff --git a/pkg/lib/iscsi/iscsi/multipath_test.go b/pkg/lib/iscsi/iscsi/multipath_test.go new file mode 100644 index 00000000..abf52498 --- /dev/null +++ b/pkg/lib/iscsi/iscsi/multipath_test.go @@ -0,0 +1,79 @@ +package iscsi + +import ( + "context" + "os/exec" + "testing" + "time" + + "github.com/prashantv/gostub" + "github.com/stretchr/testify/assert" +) + +func TestExecWithTimeout(t *testing.T) { + tests := map[string]struct { + mockedStdout string + mockedExitStatus int + wantTimeout bool + }{ + "Success": { + mockedStdout: "some output", + mockedExitStatus: 0, + wantTimeout: false, + }, + "WithError": { + mockedStdout: "some\noutput", + mockedExitStatus: 1, + wantTimeout: false, + }, + "WithTimeout": { + mockedStdout: "", + mockedExitStatus: 0, + wantTimeout: true, + }, + "WithTimeoutAndOutput": { + mockedStdout: "should not be returned", + mockedExitStatus: 0, + wantTimeout: true, + }, + "WithTimeoutAndError": { + mockedStdout: "", + mockedExitStatus: 1, + wantTimeout: true, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + timeout := time.Second + if tt.wantTimeout { + timeout = time.Millisecond * 50 + } + + defer gostub.Stub(&execCommandContext, func(ctx context.Context, command string, args ...string) *exec.Cmd { + if tt.wantTimeout { + time.Sleep(timeout + time.Millisecond*10) + } + return makeFakeExecCommandContext(tt.mockedExitStatus, tt.mockedStdout)(ctx, command, args...) + }).Reset() + + out, err := ExecWithTimeout("dummy", []string{}, timeout) + + if tt.wantTimeout || tt.mockedExitStatus != 0 { + assert.NotNil(err) + if tt.wantTimeout { + assert.Equal(context.DeadlineExceeded, err) + } + } else { + assert.Nil(err) + } + + if tt.wantTimeout { + assert.Equal("", string(out)) + } else { + assert.Equal(tt.mockedStdout, string(out)) + } + }) + } +} diff --git a/vendor/github.com/kubernetes-csi/csi-lib-iscsi/LICENSE b/vendor/github.com/kubernetes-csi/csi-lib-iscsi/LICENSE deleted file mode 100644 index 8dada3ed..00000000 --- a/vendor/github.com/kubernetes-csi/csi-lib-iscsi/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/kubernetes-csi/csi-lib-iscsi/iscsi/iscsi.go b/vendor/github.com/kubernetes-csi/csi-lib-iscsi/iscsi/iscsi.go index e49ef2dc..465ca0bd 100644 --- a/vendor/github.com/kubernetes-csi/csi-lib-iscsi/iscsi/iscsi.go +++ b/vendor/github.com/kubernetes-csi/csi-lib-iscsi/iscsi/iscsi.go @@ -30,7 +30,7 @@ var ( sleep = time.Sleep ) -// iscsiSession contains information avout an iSCSI session +// iscsiSession contains information about an iSCSI session type iscsiSession struct { Protocol string ID int32 @@ -239,7 +239,7 @@ func getMultipathDevice(devices []Device) (*Device, error) { return multipathDevice, nil } -// Connect is for backward-compatiblity with c.Connect() +// Connect is for backward-compatibility with c.Connect() func Connect(c Connector) (string, error) { return c.Connect() } @@ -477,7 +477,7 @@ func GetSCSIDevices(devicePaths []string, strict bool) ([]Device, error) { deviceInfo, err := lsblk(devicePaths, strict) if err != nil { - debug.Printf("An error occured while looking info about SCSI devices: %v", err) + debug.Printf("An error occurred while looking info about SCSI devices: %v", err) return nil, err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 79c85376..679c6a81 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -46,7 +46,7 @@ github.com/hashicorp/golang-lru/simplelru github.com/inconshreveable/mousetrap # github.com/json-iterator/go v1.1.10 github.com/json-iterator/go -# github.com/kubernetes-csi/csi-lib-iscsi v0.0.0-20211110090527-5c802c48a124 +# github.com/kubernetes-csi/csi-lib-iscsi v0.0.0-20211110090527-5c802c48a124 => ./pkg/lib/iscsi ## explicit github.com/kubernetes-csi/csi-lib-iscsi/iscsi # github.com/kubernetes-csi/csi-lib-utils v0.10.0 @@ -581,3 +581,4 @@ sigs.k8s.io/yaml # k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.21.1 # k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.21.1 # k8s.io/sample-controller => k8s.io/sample-controller v0.21.1 +# github.com/kubernetes-csi/csi-lib-iscsi => ./pkg/lib/iscsi