diff --git a/Dockerfile b/Dockerfile index 1e87c61..510a291 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,6 @@ COPY magellan /magellan COPY /bin/magellan.sh /magellan.sh -CMD [ "/magellan.sh" ] +CMD [ "/magellan" ] ENTRYPOINT [ "/sbin/tini", "--" ] diff --git a/Makefile b/Makefile index dda9c30..c21ed53 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,12 @@ diff: ## git diff git diff --exit-code RES=$$(git status --porcelain) ; if [ -n "$$RES" ]; then echo $$RES && exit 1 ; fi +.PHONY: docs +docs: ## go docs + $(call print-target) + go doc github.com/OpenCHAMI/magellan/cmd + go doc github.com/OpenCHAMI/magellan/internal + go doc github.com/OpenCHAMI/magellan/pkg/crawler define print-target @printf "Executing target: \033[36m$@\033[0m\n" diff --git a/README.md b/README.md index 49724f4..52db387 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The `magellan` CLI tool is a Redfish-based, board management controller (BMC) di [Build](#building) and [run on bare metal](#running-the-tool) or run and test with Docker using the [latest prebuilt image](#running-with-docker). For quick testing, the repository integrates a Redfish emulator that can be ran by executing the `emulator/setup.sh` script or running `make emulator`. -## Building +## Building the Executable The `magellan` tool can be built to run on bare metal. Install the required Go tools, clone the repo, and then build the binary in the root directory with the following: @@ -18,6 +18,30 @@ go mod tidy && go build And that's it. The last line should find and download all of the required dependencies to build the project. Although other versions of Go may work, the project has been tested to work with versions v1.20 and later on MacOS and Linux. +### Building on Debian 12 (Bookworm) + +Getting the `magellan` tool to work with Go 1.21 on Debian 12 may require installing the `golang-1.21` meta-package from `bookworm-backports` through `apt` along with GCC for comping the `go-sqlite3` driver. + +```bash +apt install gcc golang-1.21/bookworm-backport +``` + +The binary executable for the `golang-1.21` executable can then be found using `dpkg`. + +```bash +dpkg -L golang-1.21-go +``` + +Using the correct binary, set the `CGO_ENABLED` environment variable and build the executable with `cgo` enabled: + +```bash +export GOBIN=/usr/bin/golang-1.21/bin/go +go env -w CGO_ENABLED=1 +go mod tidy && go build +``` + +This might take some time to complete initially because of the `go-sqlite3` driver, but should be much faster for subsequent builds. + ### Docker The tool can also run using Docker. To build the Docker container, run `docker build -t magellan:testing .` in the project's directory. This is useful if you to run `magellan` on a different system through Docker desktop without having to install and build with Go (or if you can't do so for some reason). [Prebuilt images](https://github.com/OpenCHAMI/magellan/pkgs/container/magellan) are available as well on `ghcr`. Images can be pulled directly from the repository: @@ -28,6 +52,10 @@ docker pull ghcr.io/openchami/magellan:latest See the ["Running with Docker"](#running-with-docker) section below about running with the Docker container. + + + + ## Usage The sections below assume that the BMC nodes have an IP address available to query Redfish. Currently, `magellan` does not support discovery with MAC addresses although that may change in the future. @@ -114,7 +142,7 @@ To inspect the cache, use the `list` command. Make sure to point to the same dat This will print a list of node info found and stored from the scan. Like the `scan` subcommand, the output format can be set using the `--format` flag. -Finally, set the `MAGELLAN_ACCESS_TOKEN`run the `collect` command to query the node from cache and send the info to be stored into SMD: +Finally, set the `ACCESS_TOKEN`run the `collect` command to query the node from cache and send the info to be stored into SMD: ```bash ./magellan collect \ @@ -155,14 +183,14 @@ watch -n 1 "./magellan update --status --host 172.16.0.110 --user admin --pass p ### Getting an Access Token (WIP) -The `magellan` tool has a `login` subcommand that works with the [`opaal`](https://github.com/OpenCHAMI/opaal) service to obtain a token needed to access the SMD service. If the SMD instance requires authentication, set the `MAGELLAN_ACCESS_TOKEN` environment variable to have `magellan` include it in the header for HTTP requests to SMD. +The `magellan` tool has a `login` subcommand that works with the [`opaal`](https://github.com/OpenCHAMI/opaal) service to obtain a token needed to access the SMD service. If the SMD instance requires authentication, set the `ACCESS_TOKEN` environment variable to have `magellan` include it in the header for HTTP requests to SMD. ```bash # must have a running OPAAL instance ./magellan login --url https://opaal:4444/login # ...complete login flow to get token -export MAGELLAN_ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c +export ACCESS_TOKEN=eyJhbGciOiJIUzI1NiIs... ``` Alternatively, if you are running the OpenCHAMI quickstart in the [deployment recipes](https://github.com/OpenCHAMI/deployment-recipes), you can run the provided script to generate a token and set the environment variable that way. @@ -170,7 +198,7 @@ Alternatively, if you are running the OpenCHAMI quickstart in the [deployment re ```bash quickstart_dir=path/to/deployment/recipes/quickstart source $quickstart_dir/bash_functions.sh -export MAGELLAN_ACCESS_TOKEN=$(gen_access_token) +export ACCESS_TOKEN=$(gen_access_token) ``` ### Running with Docker diff --git a/cmd/collect.go b/cmd/collect.go index 2155cbe..3641a55 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -18,9 +18,17 @@ var ( forceUpdate bool ) +// The `collect` command fetches data from a collection of BMC nodes. +// This command should be ran after the `scan` to find available hosts +// on a subnet. var collectCmd = &cobra.Command{ Use: "collect", - Short: "Query information about BMC", + Short: "Collect system information by interrogating BMC node", + Long: "Send request(s) to a collection of hosts running Redfish services found stored from the 'scan' in cache.\n" + + "See the 'scan' command on how to perform a scan.\n\n" + + "Examples:\n" + + " magellan collect --cache ./assets.db --output ./logs --timeout 30 --cacert cecert.pem\n" + + " magellan collect --host smd.example.com --port 27779 --username username --password password", Run: func(cmd *cobra.Command, args []string) { // make application logger l := log.NewLogger(logrus.New(), logrus.DebugLevel) @@ -49,8 +57,8 @@ var collectCmd = &cobra.Command{ concurrency = mathutil.Clamp(len(probeStates), 1, 255) } q := &magellan.QueryParams{ - User: username, - Pass: password, + Username: username, + Password: password, Protocol: protocol, Timeout: timeout, Concurrency: concurrency, @@ -77,23 +85,26 @@ func init() { currentUser, _ = user.Current() collectCmd.PersistentFlags().StringVar(&smd.Host, "host", smd.Host, "set the host to the SMD API") collectCmd.PersistentFlags().IntVarP(&smd.Port, "port", "p", smd.Port, "set the port to the SMD API") - collectCmd.PersistentFlags().StringVar(&username, "user", "", "set the BMC user") - collectCmd.PersistentFlags().StringVar(&password, "pass", "", "set the BMC password") + collectCmd.PersistentFlags().StringVar(&username, "username", "", "set the BMC user") + collectCmd.PersistentFlags().StringVar(&password, "password", "", "set the BMC password") collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "set the protocol used to query") collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/data/", currentUser.Username+"/"), "set the path to store collection data") collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD") - collectCmd.PersistentFlags().StringVar(&cacertPath, "ca-cert", "", "path to CA cert. (defaults to system CAs)") - collectCmd.MarkFlagsRequiredTogether("user", "pass") + collectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") + // set flags to only be used together + collectCmd.MarkFlagsRequiredTogether("username", "password") + + // bind flags to config properties viper.BindPFlag("collect.driver", collectCmd.Flags().Lookup("driver")) viper.BindPFlag("collect.host", collectCmd.Flags().Lookup("host")) viper.BindPFlag("collect.port", collectCmd.Flags().Lookup("port")) - viper.BindPFlag("collect.user", collectCmd.Flags().Lookup("user")) - viper.BindPFlag("collect.pass", collectCmd.Flags().Lookup("pass")) + viper.BindPFlag("collect.username", collectCmd.Flags().Lookup("username")) + viper.BindPFlag("collect.password", collectCmd.Flags().Lookup("password")) viper.BindPFlag("collect.protocol", collectCmd.Flags().Lookup("protocol")) viper.BindPFlag("collect.output", collectCmd.Flags().Lookup("output")) viper.BindPFlag("collect.force-update", collectCmd.Flags().Lookup("force-update")) - viper.BindPFlag("collect.ca-cert", collectCmd.Flags().Lookup("secure-tls")) + viper.BindPFlag("collect.cacert", collectCmd.Flags().Lookup("secure-tls")) viper.BindPFlags(collectCmd.Flags()) rootCmd.AddCommand(collectCmd) diff --git a/cmd/crawl.go b/cmd/crawl.go index cd573ed..a4956bb 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -11,13 +11,16 @@ import ( "github.com/spf13/cobra" ) +// The `crawl` command walks a collection of Redfish endpoints to collect +// specfic inventory detail. This command only expects host names and does +// not require a scan to be performed beforehand. var crawlCmd = &cobra.Command{ - Use: "crawl [uri]", + Use: "crawl [uri]", + Short: "Crawl a single BMC for inventory information", Long: "Crawl a single BMC for inventory information\n" + "\n" + "Example:\n" + " magellan crawl https://bmc.example.com", - Short: "Crawl a single BMC for inventory information", Args: func(cmd *cobra.Command, args []string) error { // Validate that the only argument is a valid URI if err := cobra.ExactArgs(1)(cmd, args); err != nil { diff --git a/cmd/list.go b/cmd/list.go index b722c81..89cd847 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -12,9 +12,17 @@ import ( "github.com/spf13/cobra" ) +// The `list` command provides an easy way to show what was found +// and stored in a cache database from a scan. The data that's stored +// is what is consumed by the `collect` command with the --cache flag. var listCmd = &cobra.Command{ Use: "list", - Short: "List information from scan", + Short: "List information stored in cache from a scan", + Long: "Prints all of the host and associated data found from performing a scan.\n" + + "See the 'scan' command on how to perform a scan.\n\n" + + "Examples:\n" + + " magellan list\n" + + " magellan list " Run: func(cmd *cobra.Command, args []string) { probeResults, err := sqlite.GetProbeResults(cachePath) if err != nil { diff --git a/cmd/root.go b/cmd/root.go index 2d27bee..40f7407 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,3 +1,17 @@ +// The cmd package implements the interface for the magellan CLI. The files +// contained in this package only contains implementations for handling CLI +// arguments and passing them to functions within magellan's internal API. +// +// Each CLI subcommand will have at least one corresponding internal file +// with an API routine that implements the command's functionality. The main +// API routine will usually be the first function defined in the fill. +// +// For example: +// +// cmd/scan.go --> internal/scan.go ( magellan.ScanForAssets() ) +// cmd/collect.go --> internal/collect.go ( magellan.CollectAll() ) +// cmd/list.go --> none (doesn't have API call since it's simple) +// cmd/update.go --> internal/update.go ( magellan.UpdateFirmware() ) package cmd import ( @@ -30,11 +44,8 @@ var ( verbose bool ) -// TODO: discover bmc's on network (dora) -// TODO: query bmc component information and store in db (?) -// TODO: send bmc component information to smd -// TODO: set ports to scan automatically with set driver - +// The `root` command doesn't do anything on it's own except display +// a help message and then exits. var rootCmd = &cobra.Command{ Use: "magellan", Short: "Tool for BMC discovery", @@ -47,6 +58,7 @@ var rootCmd = &cobra.Command{ }, } +// This Execute() function is called from main to run the CLI. func Execute() { if err := rootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) @@ -54,9 +66,17 @@ func Execute() { } } +// LoadAccessToken() tries to load a JWT string from an environment +// variable, file, or config in that order. If loading the token +// fails with one options, it will fallback to the next option until +// all options are exhausted. +// +// Returns a token as a string with no error if successful. +// Alternatively, returns an empty string with an error if a token is +// not able to be loaded. func LoadAccessToken() (string, error) { // try to load token from env var - testToken := os.Getenv("MAGELLAN_ACCESS_TOKEN") + testToken := os.Getenv("ACCESS_TOKEN") if testToken != "" { return testToken, nil } @@ -93,12 +113,21 @@ func init() { viper.BindPFlags(rootCmd.Flags()) } +// InitializeConfig() initializes a new config object by loading it +// from a file given a non-empty string. +// +// See the 'LoadConfig' function in 'internal/config' for details. func InitializeConfig() { if configPath != "" { magellan.LoadConfig(configPath) } } +// SetDefaults() resets all of the viper properties back to their +// default values. +// +// TODO: This function should probably be moved to 'internal/config.go' +// instead of in this file. func SetDefaults() { viper.SetDefault("threads", 1) viper.SetDefault("timeout", 30) diff --git a/cmd/scan.go b/cmd/scan.go index 5bb5019..9fafdfe 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -25,9 +25,21 @@ var ( disableProbing bool ) +// The `scan` command is usually the first step to using the CLI tool. +// This command will perform a network scan over a subnet by supplying +// a list of subnets, subnet masks, and additional IP address to probe. +// +// See the `ScanForAssets()` function in 'internal/scan.go' for details +// related to the implementation. var scanCmd = &cobra.Command{ Use: "scan", Short: "Scan for BMC nodes on a network", + Long: "Perform a net scan by attempting to connect to each host and port specified and getting a response. " + + "If the '--disable-probe` flag is used, the tool will not send another request to probe for available " + + "Redfish services.\n\n" + + "Example:\n" + + " magellan scan --subnet 172.16.0.0/24 --add-host 10.0.0.101\n" + + " magellan scan --subnet 172.16.0.0 --subnet-mask 255.255.255.0 --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { var ( hostsToScan []string diff --git a/cmd/update.go b/cmd/update.go index 0a124d9..87f2282 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -18,9 +18,16 @@ var ( status bool ) +// The `update` command provides an interface to easily update firmware +// using Redfish. It also provides a simple way to check the status of +// an update in-progress. var updateCmd = &cobra.Command{ Use: "update", Short: "Update BMC node firmware", + Long: "Perform an firmware update using Redfish by providing a remote firmware URL and component.\n" + + "Examples:\n" + + " magellan update --host 172.16.0.108 --port 443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + + " magellan update --status --host 172.16.0.108 --port 443 --username bmc_username --password bmc_password", Run: func(cmd *cobra.Command, args []string) { l := log.NewLogger(logrus.New(), logrus.DebugLevel) q := &magellan.UpdateParams{ @@ -33,8 +40,8 @@ var updateCmd = &cobra.Command{ Preferred: "redfish", Protocol: protocol, Host: host, - User: username, - Pass: password, + Username: username, + Password: password, Timeout: timeout, Port: port, }, @@ -78,14 +85,14 @@ func init() { updateCmd.Flags().StringVar(&component, "component", "", "set the component to upgrade") updateCmd.Flags().BoolVar(&status, "status", false, "get the status of the update") - viper.BindPFlag("bmc-host", updateCmd.Flags().Lookup("bmc-host")) - viper.BindPFlag("bmc-port", updateCmd.Flags().Lookup("bmc-port")) - viper.BindPFlag("user", updateCmd.Flags().Lookup("user")) - viper.BindPFlag("pass", updateCmd.Flags().Lookup("pass")) + viper.BindPFlag("host", updateCmd.Flags().Lookup("host")) + viper.BindPFlag("port", updateCmd.Flags().Lookup("port")) + viper.BindPFlag("username", updateCmd.Flags().Lookup("user")) + viper.BindPFlag("password", updateCmd.Flags().Lookup("pass")) viper.BindPFlag("transfer-protocol", updateCmd.Flags().Lookup("transfer-protocol")) viper.BindPFlag("protocol", updateCmd.Flags().Lookup("protocol")) - viper.BindPFlag("firmware-url", updateCmd.Flags().Lookup("firmware-url")) - viper.BindPFlag("firmware-version", updateCmd.Flags().Lookup("firmware-version")) + viper.BindPFlag("firmware.url", updateCmd.Flags().Lookup("firmware.url")) + viper.BindPFlag("firmware.version", updateCmd.Flags().Lookup("firmware.version")) viper.BindPFlag("component", updateCmd.Flags().Lookup("component")) viper.BindPFlag("secure-tls", updateCmd.Flags().Lookup("secure-tls")) viper.BindPFlag("status", updateCmd.Flags().Lookup("status")) diff --git a/config.yaml b/config.yaml index 455fe0d..30f244c 100644 --- a/config.yaml +++ b/config.yaml @@ -11,22 +11,23 @@ scan: collect: # host: smd-host # port: smd-port - user: "admin" - pass: "password" + username: "admin" + password: "password" protocol: "https" output: "/tmp/magellan/data/" threads: 1 force-update: false - ca-cert: "cacert.pem" + cacert: "cacert.pem" update: - bmc-host: - bmc-port: 443 - user: "admin" - pass: "password" + host: + port: 443 + username: "admin" + password: "password" transfer-protocol: "HTTP" protocol: "https" - firmware-url: - firmware-version: + firmware: + url: + version: component: secure-tls: false status: false diff --git a/internal/collect.go b/internal/collect.go index f678e42..f5e6480 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -1,3 +1,4 @@ +// Package magellan implements the core routines for the tools. package magellan import ( @@ -31,25 +32,33 @@ const ( HTTPS_PORT = 443 ) -// NOTE: ...params were getting too long... +// QueryParams is a collections of common parameters passed to the CLI. +// Each CLI subcommand has a corresponding implementation function that +// takes an object as an argument. However, the implementation may not +// use all of the properties within the object. type QueryParams struct { - Host string - Port int - Protocol string - User string - Pass string - Drivers []string - Concurrency int - Preferred string - Timeout int - CaCertPath string - Verbose bool - IpmitoolPath string - OutputPath string - ForceUpdate bool - AccessToken string + Host string // set by the 'host' flag + Port int // set by the 'port' flag + Protocol string // set by the 'protocol' flag + Username string // set the BMC username with the 'username' flag + Password string // set the BMC password with the 'password' flag + Drivers []string // DEPRECATED: TO BE REMOVED!!! + Concurrency int // set the of concurrent jobs with the 'concurrency' flag + Preferred string // DEPRECATED: TO BE REMOVED!!! + Timeout int // set the timeout with the 'timeout' flag + CaCertPath string // set the cert path with the 'cacert' flag + Verbose bool // set whether to include verbose output with 'verbose' flag + IpmitoolPath string // DEPRECATED: TO BE REMOVE!!! + OutputPath string // set the path to save output with 'output' flag + ForceUpdate bool // set whether to force updating SMD with 'force-update' flag + AccessToken string // set the access token to include in request with 'access-token' flag } +// This is the main function used to collect information from the BMC nodes via Redfish. +// The function expects a list of hosts found using the `ScanForAssets()` function. +// +// Requests can be made to several of the nodes using a goroutine by setting the q.Concurrency +// property value between 1 and 255. func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) error { // check for available probe states if probeStates == nil { @@ -102,6 +111,7 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err if err != nil { l.Log.Errorf("failed to connect to BMC (%v:%v): %v", q.Host, q.Port, err) } + defer gofishClient.Logout() // data to be sent to smd data := map[string]any{ @@ -109,7 +119,7 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err "Type": "", "Name": "", "FQDN": ps.Host, - "User": q.User, + "User": q.Username, // "Password": q.Pass, "MACRequired": true, "RediscoverOnUpdate": false, @@ -218,35 +228,7 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err return nil } -func CollectMetadata(client *bmclib.Client, q *QueryParams) ([]byte, error) { - // open BMC session and update driver registry - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout)) - client.Registry.FilterForCompatible(ctx) - err := client.Open(ctx) - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to connect to bmc: %v", err) - } - - defer client.Close(ctx) - - metadata := client.GetMetadata() - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to get metadata: %v", err) - } - - // retrieve inventory data - b, err := json.MarshalIndent(metadata, "", " ") - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - ctxCancel() - return b, nil -} - +// CollectInventory() fetches inventory data from all of the BMC hosts provided. func CollectInventory(client *bmclib.Client, q *QueryParams) ([]byte, error) { // open BMC session and update driver registry ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout)) @@ -275,6 +257,7 @@ func CollectInventory(client *bmclib.Client, q *QueryParams) ([]byte, error) { return b, nil } +// TODO: DELETE ME!!! func CollectPowerState(client *bmclib.Client, q *QueryParams) ([]byte, error) { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout)) client.Registry.FilterForCompatible(ctx) @@ -303,6 +286,7 @@ func CollectPowerState(client *bmclib.Client, q *QueryParams) ([]byte, error) { } +// TODO: DELETE ME!!! func CollectUsers(client *bmclib.Client, q *QueryParams) ([]byte, error) { // open BMC session and update driver registry ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout)) @@ -333,11 +317,17 @@ func CollectUsers(client *bmclib.Client, q *QueryParams) ([]byte, error) { return b, nil } +// TODO: DELETE ME!!! func CollectBios(client *bmclib.Client, q *QueryParams) ([]byte, error) { b, err := makeRequest(client, client.GetBiosConfiguration, q.Timeout) return b, err } +// CollectEthernetInterfaces() collects all of the ethernet interfaces found +// from all systems from under the "/redfish/v1/Systems" endpoint. +// +// TODO: This function needs to be refactored entirely...if not deleted +// in favor of using crawler.CrawlBM() instead. func CollectEthernetInterfaces(c *gofish.APIClient, q *QueryParams, systemID string) ([]byte, error) { // TODO: add more endpoints to test for ethernet interfaces // /redfish/v1/Chassis/{ChassisID}/NetworkAdapters/{NetworkAdapterId}/NetworkDeviceFunctions/{NetworkDeviceFunctionId}/EthernetInterfaces/{EthernetInterfaceId} @@ -380,6 +370,12 @@ func CollectEthernetInterfaces(c *gofish.APIClient, q *QueryParams, systemID str return b, nil } +// CollectChassis() fetches all chassis related information from each node specified +// via the Redfish API. Like the other collect functions, this function uses the gofish +// library to make requests to each node. Additionally, all of the network adapters found +// are added to the output as well. +// +// Returns a map that represents a Chassis object with NetworkAdapters. func CollectChassis(c *gofish.APIClient, q *QueryParams) ([]map[string]any, error) { rfChassis, err := c.Service.Chassis() if err != nil { @@ -402,6 +398,7 @@ func CollectChassis(c *gofish.APIClient, q *QueryParams) ([]map[string]any, erro return chassis, nil } +// TODO: DELETE ME!!! func CollectStorage(c *gofish.APIClient, q *QueryParams) ([]byte, error) { systems, err := c.Service.StorageSystems() if err != nil { @@ -427,19 +424,23 @@ func CollectStorage(c *gofish.APIClient, q *QueryParams) ([]byte, error) { return b, nil } +// CollectSystems pulls system information from each BMC node via Redfish using the +// `gofish` library. +// +// The process of collecting this info is as follows: +// 1. check if system has ethernet interfaces +// 1.a. if yes, create system data and ethernet interfaces JSON +// 1.b. if no, try to get data using manager instead +// 2. check if manager has "ManagerForServices" and "EthernetInterfaces" properties +// 2.a. if yes, query both properties to use in next step +// 2.b. for each service, query its data and add the ethernet interfaces +// 2.c. add the system to list of systems to marshal and return func CollectSystems(c *gofish.APIClient, q *QueryParams) ([]map[string]any, error) { rfSystems, err := c.Service.Systems() if err != nil { return nil, fmt.Errorf("failed to get systems (%v:%v): %v", q.Host, q.Port, err) } - // 1. check if system has ethernet interfaces - // 1.a. if yes, create system data and ethernet interfaces JSON - // 1.b. if no, try to get data using manager instead - // 2. check if manager has "ManagerForServices" and "EthernetInterfaces" properties - // 2.a. if yes, query both properties to use in next step - // 2.b. for each service, query its data and add the ethernet interfaces - // 2.c. add the system to list of systems to marshal and return var systems []map[string]any for _, system := range rfSystems { @@ -605,6 +606,7 @@ func CollectSystems(c *gofish.APIClient, q *QueryParams) ([]map[string]any, erro return systems, nil } +// TODO: DELETE ME!!! func CollectRegisteries(c *gofish.APIClient, q *QueryParams) ([]byte, error) { registries, err := c.Service.Registries() if err != nil { @@ -620,6 +622,7 @@ func CollectRegisteries(c *gofish.APIClient, q *QueryParams) ([]byte, error) { return b, nil } +// TODO: MAYBE DELETE??? func CollectProcessors(q *QueryParams) ([]byte, error) { url := baseRedfishUrl(q) + "/Systems" res, body, err := util.MakeRequest(nil, url, "GET", nil, nil) @@ -699,8 +702,8 @@ func makeGofishConfig(q *QueryParams) (gofish.ClientConfig, error) { ) return gofish.ClientConfig{ Endpoint: url, - Username: q.User, - Password: q.Pass, + Username: q.Username, + Password: q.Password, Insecure: true, TLSHandshakeTimeout: q.Timeout, HTTPClient: client, @@ -739,8 +742,8 @@ func makeJson(object any) ([]byte, error) { func baseRedfishUrl(q *QueryParams) string { url := fmt.Sprintf("%s://", q.Protocol) - if q.User != "" && q.Pass != "" { - url += fmt.Sprintf("%s:%s@", q.User, q.Pass) + if q.Username != "" && q.Password != "" { + url += fmt.Sprintf("%s:%s@", q.Username, q.Password) } return fmt.Sprintf("%s%s:%d", url, q.Host, q.Port) } diff --git a/internal/config.go b/internal/config.go index 82df93c..4059269 100644 --- a/internal/config.go +++ b/internal/config.go @@ -7,21 +7,25 @@ import ( "github.com/spf13/viper" ) +// LoadConfig() will load a YAML config file at the specified path. There are some general +// considerations about how this is done with spf13/viper: +// +// 1. There are intentionally no search paths set, so config path has to be set explicitly +// 2. No data will be written to the config file from the tool +// 3. Parameters passed as CLI flags and envirnoment variables should always have +// precedence over values set in the config. func LoadConfig(path string) error { dir, filename, ext := util.SplitPathForViper(path) // fmt.Printf("dir: %s\nfilename: %s\nextension: %s\n", dir, filename, ext) viper.AddConfigPath(dir) viper.SetConfigName(filename) viper.SetConfigType(ext) - // ...no search paths set intentionally, so config has to be set explicitly - // ...also, the config file will not save anything - // ...and finally, parameters passed to CLI have precedence over config values viper.AutomaticEnv() if err := viper.ReadInConfig(); err != nil { if _, ok := err.(viper.ConfigFileNotFoundError); ok { return fmt.Errorf("config file not found: %w", err) } else { - return fmt.Errorf("failed toload config file: %w", err) + return fmt.Errorf("failed to load config file: %w", err) } } diff --git a/internal/login.go b/internal/login.go index 508a987..a78aa8d 100644 --- a/internal/login.go +++ b/internal/login.go @@ -8,6 +8,17 @@ import ( "github.com/pkg/browser" ) +// Login() initiates the process to retrieve an access token from an identity provider. +// This function is especially designed to work by OPAAL, but will propably be changed +// in the future to be more agnostic. +// +// The 'targetHost' and 'targetPort' parameters should point to the target host/port +// to create a temporary server to receive the access token. If an empty 'targetHost' +// or an invalid port range is passed, then neither of the parameters will be used +// and no server will be started. +// +// Returns an access token as a string if successful and nil error. Otherwise, returns +// an empty string with an error set. func Login(loginUrl string, targetHost string, targetPort int) (string, error) { var accessToken string diff --git a/internal/scan.go b/internal/scan.go index ef4ca41..660aede 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -19,94 +19,31 @@ type ScannedResult struct { Timestamp time.Time `json:"timestamp"` } -func rawConnect(host string, ports []int, timeout int, keepOpenOnly bool) []ScannedResult { - results := []ScannedResult{} - for _, p := range ports { - result := ScannedResult{ - Host: host, - Port: p, - Protocol: "tcp", - State: false, - Timestamp: time.Now(), - } - t := time.Second * time.Duration(timeout) - port := fmt.Sprint(p) - conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), t) - if err != nil { - result.State = false - // fmt.Println("Connecting error:", err) - } - if conn != nil { - result.State = true - defer conn.Close() - // fmt.Println("Opened", net.JoinHostPort(host, port)) - } - if keepOpenOnly { - if result.State { - results = append(results, result) - } - } else { - results = append(results, result) - } - } - - return results -} - -func GenerateHosts(subnet string, subnetMask *net.IP) []string { - if subnet == "" || subnetMask == nil { - return nil - } - - // convert subnets from string to net.IP - subnetIp := net.ParseIP(subnet) - if subnetIp == nil { - // try parse CIDR instead - ip, network, err := net.ParseCIDR(subnet) - if err != nil { - return nil - } - subnetIp = ip - if network != nil { - t := net.IP(network.Mask) - subnetMask = &t - } - } - - mask := net.IPMask(subnetMask.To4()) - - // if no subnet mask, use a default 24-bit mask (for now) - return generateHosts(&subnetIp, &mask) -} - -func generateHosts(ip *net.IP, mask *net.IPMask) []string { - // get all IP addresses in network - ones, _ := mask.Size() - hosts := []string{} - end := int(math.Pow(2, float64((32-ones)))) - 1 - for i := 0; i < end; i++ { - // ip[3] = byte(i) - ip = util.GetNextIP(ip, 1) - if ip == nil { - continue - } - // host := fmt.Sprintf("%v.%v.%v.%v", (*ip)[0], (*ip)[1], (*ip)[2], (*ip)[3]) - // fmt.Printf("host: %v\n", ip.String()) - hosts = append(hosts, ip.String()) - } - return hosts -} - -func ScanForAssets(hosts []string, ports []int, threads int, timeout int, disableProbing bool, verbose bool) []ScannedResult { +// ScanForAssets() performs a net scan on a network to find available services +// running. The function expects a list of hosts and ports to make requests. +// Note that each all ports will be used per host. +// +// This function runs in a goroutine with the "concurrency" flag setting the +// number of concurrent requests. Only one request is made to each BMC node +// at a time, but setting a value greater than 1 with enable the requests +// to be made concurrently. +// +// If the "disableProbing" flag is set, then the function will skip the extra +// HTTP request made to check if the response was from a Redfish service. +// Otherwise, not receiving a 200 OK response code from the HTTP request will +// remove the service from being stored in the list of scanned results. +// +// Returns a list of scanned results to be stored in cache (but isn't doing here). +func ScanForAssets(hosts []string, ports []int, concurrency int, timeout int, disableProbing bool, verbose bool) []ScannedResult { var ( results = make([]ScannedResult, 0, len(hosts)) - done = make(chan struct{}, threads+1) - chanHost = make(chan string, threads+1) + done = make(chan struct{}, concurrency+1) + chanHost = make(chan string, concurrency+1) ) var wg sync.WaitGroup - wg.Add(threads) - for i := 0; i < threads; i++ { + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { go func() { for { host, ok := <-chanHost @@ -161,6 +98,92 @@ func ScanForAssets(hosts []string, ports []int, threads int, timeout int, disabl return results } +// GenerateHosts() builds a list of hosts to scan using the "subnet" +// and "subnetMask" arguments passed. The function is capable of +// distinguishing between IP formats: a subnet with just an IP address (172.16.0.0) and +// a subnet with IP address and CIDR (172.16.0.0/24). +// +// NOTE: If a IP address is provided with CIDR, then the "subnetMask" +// parameter will be ignored. If neither is provided, then the default +// subnet mask will be used instead. +func GenerateHosts(subnet string, subnetMask *net.IP) []string { + if subnet == "" || subnetMask == nil { + return nil + } + + // convert subnets from string to net.IP + subnetIp := net.ParseIP(subnet) + if subnetIp == nil { + // try parse CIDR instead + ip, network, err := net.ParseCIDR(subnet) + if err != nil { + return nil + } + subnetIp = ip + if network != nil { + t := net.IP(network.Mask) + subnetMask = &t + } + } + + mask := net.IPMask(subnetMask.To4()) + + // if no subnet mask, use a default 24-bit mask (for now) + return generateHosts(&subnetIp, &mask) +} + func GetDefaultPorts() []int { return []int{HTTPS_PORT} } + +func rawConnect(host string, ports []int, timeout int, keepOpenOnly bool) []ScannedResult { + results := []ScannedResult{} + for _, p := range ports { + result := ScannedResult{ + Host: host, + Port: p, + Protocol: "tcp", + State: false, + Timestamp: time.Now(), + } + t := time.Second * time.Duration(timeout) + port := fmt.Sprint(p) + conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), t) + if err != nil { + result.State = false + // fmt.Println("Connecting error:", err) + } + if conn != nil { + result.State = true + defer conn.Close() + // fmt.Println("Opened", net.JoinHostPort(host, port)) + } + if keepOpenOnly { + if result.State { + results = append(results, result) + } + } else { + results = append(results, result) + } + } + + return results +} + +func generateHosts(ip *net.IP, mask *net.IPMask) []string { + // get all IP addresses in network + ones, _ := mask.Size() + hosts := []string{} + end := int(math.Pow(2, float64((32-ones)))) - 1 + for i := 0; i < end; i++ { + // ip[3] = byte(i) + ip = util.GetNextIP(ip, 1) + if ip == nil { + continue + } + // host := fmt.Sprintf("%v.%v.%v.%v", (*ip)[0], (*ip)[1], (*ip)[2], (*ip)[3]) + // fmt.Printf("host: %v\n", ip.String()) + hosts = append(hosts, ip.String()) + } + return hosts +} diff --git a/internal/update.go b/internal/update.go index e6ed6b3..35ed4c4 100644 --- a/internal/update.go +++ b/internal/update.go @@ -26,8 +26,14 @@ type UpdateParams struct { TransferProtocol string } -// NOTE: Does not work since OpenBMC, whic bmclib uses underneath, does not -// support multipart updates. See issue: https://github.com/bmc-toolbox/bmclib/issues/341 +// UpdateFirmware() uses 'bmc-toolbox/bmclib' to update the firmware of a BMC node. +// The function expects the firmware URL, firmware version, and component flags to be +// set from the CLI to perform a firmware update. +// +// NOTE: Multipart HTTP updating may not work since older verions of OpenBMC, which bmclib +// uses underneath, did not support support multipart updates. This was changed with the +// inclusion of support for MultipartHttpPushUri in OpenBMC (https://gerrit.openbmc.org/c/openbmc/bmcweb/+/32174). +// Also, related to bmclib: https://github.com/bmc-toolbox/bmclib/issues/341 func UpdateFirmware(client *bmclib.Client, l *log.Logger, q *UpdateParams) error { if q.Component == "" { return fmt.Errorf("component is required") diff --git a/internal/util/util.go b/internal/util/util.go index a0ba641..6817f4a 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -13,6 +13,11 @@ import ( "time" ) +// PathExists() is a wrapper function that simplifies checking +// if a file or directory already exists at the provided path. +// +// Returns whether the path exists and no error if successful, +// otherwise, it returns false with an error. func PathExists(path string) (bool, error) { _, err := os.Stat(path) if err == nil { @@ -24,6 +29,8 @@ func PathExists(path string) (bool, error) { return false, err } +// GetNextIP() returns the next IP address, but does not account +// for net masks. func GetNextIP(ip *net.IP, inc uint) *net.IP { if ip == nil { return &net.IP{} @@ -40,7 +47,14 @@ func GetNextIP(ip *net.IP, inc uint) *net.IP { return &r } -// Generic convenience function used to make HTTP requests. +// MakeRequest() is a wrapper function that condenses simple HTTP +// requests done to a single call. It expects an optional HTTP client, +// URL, HTTP method, request body, and request headers. This function +// is useful when making many requests where only these few arguments +// are changing. +// +// Returns a HTTP response object, response body as byte array, and any +// error that may have occurred with making the request. func MakeRequest(client *http.Client, url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) { // use defaults if no client provided if client == nil { @@ -69,6 +83,12 @@ func MakeRequest(client *http.Client, url string, httpMethod string, body []byte return res, b, err } +// MakeOutputDirectory() creates a new directory at the path argument if +// the path does not exist +// +// TODO: Refactor this function for hive partitioning or possibly move into +// the logging package. +// TODO: Add an option to force overwriting the path. func MakeOutputDirectory(path string) (string, error) { // get the current data + time using Go's stupid formatting t := time.Now() @@ -93,12 +113,27 @@ func MakeOutputDirectory(path string) (string, error) { return final, nil } +// SplitPathForViper() is an utility function to split a path into 3 parts: +// - directory +// - filename +// - extension +// The intent was to break a path into a format that's more easily consumable +// by spf13/viper's API. See the "LoadConfig()" function in internal/config.go +// for more details. +// +// TODO: Rename function to something more generalized. func SplitPathForViper(path string) (string, string, string) { filename := filepath.Base(path) ext := filepath.Ext(filename) return filepath.Dir(path), strings.TrimSuffix(filename, ext), strings.TrimPrefix(ext, ".") } +// FormatErrorList() is a wrapper function that unifies error list formatting +// and makes printing error lists consistent. +// +// NOTE: The error returned IS NOT an error in itself and may be a bit misleading. +// Instead, it is a single condensed error composed of all of the errors included +// in the errList argument. func FormatErrorList(errList []error) error { var err error for i, e := range errList { @@ -108,6 +143,9 @@ func FormatErrorList(errList []error) error { return err } +// HasErrors() is a simple wrapper function to check if an error list contains +// errors. Having a function that clearly states its purpose helps to improve +// readibility although it may seem pointless. func HasErrors(errList []error) bool { return len(errList) > 0 } diff --git a/tests/api_test.go b/tests/api_test.go new file mode 100644 index 0000000..558e688 --- /dev/null +++ b/tests/api_test.go @@ -0,0 +1,53 @@ +// This file contains generic tests used to confirm expected behaviors of the +// builtin APIs. This is to guarantee that our functions work as expected +// regardless of the hardware being used such as testing the `scan`, and `collect` +// functionality and `gofish` library and asserting expected outputs. +// +// These tests are meant to be ran with the emulator included in the project. +// Make sure the emulator is running before running the tests. +package tests + +import ( + "testing" + + magellan "github.com/OpenCHAMI/magellan/internal" + "github.com/OpenCHAMI/magellan/internal/log" + "github.com/sirupsen/logrus" +) + +func TestScanAndCollect(t *testing.T) { + var ( + hosts = []string{"http://127.0.0.1"} + ports = []int{5000} + l = log.NewLogger(logrus.New(), logrus.DebugLevel) + ) + // do a scan on the emulator cluster with probing disabled and check results + results := magellan.ScanForAssets(hosts, ports, 1, 30, true, false) + if len(results) <= 0 { + t.Fatal("expected to find at least one BMC node, but found none") + } + // do a scan on the emulator cluster with probing enabled + results = magellan.ScanForAssets(hosts, ports, 1, 30, false, false) + if len(results) <= 0 { + t.Fatal("expected to find at least one BMC node, but found none") + } + + // do a collect on the emulator cluster to collect Redfish info + magellan.CollectAll(results) +} + +func TestCrawlCommand(t *testing.T) { + +} + +func TestListCommand(t *testing.T) { + +} + +func TestUpdateCommand(t *testing.T) { + +} + +func TestGofishFunctions(t *testing.T) { + +} diff --git a/tests/compatibility_test.go b/tests/compatibility_test.go new file mode 100644 index 0000000..5517bd2 --- /dev/null +++ b/tests/compatibility_test.go @@ -0,0 +1,108 @@ +// This file contains a series of tests that are meant to ensure correct +// Redfish behaviors and responses across different Refish implementations +// and are expected to be ran with various hardware and firmware to test +// compatibility with the tool. These tests are meant to be used as a way +// to pinpoint exactly where an issue is occurring in a more predictable +// and reproducible manner. +package tests + +import ( + "encoding/json" + "flag" + "fmt" + "net/http" + "testing" + + "github.com/OpenCHAMI/magellan/internal/util" + "github.com/OpenCHAMI/magellan/pkg/crawler" +) + +var ( + host = flag.String("host", "localhost", "set the BMC host") + username = flag.String("username", "", "set the BMC username used for the tests") + password = flag.String("password", "", "set the BMC password used for the tests") +) + +// Simple test to fetch the base Redfish URL and assert a 200 OK response. +func TestRedfishV1Availability(t *testing.T) { + var ( + url = fmt.Sprintf("%s/redfish/v1", host) + body = []byte{} + headers = map[string]string{} + ) + res, b, err := util.MakeRequest(nil, url, http.MethodGet, body, headers) + if err != nil { + t.Fatalf("failed to make request to BMC: %v", err) + } + + // test for a 200 response code here + if res.StatusCode != http.StatusOK { + t.Fatalf("expected response code to return status code 200") + } + + // make sure the response body is not empty + if len(b) <= 0 { + t.Fatalf("expected response body to not be empty") + } + + // make sure the response body is in a JSON format + if json.Valid(b) { + t.Fatalf("expected response body to be valid JSON") + } + +} + +// Simple test to ensure an expected Redfish version minimum requirement. +func TestRedfishVersion(t *testing.T) { + var ( + url = fmt.Sprintf("%s/redfish/v1", host) + body = []byte{} + headers = map[string]string{} + ) + + util.MakeRequest(nil, url, http.MethodGet, body, headers) +} + +// Crawls a BMC node and checks that we're able to query certain properties +// that we need for Magellan to run correctly. This test differs from the +// `TestCrawlCommand` testing function as it is not checking specifically +// for functionality. +func TestExpectedProperties(t *testing.T) { + // make sure what have a valid host + if host == nil { + t.Fatal("invalid host (host is nil)") + } + + systems, err := crawler.CrawlBMC( + crawler.CrawlerConfig{ + URI: *host, + Username: *username, + Password: *password, + Insecure: true, + }, + ) + + if err != nil { + t.Fatalf("failed to crawl BMC: %v", err) + } + + // check that we got results in systems + if len(systems) <= 0 { + t.Fatal("no systems found") + } + + // check that we're getting EthernetInterfaces and NetworkInterfaces + for _, system := range systems { + // check that we have at least one CPU for each system + if system.ProcessorCount <= 0 { + t.Errorf("no processors found") + } + // we expect each system to have at least one of each interface + if len(system.EthernetInterfaces) <= 0 { + t.Errorf("no ethernet interfaces found for system '%s'", system.Name) + } + if len(system.NetworkInterfaces) <= 0 { + t.Errorf("no network interfaces found for system '%s'", system.Name) + } + } +}