diff --git a/providers/azure/connection/azureinstancesnapshot/provider.go b/providers/azure/connection/azureinstancesnapshot/provider.go index c9ee3163c0..ff50087eb8 100644 --- a/providers/azure/connection/azureinstancesnapshot/provider.go +++ b/providers/azure/connection/azureinstancesnapshot/provider.go @@ -202,5 +202,5 @@ func (c *AzureSnapshotConnection) Type() shared.ConnectionType { } func (c *AzureSnapshotConnection) Config() *inventory.Config { - return c.DeviceConnection.Conf + return c.DeviceConnection.Conf() } diff --git a/providers/gcp/connection/gcpinstancesnapshot/provider.go b/providers/gcp/connection/gcpinstancesnapshot/provider.go index 097bf875ab..675c1174b8 100644 --- a/providers/gcp/connection/gcpinstancesnapshot/provider.go +++ b/providers/gcp/connection/gcpinstancesnapshot/provider.go @@ -300,5 +300,5 @@ func (c *GcpSnapshotConnection) Type() shared.ConnectionType { } func (c *GcpSnapshotConnection) Config() *inventory.Config { - return c.DeviceConnection.Conf + return c.DeviceConnection.Conf() } diff --git a/providers/os/config/config.go b/providers/os/config/config.go index 70d04104ea..50e431ef24 100644 --- a/providers/os/config/config.go +++ b/providers/os/config/config.go @@ -288,6 +288,12 @@ var Config = plugin.Provider{ Desc: "The serial number of the block device that should be scanned. Supported only for Windows scanning. Do not use together with --device-name or --lun", Option: plugin.FlagOption_Hidden, }, + { + Long: "mount-all-partitions", + Type: plugin.FlagType_String, + Desc: "Mount all partitions of the block device", + Option: plugin.FlagOption_Hidden, + }, { Long: "platform-ids", Type: plugin.FlagType_List, diff --git a/providers/os/connection/device/device_connection.go b/providers/os/connection/device/device_connection.go index e484124964..cc1876741d 100644 --- a/providers/os/connection/device/device_connection.go +++ b/providers/os/connection/device/device_connection.go @@ -29,6 +29,8 @@ type DeviceConnection struct { plugin.Connection asset *inventory.Asset deviceManager DeviceManager + + MountedDirs []string } func getDeviceManager(conf *inventory.Config) (DeviceManager, error) { @@ -54,12 +56,9 @@ func NewDeviceConnection(connId uint32, conf *inventory.Config, asset *inventory if err != nil { return nil, err } - if len(blocks) != 1 { - // FIXME: remove this when we start scanning multiple blocks - return nil, errors.New("internal>blocks size is not equal to 1") + if len(blocks) == 0 { + return nil, errors.New("internal> no blocks found") } - block := blocks[0] - log.Debug().Str("name", block.Name).Str("type", block.FsType).Msg("identified partition for mounting") res := &DeviceConnection{ Connection: plugin.NewConnection(connId, asset), @@ -67,60 +66,96 @@ func NewDeviceConnection(connId uint32, conf *inventory.Config, asset *inventory asset: asset, } - scanDir, err := manager.Mount(block) - if err != nil { - log.Error().Err(err).Msg("unable to complete mount step") - res.Close() - return nil, err - } if conf.Options == nil { conf.Options = make(map[string]string) } - conf.Options["path"] = scanDir - // create and initialize fs provider - fsConn, err := fs.NewConnection(connId, &inventory.Config{ - Path: scanDir, - PlatformId: conf.PlatformId, - Options: conf.Options, - Type: "fs", - Record: conf.Record, - }, asset) - if err != nil { - res.Close() - return nil, err - } + for i := range blocks { + block := blocks[i] + log.Debug().Str("name", block.Name).Str("type", block.FsType).Msg("identified partition for mounting") + + scanDir, err := manager.Mount(block) + if err != nil { + log.Error().Err(err).Msg("unable to complete mount step") + res.Close() + return nil, err + } + res.MountedDirs = append(res.MountedDirs, scanDir) + + // create and initialize fs provider + conf.Options["path"] = scanDir + fsConn, err := fs.NewConnection(connId, &inventory.Config{ + Path: scanDir, + PlatformId: conf.PlatformId, + Options: conf.Options, + Type: "fs", + Record: conf.Record, + }, asset) + if err != nil { + res.Close() + return nil, err + } + + // allow injecting platform ids into the device connection. we cannot always know the asset that's being scanned, e.g. + // if we can scan an azure VM's disk we should be able to inject the platform ids of the VM + if platformIDs, ok := conf.Options[PlatformIdInject]; ok { + platformIds := strings.Split(platformIDs, ",") + if len(platformIds) > 0 { + log.Debug().Strs("platform-ids", platformIds).Msg("device connection> injecting platform ids") + conf.PlatformId = platformIds[0] + asset.PlatformIds = append(asset.PlatformIds, platformIds...) + } + } - res.FileSystemConnection = fsConn + if asset.Platform != nil { + log.Debug().Msg("device connection> platform already detected") - // allow injecting platform ids into the device connection. we cannot always know the asset that's being scanned, e.g. - // if we can scan an azure VM's disk we should be able to inject the platform ids of the VM - if platformIDs, ok := conf.Options[PlatformIdInject]; ok { - platformIds := strings.Split(platformIDs, ",") - if len(platformIds) > 0 { - log.Debug().Strs("platform-ids", platformIds).Msg("device connection> injecting platform ids") - conf.PlatformId = platformIds[0] - asset.PlatformIds = append(asset.PlatformIds, platformIds...) + // Edge case: asset platform is provided from the inventory + if res.FileSystemConnection == nil { + res.FileSystemConnection = fsConn + } + continue + } + + p, ok := detector.DetectOS(fsConn) + if !ok { + log.Debug(). + Str("block", block.Name). + Msg("device connection> cannot detect os") + continue + } + asset.Platform = p + asset.IdDetector = []string{ids.IdDetector_Hostname} + res.FileSystemConnection = fsConn + + fingerprint, p, err := id.IdentifyPlatform(res, &plugin.ConnectReq{}, asset.Platform, asset.IdDetector) + if err != nil { + log.Debug().Err(err).Msg("device connection> failed to identify platform from device") + asset.Platform = nil + } + + if err == nil { + log.Debug().Str("scan_dir", scanDir).Msg("device connection> detected platform from device") + if asset.Name == "" { + asset.Name = fingerprint.Name + } + asset.PlatformIds = append(asset.PlatformIds, fingerprint.PlatformIDs...) + asset.IdDetector = fingerprint.ActiveIdDetectors + asset.Platform = p + asset.Id = conf.Type } } - p, ok := detector.DetectOS(fsConn) - if !ok { + if asset.Platform == nil { res.Close() return nil, errors.New("failed to detect OS") } - asset.Platform = p - asset.IdDetector = []string{ids.IdDetector_Hostname} - fingerprint, p, err := id.IdentifyPlatform(res, &plugin.ConnectReq{}, asset.Platform, asset.IdDetector) - if err == nil { - if asset.Name == "" { - asset.Name = fingerprint.Name - } - asset.PlatformIds = append(asset.PlatformIds, fingerprint.PlatformIDs...) - asset.IdDetector = fingerprint.ActiveIdDetectors - asset.Platform = p - asset.Id = conf.Type + + if res.FileSystemConnection == nil { + res.Close() + return nil, errors.New("failed to create fs connection") } + return res, nil } @@ -166,3 +201,7 @@ func (p *DeviceConnection) FileSystem() afero.Fs { func (p *DeviceConnection) FileInfo(path string) (shared.FileInfoDetails, error) { return p.FileSystemConnection.FileInfo(path) } + +func (p *DeviceConnection) Conf() *inventory.Config { + return p.FileSystemConnection.Conf +} diff --git a/providers/os/connection/device/linux/device_manager.go b/providers/os/connection/device/linux/device_manager.go index abdf101693..6d6b01716a 100644 --- a/providers/os/connection/device/linux/device_manager.go +++ b/providers/os/connection/device/linux/device_manager.go @@ -12,8 +12,9 @@ import ( ) const ( - LunOption = "lun" - DeviceName = "device-name" + LunOption = "lun" + DeviceName = "device-name" + MountAllPartitions = "mount-all-partitions" ) type LinuxDeviceManager struct { @@ -52,11 +53,11 @@ func (d *LinuxDeviceManager) IdentifyMountTargets(opts map[string]string) ([]*sn return []*snapshot.PartitionInfo{pi}, nil } - pi, err := d.identifyViaDeviceName(opts[DeviceName]) + partitions, err := d.identifyViaDeviceName(opts[DeviceName], opts[MountAllPartitions] == "true") if err != nil { return nil, err } - return []*snapshot.PartitionInfo{pi}, nil + return partitions, nil } func (d *LinuxDeviceManager) Mount(pi *snapshot.PartitionInfo) (string, error) { @@ -86,9 +87,13 @@ func (d *LinuxDeviceManager) UnmountAndClose() { func validateOpts(opts map[string]string) error { lun := opts[LunOption] deviceName := opts[DeviceName] + mountAll := opts[MountAllPartitions] == "true" if lun != "" && deviceName != "" { return errors.New("both lun and device name provided") } + if deviceName == "" && mountAll { + return errors.New("mount-all-partitions requires a device name") + } return nil } @@ -123,7 +128,7 @@ func (c *LinuxDeviceManager) identifyViaLun(lun int) (*snapshot.PartitionInfo, e return device.GetMountablePartition() } -func (c *LinuxDeviceManager) identifyViaDeviceName(deviceName string) (*snapshot.PartitionInfo, error) { +func (c *LinuxDeviceManager) identifyViaDeviceName(deviceName string, mountAll bool) ([]*snapshot.PartitionInfo, error) { blockDevices, err := c.volumeMounter.CmdRunner.GetBlockDevices() if err != nil { return nil, err @@ -133,7 +138,11 @@ func (c *LinuxDeviceManager) identifyViaDeviceName(deviceName string) (*snapshot // this is a best-guess approach if deviceName == "" { // TODO: we should rename/simplify this method - return blockDevices.GetUnnamedBlockEntry() + pi, err := blockDevices.GetUnnamedBlockEntry() + if err != nil { + return nil, err + } + return []*snapshot.PartitionInfo{pi}, nil } // if we have a specific device we're looking for we can just ask only for that @@ -142,5 +151,14 @@ func (c *LinuxDeviceManager) identifyViaDeviceName(deviceName string) (*snapshot return nil, err } - return device.GetMountablePartition() + if mountAll { + log.Debug().Str("device", device.Name).Msg("mounting all partitions") + return device.GetMountablePartitions(true) + } + + pi, err := device.GetMountablePartition() + if err != nil { + return nil, err + } + return []*snapshot.PartitionInfo{pi}, nil } diff --git a/providers/os/connection/snapshot/blockdevices.go b/providers/os/connection/snapshot/blockdevices.go index 4afac10279..f72bbe879c 100644 --- a/providers/os/connection/snapshot/blockdevices.go +++ b/providers/os/connection/snapshot/blockdevices.go @@ -118,7 +118,7 @@ func (blockEntries BlockDevices) GetMountablePartitionByDevice(device string) (* } // sort the candidates by size, so we can pick the largest one - sortPartitionsBySize(partitions) + sortBlockDevicesBySize(partitions) // return the largest partition. we can extend this to be a parameter in the future devFsName := "/dev/" + partitions[0].Name @@ -151,15 +151,33 @@ func (blockEntries BlockDevices) FindDevice(name string) (BlockDevice, error) { } // Searches all the partitions in the device and finds one that can be mounted. It must be unmounted, non-boot partition -// If multiple partitions meet this criteria, the largest one is returned. -func (device BlockDevice) GetMountablePartition() (*PartitionInfo, error) { +func (device BlockDevice) GetMountablePartitions(includeAll bool) ([]*PartitionInfo, error) { log.Debug().Str("device", device.Name).Msg("get partitions for device") - partitions := []BlockDevice{} - for _, partition := range device.Children { + + blockDevices := device.Children + // sort the candidates by size, so we can pick the largest one + sortBlockDevicesBySize(blockDevices) + + filter := func(partition BlockDevice) bool { + return partition.IsNoBootVolumeAndUnmounted() + } + if includeAll { + filter = func(partition BlockDevice) bool { + return !partition.IsMounted() + } + } + + partitions := []*PartitionInfo{} + for _, partition := range blockDevices { log.Debug().Str("name", partition.Name).Int("size", partition.Size).Msg("checking partition") - if partition.IsNoBootVolumeAndUnmounted() { + if partition.FsType == "" { + log.Debug().Str("name", partition.Name).Msg("skipping partition without filesystem type") + continue + } + if filter(partition) { log.Debug().Str("name", partition.Name).Msg("found suitable partition") - partitions = append(partitions, partition) + devFsName := "/dev/" + partition.Name + partitions = append(partitions, &PartitionInfo{Name: devFsName, FsType: partition.FsType}) } } @@ -167,15 +185,21 @@ func (device BlockDevice) GetMountablePartition() (*PartitionInfo, error) { return nil, fmt.Errorf("no suitable partitions found on device %s", device.Name) } - // sort the candidates by size, so we can pick the largest one - sortPartitionsBySize(partitions) + return partitions, nil +} +// If multiple partitions meet this criteria, the largest one is returned. +func (device BlockDevice) GetMountablePartition() (*PartitionInfo, error) { // return the largest partition. we can extend this to be a parameter in the future - devFsName := "/dev/" + partitions[0].Name - return &PartitionInfo{Name: devFsName, FsType: partitions[0].FsType}, nil + partitions, err := device.GetMountablePartitions(false) + if err != nil { + return nil, err + } + + return partitions[0], nil } -func sortPartitionsBySize(partitions []BlockDevice) { +func sortBlockDevicesBySize(partitions []BlockDevice) { sort.Slice(partitions, func(i, j int) bool { return partitions[i].Size > partitions[j].Size }) @@ -197,6 +221,20 @@ func (blockEntries BlockDevices) GetUnnamedBlockEntry() (*PartitionInfo, error) return nil, errors.New("target volume not found on instance") } +func (blockEntries BlockDevices) GetDeviceWithUnmountedPartitions() (BlockDevice, error) { + log.Debug().Msg("get device with unmounted partitions") + for i := range blockEntries.BlockDevices { + d := blockEntries.BlockDevices[i] + log.Debug().Str("name", d.Name).Interface("children", d.Children).Interface("mountpoint", d.MountPoint).Msg("found block device") + if d.MountPoint != "" { // empty string means it is not mounted + continue + } + + return d, nil + } + return BlockDevice{}, errors.New("target block device not found on instance") +} + func (blockEntries BlockDevices) GetUnmountedBlockEntry() (*PartitionInfo, error) { log.Debug().Msg("get unmounted block entry") for i := range blockEntries.BlockDevices { @@ -224,7 +262,7 @@ func findVolume(children []BlockDevice) *PartitionInfo { if len(candidates) == 0 { return nil } - sortPartitionsBySize(candidates) + sortBlockDevicesBySize(candidates) return &PartitionInfo{Name: "/dev/" + candidates[0].Name, FsType: candidates[0].FsType} } diff --git a/providers/os/connection/snapshot/volumemounter.go b/providers/os/connection/snapshot/volumemounter.go index cdc9ff45af..51b41fe6bf 100644 --- a/providers/os/connection/snapshot/volumemounter.go +++ b/providers/os/connection/snapshot/volumemounter.go @@ -19,16 +19,16 @@ const ( ) type VolumeMounter struct { - // the tmp dir we create; serves as the directory we mount the volume to - ScanDir string - // where we tell AWS to attach the volume; it doesn't necessarily get attached there, but we have to reference this same location when detaching - VolumeAttachmentLoc string - CmdRunner *LocalCommandRunner + // the tmp dirs we create; serves as the directory we mount the volumes to + // maps the device name to the directory + ScanDirs map[string]string + CmdRunner *LocalCommandRunner } func NewVolumeMounter(shell []string) *VolumeMounter { return &VolumeMounter{ CmdRunner: &LocalCommandRunner{Shell: shell}, + ScanDirs: make(map[string]string), } } @@ -47,6 +47,8 @@ func (m *VolumeMounter) MountP(partition *PartitionInfo) (string, error) { if err != nil { return "", err } + m.ScanDirs[partition.Name] = dir + return dir, m.mountVolume(partition) } @@ -56,7 +58,6 @@ func (m *VolumeMounter) createScanDir() (string, error) { log.Error().Err(err).Msg("error creating directory") return "", err } - m.ScanDir = dir log.Debug().Str("dir", dir).Msg("created tmp scan dir") return dir, nil } @@ -94,24 +95,47 @@ func (m *VolumeMounter) mountVolume(fsInfo *PartitionInfo) error { opts = append(opts, "nouuid") } opts = stringx.DedupStringArray(opts) - log.Debug().Str("fstype", fsInfo.FsType).Str("device", fsInfo.Name).Str("scandir", m.ScanDir).Str("opts", strings.Join(opts, ",")).Msg("mount volume to scan dir") - return Mount(fsInfo.Name, m.ScanDir, fsInfo.FsType, opts) + scanDir := m.ScanDirs[fsInfo.Name] + log.Debug().Str("fstype", fsInfo.FsType).Str("device", fsInfo.Name).Str("scandir", scanDir).Str("opts", strings.Join(opts, ",")).Msg("mount volume to scan dir") + return Mount(fsInfo.Name, scanDir, fsInfo.FsType, opts) } func (m *VolumeMounter) UnmountVolumeFromInstance() error { - if m.ScanDir == "" { - log.Warn().Msg("no scan dir to unmount, skipping") + if len(m.ScanDirs) == 0 { + log.Warn().Msg("no scan dirs to unmount, skipping") return nil } - log.Debug().Str("dir", m.ScanDir).Msg("unmount volume") - if err := Unmount(m.ScanDir); err != nil { - log.Error().Err(err).Msg("failed to unmount dir") - return err + + var errs []error + for name, dir := range m.ScanDirs { + log.Debug(). + Str("dir", dir). + Str("name", name). + Msg("unmount volume") + if err := Unmount(dir); err != nil { + log.Error(). + Str("dir", dir). + Err(err).Msg("failed to unmount dir") + errs = append(errs, err) + } } - return nil + return errors.Join(errs...) } func (m *VolumeMounter) RemoveTempScanDir() error { - log.Debug().Str("dir", m.ScanDir).Msg("remove created dir") - return os.RemoveAll(m.ScanDir) + var errs []error + for name, dir := range m.ScanDirs { + log.Debug(). + Str("dir", dir). + Str("name", name). + Msg("remove created dir") + if err := os.RemoveAll(dir); err != nil { + log.Error().Err(err). + Str("dir", dir). + Msg("failed to remove dir") + errs = append(errs, err) + } + } + + return errors.Join(errs...) } diff --git a/providers/os/provider/provider.go b/providers/os/provider/provider.go index a2b935d885..5af8c06bb5 100644 --- a/providers/os/provider/provider.go +++ b/providers/os/provider/provider.go @@ -233,6 +233,9 @@ func (s *Service) ParseCLI(req *plugin.ParseCLIReq) (*plugin.ParseCLIRes, error) if serialNumber, ok := flags["serial-number"]; ok { conf.Options["serial-number"] = serialNumber.RawData().Value.(string) } + if mountAll, ok := flags["mount-all-partitions"]; ok { + conf.Options["mount-all-partitions"] = mountAll.RawData().Value.(string) + } if platformIDs, ok := flags["platform-ids"]; ok { platformIDs := platformIDs.Array