diff --git a/agent/driver/software/deb.go b/agent/driver/software/deb.go index a6aaf3b..f22a595 100644 --- a/agent/driver/software/deb.go +++ b/agent/driver/software/deb.go @@ -145,7 +145,7 @@ func getConfigFiles(packageName string) ([]string, error) { return configs, nil } -func GetDEBs() ([]software.DEB, error) { +func GetDEBs(showDefaultPackages bool) ([]software.DEB, error) { var DEBs []software.DEB dpkgStatusFile := "/var/lib/dpkg/status" @@ -171,5 +171,34 @@ func GetDEBs() ([]software.DEB, error) { DEBs[i].Conffiles = configs } + if !showDefaultPackages { + var filteredDEBs []software.DEB + + defaultPackages, err := GetDefaultPackages() + if err != nil { + logger.Println(logger.DEBUG, false, "DEB: Error occurred while getting default packages."+ + " ("+err.Error()+")") + } + + for _, deb := range DEBs { + var defPkgFound bool + + for _, defPkg := range defaultPackages { + if defPkg == deb.Package { + defPkgFound = true + break + } + } + + if defPkgFound { + continue + } + + filteredDEBs = append(filteredDEBs, deb) + } + + return filteredDEBs, nil + } + return DEBs, nil } diff --git a/agent/driver/software/packageFilter.go b/agent/driver/software/packageFilter.go new file mode 100644 index 0000000..2956e9b --- /dev/null +++ b/agent/driver/software/packageFilter.go @@ -0,0 +1,294 @@ +package software + +import ( + "compress/gzip" + "crypto/tls" + "encoding/xml" + "errors" + "fmt" + "github.com/jollaman999/utils/logger" + "github.com/shirou/gopsutil/v3/host" + "io" + "net/http" + "sort" + "strings" + + "github.com/ulikunitz/xz" +) + +// RepoMD represents the structure of repomd.xml for parsing +type RepoMD struct { + XMLName xml.Name `xml:"repomd"` + Data []struct { + Type string `xml:"type,attr"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + } `xml:"data"` +} + +// PackageReq represents a single package requirement in a group +type PackageReq struct { + Type string `xml:"type,attr"` + Name string `xml:",chardata"` +} + +// Group represents a group in comps.xml.xz +type Group struct { + ID string `xml:"id"` + Name string `xml:"name"` + PackageList []PackageReq `xml:"packagelist>packagereq"` +} + +// Groups represents the structure that contains multiple Group elements +type Groups struct { + XMLName xml.Name `xml:"comps"` + Groups []Group `xml:"group"` +} + +// createTransport creates an HTTP transport that ignores SSL certificate verification +func createTransport() *http.Transport { + return &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } +} + +// fetchURL fetches the content from a URL with SSL verification disabled +func fetchURL(url string) ([]byte, error) { + client := &http.Client{ + Transport: createTransport(), + } + + resp, err := client.Get(url) + if err != nil { + return nil, err + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("packageFilter: failed to fetch URL %s: %s", url, resp.Status) + logger.Println(logger.ERROR, true, errMsg) + return nil, errors.New(errMsg) + } + + var reader io.Reader + reader = resp.Body + + // Check if the file is gzipped or xz compressed + if strings.HasSuffix(url, ".gz") { + gzipReader, err := gzip.NewReader(resp.Body) + if err != nil { + return nil, err + } + defer func() { + _ = gzipReader.Close() + }() + reader = gzipReader + } else if strings.HasSuffix(url, ".xz") { + xzReader, err := xz.NewReader(resp.Body) + if err != nil { + return nil, err + } + reader = xzReader + } + + body, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + + return body, nil +} + +// parseRepoMD parses the repomd.xml file to find the group_gz or group_xz data location +func parseRepoMD(data []byte) (string, error) { + var repomd RepoMD + if err := xml.Unmarshal(data, &repomd); err != nil { + return "", err + } + + for _, data := range repomd.Data { + if data.Type == "group_gz" || data.Type == "group_xz" { + return data.Location.Href, nil + } + } + + errMsg := "no group_gz or group_xz data found in repomd.xml" + logger.Println(logger.ERROR, true, errMsg) + return "", errors.New(errMsg) +} + +// parseGroups parses the group XML data to extract mandatory and default packages +func parseGroups(data []byte) ([]string, error) { + var groups Groups + if err := xml.Unmarshal(data, &groups); err != nil { + return nil, err + } + + var packages []string + for _, group := range groups.Groups { + // Matching against "core" in a case-insensitive manner + if strings.ToLower(group.ID) == "core" || strings.ToLower(group.Name) == "core" { + for _, pkg := range group.PackageList { + if pkg.Type == "mandatory" || pkg.Type == "default" { + packages = append(packages, pkg.Name) + } + } + break + } + } + + // Sort packages in ascending order + sort.Strings(packages) + + return packages, nil +} + +// parseUbuntuManifest parses the Ubuntu minimal cloud image manifest file to extract package names +func parseUbuntuManifest(data []byte) ([]string, error) { + lines := strings.Split(string(data), "\n") + var packages []string + + for _, line := range lines { + parts := strings.SplitN(line, "\t", 2) + if len(parts) > 1 { + pkgName := strings.Split(parts[0], ":")[0] + if pkgName != "" { + packages = append(packages, pkgName) + } + } + } + + // Sort packages in ascending order + sort.Strings(packages) + + return packages, nil +} + +// getUbuntuReleaseName maps Ubuntu version to release name +func getUbuntuReleaseName(version string) string { + switch { + case strings.HasPrefix(version, "24.04"): + return "noble" + case strings.HasPrefix(version, "23.10"): + return "mantic" + case strings.HasPrefix(version, "22.04"): + return "jammy" + case strings.HasPrefix(version, "20.04"): + return "focal" + case strings.HasPrefix(version, "18.04"): + return "bionic" + case strings.HasPrefix(version, "16.04"): + return "xenial" + case strings.HasPrefix(version, "14.04"): + return "trusty" + default: + return "unknown" + } +} + +// GetDefaultPackages fetches and returns the default package list for a given OS type and version +func GetDefaultPackages() ([]string, error) { + h, err := host.Info() + if err != nil { + return nil, err + } + + osType := h.Platform + version := h.PlatformVersion + + baseURL := "" + fallbackURL := "" + switch strings.ToLower(osType) { + case "centos": + baseURL = fmt.Sprintf("https://vault.centos.org/%s/os/x86_64/repodata/repomd.xml", version) + case "redhat": + fallthrough + case "rocky": + baseURL = fmt.Sprintf("https://dl.rockylinux.org/pub/rocky/%s/BaseOS/x86_64/os/repodata/repomd.xml", version) + fallbackURL = fmt.Sprintf("https://dl.rockylinux.org/vault/rocky/%s/BaseOS/x86_64/os/repodata/repomd.xml", version) + case "ubuntu": + releaseName := getUbuntuReleaseName(version) + if releaseName == "unknown" { + errMsg := fmt.Sprintf("packageFilter: unsupported Ubuntu version: %s", version) + logger.Println(logger.ERROR, true, errMsg) + return nil, errors.New(errMsg) + } + baseURL = fmt.Sprintf("https://cloud-images.ubuntu.com/minimal/releases/%s/release/ubuntu-%s-minimal-cloudimg-amd64.manifest", releaseName, version) + default: + errMsg := fmt.Sprintf("packageFilter: unsupported OS Type: %s", osType) + logger.Println(logger.ERROR, true, errMsg) + return nil, errors.New(errMsg) + } + + if osType == "ubuntu" { + // Fetch Ubuntu manifest file + logger.Println(logger.INFO, false, "packageFilter: Fetching Ubuntu manifest from:", baseURL) + manifestData, err := fetchURL(baseURL) + if err != nil { + errMsg := "packageFilter: error fetching Ubuntu manifest: " + err.Error() + logger.Println(logger.ERROR, true, errMsg) + return nil, errors.New(errMsg) + } + + // Parse the manifest file to extract package names + packages, err := parseUbuntuManifest(manifestData) + if err != nil { + errMsg := "packageFilter: error parsing Ubuntu manifest: " + err.Error() + logger.Println(logger.ERROR, true, errMsg) + return nil, errors.New(errMsg) + } + + return packages, nil + } + + // Fetch repomd.xml for CentOS, RockyLinux, or RedHat + logger.Println(logger.INFO, false, "packageFilter: Fetching repomd.xml from:", baseURL) + repomdData, err := fetchURL(baseURL) + if err != nil && fallbackURL != "" && strings.Contains(err.Error(), "404") { + // If there's a 404 error and we have a fallback URL, try fetching from the fallback URL + logger.Println(logger.WARN, false, "packageFilter: Primary URL failed, trying fallback URL:", fallbackURL) + baseURL = fallbackURL + repomdData, err = fetchURL(baseURL) + if err != nil { + errMsg := "packageFilter: error fetching repomd.xml from fallback URL: " + err.Error() + logger.Println(logger.ERROR, true, errMsg) + return nil, errors.New(errMsg) + } + } else if err != nil { + errMsg := "packageFilter: error fetching repomd.xml: " + err.Error() + logger.Println(logger.ERROR, true, errMsg) + return nil, errors.New(errMsg) + } + + // Parse repomd.xml to find the group_gz or group_xz location + groupFileURL, err := parseRepoMD(repomdData) + if err != nil { + errMsg := "packageFilter: error parsing repomd.xml: " + err.Error() + logger.Println(logger.ERROR, true, errMsg) + return nil, errors.New(errMsg) + } + + groupFileFullURL := strings.TrimSuffix(baseURL, "repodata/repomd.xml") + groupFileURL + + // Fetch the group file and decompress it + logger.Println(logger.INFO, false, "packageFilter: Fetching and decompressing group file from:", groupFileFullURL) + groupData, err := fetchURL(groupFileFullURL) + if err != nil { + errMsg := "packageFilter: error fetching group file: " + err.Error() + logger.Println(logger.ERROR, true, errMsg) + return nil, errors.New(errMsg) + } + + // Parse the groups data to extract mandatory and default packages + packages, err := parseGroups(groupData) + if err != nil { + errMsg := "packageFilter: error parsing group data: " + err.Error() + logger.Println(logger.ERROR, true, errMsg) + return nil, errors.New(errMsg) + } + + return packages, nil +} diff --git a/agent/driver/software/rpm.go b/agent/driver/software/rpm.go index d934138..79797a5 100644 --- a/agent/driver/software/rpm.go +++ b/agent/driver/software/rpm.go @@ -4,6 +4,7 @@ import ( "github.com/cloud-barista/cm-honeybee/agent/pkg/api/rest/model/onprem/software" _ "github.com/glebarez/go-sqlite" // sqlite "github.com/hashicorp/go-multierror" + "github.com/jollaman999/utils/logger" rpmdb "github.com/knqyf263/go-rpmdb/pkg" ) @@ -30,12 +31,14 @@ func detectDB() (*rpmdb.RpmDB, error) { return nil, result } -func GetRPMs() ([]software.RPM, error) { +func GetRPMs(showDefaultPackages bool) ([]software.RPM, error) { db, err := detectDB() if err != nil { return []software.RPM{}, err } - defer db.Close() + defer func() { + _ = db.Close() + }() pkgList, err := db.ListPackages() if err != nil { return []software.RPM{}, err @@ -58,5 +61,34 @@ func GetRPMs() ([]software.RPM, error) { }) } + if !showDefaultPackages { + var filteredRPMs []software.RPM + + defaultPackages, err := GetDefaultPackages() + if err != nil { + logger.Println(logger.DEBUG, false, "DEB: Error occurred while getting default packages."+ + " ("+err.Error()+")") + } + + for _, rpm := range rpms { + var defPkgFound bool + + for _, defPkg := range defaultPackages { + if defPkg == rpm.Name { + defPkgFound = true + break + } + } + + if defPkgFound { + continue + } + + filteredRPMs = append(filteredRPMs, rpm) + } + + return filteredRPMs, nil + } + return rpms, nil } diff --git a/agent/driver/software/software.go b/agent/driver/software/software.go index 1b680e4..63b6916 100644 --- a/agent/driver/software/software.go +++ b/agent/driver/software/software.go @@ -10,7 +10,7 @@ import ( var softwareInfoLock sync.Mutex -func GetSoftwareInfo() (*software2.Software, error) { +func GetSoftwareInfo(showDefaultPackages bool) (*software2.Software, error) { if !softwareInfoLock.TryLock() { return nil, errors.New("software info collection is in progress") } @@ -28,14 +28,14 @@ func GetSoftwareInfo() (*software2.Software, error) { } if h.PlatformFamily == "debian" { - deb, err = GetDEBs() + deb, err = GetDEBs(showDefaultPackages) if err != nil { return nil, err } } if h.PlatformFamily == "fedora" || h.PlatformFamily == "rhel" { - rpm, err = GetRPMs() + rpm, err = GetRPMs(showDefaultPackages) if err != nil { return nil, err } diff --git a/agent/go.mod b/agent/go.mod index 29bb895..79a0612 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -24,6 +24,7 @@ require ( github.com/swaggo/echo-swagger v1.4.1 github.com/swaggo/swag v1.16.3 github.com/taigrr/systemctl v1.0.7 + github.com/ulikunitz/xz v0.5.12 ) require ( diff --git a/agent/go.sum b/agent/go.sum index a8a05a1..6c9615b 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -158,6 +158,8 @@ github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZ github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= diff --git a/agent/pkg/api/rest/controller/softwareInfo.go b/agent/pkg/api/rest/controller/softwareInfo.go index 67d73d6..12b63c0 100644 --- a/agent/pkg/api/rest/controller/softwareInfo.go +++ b/agent/pkg/api/rest/controller/softwareInfo.go @@ -6,6 +6,7 @@ import ( _ "github.com/cloud-barista/cm-honeybee/agent/pkg/api/rest/model/onprem/software" // Need for swag "github.com/labstack/echo/v4" "net/http" + "strconv" ) // GetSoftwareInfo godoc @@ -15,12 +16,16 @@ import ( // @Tags [Software] Get software info // @Accept json // @Produce json +// @Param show_default_packages query bool true "Enable for show all packages include default packages." // @Success 200 {object} software.Software "Successfully get information of software." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to get information of software." // @Router /honeybee-agent/software [get] func GetSoftwareInfo(c echo.Context) error { - softwareInfo, err := software.GetSoftwareInfo() + showDefaultPackagesStr := c.QueryParam("show_default_packages") + showDefaultPackages, _ := strconv.ParseBool(showDefaultPackagesStr) + + softwareInfo, err := software.GetSoftwareInfo(showDefaultPackages) if err != nil { return common.ReturnInternalError(c, err, "Failed to get information of software.") } diff --git a/agent/pkg/api/rest/docs/docs.go b/agent/pkg/api/rest/docs/docs.go index 2fbf86b..cff3648 100644 --- a/agent/pkg/api/rest/docs/docs.go +++ b/agent/pkg/api/rest/docs/docs.go @@ -92,6 +92,15 @@ const docTemplate = `{ "[Software] Get software info" ], "summary": "Get a list of software information", + "parameters": [ + { + "type": "boolean", + "description": "Enable for show all packages include default packages.", + "name": "show_default_packages", + "in": "query", + "required": true + } + ], "responses": { "200": { "description": "Successfully get information of software.", diff --git a/agent/pkg/api/rest/docs/swagger.json b/agent/pkg/api/rest/docs/swagger.json index 8929c9d..a90b568 100644 --- a/agent/pkg/api/rest/docs/swagger.json +++ b/agent/pkg/api/rest/docs/swagger.json @@ -81,6 +81,15 @@ "[Software] Get software info" ], "summary": "Get a list of software information", + "parameters": [ + { + "type": "boolean", + "description": "Enable for show all packages include default packages.", + "name": "show_default_packages", + "in": "query", + "required": true + } + ], "responses": { "200": { "description": "Successfully get information of software.", diff --git a/agent/pkg/api/rest/docs/swagger.yaml b/agent/pkg/api/rest/docs/swagger.yaml index 269a15b..eaca6cf 100644 --- a/agent/pkg/api/rest/docs/swagger.yaml +++ b/agent/pkg/api/rest/docs/swagger.yaml @@ -767,6 +767,12 @@ paths: consumes: - application/json description: Get software information. + parameters: + - description: Enable for show all packages include default packages. + in: query + name: show_default_packages + required: true + type: boolean produces: - application/json responses: