Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: recreate changed files #1762

Merged
merged 5 commits into from
Nov 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion odiglet/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ import (
"context"
"os"

detector "github.com/odigos-io/odigos/odiglet/pkg/detector"
"github.com/odigos-io/odigos/odiglet/pkg/ebpf/sdks"
"github.com/odigos-io/odigos/odiglet/pkg/instrumentation/fs"
detector "github.com/odigos-io/odigos/odiglet/pkg/detector"

"github.com/kubevirt/device-plugin-manager/pkg/dpm"
"github.com/odigos-io/odigos/common"
Expand All @@ -28,6 +28,9 @@ import (
)

func odigletInitPhase() {
if err := log.Init(); err != nil {
panic(err)
}
err := fs.CopyAgentsDirectoryToHost()
if err != nil {
log.Logger.Error(err, "Failed to copy agents directory to host")
Expand Down
76 changes: 70 additions & 6 deletions odiglet/pkg/instrumentation/fs/agents.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package fs

import (
"crypto/sha256"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"

"github.com/odigos-io/odigos/odiglet/pkg/log"
Expand All @@ -24,7 +28,7 @@ func CopyAgentsDirectoryToHost() error {
// so we will just remove it's content

// We kept the following list of files to avoid removing instrumentations that are already loaded in the process memory
filesToKeepMap := map[string]struct{}{
filesToKeep := map[string]struct{}{
"/var/odigos/nodejs-ebpf/build/Release/dtrace-injector-native.node": {},
"/var/odigos/nodejs-ebpf/build/Release/obj.target/dtrace-injector-native.node": {},
"/var/odigos/nodejs-ebpf/build/Release/.deps/Release/dtrace-injector-native.node.d": {},
Expand All @@ -34,13 +38,18 @@ func CopyAgentsDirectoryToHost() error {
"/var/odigos/python-ebpf/pythonUSDT.abi3.so": {},
}

err := removeFilesInDir(hostDir, filesToKeepMap)
err := removeChangedFilesFromKeepMap(filesToKeep, containerDir, hostDir)
if err != nil {
log.Logger.Error(err, "Error getting changed files")
}

err = removeFilesInDir(hostDir, filesToKeep)
if err != nil {
log.Logger.Error(err, "Error removing instrumentation directory from host")
return err
}

err = copyDirectories(containerDir, hostDir, filesToKeepMap)
err = copyDirectories(containerDir, hostDir, filesToKeep)
if err != nil {
log.Logger.Error(err, "Error copying instrumentation directory to host")
return err
Expand Down Expand Up @@ -73,7 +82,62 @@ func CopyAgentsDirectoryToHost() error {
return nil
}

func ShouldRecreateAllCFiles() bool {
value, exists := os.LookupEnv("RECREATE_ALL_C_FILES")
return exists && value == "true"
func removeChangedFilesFromKeepMap(filesToKeepMap map[string]struct{}, containerDir string, hostDir string) error {
for hostPath := range filesToKeepMap {
// Convert host path to container path
containerPath := strings.Replace(hostPath, hostDir, containerDir, 1)

// Check if both files exist
hostInfo, hostErr := os.Stat(hostPath)
containerInfo, containerErr := os.Stat(containerPath)

// If either file doesn't exist, mark as changed and remove from filesToKeepMap
if hostErr != nil || containerErr != nil {
delete(filesToKeepMap, hostPath)
log.Logger.V(0).Info("File marked for deletion (missing)", "file", hostPath)
continue
}

// If sizes are different, mark as changed
if hostInfo.Size() != containerInfo.Size() {
delete(filesToKeepMap, hostPath)
log.Logger.V(0).Info("File marked for deletion (size mismatch)", "file", hostPath)
continue
}

// Compare file hashes
hostHash, err := calculateFileHash(hostPath)
if err != nil {
return fmt.Errorf("error calculating hash for host file %s: %v", hostPath, err)
}

containerHash, err := calculateFileHash(containerPath)
if err != nil {
return fmt.Errorf("error calculating hash for container file %s: %v", containerPath, err)
}

// If hashes are different, mark as changed
if hostHash != containerHash {
delete(filesToKeepMap, hostPath)
log.Logger.V(0).Info("File marked for deletion (content mismatch)", "file", hostPath)
}
}

return nil
}

// calculateFileHash computes the SHA-256 hash of a file
func calculateFileHash(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()

hasher := sha256.New()
if _, err := io.Copy(hasher, file); err != nil {
return "", err
}

return fmt.Sprintf("%x", hasher.Sum(nil)), nil
}
13 changes: 6 additions & 7 deletions odiglet/pkg/instrumentation/fs/copy.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,16 @@ func getNumberOfWorkers() int {
return min(maxWorkers, max(1, runtime.NumCPU()/4))
}

func copyDirectories(srcDir string, destDir string, filesToKeepMap map[string]struct{}) error {
func copyDirectories(srcDir string, destDir string, filesToKeep map[string]struct{}) error {
start := time.Now()

hostContainEbpfDir := HostContainsEbpfDir(destDir)
shouldRecreateCFiles := ShouldRecreateAllCFiles()

// If the host directory NOT contains ebpf directories OR we should recreate C files, we copy all files
CopyCFiles := !hostContainEbpfDir || shouldRecreateCFiles
// If the host directory NOT contains ebpf directories we copy all files
CopyCFiles := !hostContainEbpfDir
log.Logger.V(0).Info("Copying instrumentation files to host", "srcDir", srcDir, "destDir", destDir, "CopyCFiles", CopyCFiles)

files, err := getFiles(srcDir, CopyCFiles, filesToKeepMap)
files, err := getFiles(srcDir, CopyCFiles, filesToKeep)
if err != nil {
return err
}
Expand Down Expand Up @@ -85,15 +84,15 @@ func worker(fileChan <-chan string, sourceDir, destDir string, wg *sync.WaitGrou
}
}

func getFiles(dir string, CopyCFiles bool, filesToKeepMap map[string]struct{}) ([]string, error) {
func getFiles(dir string, CopyCFiles bool, filesToKeep map[string]struct{}) ([]string, error) {
var files []string
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() {
if !CopyCFiles {
if _, found := filesToKeepMap[strings.Replace(path, "/instrumentations/", "/var/odigos/", 1)]; found {
if _, found := filesToKeep[strings.Replace(path, "/instrumentations/", "/var/odigos/", 1)]; found {
log.Logger.V(0).Info("Skipping copying file", "file", path)
return nil
}
Expand Down
18 changes: 7 additions & 11 deletions odiglet/pkg/instrumentation/fs/remove.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,17 @@ import (
)

func removeFilesInDir(hostDir string, filesToKeep map[string]struct{}) error {
shouldRecreateCFiles := ShouldRecreateAllCFiles()
log.Logger.V(0).Info("Removing files in the host directory", "hostDir", hostDir, "shouldRecreateCFiles", shouldRecreateCFiles)
log.Logger.V(0).Info("Removing files in the host directory", "hostDir", hostDir)

// Mark directories as protected if they contain a file that needs to be preserved.
// If C files should be recreated, skip marking any directories as protected.
protectedDirs := make(map[string]bool)
if !shouldRecreateCFiles {
for file := range filesToKeep {
dir := filepath.Dir(file)
for dir != hostDir {
protectedDirs[dir] = true
dir = filepath.Dir(dir)
}
protectedDirs[hostDir] = true // Protect the main directory
for file := range filesToKeep {
dir := filepath.Dir(file)
for dir != hostDir {
protectedDirs[dir] = true
dir = filepath.Dir(dir)
}
protectedDirs[hostDir] = true // Protect the main directory
}

return filepath.Walk(hostDir, func(path string, info os.FileInfo, err error) error {
Expand Down
Loading