From 8e4d10bacf15f0bd069cc44890485024f9d65aa0 Mon Sep 17 00:00:00 2001 From: Justin Merrell Date: Wed, 14 Feb 2024 23:18:45 -0500 Subject: [PATCH 1/3] fix: correct verbose for rsync, and dep fix --- cmd/project/functions.go | 15 +++++++++- cmd/project/ssh.go | 64 ++++++++++++++++++++++++++++++++-------- 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/cmd/project/functions.go b/cmd/project/functions.go index ede3b5b..5e2136b 100644 --- a/cmd/project/functions.go +++ b/cmd/project/functions.go @@ -295,7 +295,7 @@ func startProject(networkVolumeId string) error { fmt.Printf("Activating Python virtual environment %s on Pod %s\n", venvPath, projectPodId) sshConn.RunCommands([]string{ fmt.Sprint(` - DEPENDENCIES=("wget" "sudo" "lsof" "git" "rsync" "zstd" "jq" "inotify-tools") + DEPENDENCIES=("wget" "sudo" "lsof" "git" "rsync" "zstd" "jq") function check_and_install_dependencies() { for dep in "${DEPENDENCIES[@]}"; do @@ -311,6 +311,19 @@ func startProject(networkVolumeId string) error { fi done + # Specifically check for inotifywait command from inotify-tools package + if ! command -v inotifywait &> /dev/null; then + echo "inotifywait could not be found, attempting to install inotify-tools..." + if apt-get install -y inotify-tools; then + echo "inotify-tools installed successfully." + else + echo "Failed to install inotify-tools." + exit 1 + fi + else + echo "inotify-tools is already installed." + fi + wget -qO- cli.runpod.net | sudo bash &> /dev/null } check_and_install_dependencies`), diff --git a/cmd/project/ssh.go b/cmd/project/ssh.go index ada36d5..371ef2b 100644 --- a/cmd/project/ssh.go +++ b/cmd/project/ssh.go @@ -2,6 +2,7 @@ package project import ( "bufio" + "bytes" "cli/api" "errors" "fmt" @@ -69,7 +70,7 @@ func (sshConn *SSHConnection) getSshOptions() []string { } func (sshConn *SSHConnection) Rsync(localDir string, remoteDir string, quiet bool) error { - rsyncCmdArgs := []string{"-avz", "--no-owner", "--no-group"} + rsyncCmdArgs := []string{"--compress", "--archive", "--verbose", "--no-owner", "--no-group"} // Retrieve and apply ignore patterns patterns, err := GetIgnoreList() @@ -80,21 +81,60 @@ func (sshConn *SSHConnection) Rsync(localDir string, remoteDir string, quiet boo rsyncCmdArgs = append(rsyncCmdArgs, "--exclude", pat) } - // Add quiet flag if requested - if quiet { - rsyncCmdArgs = append(rsyncCmdArgs, "--quiet") - } + //Filter from .runpodignore + rsyncCmdArgs = append(rsyncCmdArgs, "--filter=:- .runpodignore") // Prepare SSH options for rsync sshOptions := fmt.Sprintf("ssh %s", strings.Join(sshConn.getSshOptions(), " ")) rsyncCmdArgs = append(rsyncCmdArgs, "-e", sshOptions, localDir, fmt.Sprintf("root@%s:%s", sshConn.podIp, remoteDir)) - cmd := exec.Command("rsync", rsyncCmdArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + // Perform a dry run to check if files need syncing + dryRunArgs := append(rsyncCmdArgs, "--dry-run") + dryRunCmd := exec.Command("rsync", dryRunArgs...) + var dryRunBuf bytes.Buffer + dryRunCmd.Stdout = &dryRunBuf + dryRunCmd.Stderr = &dryRunBuf + if err := dryRunCmd.Run(); err != nil { + return fmt.Errorf("running rsync dry run: %w", err) + } + dryRunOutput := dryRunBuf.String() + + // Parse the dry run output to determine if files need syncing + filesNeedSyncing := false + scanner := bufio.NewScanner(strings.NewReader(dryRunOutput)) + for scanner.Scan() { + line := scanner.Text() + + if line == "" || strings.Contains(line, "sending incremental file list") || strings.Contains(line, "total size is") || strings.Contains(line, "bytes/sec") { + continue + } + + filename := filepath.Base(line) + if filename == "" || filename == "." || strings.HasSuffix(line, "/") { + continue + } - if err := cmd.Run(); err != nil { - return fmt.Errorf("executing rsync command: %w", err) + filesNeedSyncing = true + break + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("scanning dry run output: %w", err) + } + + // Add quiet flag if requested + if quiet { + rsyncCmdArgs = append(rsyncCmdArgs, "--quiet") + } + + if filesNeedSyncing { + fmt.Println("Syncing files...") + + cmd := exec.Command("rsync", rsyncCmdArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("executing rsync command: %w", err) + } } return nil @@ -133,7 +173,7 @@ func hasChanges(localDir string, lastSyncTime time.Time) (bool, string) { func (sshConn *SSHConnection) SyncDir(localDir string, remoteDir string) { syncFiles := func() { - fmt.Println("Syncing files...") + // fmt.Println("Syncing files...") err := sshConn.Rsync(localDir, remoteDir, true) if err != nil { fmt.Printf(" error: %v\n", err) @@ -148,7 +188,7 @@ func (sshConn *SSHConnection) SyncDir(localDir string, remoteDir string) { time.Sleep(100 * time.Millisecond) hasChanged, firstModifiedFile := hasChanges(localDir, lastSyncTime) if hasChanged { - fmt.Printf("Found changes in %s\n", firstModifiedFile) + fmt.Printf("Local changes detected in %s\n", firstModifiedFile) syncFiles() lastSyncTime = time.Now() } From 934792738cdf5f8784b1e374f91c593e45e035be Mon Sep 17 00:00:00 2001 From: Nathaniel Saxe Date: Thu, 15 Feb 2024 01:14:01 -0500 Subject: [PATCH 2/3] don't print when inotify is already installed --- cmd/project/functions.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/project/functions.go b/cmd/project/functions.go index 5e2136b..d08aaec 100644 --- a/cmd/project/functions.go +++ b/cmd/project/functions.go @@ -320,8 +320,6 @@ func startProject(networkVolumeId string) error { echo "Failed to install inotify-tools." exit 1 fi - else - echo "inotify-tools is already installed." fi wget -qO- cli.runpod.net | sudo bash &> /dev/null From 3309865d9176827157e270ea7f12bbf8e6fe1dbf Mon Sep 17 00:00:00 2001 From: Justin Merrell Date: Wed, 21 Feb 2024 16:51:22 -0500 Subject: [PATCH 3/3] fix: ignore more rysnc and remove jq --- cmd/project/functions.go | 2 +- cmd/project/ssh.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/project/functions.go b/cmd/project/functions.go index d08aaec..d2af8d1 100644 --- a/cmd/project/functions.go +++ b/cmd/project/functions.go @@ -295,7 +295,7 @@ func startProject(networkVolumeId string) error { fmt.Printf("Activating Python virtual environment %s on Pod %s\n", venvPath, projectPodId) sshConn.RunCommands([]string{ fmt.Sprint(` - DEPENDENCIES=("wget" "sudo" "lsof" "git" "rsync" "zstd" "jq") + DEPENDENCIES=("wget" "sudo" "lsof" "git" "rsync" "zstd") function check_and_install_dependencies() { for dep in "${DEPENDENCIES[@]}"; do diff --git a/cmd/project/ssh.go b/cmd/project/ssh.go index 371ef2b..de96203 100644 --- a/cmd/project/ssh.go +++ b/cmd/project/ssh.go @@ -105,7 +105,7 @@ func (sshConn *SSHConnection) Rsync(localDir string, remoteDir string, quiet boo for scanner.Scan() { line := scanner.Text() - if line == "" || strings.Contains(line, "sending incremental file list") || strings.Contains(line, "total size is") || strings.Contains(line, "bytes/sec") { + if line == "" || strings.Contains(line, "sending incremental file list") || strings.Contains(line, "total size is") || strings.Contains(line, "bytes/sec") || strings.Contains(line, "building file list") { continue }