From d54977bb89302aca830877c2ce460aa675102da9 Mon Sep 17 00:00:00 2001 From: lstocchi Date: Mon, 4 Aug 2025 15:47:09 +0200 Subject: [PATCH 1/2] optimize image copy on windows this commit adds an optimization of the image copy process on Windows. It leverages robocopy, when available, which is a file replication tool built into Windows. By using it we reduce the time to copy of ~40%. Robocopy is launched with flags: - /J: Copies using unbuffered I/O (recommended for large files) - /MT: Creates multi-threaded copies with n threads. Default value of n is 8 - /R: Specifies the number of retries on failed copies. 0 in this patch - /IS Includes Same files, which forces an overwrite even if the destination file appears identical to the source. Robocopy also uses specific exit code numbers which does not follow classic values. E.g. exit code 1 means all copies has been completed successfully. All exit codes are listed at https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/robocopy#exit-return-codes If robocopy is not found on the Win system, it fallback to the old copyFile function. Signed-off-by: lstocchi --- pkg/imagepullers/noop.go | 93 ++++++++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 14 deletions(-) diff --git a/pkg/imagepullers/noop.go b/pkg/imagepullers/noop.go index 4cf5a311..79c53767 100644 --- a/pkg/imagepullers/noop.go +++ b/pkg/imagepullers/noop.go @@ -2,11 +2,15 @@ package imagepullers import ( "bufio" + "errors" "fmt" "io" "os" + "os/exec" "path/filepath" + "github.com/sirupsen/logrus" + "github.com/containers/podman/v5/pkg/machine/define" "github.com/containers/podman/v5/pkg/machine/env" @@ -91,21 +95,17 @@ func doCopyFile(src, dest string, vmType define.VMType) error { } defer srcF.Close() - destF, err := os.Create(dest) - if err != nil { - return err - } - defer destF.Close() - switch vmType { case define.AppleHvVirt, define.LibKrun: - return copyFileMac(srcF, destF) + return copyFileMac(srcF, dest) + case define.WSLVirt, define.HyperVVirt: + return copyFileWin(srcF, dest) default: - return copyFile(srcF, destF) + return copyFile(srcF, dest) } } -func copyFileMac(src, dest *os.File) error { +func copyFileMac(src *os.File, dest string) error { srcImg, err := qcow2reader.Open(src) if err != nil { return err @@ -124,18 +124,83 @@ func copyFileMac(src, dest *os.File) error { } } -func convertToRaw(srcImg image.Image, dest *os.File) error { +func copyFileWin(srcF *os.File, dest string) error { + binary, err := exec.LookPath("robocopy") + if err != nil { + logrus.Debugf("warning: failed to get robocopy binary: %v. Falling back to default file copy between %s and %s\n", err, srcF.Name(), dest) + return copyFile(srcF, dest) + } + + srcDir, srcFile := filepath.Split(srcF.Name()) + destDir := filepath.Dir(dest) + + // Executes the robocopy command with options optimized for a fast, single-file copy. + // /J: Copies using unbuffered I/O (better for large files). + // /MT: Enables multi-threaded copying for improved performance. + // /R:0: Sets retries on failed copies to 0 to avoid long waits. + // /IS: Includes Same files, which forces an overwrite even if the destination + // file appears identical to the source. + cmd := exec.Command(binary, "/J", "/MT", "/R:0", "/IS", srcDir, destDir, srcFile) + if logrus.IsLevelEnabled(logrus.DebugLevel) { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } else { + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + } + + err = cmd.Run() + if err != nil { + // robocopy does not use classic exit codes like linux commands, so we need to check for specific errors + // Only exit code 8 is considered an error, all other exit codes are considered successful with exceptions + // see https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/robocopy + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitCode := exitErr.ExitCode() + if exitCode >= 8 { + return fmt.Errorf("failed to run robocopy: %w", err) + } + } else { + return fmt.Errorf("failed to run robocopy: %w", err) + } + } + + if err := os.Remove(dest); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to remove existing destination file: %w", err) + } + + err = os.Rename(filepath.Join(destDir, srcFile), dest) + if err != nil { + return fmt.Errorf("failed to rename file: %w", err) + } + + return nil +} + +func convertToRaw(srcImg image.Image, dest string) error { + destF, err := os.Create(dest) + if err != nil { + return err + } + defer destF.Close() + if err := srcImg.Readable(); err != nil { return fmt.Errorf("source image is not readable: %w", err) } - return convert.Convert(dest, srcImg, convert.Options{}) + return convert.Convert(destF, srcImg, convert.Options{}) } -func copyFile(src, dst *os.File) error { - bufferedWriter := bufio.NewWriter(dst) +func copyFile(src *os.File, dest string) error { + destF, err := os.Create(dest) + if err != nil { + return err + } + defer destF.Close() + + bufferedWriter := bufio.NewWriter(destF) defer bufferedWriter.Flush() - _, err := io.Copy(bufferedWriter, src) + _, err = io.Copy(bufferedWriter, src) return err } From 9fa6017a9411a5bcf05ef97de3315ccc9cd04cd7 Mon Sep 17 00:00:00 2001 From: Luca Stocchi Date: Thu, 14 Aug 2025 17:15:39 +0200 Subject: [PATCH 2/2] separate noop into OS specific code noop contains funcs that are specific to different OS. This commit creates OS specific noop file that implements their logic. This cleans the noop file of all switches/if. This is a simple refactor, no logic is changed Signed-off-by: Luca Stocchi --- pkg/imagepullers/noop.go | 128 +------------------------------ pkg/imagepullers/noop_darwin.go | 51 ++++++++++++ pkg/imagepullers/noop_unix.go | 18 +++++ pkg/imagepullers/noop_windows.go | 81 +++++++++++++++++++ 4 files changed, 153 insertions(+), 125 deletions(-) create mode 100644 pkg/imagepullers/noop_darwin.go create mode 100644 pkg/imagepullers/noop_unix.go create mode 100644 pkg/imagepullers/noop_windows.go diff --git a/pkg/imagepullers/noop.go b/pkg/imagepullers/noop.go index 79c53767..261a08fc 100644 --- a/pkg/imagepullers/noop.go +++ b/pkg/imagepullers/noop.go @@ -2,24 +2,13 @@ package imagepullers import ( "bufio" - "errors" "fmt" "io" "os" - "os/exec" - "path/filepath" - - "github.com/sirupsen/logrus" "github.com/containers/podman/v5/pkg/machine/define" "github.com/containers/podman/v5/pkg/machine/env" - - "github.com/lima-vm/go-qcow2reader" - "github.com/lima-vm/go-qcow2reader/convert" - "github.com/lima-vm/go-qcow2reader/image" - "github.com/lima-vm/go-qcow2reader/image/qcow2" - "github.com/lima-vm/go-qcow2reader/image/raw" ) type NoopImagePuller struct { @@ -40,21 +29,6 @@ func (puller *NoopImagePuller) SetSourceURI(sourcePath string) { puller.sourceURI = sourcePath } -func imageExtension(vmType define.VMType, sourceURI string) string { - switch vmType { - case define.WSLVirt: - ext := filepath.Ext(sourceURI) - if ext == ".wsl" { - return ".wsl" - } - return ".tar.gz" - case define.QemuVirt, define.HyperVVirt: - return filepath.Ext(sourceURI) - default: - return "." + vmType.ImageFormat().Kind() - } -} - func (puller *NoopImagePuller) LocalPath() (*define.VMFile, error) { // if localPath has already been calculated returns it if puller.localPath != nil { @@ -85,110 +59,14 @@ func (puller *NoopImagePuller) Download() error { if err != nil { return err } - return doCopyFile(puller.sourceURI, localPath.Path, puller.vmType) -} - -func doCopyFile(src, dest string, vmType define.VMType) error { - srcF, err := os.Open(src) - if err != nil { - return err - } - defer srcF.Close() - - switch vmType { - case define.AppleHvVirt, define.LibKrun: - return copyFileMac(srcF, dest) - case define.WSLVirt, define.HyperVVirt: - return copyFileWin(srcF, dest) - default: - return copyFile(srcF, dest) - } -} - -func copyFileMac(src *os.File, dest string) error { - srcImg, err := qcow2reader.Open(src) - if err != nil { - return err - } - defer srcImg.Close() - - switch srcImg.Type() { - case raw.Type: - // if the image is raw it performs a simple copy - return copyFile(src, dest) - case qcow2.Type: - // if the image is qcow2 it performs a conversion to raw - return convertToRaw(srcImg, dest) - default: - return fmt.Errorf("%s format not supported for conversion to raw", srcImg.Type()) - } -} - -func copyFileWin(srcF *os.File, dest string) error { - binary, err := exec.LookPath("robocopy") - if err != nil { - logrus.Debugf("warning: failed to get robocopy binary: %v. Falling back to default file copy between %s and %s\n", err, srcF.Name(), dest) - return copyFile(srcF, dest) - } - - srcDir, srcFile := filepath.Split(srcF.Name()) - destDir := filepath.Dir(dest) - - // Executes the robocopy command with options optimized for a fast, single-file copy. - // /J: Copies using unbuffered I/O (better for large files). - // /MT: Enables multi-threaded copying for improved performance. - // /R:0: Sets retries on failed copies to 0 to avoid long waits. - // /IS: Includes Same files, which forces an overwrite even if the destination - // file appears identical to the source. - cmd := exec.Command(binary, "/J", "/MT", "/R:0", "/IS", srcDir, destDir, srcFile) - if logrus.IsLevelEnabled(logrus.DebugLevel) { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } else { - cmd.Stdout = io.Discard - cmd.Stderr = io.Discard - } - - err = cmd.Run() - if err != nil { - // robocopy does not use classic exit codes like linux commands, so we need to check for specific errors - // Only exit code 8 is considered an error, all other exit codes are considered successful with exceptions - // see https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/robocopy - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - exitCode := exitErr.ExitCode() - if exitCode >= 8 { - return fmt.Errorf("failed to run robocopy: %w", err) - } - } else { - return fmt.Errorf("failed to run robocopy: %w", err) - } - } - - if err := os.Remove(dest); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("failed to remove existing destination file: %w", err) - } - - err = os.Rename(filepath.Join(destDir, srcFile), dest) - if err != nil { - return fmt.Errorf("failed to rename file: %w", err) - } - - return nil -} -func convertToRaw(srcImg image.Image, dest string) error { - destF, err := os.Create(dest) + src, err := os.Open(puller.sourceURI) if err != nil { return err } - defer destF.Close() - - if err := srcImg.Readable(); err != nil { - return fmt.Errorf("source image is not readable: %w", err) - } + defer src.Close() - return convert.Convert(destF, srcImg, convert.Options{}) + return doCopyFile(src, localPath.Path) } func copyFile(src *os.File, dest string) error { diff --git a/pkg/imagepullers/noop_darwin.go b/pkg/imagepullers/noop_darwin.go new file mode 100644 index 00000000..70d46819 --- /dev/null +++ b/pkg/imagepullers/noop_darwin.go @@ -0,0 +1,51 @@ +//go:build darwin + +package imagepullers + +import ( + "fmt" + "os" + + "github.com/containers/podman/v5/pkg/machine/define" + + "github.com/lima-vm/go-qcow2reader" + "github.com/lima-vm/go-qcow2reader/convert" + "github.com/lima-vm/go-qcow2reader/image" + "github.com/lima-vm/go-qcow2reader/image/qcow2" + "github.com/lima-vm/go-qcow2reader/image/raw" +) + +func imageExtension(vmType define.VMType, _ string) string { + return "." + vmType.ImageFormat().Kind() +} + +func doCopyFile(src *os.File, dest string) error { + srcImg, err := qcow2reader.Open(src) + if err != nil { + return err + } + defer srcImg.Close() + + switch srcImg.Type() { + case raw.Type: + return copyFile(src, dest) + case qcow2.Type: + return convertToRaw(srcImg, dest) + default: + return fmt.Errorf("%s format not supported for conversion to raw", srcImg.Type()) + } +} + +func convertToRaw(srcImg image.Image, dest string) error { + destF, err := os.Create(dest) + if err != nil { + return err + } + defer destF.Close() + + if err := srcImg.Readable(); err != nil { + return fmt.Errorf("source image is not readable: %w", err) + } + + return convert.Convert(destF, srcImg, convert.Options{}) +} diff --git a/pkg/imagepullers/noop_unix.go b/pkg/imagepullers/noop_unix.go new file mode 100644 index 00000000..60ebfca3 --- /dev/null +++ b/pkg/imagepullers/noop_unix.go @@ -0,0 +1,18 @@ +//go:build !windows && !darwin + +package imagepullers + +import ( + "os" + "path/filepath" + + "github.com/containers/podman/v5/pkg/machine/define" +) + +func imageExtension(_ define.VMType, sourceURI string) string { + return filepath.Ext(sourceURI) +} + +func doCopyFile(src *os.File, dest string) error { + return copyFile(src, dest) +} diff --git a/pkg/imagepullers/noop_windows.go b/pkg/imagepullers/noop_windows.go new file mode 100644 index 00000000..45cbd754 --- /dev/null +++ b/pkg/imagepullers/noop_windows.go @@ -0,0 +1,81 @@ +//go:build windows + +package imagepullers + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + + "github.com/containers/podman/v5/pkg/machine/define" + "github.com/sirupsen/logrus" +) + +func imageExtension(vmType define.VMType, sourceURI string) string { + switch vmType { + case define.WSLVirt: + ext := filepath.Ext(sourceURI) + if ext == ".wsl" { + return ".wsl" + } + return ".tar.gz" + default: + return filepath.Ext(sourceURI) + } +} + +func doCopyFile(src *os.File, dest string) error { + binary, err := exec.LookPath("robocopy") + if err != nil { + logrus.Debugf("warning: failed to get robocopy binary: %v. Falling back to default file copy between %s and %s\n", err, src.Name(), dest) + return copyFile(src, dest) + } + + srcDir, srcFile := filepath.Split(src.Name()) + destDir := filepath.Dir(dest) + + // Executes the robocopy command with options optimized for a fast, single-file copy. + // /J: Copies using unbuffered I/O (better for large files). + // /MT: Enables multi-threaded copying for improved performance. + // /R:0: Sets retries on failed copies to 0 to avoid long waits. + // /IS: Includes Same files, which forces an overwrite even if the destination + // file appears identical to the source. + cmd := exec.Command(binary, "/J", "/MT", "/R:0", "/IS", srcDir, destDir, srcFile) + if logrus.IsLevelEnabled(logrus.DebugLevel) { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } else { + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + } + + err = cmd.Run() + if err != nil { + // robocopy does not use classic exit codes like linux commands, so we need to check for specific errors + // Only exit code 8 is considered an error, all other exit codes are considered successful with exceptions + // see https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/robocopy + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitCode := exitErr.ExitCode() + if exitCode >= 8 { + return fmt.Errorf("failed to run robocopy: %w", err) + } + } else { + return fmt.Errorf("failed to run robocopy: %w", err) + } + } + + if err := os.Remove(dest); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to remove existing destination file: %w", err) + } + + err = os.Rename(filepath.Join(destDir, srcFile), dest) + if err != nil { + return fmt.Errorf("failed to rename file: %w", err) + } + + return nil +}