feat: upload qcow2 images

It is now possible to upload qcow2 images directly. These images will be
converted to raw disk images directly on the cloud server.

In the CLI you can use the new `--format=qcow2` flag to upload qcow2
images. In the library you can set `UploadOptions.ImageFormat` to `FormatQCOW2`.

Because of the underlying process, qcow2 images need to be written to a
file first. This limits their size to 960 MB at the moment. The CLI
automatically checks the file size (if possible) and shows a warning if
this limit would be triggered. The library accepts an input with the
file size and logs a warning if the limit would be triggered.

cmd/upload.go

@@ -1,7 +1,9 @@
 package cmd
 
 import (
+	_ "embed"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 
@@ -16,20 +18,24 @@ const (
 	uploadFlagImageURL     = "image-url"
 	uploadFlagImagePath    = "image-path"
 	uploadFlagCompression  = "compression"
+	uploadFlagFormat       = "format"
 	uploadFlagArchitecture = "architecture"
 	uploadFlagServerType   = "server-type"
 	uploadFlagDescription  = "description"
 	uploadFlagLabels       = "labels"
 )
 
+//go:embed upload.md
+var longDescription string
+
 // uploadCmd represents the upload command
 var uploadCmd = &cobra.Command{
 	Use:   "upload (--image-path=<local-path> | --image-url=<url>) --architecture=<x86|arm>",
 	Short: "Upload the specified disk image into your Hetzner Cloud project.",
-	Long: `This command implements a fake "upload", by going through a real server and snapshots.
-This does cost a bit of money for the server.`,
+	Long:  longDescription,
 	Example: `  hcloud-upload-image upload --image-path /home/you/images/custom-linux-image-x86.bz2 --architecture x86 --compression bz2 --description "My super duper custom linux"
-  hcloud-upload-image upload --image-url https://examples.com/image-arm.raw --architecture arm --labels foo=bar,version=latest`,
+  hcloud-upload-image upload --image-url https://examples.com/image-arm.raw --architecture arm --labels foo=bar,version=latest
+  hcloud-upload-image upload --image-url https://examples.com/image-x86.qcow2 --architecture x86 --format qcow2`,
 	DisableAutoGenTag: true,
 
 	GroupID: "primary",
@@ -43,6 +49,7 @@ This does cost a bit of money for the server.`,
 		imageURLString, _ := cmd.Flags().GetString(uploadFlagImageURL)
 		imagePathString, _ := cmd.Flags().GetString(uploadFlagImagePath)
 		imageCompression, _ := cmd.Flags().GetString(uploadFlagCompression)
+		imageFormat, _ := cmd.Flags().GetString(uploadFlagFormat)
 		architecture, _ := cmd.Flags().GetString(uploadFlagArchitecture)
 		serverType, _ := cmd.Flags().GetString(uploadFlagServerType)
 		description, _ := cmd.Flags().GetString(uploadFlagDescription)
@@ -50,6 +57,7 @@ This does cost a bit of money for the server.`,
 
 		options := hcloudimages.UploadOptions{
 			ImageCompression: hcloudimages.Compression(imageCompression),
+			ImageFormat:      hcloudimages.Format(imageFormat),
 			Description:      hcloud.Ptr(description),
 			Labels:           labels,
 		}
@@ -60,8 +68,26 @@ This does cost a bit of money for the server.`,
 				return fmt.Errorf("unable to parse url from --%s=%q: %w", uploadFlagImageURL, imageURLString, err)
 			}
 
+			// Check for image size
+			resp, err := http.Head(imageURL.String())
+			switch {
+			case err != nil:
+				logger.DebugContext(ctx, "failed to check for file size, error on request", "err", err)
+			case resp.ContentLength == -1:
+				logger.DebugContext(ctx, "failed to check for file size, server did not set the Content-Length", "err", err)
+			default:
+				options.ImageSize = resp.ContentLength
+			}
+
 			options.ImageURL = imageURL
 		} else if imagePathString != "" {
+			stat, err := os.Stat(imagePathString)
+			if err != nil {
+				logger.DebugContext(ctx, "failed to check for file size, error on stat", "err", err)
+			} else {
+				options.ImageSize = stat.Size()
+			}
+
 			imageFile, err := os.Open(imagePathString)
 			if err != nil {
 				return fmt.Errorf("unable to read file from --%s=%q: %w", uploadFlagImagePath, imagePathString, err)
@@ -101,6 +127,12 @@ func init() {
 		cobra.FixedCompletions([]string{string(hcloudimages.CompressionBZ2), string(hcloudimages.CompressionXZ)}, cobra.ShellCompDirectiveNoFileComp),
 	)
 
+	uploadCmd.Flags().String(uploadFlagFormat, "", "Format of the image. [choices: qcow2]")
+	_ = uploadCmd.RegisterFlagCompletionFunc(
+		uploadFlagFormat,
+		cobra.FixedCompletions([]string{string(hcloudimages.FormatQCOW2)}, cobra.ShellCompDirectiveNoFileComp),
+	)
+
 	uploadCmd.Flags().String(uploadFlagArchitecture, "", "CPU architecture of the disk image [choices: x86, arm]")
 	_ = uploadCmd.RegisterFlagCompletionFunc(
 		uploadFlagArchitecture,

cmd/upload.md

@@ -0,0 +1,12 @@
+This command implements a fake "upload", by going through a real server and
+snapshots. This does cost a bit of money for the server.
+
+#### Image Size
+
+The image size for raw disk images is only limited by the servers root disk.
+
+The image size for qcow2 images is limited to the rescue systems root disk.
+This is currently a memory-backed file system with **960 MB** of space. A qcow2
+image not be larger than this size, or the process will error. There is a
+warning being logged if hcloud-upload-image can detect that your file is larger
+than this size.

docs/cli/hcloud-upload-image_upload.md

@@ -4,8 +4,19 @@ Upload the specified disk image into your Hetzner Cloud project.
 
 ### Synopsis
 
-This command implements a fake "upload", by going through a real server and snapshots.
-This does cost a bit of money for the server.
+This command implements a fake "upload", by going through a real server and
+snapshots. This does cost a bit of money for the server.
+
+#### Image Size
+
+The image size for raw disk images is only limited by the servers root disk.
+
+The image size for qcow2 images is limited to the rescue systems root disk.
+This is currently a memory-backed file system with **960 MB** of space. A qcow2
+image not be larger than this size, or the process will error. There is a
+warning being logged if hcloud-upload-image can detect that your file is larger
+than this size.
+
 
 ```
 hcloud-upload-image upload (--image-path=<local-path> | --image-url=<url>) --architecture=<x86|arm> [flags]
@@ -16,6 +27,7 @@ hcloud-upload-image upload (--image-path=<local-path> | --image-url=<url>) --arc
 ```
   hcloud-upload-image upload --image-path /home/you/images/custom-linux-image-x86.bz2 --architecture x86 --compression bz2 --description "My super duper custom linux"
   hcloud-upload-image upload --image-url https://examples.com/image-arm.raw --architecture arm --labels foo=bar,version=latest
+  hcloud-upload-image upload --image-url https://examples.com/image-x86.qcow2 --architecture x86 --format qcow2
 ```
 
 ### Options
@@ -24,6 +36,7 @@ hcloud-upload-image upload (--image-path=<local-path> | --image-url=<url>) --arc
       --architecture string     CPU architecture of the disk image [choices: x86, arm]
       --compression string      Type of compression that was used on the disk image [choices: bz2, xz]
       --description string      Description for the resulting image
+      --format string           Format of the image. [choices: qcow2]
   -h, --help                    help for upload
       --image-path string       Local path to the disk image that should be uploaded
       --image-url string        Remote URL of the disk image that should be uploaded

go.work.sum

@@ -92,6 +92,8 @@ golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
 golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

hcloudimages/client.go

@@ -43,6 +43,10 @@ var (
 	defaultRescueType = hcloud.ServerRescueTypeLinux64
 
 	defaultSSHDialTimeout = 1 * time.Minute
+
+	// Size observed on x86, 2025-05-03, no idea if that changes.
+	// Might be able to extends this to more of the available memory.
+	rescueSystemRootDiskSizeMB int64 = 960
 )
 
 type UploadOptions struct {
@@ -56,10 +60,14 @@ type UploadOptions struct {
 	// set to anything else, the file will be decompressed before written to the disk.
 	ImageCompression Compression
 
+	ImageFormat Format
+
+	// Can be optionally set to make the client validate that the image can be written to the server.
+	ImageSize int64
+
 	// Possible future additions:
 	// ImageSignatureVerification
 	// ImageLocalPath
-	// ImageType (RawDiskImage, ISO, qcow2, ...)
 
 	// Architecture should match the architecture of the Image. This decides if the Snapshot can later be
 	// used with [hcloud.ArchitectureX86] or [hcloud.ArchitectureARM] servers.
@@ -101,6 +109,19 @@ const (
 	// zip,zstd
 )
 
+type Format string
+
+const (
+	FormatRaw Format = ""
+
+	// FormatQCOW2 allows to upload images in the qcow2 format directly.
+	//
+	// The qcow2 image must fit on the disk available in the rescue system. "qemu-img dd", which is used to convert
+	// qcow2 to raw, requires a file as an input. If [UploadOption.ImageSize] is set and FormatQCOW2 is used, there is a
+	// warning message displayed if there is a high probability of issues.
+	FormatQCOW2 Format = "qcow2"
+)
+
 // NewClient instantiates a new client. It requires a working [*hcloud.Client] to interact with the Hetzner Cloud API.
 func NewClient(c *hcloud.Client) *Client {
 	return &Client{
@@ -134,6 +155,19 @@ func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Ima
 	resourceName := resourcePrefix + id
 	labels := labelutil.Merge(DefaultLabels, options.Labels)
 
+	// 0. Validations
+	if options.ImageFormat == FormatQCOW2 && options.ImageSize > 0 {
+		if options.ImageSize > rescueSystemRootDiskSizeMB*1024*1024 {
+			// Just a warning, because the size might change with time.
+			// Alternatively one could add an override flag for the check and make this an error.
+			logger.WarnContext(ctx,
+				fmt.Sprintf("image must be smaller than %d MB (rescue system root disk) for qcow2", rescueSystemRootDiskSizeMB),
+				"maximum-size", rescueSystemRootDiskSizeMB,
+				"actual-size", options.ImageSize/(1024*1024),
+			)
+		}
+	}
+
 	// 1. Create SSH Key
 	logger.InfoContext(ctx, "# Step 1: Generating SSH Key")
 	privateKey, publicKey, err := sshutil.GenerateKeyPair()
@@ -299,21 +333,28 @@ func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Ima
 	logger.InfoContext(ctx, "# Step 6: Downloading image and writing to disk")
 	cmd := ""
 	if options.ImageURL != nil {
-		cmd += fmt.Sprintf("wget --no-verbose -O - %q | ", options.ImageURL.String())
+		cmd += fmt.Sprintf("wget --no-verbose -O - %q", options.ImageURL.String())
 	}
 
 	if options.ImageCompression != CompressionNone {
 		switch options.ImageCompression {
 		case CompressionBZ2:
-			cmd += "bzip2 -cd | "
+			cmd += " | bzip2 -cd"
 		case CompressionXZ:
-			cmd += "xz -cd | "
+			cmd += " | xz -cd"
 		default:
 			return nil, fmt.Errorf("unknown compression: %q", options.ImageCompression)
 		}
 	}
 
-	cmd += "dd of=/dev/sda bs=4M && sync"
+	switch options.ImageFormat {
+	case FormatRaw:
+		cmd += " | dd of=/dev/sda bs=4M"
+	case FormatQCOW2:
+		cmd += " > image.qcow2 && qemu-img dd -f qcow2 -O raw if=image.qcow2 of=/dev/sda bs=4M"
+	}
+
+	cmd += " && sync"
 
 	// Make sure that we fail early, ie. if the image url does not work.
 	// the pipefail does not work correctly without wrapping in bash.