feat: upload qcow2 images (#69)

It is now possible to upload qcow2 images. These images will be
converted to raw disk images on the cloud server.

In the CLI you can use the new `--format=qcow2` flag to upload qcow2
images. In the library you can set `UploadOptions.ImageFormat` to
`FormatQCOW2`.

Because of the underlying process, qcow2 images need to be written to a
file first. This limits their size to 960 MB at the moment. The CLI
automatically checks the file size (if possible) and shows a warning if
this limit would be triggered. The library accepts an input with the
file size and logs a warning if the limit would be triggered.

Closes #44

cmd/upload.go

@@ -1,7 +1,9 @@
 package cmd
 
 import (
+	_ "embed"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 
@@ -16,20 +18,24 @@ const (
 	uploadFlagImageURL     = "image-url"
 	uploadFlagImagePath    = "image-path"
 	uploadFlagCompression  = "compression"
+	uploadFlagFormat       = "format"
 	uploadFlagArchitecture = "architecture"
 	uploadFlagServerType   = "server-type"
 	uploadFlagDescription  = "description"
 	uploadFlagLabels       = "labels"
 )
 
+//go:embed upload.md
+var longDescription string
+
 // uploadCmd represents the upload command
 var uploadCmd = &cobra.Command{
 	Use:   "upload (--image-path=<local-path> | --image-url=<url>) --architecture=<x86|arm>",
 	Short: "Upload the specified disk image into your Hetzner Cloud project.",
-	Long: `This command implements a fake "upload", by going through a real server and snapshots.
-This does cost a bit of money for the server.`,
+	Long:  longDescription,
 	Example: `  hcloud-upload-image upload --image-path /home/you/images/custom-linux-image-x86.bz2 --architecture x86 --compression bz2 --description "My super duper custom linux"
-  hcloud-upload-image upload --image-url https://examples.com/image-arm.raw --architecture arm --labels foo=bar,version=latest`,
+  hcloud-upload-image upload --image-url https://examples.com/image-arm.raw --architecture arm --labels foo=bar,version=latest
+  hcloud-upload-image upload --image-url https://examples.com/image-x86.qcow2 --architecture x86 --format qcow2`,
 	DisableAutoGenTag: true,
 
 	GroupID: "primary",
@@ -43,6 +49,7 @@ This does cost a bit of money for the server.`,
 		imageURLString, _ := cmd.Flags().GetString(uploadFlagImageURL)
 		imagePathString, _ := cmd.Flags().GetString(uploadFlagImagePath)
 		imageCompression, _ := cmd.Flags().GetString(uploadFlagCompression)
+		imageFormat, _ := cmd.Flags().GetString(uploadFlagFormat)
 		architecture, _ := cmd.Flags().GetString(uploadFlagArchitecture)
 		serverType, _ := cmd.Flags().GetString(uploadFlagServerType)
 		description, _ := cmd.Flags().GetString(uploadFlagDescription)
@@ -50,6 +57,7 @@ This does cost a bit of money for the server.`,
 
 		options := hcloudimages.UploadOptions{
 			ImageCompression: hcloudimages.Compression(imageCompression),
+			ImageFormat:      hcloudimages.Format(imageFormat),
 			Description:      hcloud.Ptr(description),
 			Labels:           labels,
 		}
@@ -60,8 +68,26 @@ This does cost a bit of money for the server.`,
 				return fmt.Errorf("unable to parse url from --%s=%q: %w", uploadFlagImageURL, imageURLString, err)
 			}
 
+			// Check for image size
+			resp, err := http.Head(imageURL.String())
+			switch {
+			case err != nil:
+				logger.DebugContext(ctx, "failed to check for file size, error on request", "err", err)
+			case resp.ContentLength == -1:
+				logger.DebugContext(ctx, "failed to check for file size, server did not set the Content-Length", "err", err)
+			default:
+				options.ImageSize = resp.ContentLength
+			}
+
 			options.ImageURL = imageURL
 		} else if imagePathString != "" {
+			stat, err := os.Stat(imagePathString)
+			if err != nil {
+				logger.DebugContext(ctx, "failed to check for file size, error on stat", "err", err)
+			} else {
+				options.ImageSize = stat.Size()
+			}
+
 			imageFile, err := os.Open(imagePathString)
 			if err != nil {
 				return fmt.Errorf("unable to read file from --%s=%q: %w", uploadFlagImagePath, imagePathString, err)
@@ -101,6 +127,12 @@ func init() {
 		cobra.FixedCompletions([]string{string(hcloudimages.CompressionBZ2), string(hcloudimages.CompressionXZ)}, cobra.ShellCompDirectiveNoFileComp),
 	)
 
+	uploadCmd.Flags().String(uploadFlagFormat, "", "Format of the image. [choices: qcow2]")
+	_ = uploadCmd.RegisterFlagCompletionFunc(
+		uploadFlagFormat,
+		cobra.FixedCompletions([]string{string(hcloudimages.FormatQCOW2)}, cobra.ShellCompDirectiveNoFileComp),
+	)
+
 	uploadCmd.Flags().String(uploadFlagArchitecture, "", "CPU architecture of the disk image [choices: x86, arm]")
 	_ = uploadCmd.RegisterFlagCompletionFunc(
 		uploadFlagArchitecture,

cmd/upload.md

@@ -0,0 +1,12 @@
+This command implements a fake "upload", by going through a real server and
+snapshots. This does cost a bit of money for the server.
+
+#### Image Size
+
+The image size for raw disk images is only limited by the servers root disk.
+
+The image size for qcow2 images is limited to the rescue systems root disk.
+This is currently a memory-backed file system with **960 MB** of space. A qcow2
+image not be larger than this size, or the process will error. There is a
+warning being logged if hcloud-upload-image can detect that your file is larger
+than this size.

docs/cli/hcloud-upload-image_upload.md

@@ -4,8 +4,19 @@ Upload the specified disk image into your Hetzner Cloud project.
 
 ### Synopsis
 
-This command implements a fake "upload", by going through a real server and snapshots.
-This does cost a bit of money for the server.
+This command implements a fake "upload", by going through a real server and
+snapshots. This does cost a bit of money for the server.
+
+#### Image Size
+
+The image size for raw disk images is only limited by the servers root disk.
+
+The image size for qcow2 images is limited to the rescue systems root disk.
+This is currently a memory-backed file system with **960 MB** of space. A qcow2
+image not be larger than this size, or the process will error. There is a
+warning being logged if hcloud-upload-image can detect that your file is larger
+than this size.
+
 
 ```
 hcloud-upload-image upload (--image-path=<local-path> | --image-url=<url>) --architecture=<x86|arm> [flags]
@@ -16,6 +27,7 @@ hcloud-upload-image upload (--image-path=<local-path> | --image-url=<url>) --arc
 ```
   hcloud-upload-image upload --image-path /home/you/images/custom-linux-image-x86.bz2 --architecture x86 --compression bz2 --description "My super duper custom linux"
   hcloud-upload-image upload --image-url https://examples.com/image-arm.raw --architecture arm --labels foo=bar,version=latest
+  hcloud-upload-image upload --image-url https://examples.com/image-x86.qcow2 --architecture x86 --format qcow2
 ```
 
 ### Options
@@ -24,6 +36,7 @@ hcloud-upload-image upload (--image-path=<local-path> | --image-url=<url>) --arc
       --architecture string     CPU architecture of the disk image [choices: x86, arm]
       --compression string      Type of compression that was used on the disk image [choices: bz2, xz]
       --description string      Description for the resulting image
+      --format string           Format of the image. [choices: qcow2]
   -h, --help                    help for upload
       --image-path string       Local path to the disk image that should be uploaded
       --image-url string        Remote URL of the disk image that should be uploaded

hcloudimages/client.go

@@ -43,6 +43,10 @@ var (
 	defaultRescueType = hcloud.ServerRescueTypeLinux64
 
 	defaultSSHDialTimeout = 1 * time.Minute
+
+	// Size observed on x86, 2025-05-03, no idea if that changes.
+	// Might be able to extends this to more of the available memory.
+	rescueSystemRootDiskSizeMB int64 = 960
 )
 
 type UploadOptions struct {
@@ -56,10 +60,14 @@ type UploadOptions struct {
 	// set to anything else, the file will be decompressed before written to the disk.
 	ImageCompression Compression
 
+	ImageFormat Format
+
+	// Can be optionally set to make the client validate that the image can be written to the server.
+	ImageSize int64
+
 	// Possible future additions:
 	// ImageSignatureVerification
 	// ImageLocalPath
-	// ImageType (RawDiskImage, ISO, qcow2, ...)
 
 	// Architecture should match the architecture of the Image. This decides if the Snapshot can later be
 	// used with [hcloud.ArchitectureX86] or [hcloud.ArchitectureARM] servers.
@@ -101,6 +109,19 @@ const (
 	// zip,zstd
 )
 
+type Format string
+
+const (
+	FormatRaw Format = ""
+
+	// FormatQCOW2 allows to upload images in the qcow2 format directly.
+	//
+	// The qcow2 image must fit on the disk available in the rescue system. "qemu-img dd", which is used to convert
+	// qcow2 to raw, requires a file as an input. If [UploadOption.ImageSize] is set and FormatQCOW2 is used, there is a
+	// warning message displayed if there is a high probability of issues.
+	FormatQCOW2 Format = "qcow2"
+)
+
 // NewClient instantiates a new client. It requires a working [*hcloud.Client] to interact with the Hetzner Cloud API.
 func NewClient(c *hcloud.Client) *Client {
 	return &Client{
@@ -134,6 +155,19 @@ func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Ima
 	resourceName := resourcePrefix + id
 	labels := labelutil.Merge(DefaultLabels, options.Labels)
 
+	// 0. Validations
+	if options.ImageFormat == FormatQCOW2 && options.ImageSize > 0 {
+		if options.ImageSize > rescueSystemRootDiskSizeMB*1024*1024 {
+			// Just a warning, because the size might change with time.
+			// Alternatively one could add an override flag for the check and make this an error.
+			logger.WarnContext(ctx,
+				fmt.Sprintf("image must be smaller than %d MB (rescue system root disk) for qcow2", rescueSystemRootDiskSizeMB),
+				"maximum-size", rescueSystemRootDiskSizeMB,
+				"actual-size", options.ImageSize/(1024*1024),
+			)
+		}
+	}
+
 	// 1. Create SSH Key
 	logger.InfoContext(ctx, "# Step 1: Generating SSH Key")
 	privateKey, publicKey, err := sshutil.GenerateKeyPair()
@@ -299,21 +333,28 @@ func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Ima
 	logger.InfoContext(ctx, "# Step 6: Downloading image and writing to disk")
 	cmd := ""
 	if options.ImageURL != nil {
-		cmd += fmt.Sprintf("wget --no-verbose -O - %q | ", options.ImageURL.String())
+		cmd += fmt.Sprintf("wget --no-verbose -O - %q", options.ImageURL.String())
 	}
 
 	if options.ImageCompression != CompressionNone {
 		switch options.ImageCompression {
 		case CompressionBZ2:
-			cmd += "bzip2 -cd | "
+			cmd += " | bzip2 -cd"
 		case CompressionXZ:
-			cmd += "xz -cd | "
+			cmd += " | xz -cd"
 		default:
 			return nil, fmt.Errorf("unknown compression: %q", options.ImageCompression)
 		}
 	}
 
-	cmd += "dd of=/dev/sda bs=4M && sync"
+	switch options.ImageFormat {
+	case FormatRaw:
+		cmd += " | dd of=/dev/sda bs=4M"
+	case FormatQCOW2:
+		cmd += " > image.qcow2 && qemu-img dd -f qcow2 -O raw if=image.qcow2 of=/dev/sda bs=4M"
+	}
+
+	cmd += " && sync"
 
 	// Make sure that we fail early, ie. if the image url does not work.
 	// the pipefail does not work correctly without wrapping in bash.