diff --git a/cmd/upload.go b/cmd/upload.go index 9196900..ddc1a6a 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -1,7 +1,9 @@ package cmd import ( + _ "embed" "fmt" + "net/http" "net/url" "os" @@ -16,20 +18,24 @@ const ( uploadFlagImageURL = "image-url" uploadFlagImagePath = "image-path" uploadFlagCompression = "compression" + uploadFlagFormat = "format" uploadFlagArchitecture = "architecture" uploadFlagServerType = "server-type" uploadFlagDescription = "description" uploadFlagLabels = "labels" ) +//go:embed upload.md +var longDescription string + // uploadCmd represents the upload command var uploadCmd = &cobra.Command{ Use: "upload (--image-path= | --image-url=) --architecture=", Short: "Upload the specified disk image into your Hetzner Cloud project.", - Long: `This command implements a fake "upload", by going through a real server and snapshots. -This does cost a bit of money for the server.`, + Long: longDescription, Example: ` hcloud-upload-image upload --image-path /home/you/images/custom-linux-image-x86.bz2 --architecture x86 --compression bz2 --description "My super duper custom linux" - hcloud-upload-image upload --image-url https://examples.com/image-arm.raw --architecture arm --labels foo=bar,version=latest`, + hcloud-upload-image upload --image-url https://examples.com/image-arm.raw --architecture arm --labels foo=bar,version=latest + hcloud-upload-image upload --image-url https://examples.com/image-x86.qcow2 --architecture x86 --format qcow2`, DisableAutoGenTag: true, GroupID: "primary", @@ -43,6 +49,7 @@ This does cost a bit of money for the server.`, imageURLString, _ := cmd.Flags().GetString(uploadFlagImageURL) imagePathString, _ := cmd.Flags().GetString(uploadFlagImagePath) imageCompression, _ := cmd.Flags().GetString(uploadFlagCompression) + imageFormat, _ := cmd.Flags().GetString(uploadFlagFormat) architecture, _ := cmd.Flags().GetString(uploadFlagArchitecture) serverType, _ := cmd.Flags().GetString(uploadFlagServerType) description, _ := cmd.Flags().GetString(uploadFlagDescription) @@ -50,6 +57,7 @@ This does cost a bit of money for the server.`, options := hcloudimages.UploadOptions{ ImageCompression: hcloudimages.Compression(imageCompression), + ImageFormat: hcloudimages.Format(imageFormat), Description: hcloud.Ptr(description), Labels: labels, } @@ -60,8 +68,26 @@ This does cost a bit of money for the server.`, return fmt.Errorf("unable to parse url from --%s=%q: %w", uploadFlagImageURL, imageURLString, err) } + // Check for image size + resp, err := http.Head(imageURL.String()) + switch { + case err != nil: + logger.DebugContext(ctx, "failed to check for file size, error on request", "err", err) + case resp.ContentLength == -1: + logger.DebugContext(ctx, "failed to check for file size, server did not set the Content-Length", "err", err) + default: + options.ImageSize = resp.ContentLength + } + options.ImageURL = imageURL } else if imagePathString != "" { + stat, err := os.Stat(imagePathString) + if err != nil { + logger.DebugContext(ctx, "failed to check for file size, error on stat", "err", err) + } else { + options.ImageSize = stat.Size() + } + imageFile, err := os.Open(imagePathString) if err != nil { return fmt.Errorf("unable to read file from --%s=%q: %w", uploadFlagImagePath, imagePathString, err) @@ -101,6 +127,12 @@ func init() { cobra.FixedCompletions([]string{string(hcloudimages.CompressionBZ2), string(hcloudimages.CompressionXZ)}, cobra.ShellCompDirectiveNoFileComp), ) + uploadCmd.Flags().String(uploadFlagFormat, "", "Format of the image. [choices: qcow2]") + _ = uploadCmd.RegisterFlagCompletionFunc( + uploadFlagFormat, + cobra.FixedCompletions([]string{string(hcloudimages.FormatQCOW2)}, cobra.ShellCompDirectiveNoFileComp), + ) + uploadCmd.Flags().String(uploadFlagArchitecture, "", "CPU architecture of the disk image [choices: x86, arm]") _ = uploadCmd.RegisterFlagCompletionFunc( uploadFlagArchitecture, diff --git a/cmd/upload.md b/cmd/upload.md new file mode 100644 index 0000000..0d8c43c --- /dev/null +++ b/cmd/upload.md @@ -0,0 +1,12 @@ +This command implements a fake "upload", by going through a real server and +snapshots. This does cost a bit of money for the server. + +#### Image Size + +The image size for raw disk images is only limited by the servers root disk. + +The image size for qcow2 images is limited to the rescue systems root disk. +This is currently a memory-backed file system with **960 MB** of space. A qcow2 +image not be larger than this size, or the process will error. There is a +warning being logged if hcloud-upload-image can detect that your file is larger +than this size. diff --git a/docs/cli/hcloud-upload-image_upload.md b/docs/cli/hcloud-upload-image_upload.md index e1dd434..35ec884 100644 --- a/docs/cli/hcloud-upload-image_upload.md +++ b/docs/cli/hcloud-upload-image_upload.md @@ -4,8 +4,19 @@ Upload the specified disk image into your Hetzner Cloud project. ### Synopsis -This command implements a fake "upload", by going through a real server and snapshots. -This does cost a bit of money for the server. +This command implements a fake "upload", by going through a real server and +snapshots. This does cost a bit of money for the server. + +#### Image Size + +The image size for raw disk images is only limited by the servers root disk. + +The image size for qcow2 images is limited to the rescue systems root disk. +This is currently a memory-backed file system with **960 MB** of space. A qcow2 +image not be larger than this size, or the process will error. There is a +warning being logged if hcloud-upload-image can detect that your file is larger +than this size. + ``` hcloud-upload-image upload (--image-path= | --image-url=) --architecture= [flags] @@ -16,6 +27,7 @@ hcloud-upload-image upload (--image-path= | --image-url=) --arc ``` hcloud-upload-image upload --image-path /home/you/images/custom-linux-image-x86.bz2 --architecture x86 --compression bz2 --description "My super duper custom linux" hcloud-upload-image upload --image-url https://examples.com/image-arm.raw --architecture arm --labels foo=bar,version=latest + hcloud-upload-image upload --image-url https://examples.com/image-x86.qcow2 --architecture x86 --format qcow2 ``` ### Options @@ -24,6 +36,7 @@ hcloud-upload-image upload (--image-path= | --image-url=) --arc --architecture string CPU architecture of the disk image [choices: x86, arm] --compression string Type of compression that was used on the disk image [choices: bz2, xz] --description string Description for the resulting image + --format string Format of the image. [choices: qcow2] -h, --help help for upload --image-path string Local path to the disk image that should be uploaded --image-url string Remote URL of the disk image that should be uploaded diff --git a/hcloudimages/client.go b/hcloudimages/client.go index 9c6db28..d10072b 100644 --- a/hcloudimages/client.go +++ b/hcloudimages/client.go @@ -43,6 +43,10 @@ var ( defaultRescueType = hcloud.ServerRescueTypeLinux64 defaultSSHDialTimeout = 1 * time.Minute + + // Size observed on x86, 2025-05-03, no idea if that changes. + // Might be able to extends this to more of the available memory. + rescueSystemRootDiskSizeMB int64 = 960 ) type UploadOptions struct { @@ -56,10 +60,14 @@ type UploadOptions struct { // set to anything else, the file will be decompressed before written to the disk. ImageCompression Compression + ImageFormat Format + + // Can be optionally set to make the client validate that the image can be written to the server. + ImageSize int64 + // Possible future additions: // ImageSignatureVerification // ImageLocalPath - // ImageType (RawDiskImage, ISO, qcow2, ...) // Architecture should match the architecture of the Image. This decides if the Snapshot can later be // used with [hcloud.ArchitectureX86] or [hcloud.ArchitectureARM] servers. @@ -101,6 +109,19 @@ const ( // zip,zstd ) +type Format string + +const ( + FormatRaw Format = "" + + // FormatQCOW2 allows to upload images in the qcow2 format directly. + // + // The qcow2 image must fit on the disk available in the rescue system. "qemu-img dd", which is used to convert + // qcow2 to raw, requires a file as an input. If [UploadOption.ImageSize] is set and FormatQCOW2 is used, there is a + // warning message displayed if there is a high probability of issues. + FormatQCOW2 Format = "qcow2" +) + // NewClient instantiates a new client. It requires a working [*hcloud.Client] to interact with the Hetzner Cloud API. func NewClient(c *hcloud.Client) *Client { return &Client{ @@ -134,6 +155,19 @@ func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Ima resourceName := resourcePrefix + id labels := labelutil.Merge(DefaultLabels, options.Labels) + // 0. Validations + if options.ImageFormat == FormatQCOW2 && options.ImageSize > 0 { + if options.ImageSize > rescueSystemRootDiskSizeMB*1024*1024 { + // Just a warning, because the size might change with time. + // Alternatively one could add an override flag for the check and make this an error. + logger.WarnContext(ctx, + fmt.Sprintf("image must be smaller than %d MB (rescue system root disk) for qcow2", rescueSystemRootDiskSizeMB), + "maximum-size", rescueSystemRootDiskSizeMB, + "actual-size", options.ImageSize/(1024*1024), + ) + } + } + // 1. Create SSH Key logger.InfoContext(ctx, "# Step 1: Generating SSH Key") privateKey, publicKey, err := sshutil.GenerateKeyPair() @@ -299,21 +333,28 @@ func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Ima logger.InfoContext(ctx, "# Step 6: Downloading image and writing to disk") cmd := "" if options.ImageURL != nil { - cmd += fmt.Sprintf("wget --no-verbose -O - %q | ", options.ImageURL.String()) + cmd += fmt.Sprintf("wget --no-verbose -O - %q", options.ImageURL.String()) } if options.ImageCompression != CompressionNone { switch options.ImageCompression { case CompressionBZ2: - cmd += "bzip2 -cd | " + cmd += " | bzip2 -cd" case CompressionXZ: - cmd += "xz -cd | " + cmd += " | xz -cd" default: return nil, fmt.Errorf("unknown compression: %q", options.ImageCompression) } } - cmd += "dd of=/dev/sda bs=4M && sync" + switch options.ImageFormat { + case FormatRaw: + cmd += " | dd of=/dev/sda bs=4M" + case FormatQCOW2: + cmd += " > image.qcow2 && qemu-img dd -f qcow2 -O raw if=image.qcow2 of=/dev/sda bs=4M" + } + + cmd += " && sync" // Make sure that we fail early, ie. if the image url does not work. // the pipefail does not work correctly without wrapping in bash.