From 19f1e085e9ff1a2acdfdfdb70c50db879417f3a0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 16:03:07 +0200 Subject: [PATCH 01/14] chore(deps): update dependency go to v1.24.3 (#91) --- go.mod | 2 +- hcloudimages/go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 69de163..62c07fc 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/apricote/hcloud-upload-image go 1.23.0 -toolchain go1.24.2 +toolchain go1.24.3 require ( github.com/apricote/hcloud-upload-image/hcloudimages v1.0.0 diff --git a/hcloudimages/go.mod b/hcloudimages/go.mod index fdbf330..bd5e39a 100644 --- a/hcloudimages/go.mod +++ b/hcloudimages/go.mod @@ -2,7 +2,7 @@ module github.com/apricote/hcloud-upload-image/hcloudimages go 1.23.0 -toolchain go1.24.2 +toolchain go1.24.3 require ( github.com/hetznercloud/hcloud-go/v2 v2.21.0 From 7cffed4ba7b58d60d2d1244397136180345a2cb3 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 16:03:18 +0200 Subject: [PATCH 02/14] chore(deps): update dependency rust-lang/mdbook to v0.4.49 (#87) --- .github/workflows/docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 5d7bc55..f8d6e93 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -19,7 +19,7 @@ jobs: - uses: ./.github/actions/setup-mdbook with: - version: v0.4.48 # renovate: datasource=github-releases depName=rust-lang/mdbook + version: v0.4.49 # renovate: datasource=github-releases depName=rust-lang/mdbook - name: Build Book working-directory: docs From f12b6b076b0ec798d4d669cad48053abc88c3962 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 16:03:33 +0200 Subject: [PATCH 03/14] chore(deps): update dependency golangci/golangci-lint to v2.1.6 (#85) --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 124abc4..975935b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,13 +20,13 @@ jobs: - name: Run golangci-lint (CLI) uses: golangci/golangci-lint-action@v7 with: - version: v2.1.5 # renovate: datasource=github-releases depName=golangci/golangci-lint + version: v2.1.6 # renovate: datasource=github-releases depName=golangci/golangci-lint args: --timeout 5m - name: Run golangci-lint (Lib) uses: golangci/golangci-lint-action@v7 with: - version: v2.1.5 # renovate: datasource=github-releases depName=golangci/golangci-lint + version: v2.1.6 # renovate: datasource=github-releases depName=golangci/golangci-lint args: --timeout 5m working-directory: hcloudimages From b093e1eda8849b0efed400dd731c3904783036ef Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 16:03:49 +0200 Subject: [PATCH 04/14] chore(deps): update golangci/golangci-lint-action action to v8 (#86) --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 975935b..78b427c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,13 +18,13 @@ jobs: go-version-file: go.mod - name: Run golangci-lint (CLI) - uses: golangci/golangci-lint-action@v7 + uses: golangci/golangci-lint-action@v8 with: version: v2.1.6 # renovate: datasource=github-releases depName=golangci/golangci-lint args: --timeout 5m - name: Run golangci-lint (Lib) - uses: golangci/golangci-lint-action@v7 + uses: golangci/golangci-lint-action@v8 with: version: v2.1.6 # renovate: datasource=github-releases depName=golangci/golangci-lint args: --timeout 5m From e490b9a7f394e268fa1946ca51aa998c78c3d46a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Fri, 9 May 2025 16:15:07 +0200 Subject: [PATCH 05/14] fix: timeout while waiting for SSH to become available (#92) In #68 I reduced the general limits for the back off, thinking that it would speed up the upload on average because it was retrying faster. But because it was retrying faster, the 10 available retries were used up before SSH became available. The new 100 retries match the 3 minutes of total timeout that the previous solution had, and should fix all issues. In addition, I discovered that my implementation in `hcloudimages/backoff.ExponentialBackoffWithLimit` has a bug where the calculated offset could overflow before the limit was applied, resulting in negative durations. I did not fix the issue because `hcloud-go` provides such a method natively nowadays. Instead, I marked the method as deprecated, to be removed in a later release. --- cmd/root.go | 3 +-- hcloudimages/backoff/backoff.go | 4 ++++ hcloudimages/client.go | 2 +- hcloudimages/internal/control/retry.go | 5 +++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 86cfe89..98a25d3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,7 +9,6 @@ import ( "github.com/spf13/cobra" "github.com/apricote/hcloud-upload-image/hcloudimages" - "github.com/apricote/hcloud-upload-image/hcloudimages/backoff" "github.com/apricote/hcloud-upload-image/hcloudimages/contextlogger" "github.com/apricote/hcloud-upload-image/internal/ui" "github.com/apricote/hcloud-upload-image/internal/version" @@ -89,7 +88,7 @@ func initClient(cmd *cobra.Command, _ []string) { opts := []hcloud.ClientOption{ hcloud.WithToken(os.Getenv("HCLOUD_TOKEN")), hcloud.WithApplication("hcloud-upload-image", version.Version), - hcloud.WithPollOpts(hcloud.PollOpts{BackoffFunc: backoff.ExponentialBackoffWithLimit(2, 1*time.Second, 30*time.Second)}), + hcloud.WithPollOpts(hcloud.PollOpts{BackoffFunc: hcloud.ExponentialBackoffWithOpts(hcloud.ExponentialBackoffOpts{Multiplier: 2, Base: 1 * time.Second, Cap: 30 * time.Second})}), } if os.Getenv("HCLOUD_DEBUG") != "" || verbose >= 2 { diff --git a/hcloudimages/backoff/backoff.go b/hcloudimages/backoff/backoff.go index 310312a..346fb8f 100644 --- a/hcloudimages/backoff/backoff.go +++ b/hcloudimages/backoff/backoff.go @@ -16,6 +16,10 @@ import ( // It uses the formula: // // min(b^retries * d, limit) +// +// This function has a known overflow issue and should not be used anymore. +// +// Deprecated: Use BackoffFuncWithOpts from github.com/hetznercloud/hcloud-go/v2/hcloud instead. func ExponentialBackoffWithLimit(b float64, d time.Duration, limit time.Duration) hcloud.BackoffFunc { return func(retries int) time.Duration { current := time.Duration(math.Pow(b, float64(retries))) * d diff --git a/hcloudimages/client.go b/hcloudimages/client.go index 62e218a..c30cef9 100644 --- a/hcloudimages/client.go +++ b/hcloudimages/client.go @@ -316,7 +316,7 @@ func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Ima err = control.Retry( contextlogger.New(ctx, logger.With("operation", "ssh")), - 10, + 100, // ~ 3 minutes func() error { var err error logger.DebugContext(ctx, "trying to connect to server", "ip", server.PublicNet.IPv4.IP) diff --git a/hcloudimages/internal/control/retry.go b/hcloudimages/internal/control/retry.go index eb56721..44cfe55 100644 --- a/hcloudimages/internal/control/retry.go +++ b/hcloudimages/internal/control/retry.go @@ -8,7 +8,8 @@ import ( "context" "time" - "github.com/apricote/hcloud-upload-image/hcloudimages/backoff" + "github.com/hetznercloud/hcloud-go/v2/hcloud" + "github.com/apricote/hcloud-upload-image/hcloudimages/contextlogger" ) @@ -18,7 +19,7 @@ func Retry(ctx context.Context, maxTries int, f func() error) error { var err error - backoffFunc := backoff.ExponentialBackoffWithLimit(2, 200*time.Millisecond, 2*time.Second) + backoffFunc := hcloud.ExponentialBackoffWithOpts(hcloud.ExponentialBackoffOpts{Multiplier: 2, Base: 200 * time.Millisecond, Cap: 2 * time.Second}) for try := 0; try < maxTries; try++ { if ctx.Err() != nil { From 7f10f7b2530ad9762b3f63658eb3ff27472d19be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Fri, 9 May 2025 16:17:27 +0200 Subject: [PATCH 06/14] chore(main): release hcloudimages 1.0.1 (#94) ## 1.0.1 (2025-05-09) ### Bug Fixes * timeout while waiting for SSH to become available (#92) (e490b9a) --- .github/release-please-manifest.json | 2 +- hcloudimages/CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/release-please-manifest.json b/.github/release-please-manifest.json index db709a3..f2454d7 100644 --- a/.github/release-please-manifest.json +++ b/.github/release-please-manifest.json @@ -1 +1 @@ -{".":"1.0.0","hcloudimages":"1.0.0"} +{".":"1.0.0","hcloudimages":"1.0.1"} diff --git a/hcloudimages/CHANGELOG.md b/hcloudimages/CHANGELOG.md index 076256f..88d4c64 100644 --- a/hcloudimages/CHANGELOG.md +++ b/hcloudimages/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.0.1](https://github.com/apricote/hcloud-upload-image/compare/hcloudimages/v1.0.0...hcloudimages/v1.0.1) (2025-05-09) + + +### Bug Fixes + +* timeout while waiting for SSH to become available ([#92](https://github.com/apricote/hcloud-upload-image/issues/92)) ([e490b9a](https://github.com/apricote/hcloud-upload-image/commit/e490b9a7f394e268fa1946ca51aa998c78c3d46a)) + ## [1.0.0](https://github.com/apricote/hcloud-upload-image/compare/hcloudimages/v0.3.1...hcloudimages/v1.0.0) (2025-05-04) From 4ff1883fd0a7e82370289481c37419671d43260b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Fri, 9 May 2025 16:29:27 +0200 Subject: [PATCH 07/14] chore(deps): update module github.com/apricote/hcloud-upload-image/hcloudimages to v1.0.1 (#95) --- go.mod | 2 +- go.sum | 4 ++-- go.work.sum | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 62c07fc..40b6c5d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.0 toolchain go1.24.3 require ( - github.com/apricote/hcloud-upload-image/hcloudimages v1.0.0 + github.com/apricote/hcloud-upload-image/hcloudimages v1.0.1 github.com/hetznercloud/hcloud-go/v2 v2.21.0 github.com/spf13/cobra v1.9.1 ) diff --git a/go.sum b/go.sum index 3d020ac..3ca98c7 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/apricote/hcloud-upload-image/hcloudimages v1.0.0 h1:Gq0SSuPCiZFApGQ3SIEEQqD8btD6tRwfOxr+cky7oTo= -github.com/apricote/hcloud-upload-image/hcloudimages v1.0.0/go.mod h1:MwBmhSPlwS3S3ynOsFXPbco40Iv3udqhXKz2EXKSpVU= +github.com/apricote/hcloud-upload-image/hcloudimages v1.0.1 h1:ZljuANGyGGxUF+afF6HFha88B9ln60gzVPnndDO/1OU= +github.com/apricote/hcloud-upload-image/hcloudimages v1.0.1/go.mod h1:iJ95BaLfISZBY9X8K2Y2A5a49dI0RLjAuq+4BqlOSgA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/go.work.sum b/go.work.sum index 1f0124a..dd30c3a 100644 --- a/go.work.sum +++ b/go.work.sum @@ -3,7 +3,7 @@ github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjH github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/apricote/hcloud-upload-image/hcloudimages v1.0.0/go.mod h1:MwBmhSPlwS3S3ynOsFXPbco40Iv3udqhXKz2EXKSpVU= +github.com/apricote/hcloud-upload-image/hcloudimages v1.0.1/go.mod h1:iJ95BaLfISZBY9X8K2Y2A5a49dI0RLjAuq+4BqlOSgA= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= From 28bf5380f300a3851d41ff1f65c16afc692237a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Fri, 9 May 2025 17:24:55 +0200 Subject: [PATCH 08/14] chore(main): release 1.0.1 (#93) ## 1.0.1 (2025-05-09) ### Bug Fixes * timeout while waiting for SSH to become available (#92) (e490b9a) --- .github/release-please-manifest.json | 2 +- CHANGELOG.md | 7 +++++++ internal/version/version.go | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/release-please-manifest.json b/.github/release-please-manifest.json index f2454d7..5036936 100644 --- a/.github/release-please-manifest.json +++ b/.github/release-please-manifest.json @@ -1 +1 @@ -{".":"1.0.0","hcloudimages":"1.0.1"} +{".":"1.0.1","hcloudimages":"1.0.1"} diff --git a/CHANGELOG.md b/CHANGELOG.md index c9c3fe4..42301dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.0.1](https://github.com/apricote/hcloud-upload-image/compare/v1.0.0...v1.0.1) (2025-05-09) + + +### Bug Fixes + +* timeout while waiting for SSH to become available ([#92](https://github.com/apricote/hcloud-upload-image/issues/92)) ([e490b9a](https://github.com/apricote/hcloud-upload-image/commit/e490b9a7f394e268fa1946ca51aa998c78c3d46a)) + ## [1.0.0](https://github.com/apricote/hcloud-upload-image/compare/v0.3.1...v1.0.0) (2025-05-04) diff --git a/internal/version/version.go b/internal/version/version.go index 5d6e08f..14bc2dc 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -2,7 +2,7 @@ package version var ( // version is a semver version (https://semver.org). - version = "1.0.0" // x-release-please-version + version = "1.0.1" // x-release-please-version // versionPrerelease is a semver version pre-release identifier (https://semver.org). // From 420dcf94c965ee470602db6c9c23c777fda91222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Fri, 9 May 2025 23:22:24 +0200 Subject: [PATCH 09/14] fix: upload from local image generates broken command (#98) While adding support for qcow2 images in #69 I broke support for local images. Building a shell pipeline through string concatenation is not a good idea... The specific issue was fixed and I also moved building the shell pipeline to a separate function and added unit tests for all cases, so it should be easier to spot these issues in the future. Closes #97 --- hcloudimages/client.go | 66 +++++++++++-------- hcloudimages/client_test.go | 123 +++++++++++++++++++++++++++++------- hcloudimages/doc_test.go | 33 ++++++++++ 3 files changed, 173 insertions(+), 49 deletions(-) create mode 100644 hcloudimages/doc_test.go diff --git a/hcloudimages/client.go b/hcloudimages/client.go index c30cef9..fa51b55 100644 --- a/hcloudimages/client.go +++ b/hcloudimages/client.go @@ -331,34 +331,12 @@ func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Ima // 6. SSH On Server: Download Image, Decompress, Write to Root Disk logger.InfoContext(ctx, "# Step 6: Downloading image and writing to disk") - cmd := "" - if options.ImageURL != nil { - cmd += fmt.Sprintf("wget --no-verbose -O - %q", options.ImageURL.String()) + + cmd, err := assembleCommand(options) + if err != nil { + return nil, err } - if options.ImageCompression != CompressionNone { - switch options.ImageCompression { - case CompressionBZ2: - cmd += " | bzip2 -cd" - case CompressionXZ: - cmd += " | xz -cd" - default: - return nil, fmt.Errorf("unknown compression: %q", options.ImageCompression) - } - } - - switch options.ImageFormat { - case FormatRaw: - cmd += " | dd of=/dev/sda bs=4M" - case FormatQCOW2: - cmd += " > image.qcow2 && qemu-img dd -f qcow2 -O raw if=image.qcow2 of=/dev/sda bs=4M" - } - - cmd += " && sync" - - // Make sure that we fail early, ie. if the image url does not work. - // the pipefail does not work correctly without wrapping in bash. - cmd = fmt.Sprintf("bash -c 'set -euo pipefail && %s'", cmd) logger.DebugContext(ctx, "running download, decompress and write to disk command", "cmd", cmd) output, err := sshsession.Run(sshClient, cmd, options.ImageReader) @@ -522,3 +500,39 @@ func (s *Client) cleanupTempSSHKeys(ctx context.Context, logger *slog.Logger, se return nil } + +func assembleCommand(options UploadOptions) (string, error) { + // Make sure that we fail early, ie. if the image url does not work + cmd := "set -euo pipefail && " + + if options.ImageURL != nil { + cmd += fmt.Sprintf("wget --no-verbose -O - %q | ", options.ImageURL.String()) + } + + if options.ImageCompression != CompressionNone { + switch options.ImageCompression { + case CompressionBZ2: + cmd += "bzip2 -cd | " + case CompressionXZ: + cmd += "xz -cd | " + default: + return "", fmt.Errorf("unknown compression: %q", options.ImageCompression) + } + } + + switch options.ImageFormat { + case FormatRaw: + cmd += "dd of=/dev/sda bs=4M" + case FormatQCOW2: + cmd += "tee image.qcow2 > /dev/null && qemu-img dd -f qcow2 -O raw if=image.qcow2 of=/dev/sda bs=4M" + default: + return "", fmt.Errorf("unknown format: %q", options.ImageFormat) + } + + cmd += " && sync" + + // the pipefail does not work correctly without wrapping in bash. + cmd = fmt.Sprintf("bash -c '%s'", cmd) + + return cmd, nil +} diff --git a/hcloudimages/client_test.go b/hcloudimages/client_test.go index 4905bdf..86f8961 100644 --- a/hcloudimages/client_test.go +++ b/hcloudimages/client_test.go @@ -1,33 +1,110 @@ -package hcloudimages_test +package hcloudimages import ( - "context" - "fmt" "net/url" - - "github.com/hetznercloud/hcloud-go/v2/hcloud" - - "github.com/apricote/hcloud-upload-image/hcloudimages" + "testing" ) -func ExampleClient_Upload() { - client := hcloudimages.NewClient( - hcloud.NewClient(hcloud.WithToken("")), - ) - - imageURL, err := url.Parse("https://example.com/disk-image.raw.bz2") +func mustParseURL(s string) *url.URL { + u, err := url.Parse(s) if err != nil { panic(err) } - image, err := client.Upload(context.TODO(), hcloudimages.UploadOptions{ - ImageURL: imageURL, - ImageCompression: hcloudimages.CompressionBZ2, - Architecture: hcloud.ArchitectureX86, - }) - if err != nil { - panic(err) - } - - fmt.Printf("Uploaded Image: %d", image.ID) + return u +} + +func TestAssembleCommand(t *testing.T) { + tests := []struct { + name string + options UploadOptions + want string + wantErr bool + }{ + { + name: "local raw", + options: UploadOptions{}, + want: "bash -c 'set -euo pipefail && dd of=/dev/sda bs=4M && sync'", + }, + { + name: "remote raw", + options: UploadOptions{ + ImageURL: mustParseURL("https://example.com/image.xz"), + }, + want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.xz\" | dd of=/dev/sda bs=4M && sync'", + }, + { + name: "local xz", + options: UploadOptions{ + ImageCompression: CompressionXZ, + }, + want: "bash -c 'set -euo pipefail && xz -cd | dd of=/dev/sda bs=4M && sync'", + }, + { + name: "remote xz", + options: UploadOptions{ + ImageURL: mustParseURL("https://example.com/image.xz"), + ImageCompression: CompressionXZ, + }, + want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.xz\" | xz -cd | dd of=/dev/sda bs=4M && sync'", + }, + { + name: "local bz2", + options: UploadOptions{ + ImageCompression: CompressionBZ2, + }, + want: "bash -c 'set -euo pipefail && bzip2 -cd | dd of=/dev/sda bs=4M && sync'", + }, + { + name: "remote bz2", + options: UploadOptions{ + ImageURL: mustParseURL("https://example.com/image.bz2"), + ImageCompression: CompressionXZ, + }, + want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.bz2\" | xz -cd | dd of=/dev/sda bs=4M && sync'", + }, + { + name: "local qcow2", + options: UploadOptions{ + ImageFormat: FormatQCOW2, + }, + want: "bash -c 'set -euo pipefail && tee image.qcow2 > /dev/null && qemu-img dd -f qcow2 -O raw if=image.qcow2 of=/dev/sda bs=4M && sync'", + }, + { + name: "remote qcow2", + options: UploadOptions{ + ImageURL: mustParseURL("https://example.com/image.qcow2"), + ImageFormat: FormatQCOW2, + }, + want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.qcow2\" | tee image.qcow2 > /dev/null && qemu-img dd -f qcow2 -O raw if=image.qcow2 of=/dev/sda bs=4M && sync'", + }, + + { + name: "unknown compression", + options: UploadOptions{ + ImageCompression: "noodle", + }, + wantErr: true, + }, + + { + name: "unknown format", + options: UploadOptions{ + ImageFormat: "poodle", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := assembleCommand(tt.options) + if (err != nil) != tt.wantErr { + t.Errorf("assembleCommand() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("assembleCommand() got = %v, want %v", got, tt.want) + } + }) + } } diff --git a/hcloudimages/doc_test.go b/hcloudimages/doc_test.go new file mode 100644 index 0000000..4905bdf --- /dev/null +++ b/hcloudimages/doc_test.go @@ -0,0 +1,33 @@ +package hcloudimages_test + +import ( + "context" + "fmt" + "net/url" + + "github.com/hetznercloud/hcloud-go/v2/hcloud" + + "github.com/apricote/hcloud-upload-image/hcloudimages" +) + +func ExampleClient_Upload() { + client := hcloudimages.NewClient( + hcloud.NewClient(hcloud.WithToken("")), + ) + + imageURL, err := url.Parse("https://example.com/disk-image.raw.bz2") + if err != nil { + panic(err) + } + + image, err := client.Upload(context.TODO(), hcloudimages.UploadOptions{ + ImageURL: imageURL, + ImageCompression: hcloudimages.CompressionBZ2, + Architecture: hcloud.ArchitectureX86, + }) + if err != nil { + panic(err) + } + + fmt.Printf("Uploaded Image: %d", image.ID) +} From fdfb284533d3154806b0936c08015fd5cc64b0fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Sat, 10 May 2025 14:21:31 +0200 Subject: [PATCH 10/14] feat: smaller snapshots by zeroing disk first (#101) The base image used requires ~0.42Gi. Even if the uploaded image is smaller, those bytes are currently not overwritten and still part of the stored snapshot. By zeroing the root disk first, those unwanted bytes are removed and not stored with the snapshot. This has two benefits: 1. Snapshots are billed by their compressed (shown) size, so small images are now a bit cheaper. 2. The time it takes to create a server from the snapshot scales with the snapshot size, so smaller snapshots means the server can start more quickly. This reduces the size of an example Talos x86 image from 0.42Gi before, to 0.2Gi afterwards. An example Flatcar image was 0.47Gi before, and still has that size with this patch. There are two ways to zero out the disk: - `dd if=/dev/zero of=/dev/sda` actually writes zeroes to every block on the device. This takes around a minute to do. - `blkdiscard /dev/sda` talks to the disk direclty and instructs it to discard all blocks. This only takes around 5 seconds. As both have the same effect on image size, but `blkdiscard` is SO MUCH faster, I have decided to use it. Even though only small images benefit from this, this is now enabled by default as the downside (5 second slower upload) does not justify additional flags or options to enable/disable this. Closes #96 --- hcloudimages/client.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/hcloudimages/client.go b/hcloudimages/client.go index fa51b55..9652854 100644 --- a/hcloudimages/client.go +++ b/hcloudimages/client.go @@ -329,8 +329,17 @@ func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Ima } defer func() { _ = sshClient.Close() }() - // 6. SSH On Server: Download Image, Decompress, Write to Root Disk - logger.InfoContext(ctx, "# Step 6: Downloading image and writing to disk") + // 6. Wipe existing disk, to avoid storing any bytes from it in the snapshot + logger.InfoContext(ctx, "# Step 6: Cleaning existing disk") + + output, err := sshsession.Run(sshClient, "blkdiscard /dev/sda", nil) + logger.DebugContext(ctx, string(output)) + if err != nil { + return nil, fmt.Errorf("failed to clean existing disk: %w", err) + } + + // 7. SSH On Server: Download Image, Decompress, Write to Root Disk + logger.InfoContext(ctx, "# Step 7: Downloading image and writing to disk") cmd, err := assembleCommand(options) if err != nil { @@ -339,23 +348,23 @@ func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Ima logger.DebugContext(ctx, "running download, decompress and write to disk command", "cmd", cmd) - output, err := sshsession.Run(sshClient, cmd, options.ImageReader) - logger.InfoContext(ctx, "# Step 6: Finished writing image to disk") + output, err = sshsession.Run(sshClient, cmd, options.ImageReader) + logger.InfoContext(ctx, "# Step 7: Finished writing image to disk") logger.DebugContext(ctx, string(output)) if err != nil { return nil, fmt.Errorf("failed to download and write the image: %w", err) } - // 7. SSH On Server: Shutdown - logger.InfoContext(ctx, "# Step 7: Shutting down server") + // 8. SSH On Server: Shutdown + logger.InfoContext(ctx, "# Step 8: Shutting down server") _, err = sshsession.Run(sshClient, "shutdown now", nil) if err != nil { // TODO Verify if shutdown error, otherwise return logger.WarnContext(ctx, "shutdown returned error", "err", err) } - // 8. Create Image from Server - logger.InfoContext(ctx, "# Step 8: Creating Image") + // 9. Create Image from Server + logger.InfoContext(ctx, "# Step 9: Creating Image") createImageResult, _, err := s.c.Server.CreateImage(ctx, server, &hcloud.ServerCreateImageOpts{ Type: hcloud.ImageTypeSnapshot, Description: options.Description, From 03f08da8a3a694a0a0ba95a1d10bb5731964ff48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Sat, 10 May 2025 14:24:05 +0200 Subject: [PATCH 11/14] chore(main): release hcloudimages 1.1.0 (#100) ## 1.1.0 (2025-05-10) ### Features * smaller snapshots by zeroing disk first (#101) (fdfb284) ### Bug Fixes * upload from local image generates broken command (#98) (420dcf9) --- .github/release-please-manifest.json | 2 +- hcloudimages/CHANGELOG.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/release-please-manifest.json b/.github/release-please-manifest.json index 5036936..18fc7b6 100644 --- a/.github/release-please-manifest.json +++ b/.github/release-please-manifest.json @@ -1 +1 @@ -{".":"1.0.1","hcloudimages":"1.0.1"} +{".":"1.0.1","hcloudimages":"1.1.0"} diff --git a/hcloudimages/CHANGELOG.md b/hcloudimages/CHANGELOG.md index 88d4c64..3d7b6a0 100644 --- a/hcloudimages/CHANGELOG.md +++ b/hcloudimages/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.1.0](https://github.com/apricote/hcloud-upload-image/compare/hcloudimages/v1.0.1...hcloudimages/v1.1.0) (2025-05-10) + + +### Features + +* smaller snapshots by zeroing disk first ([#101](https://github.com/apricote/hcloud-upload-image/issues/101)) ([fdfb284](https://github.com/apricote/hcloud-upload-image/commit/fdfb284533d3154806b0936c08015fd5cc64b0fb)), closes [#96](https://github.com/apricote/hcloud-upload-image/issues/96) + + +### Bug Fixes + +* upload from local image generates broken command ([#98](https://github.com/apricote/hcloud-upload-image/issues/98)) ([420dcf9](https://github.com/apricote/hcloud-upload-image/commit/420dcf94c965ee470602db6c9c23c777fda91222)), closes [#97](https://github.com/apricote/hcloud-upload-image/issues/97) + ## [1.0.1](https://github.com/apricote/hcloud-upload-image/compare/hcloudimages/v1.0.0...hcloudimages/v1.0.1) (2025-05-09) From 6f949a2ba773fe9f57c83e2e761dea67fe16e113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Sat, 10 May 2025 14:27:03 +0200 Subject: [PATCH 12/14] chore(deps): update module github.com/apricote/hcloud-upload-image/hcloudimages to v1.1.0 (#102) --- go.mod | 2 +- go.sum | 4 ++-- go.work.sum | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 40b6c5d..75c60bc 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.0 toolchain go1.24.3 require ( - github.com/apricote/hcloud-upload-image/hcloudimages v1.0.1 + github.com/apricote/hcloud-upload-image/hcloudimages v1.1.0 github.com/hetznercloud/hcloud-go/v2 v2.21.0 github.com/spf13/cobra v1.9.1 ) diff --git a/go.sum b/go.sum index 3ca98c7..6ca002d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/apricote/hcloud-upload-image/hcloudimages v1.0.1 h1:ZljuANGyGGxUF+afF6HFha88B9ln60gzVPnndDO/1OU= -github.com/apricote/hcloud-upload-image/hcloudimages v1.0.1/go.mod h1:iJ95BaLfISZBY9X8K2Y2A5a49dI0RLjAuq+4BqlOSgA= +github.com/apricote/hcloud-upload-image/hcloudimages v1.1.0 h1:1guW0IO2/070qbaP6zzNJJ8XsGLKPpxyE1W6fyf7MPc= +github.com/apricote/hcloud-upload-image/hcloudimages v1.1.0/go.mod h1:iJ95BaLfISZBY9X8K2Y2A5a49dI0RLjAuq+4BqlOSgA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/go.work.sum b/go.work.sum index dd30c3a..a13bce8 100644 --- a/go.work.sum +++ b/go.work.sum @@ -3,7 +3,7 @@ github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjH github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/apricote/hcloud-upload-image/hcloudimages v1.0.1/go.mod h1:iJ95BaLfISZBY9X8K2Y2A5a49dI0RLjAuq+4BqlOSgA= +github.com/apricote/hcloud-upload-image/hcloudimages v1.1.0/go.mod h1:iJ95BaLfISZBY9X8K2Y2A5a49dI0RLjAuq+4BqlOSgA= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= From b9af8855d5c0c1e40ddfaa670296731b2ed0c53e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Sat, 10 May 2025 14:27:47 +0200 Subject: [PATCH 13/14] chore(main): release 1.1.0 (#99) ## 1.1.0 (2025-05-10) ### Features * smaller snapshots by zeroing disk first (#101) (fdfb284) ### Bug Fixes * upload from local image generates broken command (#98) (420dcf9) --- .github/release-please-manifest.json | 2 +- CHANGELOG.md | 12 ++++++++++++ internal/version/version.go | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/release-please-manifest.json b/.github/release-please-manifest.json index 18fc7b6..2b6f241 100644 --- a/.github/release-please-manifest.json +++ b/.github/release-please-manifest.json @@ -1 +1 @@ -{".":"1.0.1","hcloudimages":"1.1.0"} +{".":"1.1.0","hcloudimages":"1.1.0"} diff --git a/CHANGELOG.md b/CHANGELOG.md index 42301dc..e0ca817 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.1.0](https://github.com/apricote/hcloud-upload-image/compare/v1.0.1...v1.1.0) (2025-05-10) + + +### Features + +* smaller snapshots by zeroing disk first ([#101](https://github.com/apricote/hcloud-upload-image/issues/101)) ([fdfb284](https://github.com/apricote/hcloud-upload-image/commit/fdfb284533d3154806b0936c08015fd5cc64b0fb)), closes [#96](https://github.com/apricote/hcloud-upload-image/issues/96) + + +### Bug Fixes + +* upload from local image generates broken command ([#98](https://github.com/apricote/hcloud-upload-image/issues/98)) ([420dcf9](https://github.com/apricote/hcloud-upload-image/commit/420dcf94c965ee470602db6c9c23c777fda91222)), closes [#97](https://github.com/apricote/hcloud-upload-image/issues/97) + ## [1.0.1](https://github.com/apricote/hcloud-upload-image/compare/v1.0.0...v1.0.1) (2025-05-09) diff --git a/internal/version/version.go b/internal/version/version.go index 14bc2dc..f8df129 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -2,7 +2,7 @@ package version var ( // version is a semver version (https://semver.org). - version = "1.0.1" // x-release-please-version + version = "1.1.0" // x-release-please-version // versionPrerelease is a semver version pre-release identifier (https://semver.org). // From 56bd8400100cc86bc58868b7574d806c4dde01f2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Jun 2025 19:23:17 +0000 Subject: [PATCH 14/14] chore(deps): update module golang.org/x/crypto to v0.39.0 --- hcloudimages/go.mod | 6 +++--- hcloudimages/go.sum | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/hcloudimages/go.mod b/hcloudimages/go.mod index bd5e39a..a1f6792 100644 --- a/hcloudimages/go.mod +++ b/hcloudimages/go.mod @@ -7,7 +7,7 @@ toolchain go1.24.3 require ( github.com/hetznercloud/hcloud-go/v2 v2.21.0 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.37.0 + golang.org/x/crypto v0.39.0 ) require ( @@ -22,8 +22,8 @@ require ( github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect google.golang.org/protobuf v1.36.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/hcloudimages/go.sum b/hcloudimages/go.sum index b69ed34..8540767 100644 --- a/hcloudimages/go.sum +++ b/hcloudimages/go.sum @@ -32,16 +32,16 @@ github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDN github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= -golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=