feat: initial library code

This commit is contained in:
Julian Tölle 2024-04-29 21:00:04 +02:00
parent b331ddba81
commit 4f57df5b66
10 changed files with 575 additions and 0 deletions

27
go.mod Normal file
View file

@ -0,0 +1,27 @@
module github.com/apricote/hcloud-upload-image
go 1.22.2
require (
github.com/hetznercloud/hcloud-go/v2 v2.7.1-0.20240418173523-ca1e95747811
github.com/stretchr/testify v1.9.0
golang.org/x/crypto v0.22.0
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/samber/lo v1.39.0 // indirect
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/protobuf v1.32.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

49
go.sum Normal file
View file

@ -0,0 +1,49 @@
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/hetznercloud/hcloud-go/v2 v2.7.1-0.20240418173523-ca1e95747811 h1:rmDCSIDubTNJDJNYzHVqcVNYp3RzPuA3gy6/qtkZAEo=
github.com/hetznercloud/hcloud-go/v2 v2.7.1-0.20240418173523-ca1e95747811/go.mod h1:58Ka180ZDH5wifuxRaoRsZXVlo4lK7R+vxTnAqWuoOs=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

55
interface.go Normal file
View file

@ -0,0 +1,55 @@
package hcloud_upload_image
import (
"context"
"net/url"
"github.com/hetznercloud/hcloud-go/v2/hcloud"
)
type SnapshotClient interface {
// Upload the specified image into a snapshot on Hetzner Cloud.
//
// As the Hetzner Cloud API has no direct way to upload images, we create a temporary server,
// overwrite the root disk and take a snapshot of that disk instead.
Upload(ctx context.Context, options UploadOptions) (*hcloud.Image, error)
// Possible future additions:
// List(ctx context.Context) []*hcloud.Image
// Delete(ctx context.Context, image *hcloud.Image) error
// CleanupTempResources(ctx context.Context) error
}
type UploadOptions struct {
// ImageURL must be publicly available. The instance will download the image from this endpoint.
ImageURL *url.URL
ImageCompression Compression
// ImageSignatureVerification
// Architecture should match the architecture of the Image. This decides if the Snapshot can later be
// used with [hcloud.ArchitectureX86] or [hcloud.ArchitectureARM] servers.
//
// Internally this decides what server type is used for the temporary server.
Architecture hcloud.Architecture
// Description is an optional description that the resulting image (snapshot) will have. There is no way to
// select images by its description, you should use Labels if you need to identify your image later.
Description *string
// Labels will be added to the resulting image (snapshot). Use these to filter the image list if you
// need to identify the image later on.
//
// We also always add a label `apricote.de/created-by=hcloud-image-upload` ([CreatedByLabel], [CreatedByValue]).
Labels map[string]string
// DebugSkipResourceCleanup will skip the cleanup of the temporary SSH Key and Server.
DebugSkipResourceCleanup bool
}
type Compression int
const (
CompressionNone Compression = iota
CompressionBZ2
// zip,xz,zstd
)

243
snapshot.go Normal file
View file

@ -0,0 +1,243 @@
package hcloud_upload_image
import (
"context"
"fmt"
"log"
"time"
"github.com/hetznercloud/hcloud-go/v2/hcloud"
"golang.org/x/crypto/ssh"
"github.com/apricote/hcloud-upload-image/util/control"
"github.com/apricote/hcloud-upload-image/util/randomid"
"github.com/apricote/hcloud-upload-image/util/sshkey"
"github.com/apricote/hcloud-upload-image/util/sshsession"
)
const (
CreatedByLabel = "apricote.de/created-by"
CreatedByValue = "hcloud-upload-image"
resourcePrefix = "hcloud-upload-image-"
)
var (
DefaultLabels = map[string]string{
CreatedByLabel: CreatedByValue,
}
serverTypePerArchitecture = map[hcloud.Architecture]*hcloud.ServerType{
hcloud.ArchitectureX86: {Name: "cx11"},
hcloud.ArchitectureARM: {Name: "cax11"},
}
defaultImage = &hcloud.Image{Name: "ubuntu-22.04"}
defaultLocation = &hcloud.Location{Name: "fsn1"}
defaultRescueType = hcloud.ServerRescueTypeLinux64
defaultSSHDialTimeout = 1 * time.Minute
)
func New(client *hcloud.Client) SnapshotClient {
return &snapshotClient{
client: client,
}
}
type snapshotClient struct {
client *hcloud.Client
}
func (s snapshotClient) Upload(ctx context.Context, options UploadOptions) (*hcloud.Image, error) {
id, err := randomid.Generate()
if err != nil {
return nil, err
}
// For simplicity, we use the name random name for SSH Key + Server
resourceName := resourcePrefix + id
// 1. Create SSH Key
log.Print("Step 1: Generating SSH Key")
publicKey, privateKey, err := sshkey.GenerateKeyPair()
if err != nil {
return nil, fmt.Errorf("failed to generate temporary ssh key pair: %w", err)
}
key, _, err := s.client.SSHKey.Create(ctx, hcloud.SSHKeyCreateOpts{
Name: resourceName,
PublicKey: string(publicKey),
Labels: fullLabels(options.Labels),
})
if err != nil {
return nil, fmt.Errorf("failed to submit temporary ssh key to API: %w", err)
}
defer func() {
// Cleanup SSH Key
if options.DebugSkipResourceCleanup {
return
}
_, err := s.client.SSHKey.Delete(ctx, key)
if err != nil {
// TODO
}
}()
// 2. Create Server
log.Print("Step 2: Creating Server")
serverType, ok := serverTypePerArchitecture[options.Architecture]
if !ok {
return nil, fmt.Errorf("unknown architecture %q, valid options: %q, %q", options.Architecture, hcloud.ArchitectureX86, hcloud.ArchitectureARM)
}
serverCreateResult, _, err := s.client.Server.Create(ctx, hcloud.ServerCreateOpts{
Name: resourceName,
ServerType: serverType,
// Not used, but without this the user receives an email with a password for every created server
SSHKeys: []*hcloud.SSHKey{key},
// We need to enable rescue system first
StartAfterCreate: hcloud.Ptr(false),
// Image will never be booted, we only boot into rescue system
Image: defaultImage,
Location: defaultLocation,
Labels: fullLabels(options.Labels),
})
if err != nil {
return nil, fmt.Errorf("creating the temporary server failed: %w", err)
}
_, err = s.client.Action.WaitForActions(ctx, append(serverCreateResult.NextActions, serverCreateResult.Action)...)
if err != nil {
return nil, fmt.Errorf("creating the temporary server failed: %w", err)
}
server := serverCreateResult.Server
defer func() {
// Cleanup Server
if options.DebugSkipResourceCleanup {
return
}
_, _, err := s.client.Server.DeleteWithResult(ctx, server)
if err != nil {
// TODO
}
}()
// 3. Activate Rescue System
log.Print("Step 3: Activating Rescue System")
enableRescueResult, _, err := s.client.Server.EnableRescue(ctx, server, hcloud.ServerEnableRescueOpts{
Type: defaultRescueType,
SSHKeys: []*hcloud.SSHKey{key},
})
if err != nil {
return nil, fmt.Errorf("enabling the rescue system on the temporary server failed: %w", err)
}
_, err = s.client.Action.WaitForAction(ctx, enableRescueResult.Action)
if err != nil {
return nil, fmt.Errorf("enabling the rescue system on the temporary server failed: %w", err)
}
// 4. Boot Server
log.Print("Step 4: Booting Server")
powerOnAction, _, err := s.client.Server.Poweron(ctx, server)
if err != nil {
return nil, fmt.Errorf("starting the temporary server failed: %w", err)
}
_, err = s.client.Action.WaitForAction(ctx, powerOnAction)
if err != nil {
return nil, fmt.Errorf("starting the temporary server failed: %w", err)
}
// 5. Open SSH Session
log.Print("Step 5: Opening SSH Connection")
signer, err := ssh.ParsePrivateKey(privateKey)
if err != nil {
return nil, fmt.Errorf("parsing the automatically generated temporary private key failed: %w", err)
}
sshClientConfig := &ssh.ClientConfig{
User: "root",
Auth: []ssh.AuthMethod{
ssh.PublicKeys(signer),
},
// There is no way to get the host key of the rescue system beforehand
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: defaultSSHDialTimeout,
}
// the server needs some time until its properly started and ssh is available
var sshClient *ssh.Client
err = control.Retry(ctx, 10, func() error {
var err error
sshClient, err = ssh.Dial("tcp", server.PublicNet.IPv4.IP.String()+":ssh", sshClientConfig)
return err
})
if err != nil {
return nil, fmt.Errorf("failed to ssh into temporary server: %w", err)
}
defer sshClient.Close()
// 6. SSH On Server: Download Image, Decompress, Write to Root Disk
log.Print("Step 6: Downloading image and writing to disk")
decompressionCommand := ""
if options.ImageCompression != CompressionNone {
switch options.ImageCompression {
case CompressionBZ2:
decompressionCommand += "| bzip2 -cd"
default:
return nil, fmt.Errorf("unknown compression: %q", options.ImageCompression)
}
}
output, err := sshsession.Run(sshClient, fmt.Sprintf("wget --no-verbose -O - %q %s | dd of=/dev/sda && sync", options.ImageURL.String(), decompressionCommand))
log.Print(string(output))
if err != nil {
return nil, fmt.Errorf("failed to download and write the image: %w", err)
}
// 7. SSH On Server: Shutdown
log.Print("Step 7: Shutting down server")
_, err = sshsession.Run(sshClient, "shutdown now")
if err != nil {
// TODO Verify if shutdown error, otherwise return
fmt.Printf("shutdown error: %+v", err)
}
// 8. Create Image from Server
log.Print("Step 7: Creating Image")
createImageResult, _, err := s.client.Server.CreateImage(ctx, server, &hcloud.ServerCreateImageOpts{
Type: hcloud.ImageTypeSnapshot,
Description: options.Description,
Labels: fullLabels(options.Labels),
})
if err != nil {
return nil, fmt.Errorf("failed to create snapshot: %w", err)
}
_, err = s.client.Action.WaitForAction(ctx, createImageResult.Action)
if err != nil {
return nil, fmt.Errorf("failed to create snapshot: %w", err)
}
image := createImageResult.Image
// Resource cleanup is happening in `defer`
return image, nil
}
func fullLabels(userLabels map[string]string) map[string]string {
if userLabels == nil {
userLabels = make(map[string]string, len(DefaultLabels))
}
for k, v := range DefaultLabels {
userLabels[k] = v
}
return userLabels
}

85
util/control/retry.go Normal file
View file

@ -0,0 +1,85 @@
package control
import (
"context"
"errors"
"math"
"time"
"github.com/hetznercloud/hcloud-go/v2/hcloud"
)
// From https://github.com/hetznercloud/terraform-provider-hcloud/blob/v1.46.1/internal/control/retry.go
// ExponentialBackoffWithLimit returns a [hcloud.BackoffFunc] which implements an exponential
// backoff.
// It uses the formula:
//
// min(b^retries * d, limit)
func ExponentialBackoffWithLimit(b float64, d time.Duration, limit time.Duration) hcloud.BackoffFunc {
return func(retries int) time.Duration {
current := time.Duration(math.Pow(b, float64(retries))) * d
if current > limit {
return limit
} else {
return current
}
}
}
// DefaultRetries is a constant for the maximum number of retries we usually do.
// However, callers of Retry are free to choose a different number.
const DefaultRetries = 5
type abortErr struct {
Err error
}
func (e abortErr) Error() string {
return e.Err.Error()
}
func (e abortErr) Unwrap() error {
return e.Err
}
// AbortRetry aborts any further attempts of retrying an operation.
//
// If err is passed Retry returns the passed error. If nil is passed, Retry
// returns nil.
func AbortRetry(err error) error {
if err == nil {
return nil
}
return abortErr{Err: err}
}
// Retry executes f at most maxTries times.
func Retry(ctx context.Context, maxTries int, f func() error) error {
var err error
backoff := ExponentialBackoffWithLimit(2, 1*time.Second, 30*time.Second)
for try := 0; try < maxTries; try++ {
if ctx.Err() != nil {
return ctx.Err()
}
var aerr abortErr
err = f()
if errors.As(err, &aerr) {
return aerr.Err
}
if err != nil {
sleep := backoff(try)
time.Sleep(sleep)
continue
}
return nil
}
return err
}

16
util/randomid/randomid.go Normal file
View file

@ -0,0 +1,16 @@
package randomid
import (
"crypto/rand"
"encoding/hex"
"fmt"
)
func Generate() (string, error) {
b := make([]byte, 4)
_, err := rand.Read(b)
if err != nil {
return "", fmt.Errorf("failed to generate random string: %w", err)
}
return hex.EncodeToString(b), nil
}

View file

@ -0,0 +1,18 @@
package randomid
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGenerateRandomID(t *testing.T) {
found1, err := Generate()
assert.NoError(t, err)
found2, err := Generate()
assert.NoError(t, err)
assert.Len(t, found1, 8)
assert.Len(t, found2, 8)
assert.NotEqual(t, found1, found2)
}

45
util/sshkey/ssh_key.go Normal file
View file

@ -0,0 +1,45 @@
package sshkey
import (
"crypto/ed25519"
"encoding/pem"
"golang.org/x/crypto/ssh"
)
func GenerateKeyPair() ([]byte, []byte, error) {
pub, priv, err := ed25519.GenerateKey(nil)
if err != nil {
return nil, nil, err
}
pubBytes, err := encodePublicKey(pub)
if err != nil {
return nil, nil, err
}
privBytes, err := encodePrivateKey(priv)
if err != nil {
return nil, nil, err
}
return pubBytes, privBytes, nil
}
func encodePublicKey(pub ed25519.PublicKey) ([]byte, error) {
sshPub, err := ssh.NewPublicKey(pub)
if err != nil {
return nil, err
}
return ssh.MarshalAuthorizedKey(sshPub), nil
}
func encodePrivateKey(priv ed25519.PrivateKey) ([]byte, error) {
privPem, err := ssh.MarshalPrivateKey(priv, "")
if err != nil {
return nil, err
}
return pem.EncodeToMemory(privPem), nil
}

View file

@ -0,0 +1,25 @@
package sshkey
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGenerateSSHKeyPair(t *testing.T) {
pubBytes, privBytes, err := GenerateKeyPair()
assert.Nil(t, err)
pub := string(pubBytes)
priv := string(privBytes)
if !(strings.HasPrefix(priv, "-----BEGIN OPENSSH PRIVATE KEY-----\n") &&
strings.HasSuffix(priv, "-----END OPENSSH PRIVATE KEY-----\n")) {
assert.Fail(t, "private key is invalid", priv)
}
if !strings.HasPrefix(pub, "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA") {
assert.Fail(t, "public key is invalid", pub)
}
}

View file

@ -0,0 +1,12 @@
package sshsession
import "golang.org/x/crypto/ssh"
func Run(client *ssh.Client, cmd string) ([]byte, error) {
sess, err := client.NewSession()
if err != nil {
return nil, err
}
defer sess.Close()
return sess.CombinedOutput(cmd)
}