diff --git a/cmd/cleanup.go b/cmd/cleanup.go index b11b085..a9b198b 100644 --- a/cmd/cleanup.go +++ b/cmd/cleanup.go @@ -1,40 +1,43 @@ -/* -Copyright © 2024 NAME HERE - -*/ package cmd import ( "fmt" "github.com/spf13/cobra" + + "github.com/apricote/hcloud-upload-image/hcloudimages/contextlogger" ) // cleanupCmd represents the cleanup command var cleanupCmd = &cobra.Command{ Use: "cleanup", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: + Short: "Remove any temporary resources that were left over", + Long: `If the upload fails at any point, there might still exist a server or +ssh key in your Hetzner Cloud project. This command cleans up any resources +that match the label "apricote.de/created-by=hcloud-upload-image". -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("cleanup called") +If you want to see a preview of what would be removed, you can use the official hcloud CLI and run: + +$ hcloud server list -l apricote.de/created-by=hcloud-upload-image +$ hcloud ssh-key list -l apricote.de/created-by=hcloud-upload-image + +This command does not handle any parallel executions of hcloud-upload-image +and will remove in-use resources if called at the same time.`, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + logger := contextlogger.From(ctx) + + err := client.CleanupTempResources(ctx) + if err != nil { + return fmt.Errorf("failed to clean up temporary resources: %w", err) + } + + logger.InfoContext(ctx, "Successfully cleaned up all temporary resources!") + + return nil }, } func init() { rootCmd.AddCommand(cleanupCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // cleanupCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // cleanupCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } diff --git a/cmd/list.go b/cmd/list.go deleted file mode 100644 index b8e09c2..0000000 --- a/cmd/list.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright © 2024 NAME HERE - -*/ -package cmd - -import ( - "fmt" - - "github.com/spf13/cobra" -) - -// listCmd represents the list command -var listCmd = &cobra.Command{ - Use: "list", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("list called") - }, -} - -func init() { - rootCmd.AddCommand(listCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // listCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // listCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} diff --git a/cmd/root.go b/cmd/root.go index bb05b4e..feacb5c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "fmt" "log/slog" "os" "time" @@ -13,10 +12,21 @@ import ( "github.com/apricote/hcloud-upload-image/hcloudimages" "github.com/apricote/hcloud-upload-image/hcloudimages/backoff" "github.com/apricote/hcloud-upload-image/hcloudimages/contextlogger" + "github.com/apricote/hcloud-upload-image/internal/ui" +) + +const ( + flagVerbose = "verbose" +) + +var ( + // 1 activates slog debug output + // 2 activates hcloud-go debug output + verbose int ) // The pre-authenticated client. Set in the root command PersistentPreRun -var client hcloudimages.Client +var client *hcloudimages.Client // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ @@ -27,6 +37,14 @@ var rootCmd = &cobra.Command{ PersistentPreRun: func(cmd *cobra.Command, _ []string) { ctx := cmd.Context() + logLevel := slog.LevelInfo + if verbose >= 1 { + logLevel = slog.LevelDebug + } + slog.SetDefault(slog.New(ui.NewHandler(os.Stdout, &ui.HandlerOptions{ + Level: logLevel, + }))) + // Add logger to command context logger := slog.Default() ctx = contextlogger.New(ctx, logger) @@ -36,7 +54,7 @@ var rootCmd = &cobra.Command{ }, } -func newClient(ctx context.Context) hcloudimages.Client { +func newClient(ctx context.Context) *hcloudimages.Client { logger := contextlogger.From(ctx) // Build hcloud-go client if os.Getenv("HCLOUD_TOKEN") == "" { @@ -50,7 +68,7 @@ func newClient(ctx context.Context) hcloudimages.Client { hcloud.WithPollBackoffFunc(backoff.ExponentialBackoffWithLimit(2, 1*time.Second, 30*time.Second)), } - if os.Getenv("HCLOUD_DEBUG") != "" { + if os.Getenv("HCLOUD_DEBUG") != "" || verbose >= 2 { opts = append(opts, hcloud.WithDebugWriter(os.Stderr)) } @@ -66,7 +84,6 @@ func Execute() { func init() { rootCmd.SetErrPrefix("\033[1;31mError:") - rootCmd.SetFlagErrorFunc(func(command *cobra.Command, err error) error { - return fmt.Errorf("fooo") - }) + + rootCmd.PersistentFlags().CountVarP(&verbose, flagVerbose, "v", "verbose debug output, can be specified up to 2 times") } diff --git a/cmd/upload.go b/cmd/upload.go index 629f77d..6692dc8 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -7,7 +7,7 @@ import ( "github.com/hetznercloud/hcloud-go/v2/hcloud" "github.com/spf13/cobra" - hcloud_upload_image "github.com/apricote/hcloud-upload-image/hcloudimages" + "github.com/apricote/hcloud-upload-image/hcloudimages" "github.com/apricote/hcloud-upload-image/hcloudimages/contextlogger" ) @@ -41,9 +41,9 @@ This does cost a bit of money for the server.`, return fmt.Errorf("unable to parse url from --%s=%q: %w", uploadFlagImageURL, imageURLString, err) } - image, err := client.Upload(ctx, hcloud_upload_image.UploadOptions{ + image, err := client.Upload(ctx, hcloudimages.UploadOptions{ ImageURL: imageURL, - ImageCompression: hcloud_upload_image.Compression(imageCompression), + ImageCompression: hcloudimages.Compression(imageCompression), Architecture: hcloud.Architecture(architecture), Description: hcloud.Ptr(description), Labels: labels, @@ -67,7 +67,7 @@ func init() { uploadCmd.Flags().String(uploadFlagCompression, "", "Type of compression that was used on the disk image") _ = uploadCmd.RegisterFlagCompletionFunc( uploadFlagCompression, - cobra.FixedCompletions([]string{string(hcloud_upload_image.CompressionBZ2)}, cobra.ShellCompDirectiveNoFileComp), + cobra.FixedCompletions([]string{string(hcloudimages.CompressionBZ2)}, cobra.ShellCompDirectiveNoFileComp), ) uploadCmd.Flags().String(uploadFlagArchitecture, "", "CPU Architecture of the disk image. Choices: x86|arm") diff --git a/go.mod b/go.mod index 32731c0..e0d85e8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.2 require ( github.com/apricote/hcloud-upload-image/hcloudimages v0.0.0 - github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240430130644-7bb1a7b9ae5f + github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240503164107-1e3fa7033d8a github.com/spf13/cobra v1.8.0 ) diff --git a/go.sum b/go.sum index 839a159..0228b49 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240430130644-7bb1a7b9ae5f h1:c1ahn6OKXkSqwOfCoqyFrjVh14BEC9rD3ok0dehbCno= -github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240430130644-7bb1a7b9ae5f/go.mod h1:jvpP3qAWMIZ3WQwQLYa97ia6t98iPCgsJNwRts+Jnrk= +github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240503164107-1e3fa7033d8a h1:4L8VwfLGtlSBNPnsLINAqOEDde+vXi3AvZpTVtv+vs0= +github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240503164107-1e3fa7033d8a/go.mod h1:jvpP3qAWMIZ3WQwQLYa97ia6t98iPCgsJNwRts+Jnrk= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/go.work.sum b/go.work.sum index dc1f0c7..3b3d09d 100644 --- a/go.work.sum +++ b/go.work.sum @@ -3,8 +3,24 @@ github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HR github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 h1:YI1gOOdmMk3xodBao7fehcvoZsEeOyy/cfhlpCSPgM4= +github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= +github.com/dave/brenda v1.1.0 h1:Sl1LlwXnbw7xMhq3y2x11McFu43AjDcwkllxxgZ3EZw= +github.com/dave/brenda v1.1.0/go.mod h1:4wCUr6gSlu5/1Tk7akE5X7UorwiQ8Rij0SKH3/BGMOM= +github.com/dave/courtney v0.3.0 h1:8aR1os2ImdIQf3Zj4oro+lD/L4Srb5VwGefqZ/jzz7U= +github.com/dave/courtney v0.3.0/go.mod h1:BAv3hA06AYfNUjfjQr+5gc6vxeBVOupLqrColj+QSD8= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e h1:l99YKCdrK4Lvb/zTupt0GMPfNbncAGf8Cv/t1sYLOg0= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= github.com/dave/jennifer v1.6.0 h1:MQ/6emI2xM7wt0tJzJzyUik2Q3Tcn2eE0vtYgh4GPVI= github.com/dave/jennifer v1.6.0/go.mod h1:AxTG893FiZKqxy3FP1kL80VMshSMuz2G+EgvszgGRnk= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e h1:xURkGi4RydhyaYR6PzcyHTueQudxY4LgxN1oYEPJHa0= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= +github.com/dave/patsy v0.0.0-20210517141501-957256f50cba h1:1o36L4EKbZzazMk8iGC4kXpVnZ6TPxR2mZ9qVKjNNAs= +github.com/dave/patsy v0.0.0-20210517141501-957256f50cba/go.mod h1:qfR88CgEGLoiqDaE+xxDCi5QA5v4vUoW0UCX2Nd5Tlc= +github.com/dave/rebecca v0.9.1 h1:jxVfdOxRirbXL28vXMvUvJ1in3djwkVKXCq339qhBL0= +github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= @@ -34,16 +50,22 @@ github.com/vburenin/ifacemaker v1.2.1 h1:3Vq8B/bfBgjWTkv+jDg4dVL1KHt3k1K4lO7XRxY github.com/vburenin/ifacemaker v1.2.1/go.mod h1:5WqrzX2aD7/hi+okBjcaEQJMg4lDGrpuEX3B8L4Wgrs= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/hcloudimages/snapshot.go b/hcloudimages/client.go similarity index 57% rename from hcloudimages/snapshot.go rename to hcloudimages/client.go index 24a6d12..bf220cc 100644 --- a/hcloudimages/snapshot.go +++ b/hcloudimages/client.go @@ -2,14 +2,19 @@ package hcloudimages import ( "context" + "errors" "fmt" + "log/slog" + "net/url" "time" "github.com/hetznercloud/hcloud-go/v2/hcloud" "golang.org/x/crypto/ssh" "github.com/apricote/hcloud-upload-image/hcloudimages/contextlogger" + "github.com/apricote/hcloud-upload-image/hcloudimages/internal/actionutil" "github.com/apricote/hcloud-upload-image/hcloudimages/internal/control" + "github.com/apricote/hcloud-upload-image/hcloudimages/internal/labelutil" "github.com/apricote/hcloud-upload-image/hcloudimages/internal/randomid" "github.com/apricote/hcloud-upload-image/hcloudimages/internal/sshkey" "github.com/apricote/hcloud-upload-image/hcloudimages/internal/sshsession" @@ -39,19 +44,69 @@ var ( defaultSSHDialTimeout = 1 * time.Minute ) -func NewClient(c *hcloud.Client) Client { - return &client{ +type UploadOptions struct { + // ImageURL must be publicly available. The instance will download the image from this endpoint. + ImageURL *url.URL + // ImageCompression describes the compression of the referenced image file. It defaults to [CompressionNone]. If + // set to anything else, the file will be decompressed before written to the disk. + ImageCompression Compression + + // Possible future additions: + // ImageSignatureVerification + // ImageLocalPath + // ImageType (RawDiskImage, ISO, qcow2, ...) + + // Architecture should match the architecture of the Image. This decides if the Snapshot can later be + // used with [hcloud.ArchitectureX86] or [hcloud.ArchitectureARM] servers. + // + // Internally this decides what server type is used for the temporary server. + Architecture hcloud.Architecture + + // Description is an optional description that the resulting image (snapshot) will have. There is no way to + // select images by its description, you should use Labels if you need to identify your image later. + Description *string + + // Labels will be added to the resulting image (snapshot). Use these to filter the image list if you + // need to identify the image later on. + // + // We also always add a label `apricote.de/created-by=hcloud-image-upload` ([CreatedByLabel], [CreatedByValue]). + Labels map[string]string + + // DebugSkipResourceCleanup will skip the cleanup of the temporary SSH Key and Server. + DebugSkipResourceCleanup bool +} + +type Compression string + +const ( + CompressionNone Compression = "" + CompressionBZ2 Compression = "bz2" + + // Possible future additions: + // zip,xz,zstd +) + +// NewClient instantiates a new client. It requires a working [*hcloud.Client] to interact with the Hetzner Cloud API. +func NewClient(c *hcloud.Client) *Client { + return &Client{ c: c, } } -type client struct { +type Client struct { c *hcloud.Client } -func (s client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Image, error) { +// Upload the specified image into a snapshot on Hetzner Cloud. +// +// As the Hetzner Cloud API has no direct way to upload images, we create a temporary server, +// overwrite the root disk and take a snapshot of that disk instead. +// +// The temporary server costs money. If the upload fails, we might be unable to delete the server. Check out +// CleanupTempResources for a helper in this case. +func (s *Client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Image, error) { logger := contextlogger.From(ctx).With( - "library", "hcloud-upload-image", + "library", "hcloudimages", "method", "upload", ) @@ -62,6 +117,7 @@ func (s client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Imag logger = logger.With("run-id", id) // For simplicity, we use the name random name for SSH Key + Server resourceName := resourcePrefix + id + labels := labelutil.Merge(DefaultLabels, options.Labels) // 1. Create SSH Key logger.InfoContext(ctx, "# Step 1: Generating SSH Key") @@ -73,7 +129,7 @@ func (s client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Imag key, _, err := s.c.SSHKey.Create(ctx, hcloud.SSHKeyCreateOpts{ Name: resourceName, PublicKey: string(publicKey), - Labels: fullLabels(options.Labels), + Labels: labels, }) if err != nil { return nil, fmt.Errorf("failed to submit temporary ssh key to API: %w", err) @@ -119,7 +175,7 @@ func (s client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Imag // Image will never be booted, we only boot into rescue system Image: defaultImage, Location: defaultLocation, - Labels: fullLabels(options.Labels), + Labels: labels, }) if err != nil { return nil, fmt.Errorf("creating the temporary server failed: %w", err) @@ -253,7 +309,7 @@ func (s client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Imag createImageResult, _, err := s.c.Server.CreateImage(ctx, server, &hcloud.ServerCreateImageOpts{ Type: hcloud.ImageTypeSnapshot, Description: options.Description, - Labels: fullLabels(options.Labels), + Labels: labels, }) if err != nil { return nil, fmt.Errorf("failed to create snapshot: %w", err) @@ -273,13 +329,124 @@ func (s client) Upload(ctx context.Context, options UploadOptions) (*hcloud.Imag return image, nil } -func fullLabels(userLabels map[string]string) map[string]string { - if userLabels == nil { - userLabels = make(map[string]string, len(DefaultLabels)) +// CleanupTempResources tries to delete any resources that were left over from previous calls to [Client.Upload]. +// Upload tries to clean up any temporary resources it created at runtime, but might fail at any point. +// You can then use this command to make sure that all temporary resources are removed from your project. +// +// This method tries to delete any server or ssh keys that match the [DefaultLabels] +func (s *Client) CleanupTempResources(ctx context.Context) error { + logger := contextlogger.From(ctx).With( + "library", "hcloudimages", + "method", "cleanup", + ) + + selector := labelutil.Selector(DefaultLabels) + logger = logger.With("selector", selector) + + logger.InfoContext(ctx, "# Cleaning up Servers") + err := s.cleanupTempServers(ctx, logger, selector) + if err != nil { + return fmt.Errorf("failed to clean up all servers: %w", err) } - for k, v := range DefaultLabels { - userLabels[k] = v + logger.DebugContext(ctx, "cleaned up all servers") + + logger.InfoContext(ctx, "# Cleaning up SSH Keys") + err = s.cleanupTempSSHKeys(ctx, logger, selector) + if err != nil { + return fmt.Errorf("failed to clean up all ssh keys: %w", err) + } + logger.DebugContext(ctx, "cleaned up all ssh keys") + + return nil +} + +func (s *Client) cleanupTempServers(ctx context.Context, logger *slog.Logger, selector string) error { + servers, err := s.c.Server.AllWithOpts(ctx, hcloud.ServerListOpts{ListOpts: hcloud.ListOpts{ + LabelSelector: selector, + }}) + if err != nil { + return fmt.Errorf("failed to list servers: %w", err) } - return userLabels + if len(servers) == 0 { + logger.InfoContext(ctx, "No servers found") + return nil + } + logger.InfoContext(ctx, "removing servers", "count", len(servers)) + + errs := []error{} + actions := make([]*hcloud.Action, 0, len(servers)) + + for _, server := range servers { + result, _, err := s.c.Server.DeleteWithResult(ctx, server) + if err != nil { + errs = append(errs, err) + logger.WarnContext(ctx, "failed to delete server", "server", server.ID, "error", err) + continue + } + + actions = append(actions, result.Action) + } + + successActions, errorActions, err := actionutil.Settle(ctx, &s.c.Action, actions...) + if err != nil { + return fmt.Errorf("failed to wait for server delete: %w", err) + } + + if len(successActions) > 0 { + ids := make([]int64, 0, len(successActions)) + for _, action := range successActions { + for _, resource := range action.Resources { + if resource.Type == hcloud.ActionResourceTypeServer { + ids = append(ids, resource.ID) + } + } + } + + logger.InfoContext(ctx, "successfully deleted servers", "servers", ids) + } + + if len(errorActions) > 0 { + for _, action := range errorActions { + errs = append(errs, action.Error()) + } + } + + if len(errs) > 0 { + // The returned message contains no info about the server IDs which failed + return fmt.Errorf("failed to delete some of the servers: %w", errors.Join(errs...)) + } + + return nil +} + +func (s *Client) cleanupTempSSHKeys(ctx context.Context, logger *slog.Logger, selector string) error { + keys, _, err := s.c.SSHKey.List(ctx, hcloud.SSHKeyListOpts{ListOpts: hcloud.ListOpts{ + LabelSelector: selector, + }}) + if err != nil { + return fmt.Errorf("failed to list keys: %w", err) + } + + if len(keys) == 0 { + logger.InfoContext(ctx, "No ssh keys found") + return nil + } + + errs := []error{} + for _, key := range keys { + _, err := s.c.SSHKey.Delete(ctx, key) + if err != nil { + errs = append(errs, err) + logger.WarnContext(ctx, "failed to delete ssh key", "ssh-key", key.ID, "error", err) + continue + } + } + + if len(errs) > 0 { + // The returned message contains no info about the server IDs which failed + return fmt.Errorf("failed to delete some of the ssh keys: %w", errors.Join(errs...)) + } + + return nil } diff --git a/hcloudimages/client_test.go b/hcloudimages/client_test.go new file mode 100644 index 0000000..4905bdf --- /dev/null +++ b/hcloudimages/client_test.go @@ -0,0 +1,33 @@ +package hcloudimages_test + +import ( + "context" + "fmt" + "net/url" + + "github.com/hetznercloud/hcloud-go/v2/hcloud" + + "github.com/apricote/hcloud-upload-image/hcloudimages" +) + +func ExampleClient_Upload() { + client := hcloudimages.NewClient( + hcloud.NewClient(hcloud.WithToken("")), + ) + + imageURL, err := url.Parse("https://example.com/disk-image.raw.bz2") + if err != nil { + panic(err) + } + + image, err := client.Upload(context.TODO(), hcloudimages.UploadOptions{ + ImageURL: imageURL, + ImageCompression: hcloudimages.CompressionBZ2, + Architecture: hcloud.ArchitectureX86, + }) + if err != nil { + panic(err) + } + + fmt.Printf("Uploaded Image: %d", image.ID) +} diff --git a/hcloudimages/contextlogger/context.go b/hcloudimages/contextlogger/context.go index 811e29a..708478a 100644 --- a/hcloudimages/contextlogger/context.go +++ b/hcloudimages/contextlogger/context.go @@ -9,10 +9,13 @@ type key int var loggerKey key +// New saves the logger as a value to the context. This can then be retrieved through [From]. func New(ctx context.Context, logger *slog.Logger) context.Context { return context.WithValue(ctx, loggerKey, logger) } +// From returns the [*slog.Logger] set on the context by [New]. If there is none, +// it returns a no-op logger that discards any output it receives. func From(ctx context.Context) *slog.Logger { if ctxLogger := ctx.Value(loggerKey); ctxLogger != nil { if logger, ok := ctxLogger.(*slog.Logger); ok { diff --git a/hcloudimages/doc.go b/hcloudimages/doc.go new file mode 100644 index 0000000..786ba74 --- /dev/null +++ b/hcloudimages/doc.go @@ -0,0 +1,42 @@ +// Package hcloudimages is a library to upload Disk Images into your Hetzner Cloud project. +// +// # Overview +// +// The Hetzner Cloud API does not support uploading disk images directly, and it only provides a limited set of default +// images. The only option for custom disk images that users have is by taking a "snapshot" of an existing servers root +// disk. These can then be used to create new servers. +// +// To create a completely custom disk image, users have to follow these steps: +// +// 1. Create server with the correct server type +// 2. Enable rescue system for the server +// 3. Boot the server +// 4. Download the disk image from within the rescue system +// 5. Write disk image to servers root disk +// 6. Shut down the server +// 7. Take a snapshot of the servers root disk +// 8. Delete the server +// +// This is an annoyingly long process. Many users have automated this with Packer before, but Packer offers a lot of +// additional complexity to understand. +// +// This library is a single call to do the above: [Client.Upload] +// +// # Costs +// +// The temporary server and the snapshot itself cost money. See the [Hetzner Cloud website] for up-to-date pricing +// information. +// +// Usually the upload takes no more than a few minutes of server time, so you will only be billed for the first hour +// (<1ct for most cases). If this process fails, the server might stay around until you manually delete it. In that case +// it continues to cost its hourly price. There is a utility [Client.CleanupTemporaryResources] that removes any +// leftover resources. +// +// # Logging +// +// By default, nothing is logged. As the update process takes a bit of time you might want to gain some insight into +// the process. For this we provide optional logs through [log/slog]. You can set a [log/slog.Logger] in the +// [context.Context] through [github.com/apricote/hcloud-upload-image/hcloudimages/contextlogger.New]. +// +// [Hetzner Cloud website]: https://www.hetzner.com/cloud/ +package hcloudimages diff --git a/hcloudimages/go.mod b/hcloudimages/go.mod index 4a8dbb4..6630fff 100644 --- a/hcloudimages/go.mod +++ b/hcloudimages/go.mod @@ -3,7 +3,7 @@ module github.com/apricote/hcloud-upload-image/hcloudimages go 1.22.2 require ( - github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240430130644-7bb1a7b9ae5f + github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240503164107-1e3fa7033d8a github.com/stretchr/testify v1.9.0 golang.org/x/crypto v0.22.0 ) diff --git a/hcloudimages/go.sum b/hcloudimages/go.sum index 938dc0d..30ef313 100644 --- a/hcloudimages/go.sum +++ b/hcloudimages/go.sum @@ -6,8 +6,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240430130644-7bb1a7b9ae5f h1:c1ahn6OKXkSqwOfCoqyFrjVh14BEC9rD3ok0dehbCno= -github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240430130644-7bb1a7b9ae5f/go.mod h1:jvpP3qAWMIZ3WQwQLYa97ia6t98iPCgsJNwRts+Jnrk= +github.com/hetznercloud/hcloud-go/v2 v2.7.3-0.20240503164107-1e3fa7033d8a h1:4L8VwfLGtlSBNPnsLINAqOEDde+vXi3AvZpTVtv+vs0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/hcloudimages/interface.go b/hcloudimages/interface.go deleted file mode 100644 index 60bfe0d..0000000 --- a/hcloudimages/interface.go +++ /dev/null @@ -1,55 +0,0 @@ -package hcloudimages - -import ( - "context" - "net/url" - - "github.com/hetznercloud/hcloud-go/v2/hcloud" -) - -type Client interface { - // Upload the specified image into a snapshot on Hetzner Cloud. - // - // As the Hetzner Cloud API has no direct way to upload images, we create a temporary server, - // overwrite the root disk and take a snapshot of that disk instead. - Upload(ctx context.Context, options UploadOptions) (*hcloud.Image, error) - - // Possible future additions: - // List(ctx context.Context) []*hcloud.Image - // Delete(ctx context.Context, image *hcloud.Image) error - // CleanupTempResources(ctx context.Context) error -} - -type UploadOptions struct { - // ImageURL must be publicly available. The instance will download the image from this endpoint. - ImageURL *url.URL - ImageCompression Compression - // ImageSignatureVerification - - // Architecture should match the architecture of the Image. This decides if the Snapshot can later be - // used with [hcloud.ArchitectureX86] or [hcloud.ArchitectureARM] servers. - // - // Internally this decides what server type is used for the temporary server. - Architecture hcloud.Architecture - - // Description is an optional description that the resulting image (snapshot) will have. There is no way to - // select images by its description, you should use Labels if you need to identify your image later. - Description *string - - // Labels will be added to the resulting image (snapshot). Use these to filter the image list if you - // need to identify the image later on. - // - // We also always add a label `apricote.de/created-by=hcloud-image-upload` ([CreatedByLabel], [CreatedByValue]). - Labels map[string]string - - // DebugSkipResourceCleanup will skip the cleanup of the temporary SSH Key and Server. - DebugSkipResourceCleanup bool -} - -type Compression string - -const ( - CompressionNone Compression = "" - CompressionBZ2 Compression = "bz2" - // zip,xz,zstd -) diff --git a/hcloudimages/internal/actionutil/action.go b/hcloudimages/internal/actionutil/action.go new file mode 100644 index 0000000..855c3ec --- /dev/null +++ b/hcloudimages/internal/actionutil/action.go @@ -0,0 +1,25 @@ +package actionutil + +import ( + "context" + + "github.com/hetznercloud/hcloud-go/v2/hcloud" +) + +func Settle(ctx context.Context, client hcloud.IActionClient, actions ...*hcloud.Action) (successActions []*hcloud.Action, errorActions []*hcloud.Action, err error) { + err = client.WaitForFunc(ctx, func(update *hcloud.Action) error { + switch update.Status { + case hcloud.ActionStatusSuccess: + successActions = append(successActions, update) + case hcloud.ActionStatusError: + errorActions = append(errorActions, update) + } + + return nil + }, actions...) + if err != nil { + return nil, nil, err + } + + return successActions, errorActions, nil +} diff --git a/hcloudimages/internal/labelutil/labels.go b/hcloudimages/internal/labelutil/labels.go new file mode 100644 index 0000000..28e0eb9 --- /dev/null +++ b/hcloudimages/internal/labelutil/labels.go @@ -0,0 +1,30 @@ +package labelutil + +import "fmt" + +func Merge(a, b map[string]string) map[string]string { + result := make(map[string]string, len(a)+len(b)) + + for k, v := range a { + result[k] = v + } + for k, v := range b { + result[k] = v + } + + return result +} + +func Selector(labels map[string]string) string { + selector := make([]byte, 0, 64) + separator := "" + + for k, v := range labels { + selector = fmt.Appendf(selector, "%s%s=%s", separator, k, v) + + // Do not print separator on first element + separator = "," + } + + return string(selector) +}