mirrors/hcloud-upload-image
1
0
Fork 0
mirror of https://github.com/apricote/hcloud-upload-image.git synced 2026-05-05 20:16:16 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: faster writing to disk by skipping zero blocks (#165)
Some checks failed
ci / lint (push) Has been cancelled
Details
ci / test (push) Has been cancelled
Details
ci / go-mod-tidy (push) Has been cancelled
Details
ci / cli-help-pages (push) Has been cancelled
Details
ci / nix-build (push) Has been cancelled
Details
docs / deploy (push) Has been cancelled
Details
release-please / release-please (push) Has been cancelled
Details

Browse Source 
By default `dd` writes all bytes from raw images 1:1 to the disk.

Some images, like Flatcar, have a lot of zero blocks in them. They have
a large partition table with 1GB for the UEFI partition, 2GB for system
A and B each, another 1GB for OEM and 6GB for the user. Most of these
are just zero blocks, but we currently still write them to disk.

By using `dd conv=sparse`, dd automatically skips writing these blocks
to the disk, which results in a quicker process, as fewer bytes need to
be written.

The resulting image is the same, as "zero" is also the default for the
blocks after our `blkdiscard` command.

I did not benchmark this properly, so you have to trust me on this one.
This commit is contained in:
Julian Tölle 2026-03-16 15:05:40 +01:00 committed by GitHub
parent 099d65e41e
commit fc871afa8d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 12 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hcloudimages/client.go
View File

@ -543,7 +543,10 @@ func assembleCommand(options UploadOptions) (string, error) {
switch options.ImageFormat {
case FormatRaw:
cmd += "dd of=/dev/sda bs=4M"
// With conv=sparse dd will skip any zero blocks and not write them to the disk, this makes it faster if you
// have a large raw image with multiple (nearly) empty but large partitions.
// For example Flatcar has ~12 GB, with ~90% being zero blocks.
cmd += "dd of=/dev/sda bs=4M conv=sparse"
case FormatQCOW2:
cmd += "tee image.qcow2 > /dev/null && qemu-img dd -f qcow2 -O raw if=image.qcow2 of=/dev/sda bs=4M"
default:

16
hcloudimages/client_test.go
View File

@ -24,21 +24,21 @@ func TestAssembleCommand(t *testing.T) {
{
name: "local raw",
options: UploadOptions{},
want: "bash -c 'set -euo pipefail && dd of=/dev/sda bs=4M && sync'",
want: "bash -c 'set -euo pipefail && dd of=/dev/sda bs=4M conv=sparse && sync'",
},
{
name: "remote raw",
options: UploadOptions{
ImageURL: mustParseURL("https://example.com/image.xz"),
},
want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.xz\" | dd of=/dev/sda bs=4M && sync'",
want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.xz\" | dd of=/dev/sda bs=4M conv=sparse && sync'",
},
{
name: "local xz",
options: UploadOptions{
ImageCompression: CompressionXZ,
},
want: "bash -c 'set -euo pipefail && xz -cd | dd of=/dev/sda bs=4M && sync'",
want: "bash -c 'set -euo pipefail && xz -cd | dd of=/dev/sda bs=4M conv=sparse && sync'",
},
{
name: "remote xz",
@ -46,14 +46,14 @@ func TestAssembleCommand(t *testing.T) {
ImageURL: mustParseURL("https://example.com/image.xz"),
ImageCompression: CompressionXZ,
},
want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.xz\" | xz -cd | dd of=/dev/sda bs=4M && sync'",
want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.xz\" | xz -cd | dd of=/dev/sda bs=4M conv=sparse && sync'",
},
{
name: "local zstd",
options: UploadOptions{
ImageCompression: CompressionZSTD,
},
want: "bash -c 'set -euo pipefail && zstd -cd | dd of=/dev/sda bs=4M && sync'",
want: "bash -c 'set -euo pipefail && zstd -cd | dd of=/dev/sda bs=4M conv=sparse && sync'",
},
{
name: "remote zstd",
@ -61,14 +61,14 @@ func TestAssembleCommand(t *testing.T) {
ImageURL: mustParseURL("https://example.com/image.zst"),
ImageCompression: CompressionZSTD,
},
want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.zst\" | zstd -cd | dd of=/dev/sda bs=4M && sync'",
want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.zst\" | zstd -cd | dd of=/dev/sda bs=4M conv=sparse && sync'",
},
{
name: "local bz2",
options: UploadOptions{
ImageCompression: CompressionBZ2,
},
want: "bash -c 'set -euo pipefail && bzip2 -cd | dd of=/dev/sda bs=4M && sync'",
want: "bash -c 'set -euo pipefail && bzip2 -cd | dd of=/dev/sda bs=4M conv=sparse && sync'",
},
{
name: "remote bz2",
@ -76,7 +76,7 @@ func TestAssembleCommand(t *testing.T) {
ImageURL: mustParseURL("https://example.com/image.bz2"),
ImageCompression: CompressionBZ2,
},
want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.bz2\" | bzip2 -cd | dd of=/dev/sda bs=4M && sync'",
want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.bz2\" | bzip2 -cd | dd of=/dev/sda bs=4M conv=sparse && sync'",
},
{
name: "local qcow2",