From fc871afa8d9a5f468572cda0911537ecbf79fd58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Mon, 16 Mar 2026 15:05:40 +0100 Subject: [PATCH] feat: faster writing to disk by skipping zero blocks (#165) By default `dd` writes all bytes from raw images 1:1 to the disk. Some images, like Flatcar, have a lot of zero blocks in them. They have a large partition table with 1GB for the UEFI partition, 2GB for system A and B each, another 1GB for OEM and 6GB for the user. Most of these are just zero blocks, but we currently still write them to disk. By using `dd conv=sparse`, dd automatically skips writing these blocks to the disk, which results in a quicker process, as fewer bytes need to be written. The resulting image is the same, as "zero" is also the default for the blocks after our `blkdiscard` command. I did not benchmark this properly, so you have to trust me on this one. --- hcloudimages/client.go | 5 ++++- hcloudimages/client_test.go | 16 ++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/hcloudimages/client.go b/hcloudimages/client.go index 6311445..2b9559d 100644 --- a/hcloudimages/client.go +++ b/hcloudimages/client.go @@ -543,7 +543,10 @@ func assembleCommand(options UploadOptions) (string, error) { switch options.ImageFormat { case FormatRaw: - cmd += "dd of=/dev/sda bs=4M" + // With conv=sparse dd will skip any zero blocks and not write them to the disk, this makes it faster if you + // have a large raw image with multiple (nearly) empty but large partitions. + // For example Flatcar has ~12 GB, with ~90% being zero blocks. + cmd += "dd of=/dev/sda bs=4M conv=sparse" case FormatQCOW2: cmd += "tee image.qcow2 > /dev/null && qemu-img dd -f qcow2 -O raw if=image.qcow2 of=/dev/sda bs=4M" default: diff --git a/hcloudimages/client_test.go b/hcloudimages/client_test.go index d1d1d33..a9aead8 100644 --- a/hcloudimages/client_test.go +++ b/hcloudimages/client_test.go @@ -24,21 +24,21 @@ func TestAssembleCommand(t *testing.T) { { name: "local raw", options: UploadOptions{}, - want: "bash -c 'set -euo pipefail && dd of=/dev/sda bs=4M && sync'", + want: "bash -c 'set -euo pipefail && dd of=/dev/sda bs=4M conv=sparse && sync'", }, { name: "remote raw", options: UploadOptions{ ImageURL: mustParseURL("https://example.com/image.xz"), }, - want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.xz\" | dd of=/dev/sda bs=4M && sync'", + want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.xz\" | dd of=/dev/sda bs=4M conv=sparse && sync'", }, { name: "local xz", options: UploadOptions{ ImageCompression: CompressionXZ, }, - want: "bash -c 'set -euo pipefail && xz -cd | dd of=/dev/sda bs=4M && sync'", + want: "bash -c 'set -euo pipefail && xz -cd | dd of=/dev/sda bs=4M conv=sparse && sync'", }, { name: "remote xz", @@ -46,14 +46,14 @@ func TestAssembleCommand(t *testing.T) { ImageURL: mustParseURL("https://example.com/image.xz"), ImageCompression: CompressionXZ, }, - want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.xz\" | xz -cd | dd of=/dev/sda bs=4M && sync'", + want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.xz\" | xz -cd | dd of=/dev/sda bs=4M conv=sparse && sync'", }, { name: "local zstd", options: UploadOptions{ ImageCompression: CompressionZSTD, }, - want: "bash -c 'set -euo pipefail && zstd -cd | dd of=/dev/sda bs=4M && sync'", + want: "bash -c 'set -euo pipefail && zstd -cd | dd of=/dev/sda bs=4M conv=sparse && sync'", }, { name: "remote zstd", @@ -61,14 +61,14 @@ func TestAssembleCommand(t *testing.T) { ImageURL: mustParseURL("https://example.com/image.zst"), ImageCompression: CompressionZSTD, }, - want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.zst\" | zstd -cd | dd of=/dev/sda bs=4M && sync'", + want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.zst\" | zstd -cd | dd of=/dev/sda bs=4M conv=sparse && sync'", }, { name: "local bz2", options: UploadOptions{ ImageCompression: CompressionBZ2, }, - want: "bash -c 'set -euo pipefail && bzip2 -cd | dd of=/dev/sda bs=4M && sync'", + want: "bash -c 'set -euo pipefail && bzip2 -cd | dd of=/dev/sda bs=4M conv=sparse && sync'", }, { name: "remote bz2", @@ -76,7 +76,7 @@ func TestAssembleCommand(t *testing.T) { ImageURL: mustParseURL("https://example.com/image.bz2"), ImageCompression: CompressionBZ2, }, - want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.bz2\" | bzip2 -cd | dd of=/dev/sda bs=4M && sync'", + want: "bash -c 'set -euo pipefail && wget --no-verbose -O - \"https://example.com/image.bz2\" | bzip2 -cd | dd of=/dev/sda bs=4M conv=sparse && sync'", }, { name: "local qcow2",