From 4adb613f66d8068deeacec334174764c0f1cd4e3 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Fri, 16 Oct 2020 17:00:40 +0300 Subject: [PATCH] refactor: bring more control to install.Manifest execution This unifies more code paths under the control of `install.Manifest` vs. being split across the installer and manifest code. There should be no functional changes now. Signed-off-by: Andrey Smirnov --- cmd/installer/pkg/install/install.go | 161 ++----- cmd/installer/pkg/install/manifest.go | 404 ++++++++++++------ cmd/installer/pkg/install/manifest_test.go | 138 +++++- .../content/v0.7/en/configuration/v1alpha1.md | 13 - go.mod | 2 +- go.sum | 4 +- .../runtime/v1alpha1/bootloader/bootloader.go | 2 +- .../runtime/v1alpha1/bootloader/grub/grub.go | 2 +- .../v1alpha1/v1alpha1_sequencer_tasks.go | 11 +- internal/pkg/mount/iter.go | 5 + internal/pkg/mount/mount.go | 11 + pkg/machinery/config/provider.go | 1 - .../types/v1alpha1/v1alpha1_provider.go | 5 - .../config/types/v1alpha1/v1alpha1_types.go | 8 - 14 files changed, 436 insertions(+), 331 deletions(-) diff --git a/cmd/installer/pkg/install/install.go b/cmd/installer/pkg/install/install.go index 8fabf70c2..8b045a0b6 100644 --- a/cmd/installer/pkg/install/install.go +++ b/cmd/installer/pkg/install/install.go @@ -11,12 +11,8 @@ import ( "net/url" "os" "path/filepath" - "unsafe" "github.com/talos-systems/go-procfs/procfs" - "golang.org/x/sys/unix" - - "github.com/talos-systems/go-blockdevice/blockdevice/probe" "github.com/talos-systems/talos/internal/app/machined/pkg/runtime" "github.com/talos-systems/talos/internal/app/machined/pkg/runtime/v1alpha1/bootloader" @@ -79,8 +75,6 @@ type Installer struct { Current string Next string - - bootPartitionFound bool } // NewInstaller initializes and returns an Installer. @@ -95,16 +89,6 @@ func NewInstaller(cmdline *procfs.Cmdline, seq runtime.Sequence, opts *Options) }, } - var dev *probe.ProbedBlockDevice - - if dev, err = probe.DevForFileSystemLabel(opts.Disk, constants.BootPartitionLabel); err != nil { - i.bootPartitionFound = false - } else { - //nolint: errcheck - defer dev.Close() - i.bootPartitionFound = true - } - i.Current, i.Next, err = i.bootloader.Labels() if err != nil { return nil, err @@ -125,89 +109,15 @@ func NewInstaller(cmdline *procfs.Cmdline, seq runtime.Sequence, opts *Options) // // nolint: gocyclo func (i *Installer) Install(seq runtime.Sequence) (err error) { - if i.options.Force { - if i.bootPartitionFound { - var dev *probe.ProbedBlockDevice - - if dev, err = probe.DevForFileSystemLabel(i.options.Disk, constants.BootPartitionLabel); err != nil { - return err - } - - // Reset the partition table. - - if err = dev.Reset(); err != nil { - return err - } - - if err = dev.RereadPartitionTable(); err != nil { - return err - } - - if err = dev.Close(); err != nil { - return err - } - } - - // Zero the disk. - - if i.options.Zero { - if err = zero(i.manifest); err != nil { - return fmt.Errorf("failed to wipe device(s): %w", err) - } - } - - // Partition and format the block device(s). - - if err = i.manifest.ExecuteManifest(); err != nil { - return err - } - } else if !i.bootPartitionFound { - if i.options.Zero { - if err = zero(i.manifest); err != nil { - return fmt.Errorf("failed to wipe device(s): %w", err) - } - } - - if err = i.manifest.ExecuteManifest(); err != nil { - return err - } - } - - if seq == runtime.SequenceUpgrade { - var meta *bootloader.Meta - - if meta, err = bootloader.NewMeta(); err != nil { - return err - } - - //nolint: errcheck - defer meta.Close() - - if ok := meta.SetTag(bootloader.AdvUpgrade, i.Current); !ok { - return fmt.Errorf("failed to set upgrade tag: %q", i.Current) - } - - if _, err = meta.Write(); err != nil { - return err - } + if err = i.manifest.Execute(); err != nil { + return err } // Mount the partitions. - mountpoints := mount.NewMountPoints() - - for dev := range i.manifest.Targets { - var mp *mount.Points - - mp, err = mount.SystemMountPointsForDevice(dev) - if err != nil { - return err - } - - iter := mp.Iter() - for iter.Next() { - mountpoints.Set(iter.Key(), iter.Value()) - } + mountpoints, err := i.manifest.SystemMountpoints() + if err != nil { + return err } if err = mount.Mount(mountpoints); err != nil { @@ -252,7 +162,7 @@ func (i *Installer) Install(seq runtime.Sequence) (err error) { }, } - if i.bootPartitionFound && i.Current != "" { + if i.Current != "" { grubcfg.Fallback = i.Current grubcfg.Labels = append(grubcfg.Labels, &grub.Label{ @@ -263,10 +173,29 @@ func (i *Installer) Install(seq runtime.Sequence) (err error) { }) } - if err = i.bootloader.Install(i.Current, grubcfg, seq, i.bootPartitionFound); err != nil { + if err = i.bootloader.Install(i.Current, grubcfg, seq); err != nil { return err } + if seq == runtime.SequenceUpgrade { + var meta *bootloader.Meta + + if meta, err = bootloader.NewMeta(); err != nil { + return err + } + + //nolint: errcheck + defer meta.Close() + + if ok := meta.SetTag(bootloader.AdvUpgrade, i.Current); !ok { + return fmt.Errorf("failed to set upgrade tag: %q", i.Current) + } + + if _, err = meta.Write(); err != nil { + return err + } + } + if i.options.Save { u, err := url.Parse(i.options.ConfigSource) if err != nil { @@ -301,41 +230,3 @@ func (i *Installer) Install(seq runtime.Sequence) (err error) { return nil } - -func zero(manifest *Manifest) (err error) { - var zero *os.File - - if zero, err = os.Open("/dev/zero"); err != nil { - return err - } - - defer zero.Close() //nolint: errcheck - - for dev := range manifest.Targets { - if err = func(dev string) error { - var f *os.File - - if f, err = os.OpenFile(dev, os.O_RDWR, os.ModeDevice); err != nil { - return err - } - - defer f.Close() //nolint: errcheck - - var size uint64 - - if _, _, ret := unix.Syscall(unix.SYS_IOCTL, f.Fd(), unix.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))); ret != 0 { - return fmt.Errorf("failed to got block device size: %v", ret) - } - - if _, err = io.CopyN(f, zero, int64(size)); err != nil { - return err - } - - return f.Close() - }(dev); err != nil { - return err - } - } - - return zero.Close() -} diff --git a/cmd/installer/pkg/install/manifest.go b/cmd/installer/pkg/install/manifest.go index 351e30fa4..8fea11c0a 100644 --- a/cmd/installer/pkg/install/manifest.go +++ b/cmd/installer/pkg/install/manifest.go @@ -12,15 +12,19 @@ import ( "path/filepath" "strings" "time" + "unsafe" "github.com/talos-systems/go-retry/retry" + "golang.org/x/sys/unix" "github.com/talos-systems/go-blockdevice/blockdevice" + "github.com/talos-systems/go-blockdevice/blockdevice/probe" "github.com/talos-systems/go-blockdevice/blockdevice/table" "github.com/talos-systems/go-blockdevice/blockdevice/table/gpt/partition" "github.com/talos-systems/go-blockdevice/blockdevice/util" "github.com/talos-systems/talos/internal/app/machined/pkg/runtime" + "github.com/talos-systems/talos/internal/pkg/mount" "github.com/talos-systems/talos/pkg/machinery/constants" "github.com/talos-systems/talos/pkg/makefs" ) @@ -28,20 +32,35 @@ import ( // Manifest represents the instructions for preparing all block devices // for an installation. type Manifest struct { + Devices map[string]Device Targets map[string][]*Target } +// Device represents device options. +type Device struct { + Device string + + ResetPartitionTable bool + Zero bool +} + // Target represents an installation partition. +// +//nolint: go-lint type Target struct { - Label string - Device string - FileSystemType string - PartitionName string - Size uint - Force bool - Test bool - Assets []*Asset - BlockDevice *blockdevice.BlockDevice + Device string + + Label string + PartitionType PartitionType + FileSystemType FileSystemType + LegacyBIOSBootable bool + + Size uint + Force bool + Assets []*Asset + + // set during execution + PartitionName string } // Asset represents a file required by a target. @@ -50,13 +69,49 @@ type Asset struct { Destination string } +// PartitionType in partition table. +type PartitionType = string + +// GPT partition types. +// +// TODO: should be moved into the blockdevice library. +const ( + EFISystemPartition PartitionType = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" + BIOSBootPartition PartitionType = "21686148-6449-6E6F-744E-656564454649" + LinuxFilesystemData PartitionType = "0FC63DAF-8483-4772-8E79-3D69D8477DE4" +) + +// FileSystemType is used to format partitions. +type FileSystemType = string + +// Filesystem types. +const ( + FilesystemTypeNone FileSystemType = "none" + FilesystemTypeXFS FileSystemType = "xfs" + FilesystemTypeVFAT FileSystemType = "vfat" +) + +// Partition default sizes. +const ( + MiB = 1024 * 1024 + + EFISize = 100 * MiB + BIOSGrubSize = 1 * MiB + BootSize = 300 * MiB + MetaSize = 1 * MiB + StateSize = 100 * MiB +) + // NewManifest initializes and returns a Manifest. +// +//nolint: gocyclo func NewManifest(label string, sequence runtime.Sequence, opts *Options) (manifest *Manifest, err error) { if label == "" { return nil, fmt.Errorf("a label is required, got \"\"") } manifest = &Manifest{ + Devices: map[string]Device{}, Targets: map[string][]*Target{}, } @@ -72,6 +127,38 @@ func NewManifest(label string, sequence runtime.Sequence, opts *Options) (manife } } + // Verify existence of boot partition. + + var bootPartitionFound bool + + if dev, err := probe.DevForFileSystemLabel(opts.Disk, constants.BootPartitionLabel); err != nil { + bootPartitionFound = false + } else { + //nolint: errcheck + defer dev.Close() + bootPartitionFound = true + } + + // TODO: legacy, to support old Talos initramfs, assume force if boot partition not found + if !bootPartitionFound { + opts.Force = true + } + + if !opts.Force { + return nil, fmt.Errorf("installation with preserve is not supported yet") + } + + if !opts.Force && opts.Zero { + return nil, fmt.Errorf("zero option can't be used without force") + } + + manifest.Devices[opts.Disk] = Device{ + Device: opts.Disk, + + ResetPartitionTable: bootPartitionFound && opts.Force, + Zero: opts.Zero, + } + // Initialize any slices we need. Note that a boot partition is not // required. @@ -80,30 +167,34 @@ func NewManifest(label string, sequence runtime.Sequence, opts *Options) (manife } efiTarget := &Target{ - Device: opts.Disk, - Label: constants.EFIPartitionLabel, - Size: 100 * 1024 * 1024, - Force: true, - Test: false, + Device: opts.Disk, + Label: constants.EFIPartitionLabel, + PartitionType: EFISystemPartition, + FileSystemType: FilesystemTypeVFAT, + Size: EFISize, + Force: true, } biosTarget := &Target{ - Device: opts.Disk, - Label: constants.BIOSGrubPartitionLabel, - Size: 1 * 1024 * 1024, - Force: true, - Test: false, + Device: opts.Disk, + Label: constants.BIOSGrubPartitionLabel, + PartitionType: BIOSBootPartition, + FileSystemType: FilesystemTypeNone, + LegacyBIOSBootable: true, + Size: BIOSGrubSize, + Force: true, } var bootTarget *Target if opts.Bootloader { bootTarget = &Target{ - Device: opts.Disk, - Label: constants.BootPartitionLabel, - Size: 300 * 1024 * 1024, - Force: true, - Test: false, + Device: opts.Disk, + Label: constants.BootPartitionLabel, + PartitionType: LinuxFilesystemData, + FileSystemType: FilesystemTypeXFS, + Size: BootSize, + Force: true, Assets: []*Asset{ { Source: constants.KernelAssetPath, @@ -118,27 +209,30 @@ func NewManifest(label string, sequence runtime.Sequence, opts *Options) (manife } metaTarget := &Target{ - Device: opts.Disk, - Label: constants.MetaPartitionLabel, - Size: 1 * 1024 * 1024, - Force: true, - Test: false, + Device: opts.Disk, + Label: constants.MetaPartitionLabel, + PartitionType: LinuxFilesystemData, + FileSystemType: FilesystemTypeNone, + Size: MetaSize, + Force: true, } stateTarget := &Target{ - Device: opts.Disk, - Label: constants.StatePartitionLabel, - Size: 100 * 1024 * 1024, - Force: true, - Test: false, + Device: opts.Disk, + Label: constants.StatePartitionLabel, + PartitionType: LinuxFilesystemData, + FileSystemType: FilesystemTypeXFS, + Size: StateSize, + Force: true, } ephemeralTarget := &Target{ - Device: opts.Disk, - Label: constants.EphemeralPartitionLabel, - Size: 0, - Force: true, - Test: false, + Device: opts.Disk, + Label: constants.EphemeralPartitionLabel, + PartitionType: LinuxFilesystemData, + FileSystemType: FilesystemTypeXFS, + Size: 0, + Force: true, } for _, target := range []*Target{efiTarget, biosTarget, bootTarget, metaTarget, stateTarget, ephemeralTarget} { @@ -152,54 +246,130 @@ func NewManifest(label string, sequence runtime.Sequence, opts *Options) (manife return manifest, nil } -// ExecuteManifest partitions and formats all disks in a manifest. -func (m *Manifest) ExecuteManifest() (err error) { +// Execute partitions and formats all disks in a manifest. +func (m *Manifest) Execute() (err error) { for dev, targets := range m.Targets { - var bd *blockdevice.BlockDevice - - if bd, err = blockdevice.Open(dev, blockdevice.WithNewGPT(true)); err != nil { + if err = m.executeOnDevice(m.Devices[dev], targets); err != nil { return err } - - // nolint: errcheck - defer bd.Close() - - for _, target := range targets { - if err = target.Partition(bd); err != nil { - return fmt.Errorf("failed to partition device: %w", err) - } - } - - if err = bd.RereadPartitionTable(); err != nil { - log.Printf("failed to re-read partition table on %q: %s, ignoring error...", dev, err) - } - - for _, target := range targets { - target := target - - err = retry.Constant(time.Minute, retry.WithUnits(100*time.Millisecond)).Retry(func() error { - e := target.Format() - if e != nil { - if strings.Contains(e.Error(), "No such file or directory") { - // workaround problem with partition device not being visible immediately after partitioning - return retry.ExpectedError(e) - } - - return retry.UnexpectedError(e) - } - - return nil - }) - - if err != nil { - return fmt.Errorf("failed to format device: %w", err) - } - } } return nil } +//nolint: gocyclo +func (m *Manifest) executeOnDevice(device Device, targets []*Target) (err error) { + if device.Zero { + if err = m.zeroDevice(device); err != nil { + return err + } + } + + var bd *blockdevice.BlockDevice + + if bd, err = blockdevice.Open(device.Device, blockdevice.WithNewGPT(true)); err != nil { + return err + } + + if device.ResetPartitionTable { + // TODO: how should it work with zero option above? + if err = bd.Reset(); err != nil { + return err + } + + if err = bd.RereadPartitionTable(); err != nil { + return err + } + } + + // nolint: errcheck + defer bd.Close() + + for _, target := range targets { + if err = target.Partition(bd); err != nil { + return fmt.Errorf("failed to partition device: %w", err) + } + } + + if err = bd.RereadPartitionTable(); err != nil { + log.Printf("failed to re-read partition table on %q: %s, ignoring error...", device.Device, err) + } + + for _, target := range targets { + target := target + + err = retry.Constant(time.Minute, retry.WithUnits(100*time.Millisecond)).Retry(func() error { + e := target.Format() + if e != nil { + if strings.Contains(e.Error(), "No such file or directory") { + // workaround problem with partition device not being visible immediately after partitioning + return retry.ExpectedError(e) + } + + return retry.UnexpectedError(e) + } + + return nil + }) + + if err != nil { + return fmt.Errorf("failed to format device: %w", err) + } + } + + return nil +} + +// SystemMountpoints returns list of system mountpoints for the manifest. +func (m *Manifest) SystemMountpoints() (*mount.Points, error) { + mountpoints := mount.NewMountPoints() + + for dev := range m.Targets { + mp, err := mount.SystemMountPointsForDevice(dev) + if err != nil { + return nil, err + } + + iter := mp.Iter() + for iter.Next() { + mountpoints.Set(iter.Key(), iter.Value()) + } + } + + return mountpoints, nil +} + +// zeroDevice fills first block of the device with zeroes. +func (m *Manifest) zeroDevice(device Device) (err error) { + var zero *os.File + + if zero, err = os.Open("/dev/zero"); err != nil { + return err + } + + defer zero.Close() //nolint: errcheck + + var f *os.File + + if f, err = os.OpenFile(device.Device, os.O_RDWR, os.ModeDevice); err != nil { + return err + } + + defer f.Close() //nolint: errcheck + + var size uint64 + + if _, _, ret := unix.Syscall(unix.SYS_IOCTL, f.Fd(), unix.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))); ret != 0 { + return fmt.Errorf("failed to got block device size: %v", ret) + } + + if _, err = io.CopyN(f, zero, int64(size)); err != nil { + return err + } + + return f.Close() +} + // Partition creates a new partition on the specified device. // nolint: dupl, gocyclo func (t *Target) Partition(bd *blockdevice.BlockDevice) (err error) { @@ -211,33 +381,17 @@ func (t *Target) Partition(bd *blockdevice.BlockDevice) (err error) { return err } - opts := []interface{}{} + opts := []interface{}{ + partition.WithPartitionType(t.PartitionType), + partition.WithPartitionName(t.Label), + } - const ( - EFISystemPartition = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" - BIOSBootPartition = "21686148-6449-6E6F-744E-656564454649" - LinuxFilesystemData = "0FC63DAF-8483-4772-8E79-3D69D8477DE4" - ) + if t.Size == 0 { + opts = append(opts, partition.WithMaximumSize(true)) + } - switch t.Label { - case constants.EFIPartitionLabel: - opts = append(opts, partition.WithPartitionType(EFISystemPartition), partition.WithPartitionName(t.Label)) - case constants.BIOSGrubPartitionLabel: - opts = append(opts, partition.WithPartitionType(BIOSBootPartition), partition.WithPartitionName(t.Label), partition.WithLegacyBIOSBootableAttribute(true)) - case constants.BootPartitionLabel: - opts = append(opts, partition.WithPartitionType(LinuxFilesystemData), partition.WithPartitionName(t.Label)) - case constants.MetaPartitionLabel: - opts = append(opts, partition.WithPartitionType(LinuxFilesystemData), partition.WithPartitionName(t.Label)) - case constants.StatePartitionLabel: - opts = append(opts, partition.WithPartitionType(LinuxFilesystemData), partition.WithPartitionName(t.Label)) - case constants.EphemeralPartitionLabel: - opts = append(opts, partition.WithPartitionType(LinuxFilesystemData), partition.WithPartitionName(t.Label), partition.WithMaximumSize(true)) - default: - opts = append(opts, partition.WithPartitionType(LinuxFilesystemData)) - - if t.Size == 0 { - opts = append(opts, partition.WithMaximumSize(true)) - } + if t.LegacyBIOSBootable { + opts = append(opts, partition.WithLegacyBIOSBootableAttribute(true)) } part, err := pt.Add(uint64(t.Size), opts...) @@ -245,6 +399,8 @@ func (t *Target) Partition(bd *blockdevice.BlockDevice) (err error) { return err } + log.Printf("created %sp%d (%s) size %d blocks", t.Device, part.No(), t.Label, part.Length()) + if err = pt.Write(); err != nil { return err } @@ -261,43 +417,21 @@ func (t *Target) Partition(bd *blockdevice.BlockDevice) (err error) { // //nolint: gocyclo func (t *Target) Format() error { - switch t.Label { - case constants.EFIPartitionLabel: - log.Printf("formatting partition %q as %q with label %q\n", t.PartitionName, "fat", t.Label) - return makefs.VFAT(t.PartitionName, makefs.WithLabel(t.Label)) - case constants.BIOSGrubPartitionLabel: + if t.FileSystemType == FilesystemTypeNone { return nil - case constants.BootPartitionLabel: - log.Printf("formatting partition %q as %q with label %q\n", t.PartitionName, "xfs", t.Label) - opts := []makefs.Option{makefs.WithForce(t.Force)} + } - if t.Label != "" { - opts = append(opts, makefs.WithLabel(t.Label)) - } + log.Printf("formatting partition %q as %q with label %q\n", t.PartitionName, t.FileSystemType, t.Label) - return makefs.XFS(t.PartitionName, opts...) - case constants.MetaPartitionLabel: - return nil - case constants.StatePartitionLabel: - log.Printf("formatting partition %q as %q with label %q\n", t.PartitionName, "xfs", t.Label) - opts := []makefs.Option{makefs.WithForce(t.Force)} - - if t.Label != "" { - opts = append(opts, makefs.WithLabel(t.Label)) - } - - return makefs.XFS(t.PartitionName, opts...) - case constants.EphemeralPartitionLabel: - log.Printf("formatting partition %q as %q with label %q\n", t.PartitionName, "xfs", t.Label) - opts := []makefs.Option{makefs.WithForce(t.Force)} - - if t.Label != "" { - opts = append(opts, makefs.WithLabel(t.Label)) - } + opts := []makefs.Option{makefs.WithForce(t.Force), makefs.WithLabel(t.Label)} + switch t.FileSystemType { + case FilesystemTypeVFAT: + return makefs.VFAT(t.PartitionName, opts...) + case FilesystemTypeXFS: return makefs.XFS(t.PartitionName, opts...) default: - return nil + return fmt.Errorf("unsupported filesystem type: %q", t.FileSystemType) } } diff --git a/cmd/installer/pkg/install/manifest_test.go b/cmd/installer/pkg/install/manifest_test.go index 8299a8e33..66ae5d07a 100644 --- a/cmd/installer/pkg/install/manifest_test.go +++ b/cmd/installer/pkg/install/manifest_test.go @@ -6,18 +6,20 @@ package install_test import ( "io/ioutil" - "net/http" - "net/http/httptest" "os" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/suite" "github.com/talos-systems/go-blockdevice/blockdevice" + "github.com/talos-systems/go-blockdevice/blockdevice/table/gpt/partition" "github.com/talos-systems/talos/cmd/installer/pkg/install" "github.com/talos-systems/talos/internal/app/machined/pkg/runtime" "github.com/talos-systems/talos/internal/pkg/loopback" + "github.com/talos-systems/talos/internal/pkg/mount" ) // Some tests in this package cannot be run under buildkit, as buildkit doesn't propagate partition devices @@ -32,13 +34,19 @@ type manifestSuite struct { loopbackDevice *os.File } -const diskSize = 10 * 1024 * 1024 * 1024 * 1024 // 10 GiB +const ( + diskSize = 4 * 1024 * 1024 * 1024 // 4 GiB + lbaSize = 512 + gptReserved = 67 +) func TestManifestSuite(t *testing.T) { suite.Run(t, new(manifestSuite)) } func (suite *manifestSuite) SetupSuite() { + suite.skipIfNotRoot() + var err error suite.disk, err = ioutil.TempFile("", "talos") @@ -75,7 +83,13 @@ func (suite *manifestSuite) skipUnderBuildkit() { } } -func (suite *manifestSuite) verifyBlockdevice() { +func (suite *manifestSuite) skipIfNotRoot() { + if os.Getuid() != 0 { + suite.T().Skip("can't run the test as non-root") + } +} + +func (suite *manifestSuite) verifyBlockdevice(manifest *install.Manifest) { bd, err := blockdevice.Open(suite.loopbackDevice.Name()) suite.Require().NoError(err) @@ -84,23 +98,113 @@ func (suite *manifestSuite) verifyBlockdevice() { table, err := bd.PartitionTable() suite.Require().NoError(err) - suite.Assert().Len(table.Partitions(), 5) + // verify partition table + + suite.Assert().Len(table.Partitions(), 6) + + part := table.Partitions()[0] + suite.Assert().Equal(install.EFISystemPartition, strings.ToUpper(part.(*partition.Partition).Type.String())) + suite.Assert().EqualValues(0, part.(*partition.Partition).Flags) + suite.Assert().EqualValues(install.EFISize/lbaSize, part.Length()) + + part = table.Partitions()[1] + suite.Assert().Equal(install.BIOSBootPartition, strings.ToUpper(part.(*partition.Partition).Type.String())) + suite.Assert().EqualValues(4, part.(*partition.Partition).Flags) + suite.Assert().EqualValues(install.BIOSGrubSize/lbaSize, part.Length()) + + part = table.Partitions()[2] + suite.Assert().Equal(install.LinuxFilesystemData, strings.ToUpper(part.(*partition.Partition).Type.String())) + suite.Assert().EqualValues(0, part.(*partition.Partition).Flags) + suite.Assert().EqualValues(install.BootSize/lbaSize, part.Length()) + + part = table.Partitions()[3] + suite.Assert().Equal(install.LinuxFilesystemData, strings.ToUpper(part.(*partition.Partition).Type.String())) + suite.Assert().EqualValues(0, part.(*partition.Partition).Flags) + suite.Assert().EqualValues(install.MetaSize/lbaSize, part.Length()) + + part = table.Partitions()[4] + suite.Assert().Equal(install.LinuxFilesystemData, strings.ToUpper(part.(*partition.Partition).Type.String())) + suite.Assert().EqualValues(0, part.(*partition.Partition).Flags) + suite.Assert().EqualValues(install.StateSize/lbaSize, part.Length()) + + part = table.Partitions()[5] + suite.Assert().Equal(install.LinuxFilesystemData, strings.ToUpper(part.(*partition.Partition).Type.String())) + suite.Assert().EqualValues(0, part.(*partition.Partition).Flags) + suite.Assert().EqualValues((diskSize-install.EFISize-install.BIOSGrubSize-install.BootSize-install.MetaSize-install.StateSize)/lbaSize-gptReserved, part.Length()) suite.Assert().NoError(bd.Close()) + + // query mount points directly for the device + + mountpoints, err := mount.SystemMountPointsForDevice(suite.loopbackDevice.Name()) + suite.Require().NoError(err) + + suite.Assert().Equal(4, mountpoints.Len()) + + // verify filesystems by mounting and unmounting + + tempDir, err := ioutil.TempDir("", "talos") + suite.Require().NoError(err) + + defer func() { + suite.Assert().NoError(os.RemoveAll(tempDir)) + }() + + mountpoints, err = manifest.SystemMountpoints() + suite.Require().NoError(err) + + suite.Assert().Equal(4, mountpoints.Len()) + + suite.Require().NoError(mount.PrefixMountTargets(mountpoints, tempDir)) + + err = mount.Mount(mountpoints) + suite.Require().NoError(err) + + defer func() { + suite.Assert().NoError(mount.Unmount(mountpoints)) + }() } func (suite *manifestSuite) TestExecuteManifestClean() { suite.skipUnderBuildkit() manifest, err := install.NewManifest("A", runtime.SequenceInstall, &install.Options{ - Disk: suite.loopbackDevice.Name(), - Force: true, + Disk: suite.loopbackDevice.Name(), + Bootloader: true, + Force: true, }) suite.Require().NoError(err) - suite.Assert().NoError(manifest.ExecuteManifest()) + suite.Assert().NoError(manifest.Execute()) - suite.verifyBlockdevice() + suite.verifyBlockdevice(manifest) +} + +func (suite *manifestSuite) TestExecuteManifestForce() { + suite.skipUnderBuildkit() + + manifest, err := install.NewManifest("A", runtime.SequenceInstall, &install.Options{ + Disk: suite.loopbackDevice.Name(), + Bootloader: true, + Force: true, + }) + suite.Require().NoError(err) + + suite.Assert().NoError(manifest.Execute()) + + // reinstall + + manifest, err = install.NewManifest("B", runtime.SequenceInstall, &install.Options{ + Disk: suite.loopbackDevice.Name(), + Bootloader: true, + Force: true, + Zero: true, + }) + suite.Require().NoError(err) + + suite.Assert().NoError(manifest.Execute()) + + suite.verifyBlockdevice(manifest) } func (suite *manifestSuite) TestTargetInstall() { @@ -112,25 +216,19 @@ func (suite *manifestSuite) TestTargetInstall() { defer os.RemoveAll(dir) // Create a tempfile for local copy - tempfile, err := ioutil.TempFile(dir, "example") + src, err := ioutil.TempFile(dir, "example") suite.Require().NoError(err) - // Create simple http test server to serve up some content - mux := http.NewServeMux() - mux.HandleFunc("/yolo", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // nolint: errcheck - w.Write([]byte("null")) - })) + suite.Require().NoError(src.Close()) - ts := httptest.NewServer(mux) + dst := filepath.Join(dir, "dest") - defer ts.Close() // Attempt to download and copy files target := &install.Target{ Assets: []*install.Asset{ { - Source: tempfile.Name(), - Destination: "/path/relative/to/mountpoint/example", + Source: src.Name(), + Destination: dst, }, }, } diff --git a/docs/website/content/v0.7/en/configuration/v1alpha1.md b/docs/website/content/v0.7/en/configuration/v1alpha1.md index f056b60d8..635723708 100644 --- a/docs/website/content/v0.7/en/configuration/v1alpha1.md +++ b/docs/website/content/v0.7/en/configuration/v1alpha1.md @@ -866,19 +866,6 @@ Valid Values: - `false` - `no` -#### force - -Indicates if filesystems should be forcefully created. - -Type: `bool` - -Valid Values: - -- `true` -- `yes` -- `false` -- `no` - --- ### TimeConfig diff --git a/go.mod b/go.mod index cc1f0dba5..5279dbf5f 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 github.com/talos-systems/bootkube-plugin v0.0.0-20200915135634-229d57e818f3 github.com/talos-systems/crypto v0.2.0 - github.com/talos-systems/go-blockdevice v0.1.0 + github.com/talos-systems/go-blockdevice v0.1.1-0.20201009184237-ff3a8210be99 github.com/talos-systems/go-loadbalancer v0.1.0 github.com/talos-systems/go-procfs v0.0.0-20200219015357-57c7311fdd45 github.com/talos-systems/go-retry v0.1.1-0.20200922131245-752f081252cf diff --git a/go.sum b/go.sum index 277a83845..adcebdd98 100644 --- a/go.sum +++ b/go.sum @@ -754,8 +754,8 @@ github.com/talos-systems/bootkube-plugin v0.0.0-20200915135634-229d57e818f3 h1:L github.com/talos-systems/bootkube-plugin v0.0.0-20200915135634-229d57e818f3/go.mod h1:AbdJAgHK5rJNDPUN3msPTfQJSR9b4DKb5xNN07uG8/Y= github.com/talos-systems/crypto v0.2.0 h1:UwT8uhJ0eDlklY0vYwo1+LGoFgiqkPqjQnae6j8UNYE= github.com/talos-systems/crypto v0.2.0/go.mod h1:KwqG+jANKU1FNQIapmioHQ5fkovY1DJkAqMenjYBGh0= -github.com/talos-systems/go-blockdevice v0.1.0 h1:KEUqVnsFzLaVSWaLHoilEjJ8HTMG/LZGFMtanxDgHyo= -github.com/talos-systems/go-blockdevice v0.1.0/go.mod h1:z7Wgf5zZUFRiASnjKMoMwYQUr841NK03Pn/RZ4DkF/M= +github.com/talos-systems/go-blockdevice v0.1.1-0.20201009184237-ff3a8210be99 h1:2WrfCMZHBgdzM0KnfAhjhWVbhBTzkABeSIQS3/eH7bY= +github.com/talos-systems/go-blockdevice v0.1.1-0.20201009184237-ff3a8210be99/go.mod h1:efEE9wjtgxiovqsZAV39xlOd/AOI/0sLuZqb5jEgeqo= github.com/talos-systems/go-loadbalancer v0.1.0 h1:MQFONvSjoleU8RrKq1O1Z8CyTCJGd4SLqdAHDlR6o9s= github.com/talos-systems/go-loadbalancer v0.1.0/go.mod h1:D5Qjfz+29WVjONWECZvOkmaLsBb3f5YeWME0u/5HmIc= github.com/talos-systems/go-procfs v0.0.0-20200219015357-57c7311fdd45 h1:FND/LgzFHTBdJBOeZVzdO6B47kxQZvSIzb9AMIXYotg= diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/bootloader.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/bootloader.go index 8f2639a89..aa2b1a6f2 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/bootloader.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/bootloader.go @@ -11,6 +11,6 @@ import ( // Bootloader describes a bootloader. type Bootloader interface { Labels() (string, string, error) - Install(string, interface{}, runtime.Sequence, bool) error + Install(string, interface{}, runtime.Sequence) error Default(string) error } diff --git a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub.go b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub.go index ba1a01123..e7e6fd698 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/grub.go @@ -105,7 +105,7 @@ func (g *Grub) Labels() (current, next string, err error) { // specified kernel parameters. // // nolint: gocyclo -func (g *Grub) Install(fallback string, config interface{}, sequence runtime.Sequence, bootPartitionFound bool) (err error) { +func (g *Grub) Install(fallback string, config interface{}, sequence runtime.Sequence) (err error) { grubcfg, ok := config.(*Cfg) if !ok { return errors.New("expected a grub config") diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go index 2b145d836..b245308a7 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go @@ -815,14 +815,13 @@ func partitionAndFormatDisks(logger *log.Logger, r runtime.Runtime) (err error) Device: disk.Device(), Size: part.Size(), Force: true, - Test: false, } m.Targets[disk.Device()] = append(m.Targets[disk.Device()], extraTarget) } } - if err = m.ExecuteManifest(); err != nil { + if err = m.Execute(); err != nil { return err } @@ -1229,12 +1228,6 @@ func ResetSystemDisk(seq runtime.Sequence, data interface{}) (runtime.TaskExecut // disk is not in use. func VerifyDiskAvailability(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) { return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { - // We only need to verify system disk availability if we are going to - // reformat the ephemeral partition. - if !r.Config().Machine().Install().Force() { - return nil - } - devname := r.State().Machine().Disk().BlockDevice.Device().Name() // We MUST close this in order to avoid EBUSY. @@ -1552,7 +1545,7 @@ func Install(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, r.State().Platform().Name(), installerImage, r.Config().Machine().Registries(), - install.WithForce(r.Config().Machine().Install().Force()), + install.WithForce(true), install.WithZero(r.Config().Machine().Install().Zero()), install.WithExtraKernelArgs(r.Config().Machine().Install().ExtraKernelArgs()), ) diff --git a/internal/pkg/mount/iter.go b/internal/pkg/mount/iter.go index e7b525854..86c7b255d 100644 --- a/internal/pkg/mount/iter.go +++ b/internal/pkg/mount/iter.go @@ -60,6 +60,11 @@ func (p *Points) Get(key string) (value *Point, ok bool) { return nil, false } +// Len returns number of mount points. +func (p *Points) Len() int { + return len(p.points) +} + // Key returns the current key. func (i *PointsIterator) Key() string { return i.key diff --git a/internal/pkg/mount/mount.go b/internal/pkg/mount/mount.go index eb9bf17b6..8c5b9bc7c 100644 --- a/internal/pkg/mount/mount.go +++ b/internal/pkg/mount/mount.go @@ -116,6 +116,17 @@ func Move(mountpoints *Points, prefix string) (err error) { return nil } +// PrefixMountTargets prefixes all mountpoints targets with fixed path. +func PrefixMountTargets(mountpoints *Points, targetPrefix string) error { + iter := mountpoints.Iter() + for iter.Next() { + mountpoint := iter.Value() + mountpoint.target = filepath.Join(targetPrefix, mountpoint.target) + } + + return iter.Err() +} + func mountRetry(f RetryFunc, p *Point) (err error) { err = retry.Constant(5*time.Second, retry.WithUnits(50*time.Millisecond)).Retry(func() error { if err = f(p); err != nil { diff --git a/pkg/machinery/config/provider.go b/pkg/machinery/config/provider.go index 0e45b3a31..7408dcbe1 100644 --- a/pkg/machinery/config/provider.go +++ b/pkg/machinery/config/provider.go @@ -75,7 +75,6 @@ type Install interface { Disk() string ExtraKernelArgs() []string Zero() bool - Force() bool WithBootloader() bool } diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go index 4842efcce..d779514a3 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go @@ -979,11 +979,6 @@ func (i *InstallConfig) Zero() bool { return i.InstallWipe } -// Force implements the config.Provider interface. -func (i *InstallConfig) Force() bool { - return i.InstallForce -} - // WithBootloader implements the config.Provider interface. func (i *InstallConfig) WithBootloader() bool { return i.InstallBootloader diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go index b8846a30b..3f5275f6e 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go @@ -525,14 +525,6 @@ type InstallConfig struct { // - false // - no InstallWipe bool `yaml:"wipe"` - // description: | - // Indicates if filesystems should be forcefully created. - // values: - // - true - // - yes - // - false - // - no - InstallForce bool `yaml:"force"` } // TimeConfig represents the options for configuring time on a node.