diff --git a/go.mod b/go.mod index 1f80740ef..0f25c8c13 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/ryanuber/columnize v2.1.0+incompatible github.com/smira/go-xz v0.0.0-20150414201226-0c531f070014 github.com/spf13/cobra v0.0.5 - github.com/stretchr/testify v1.5.0 + github.com/stretchr/testify v1.5.1 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 github.com/talos-systems/go-procfs v0.0.0-20200219015357-57c7311fdd45 github.com/talos-systems/go-smbios v0.0.0-20200219201045-94b8c4e489ee diff --git a/go.sum b/go.sum index 700014af5..7d4ed6d81 100644 --- a/go.sum +++ b/go.sum @@ -535,15 +535,17 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.0 h1:DMOzIV76tmoDNE9pX6RSN0aDtCYeCg5VueieJaAo1uw= github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/talos-systems/bootkube v0.14.1-0.20200131192519-720c01d02032 h1:Ruxuw1AX8iarsbGiQQsDRwqtrGN0ZTRBqWf5xFXVtec= github.com/talos-systems/bootkube v0.14.1-0.20200131192519-720c01d02032/go.mod h1:zLrZfPQ49k0O6x6QN0pDSJn9iD0EMyj6J+5x1vqJSFw= -github.com/talos-systems/go-smbios v0.0.0-20200219201045-94b8c4e489ee h1:9i0ZFsjZ0wY8UUn/tk2MQshLBC0PNFJe3+84AUqzzyw= -github.com/talos-systems/go-smbios v0.0.0-20200219201045-94b8c4e489ee/go.mod h1:HxhrzAoTZ7ed5Z5VvtCvnCIrOxyXDS7V2B5hCetAMW8= github.com/talos-systems/go-procfs v0.0.0-20200219015357-57c7311fdd45 h1:FND/LgzFHTBdJBOeZVzdO6B47kxQZvSIzb9AMIXYotg= github.com/talos-systems/go-procfs v0.0.0-20200219015357-57c7311fdd45/go.mod h1:ATyUGFQIW8OnbnmvqefZWVPgL9g+CAmXHfkgny21xX8= +github.com/talos-systems/go-smbios v0.0.0-20200219201045-94b8c4e489ee h1:9i0ZFsjZ0wY8UUn/tk2MQshLBC0PNFJe3+84AUqzzyw= +github.com/talos-systems/go-smbios v0.0.0-20200219201045-94b8c4e489ee/go.mod h1:HxhrzAoTZ7ed5Z5VvtCvnCIrOxyXDS7V2B5hCetAMW8= github.com/talos-systems/grpc-proxy v0.2.0 h1:DN75bLfaW4xfhq0r0mwFRnfGhSB+HPhK1LNzuMEs9Pw= github.com/talos-systems/grpc-proxy v0.2.0/go.mod h1:sm97Vc/z2cok3pu6ruNeszQej4KDxFrDgfWs4C1mtC4= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= diff --git a/hack/test/provision-tests.sh b/hack/test/provision-tests.sh index ea2345fc8..2bac59ba2 100755 --- a/hack/test/provision-tests.sh +++ b/hack/test/provision-tests.sh @@ -13,4 +13,4 @@ case "${REGISTRY:-false}" in esac -"${INTEGRATION_TEST}" -test.v -talos.osctlpath "${OSCTL}" ${INTEGRATION_TEST_FLAGS} +"${INTEGRATION_TEST}" -test.v -talos.osctlpath "${OSCTL}" -talos.provision.mem 2048 -talos.provision.cpu 2 ${INTEGRATION_TEST_FLAGS} diff --git a/internal/integration/provision/upgrade.go b/internal/integration/provision/upgrade.go index 9672d6bec..408b21628 100644 --- a/internal/integration/provision/upgrade.go +++ b/internal/integration/provision/upgrade.go @@ -8,6 +8,7 @@ package provision import ( "context" + "crypto/sha256" "fmt" "io/ioutil" "net" @@ -54,7 +55,7 @@ type upgradeSpec struct { const ( talos03Version = "v0.3.2-1-g71ac6696" - talos04Version = "v0.4.0-alpha.5" + talos04Version = "v0.4.0-alpha.5-19-g8913d9df" ) var ( @@ -145,6 +146,12 @@ func (suite *UpgradeSuite) SetupSuite() { // TearDownSuite ... func (suite *UpgradeSuite) TearDownSuite() { + if suite.T().Failed() && suite.Cluster != nil { + // for failed tests, produce crash dump for easier debugging, + // as cluster is going to be torn down below + suite.provisioner.CrashDump(suite.ctx, suite.Cluster, os.Stderr) + } + if suite.clusterAccess != nil { suite.Assert().NoError(suite.clusterAccess.Close()) } @@ -166,7 +173,8 @@ func (suite *UpgradeSuite) TearDownSuite() { // setupCluster provisions source clusters and waits for health func (suite *UpgradeSuite) setupCluster() { - clusterName := fmt.Sprintf("upgrade.%s", suite.spec.ShortName) + shortNameHash := sha256.Sum256([]byte(suite.spec.ShortName)) + clusterName := fmt.Sprintf("upgrade.%x", shortNameHash[:8]) _, cidr, err := net.ParseCIDR(DefaultSettings.CIDR) suite.Require().NoError(err) @@ -396,7 +404,6 @@ func (suite *UpgradeSuite) SuiteName() string { func init() { allSuites = append(allSuites, &UpgradeSuite{specGen: upgradeZeroThreeToZeroFour}, - // disabled until root cause can be figured out: - // &UpgradeSuite{specGen: upgradeZeroFourToCurrent}, + &UpgradeSuite{specGen: upgradeZeroFourToCurrent}, ) } diff --git a/internal/pkg/provision/providers/docker/crashdump.go b/internal/pkg/provision/providers/docker/crashdump.go new file mode 100644 index 000000000..66b062046 --- /dev/null +++ b/internal/pkg/provision/providers/docker/crashdump.go @@ -0,0 +1,42 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package docker + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/docker/docker/api/types" + + "github.com/talos-systems/talos/internal/pkg/provision" +) + +// CrashDump produces debug information to help with debugging failures. +func (p *provisioner) CrashDump(ctx context.Context, cluster provision.Cluster, out io.Writer) { + containers, err := p.listNodes(ctx, cluster.Info().ClusterName) + if err != nil { + fmt.Fprintf(out, "error listing containers: %s\n", err) + return + } + + for _, container := range containers { + name := container.Names[0][:1] + fmt.Fprintf(out, "%s\n%s\n\n", name, strings.Repeat("=", len(name))) + + logs, err := p.client.ContainerLogs(ctx, container.ID, types.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + Tail: "1000", + }) + if err != nil { + fmt.Fprintf(out, "error querying container logs: %s\n", err) + continue + } + + _, _ = io.Copy(out, logs) //nolint: errcheck + } +} diff --git a/internal/pkg/provision/providers/firecracker/crashdump.go b/internal/pkg/provision/providers/firecracker/crashdump.go new file mode 100644 index 000000000..cc20bc3ad --- /dev/null +++ b/internal/pkg/provision/providers/firecracker/crashdump.go @@ -0,0 +1,58 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package firecracker + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/talos-systems/talos/internal/pkg/provision" + "github.com/talos-systems/talos/internal/pkg/tail" +) + +// CrashDump produces debug information to help with debugging failures. +func (p *provisioner) CrashDump(ctx context.Context, cluster provision.Cluster, out io.Writer) { + state, ok := cluster.(*state) + if !ok { + fmt.Fprintf(out, "error inspecting firecracker state, %#+v\n", cluster) + return + } + + statePath, err := state.StatePath() + if err != nil { + fmt.Fprintf(out, "error getting cluster state path: %s", err) + return + } + + logFiles, err := filepath.Glob(filepath.Join(statePath, "*.log")) + if err != nil { + fmt.Fprintf(out, "error finding log paths: %s\n", err) + return + } + + for _, logFile := range logFiles { + name := filepath.Base(logFile) + + fmt.Fprintf(out, "%s\n%s\n\n", name, strings.Repeat("=", len(name))) + + f, err := os.Open(logFile) + if err != nil { + fmt.Fprintf(out, "error opening file: %s\n", err) + continue + } + + if err = tail.SeekLines(f, 1000); err != nil { + fmt.Fprintf(out, "error seeking to the tail: %s\n", err) + } + + _, _ = io.Copy(out, f) //nolint: errcheck + + f.Close() //nolint: errcheck + } +} diff --git a/internal/pkg/provision/provision.go b/internal/pkg/provision/provision.go index b0d22500c..3b2ea66ee 100644 --- a/internal/pkg/provision/provision.go +++ b/internal/pkg/provision/provision.go @@ -7,6 +7,7 @@ package provision import ( "context" + "io" "github.com/talos-systems/talos/pkg/config/types/v1alpha1/generate" ) @@ -16,6 +17,8 @@ type Provisioner interface { Create(context.Context, ClusterRequest, ...Option) (Cluster, error) Destroy(context.Context, Cluster, ...Option) error + CrashDump(context.Context, Cluster, io.Writer) + Reflect(ctx context.Context, clusterName, stateDirectory string) (Cluster, error) GenOptions(NetworkRequest) []generate.GenOption