minio/docs/debugging/s3-verify/main.go

// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bytes"
	"context"
	"crypto/sha256"
	"flag"
	"fmt"
	"hash"
	"io"
	"log"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
)

var (
	sourceEndpoint, sourceAccessKey, sourceSecretKey string
	sourceBucket, sourcePrefix                       string
	targetEndpoint, targetAccessKey, targetSecretKey string
	targetBucket, targetPrefix                       string
	minimumObjectAge                                 string
	debug                                            bool
	insecure                                         bool
)

func buildS3Client(endpoint, accessKey, secretKey string, insecure bool) (*minio.Client, error) {
	u, err := url.Parse(endpoint)
	if err != nil {
		return nil, err
	}

	secure := strings.EqualFold(u.Scheme, "https")
	transport, err := minio.DefaultTransport(secure)
	if err != nil {
		return nil, err
	}
	if insecure {
		// skip TLS verification
		transport.TLSClientConfig.InsecureSkipVerify = true
	}

	clnt, err := minio.New(u.Host, &minio.Options{
		Creds:     credentials.NewStaticV4(accessKey, secretKey, ""),
		Secure:    secure,
		Transport: transport,
	})
	if err != nil {
		return nil, err
	}

	return clnt, nil
}

func main() {
	flag.StringVar(&sourceEndpoint, "source-endpoint", "https://play.min.io", "S3 endpoint URL")
	flag.StringVar(&sourceAccessKey, "source-access-key", "Q3AM3UQ867SPQQA43P2F", "S3 Access Key")
	flag.StringVar(&sourceSecretKey, "source-secret-key", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "S3 Secret Key")
	flag.StringVar(&sourceBucket, "source-bucket", "", "Select a specific bucket")
	flag.StringVar(&sourcePrefix, "source-prefix", "", "Select a prefix")

	flag.StringVar(&targetEndpoint, "target-endpoint", "https://play.min.io", "S3 endpoint URL")
	flag.StringVar(&targetAccessKey, "target-access-key", "Q3AM3UQ867SPQQA43P2F", "S3 Access Key")
	flag.StringVar(&targetSecretKey, "target-secret-key", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "S3 Secret Key")
	flag.StringVar(&targetBucket, "target-bucket", "", "Select a specific bucket")
	flag.StringVar(&targetPrefix, "target-prefix", "", "Select a prefix")

	flag.StringVar(&minimumObjectAge, "minimum-object-age", "0s", "Ignore objects younger than the specified age")
	flag.BoolVar(&debug, "debug", false, "Prints HTTP network calls to S3 endpoint")
	flag.BoolVar(&insecure, "insecure", false, "Disable TLS verification")
	flag.Parse()

	if sourceEndpoint == "" {
		log.Fatalln("source Endpoint is not provided")
	}

	if sourceAccessKey == "" {
		log.Fatalln("source Access key is not provided")
	}

	if sourceSecretKey == "" {
		log.Fatalln("source Secret key is not provided")
	}

	if sourceBucket == "" && sourcePrefix != "" {
		log.Fatalln("--source-prefix is specified without --source-bucket.")
	}

	if targetEndpoint == "" {
		log.Fatalln("target Endpoint is not provided")
	}

	if targetAccessKey == "" {
		log.Fatalln("target Access key is not provided")
	}

	if targetSecretKey == "" {
		log.Fatalln("target Secret key is not provided")
	}

	if targetBucket == "" && targetPrefix != "" {
		log.Fatalln("--target-prefix is specified without --target-bucket.")
	}

	srcU, err := url.Parse(sourceEndpoint)
	if err != nil {
		log.Fatalln(err)
	}

	ssecure := strings.EqualFold(srcU.Scheme, "https")
	stransport, err := minio.DefaultTransport(ssecure)
	if err != nil {
		log.Fatalln(err)
	}
	if insecure {
		// skip TLS verification
		stransport.TLSClientConfig.InsecureSkipVerify = true
	}

	ageDelta, err := time.ParseDuration(minimumObjectAge)
	if err != nil {
		log.Fatalln(err)
	}

	maxObjectModTime := time.Now().Add(-ageDelta)

	// Next object is used to ignore new objects in the source & target
	nextObject := func(ch <-chan minio.ObjectInfo) (ctnt minio.ObjectInfo, ok bool) {
		for {
			ctnt, ok := <-ch
			if !ok {
				return minio.ObjectInfo{}, false
			}
			if ctnt.LastModified.Before(maxObjectModTime) {
				return ctnt, ok
			}
		}
	}

	sclnt, err := buildS3Client(sourceEndpoint, sourceAccessKey, sourceSecretKey, insecure)
	if err != nil {
		log.Fatalln(err)
	}

	tclnt, err := buildS3Client(targetEndpoint, targetAccessKey, targetSecretKey, insecure)
	if err != nil {
		log.Fatalln(err)
	}

	if debug {
		sclnt.TraceOn(os.Stderr)
		tclnt.TraceOn(os.Stderr)
	}

	sopts := minio.ListObjectsOptions{
		Recursive: true,
		Prefix:    sourcePrefix,
	}

	topts := minio.ListObjectsOptions{
		Recursive: true,
		Prefix:    targetPrefix,
	}

	srcCh := sclnt.ListObjects(context.Background(), sourceBucket, sopts)
	tgtCh := tclnt.ListObjects(context.Background(), targetBucket, topts)

	srcCtnt, srcOk := nextObject(srcCh)
	tgtCtnt, tgtOk := nextObject(tgtCh)

	var srcEOF, tgtEOF bool

	srcSha256 := sha256.New()
	tgtSha256 := sha256.New()

	for {
		srcSha256.Reset()
		tgtSha256.Reset()

		srcEOF = !srcOk
		tgtEOF = !tgtOk

		// No objects from source AND target: Finish
		if srcEOF && tgtEOF {
			break
		}

		if !srcEOF && srcCtnt.Err != nil {
			log.Fatal(srcCtnt.Err)
		}

		if !tgtEOF && tgtCtnt.Err != nil {
			log.Fatal(tgtCtnt.Err)
		}

		// If source doesn't have objects anymore, comparison becomes obvious
		if srcEOF {
			fmt.Printf("only in target: %s\n", tgtCtnt.Key)
			tgtCtnt, tgtOk = <-tgtCh
			continue
		}

		// The same for target
		if tgtEOF {
			fmt.Printf("only in source: %s\n", srcCtnt.Key)
			srcCtnt, srcOk = nextObject(srcCh)
			continue
		}

		if srcCtnt.Key < tgtCtnt.Key {
			fmt.Printf("only in source: %s\n", srcCtnt.Key)
			srcCtnt, srcOk = nextObject(srcCh)
			continue
		}

		if srcCtnt.Key == tgtCtnt.Key {
			if verifyChecksum(sclnt, srcSha256, tgtSha256, srcCtnt, tgtCtnt) {
				fmt.Printf("all readable source and target: %s -> %s\n", srcCtnt.Key, tgtCtnt.Key)
			}

			srcCtnt, srcOk = nextObject(srcCh)
			tgtCtnt, tgtOk = nextObject(tgtCh)
			continue
		}

		fmt.Printf("only in target: %s\n", tgtCtnt.Key)
		tgtCtnt, tgtOk = nextObject(tgtCh)
	}
}

func verifyChecksum(sclnt *minio.Client, srcSha256, tgtSha256 hash.Hash, srcCtnt, tgtCtnt minio.ObjectInfo) (allgood bool) {
	opts := minio.GetObjectOptions{}
	if srcCtnt.Size != tgtCtnt.Size {
		fmt.Printf("differ in size sourceSize: %d, targetSize: %d\n", srcCtnt.Size, tgtCtnt.Size)
		return false
	} else if srcCtnt.ContentType != tgtCtnt.ContentType {
		fmt.Printf("differ in contentType source: %s, target: %s\n", srcCtnt.ContentType, tgtCtnt.ContentType)
		return false
	}

	core := minio.Core{Client: sclnt}
	sobj, _, _, err := core.GetObject(context.Background(), sourceBucket, srcCtnt.Key, opts)
	if err != nil {
		fmt.Printf("unreadable on source: %s (%s)\n", srcCtnt.Key, err)
		return false
	}

	tobj, _, _, err := core.GetObject(context.Background(), targetBucket, tgtCtnt.Key, opts)
	if err != nil {
		fmt.Printf("unreadable on target: %s (%s)\n", tgtCtnt.Key, err)
		return false
	}

	var sourceFailed, targetFailed bool
	var wg sync.WaitGroup

	wg.Add(2)
	go func() {
		defer wg.Done()
		srcSize, err := io.Copy(srcSha256, sobj)
		if err != nil {
			fmt.Printf("unreadable on source: %s (%s)\n", srcCtnt.Key, err)
			sourceFailed = true
		} else if srcSize != srcCtnt.Size {
			fmt.Printf("unreadable on source - size differs upon read: %s\n", srcCtnt.Key)
			sourceFailed = true
		}
	}()
	go func() {
		defer wg.Done()
		tgtSize, err := io.Copy(tgtSha256, tobj)
		if err != nil {
			fmt.Printf("unreadable on target: %s (%s)\n", tgtCtnt.Key, err)
			targetFailed = true
		} else if tgtSize != tgtCtnt.Size {
			fmt.Printf("unreadable on target - size differs upon read: %s\n", tgtCtnt.Key)
			targetFailed = true
		}
	}()
	wg.Wait()

	sobj.Close()
	tobj.Close()

	if !sourceFailed && !targetFailed {
		ssum := srcSha256.Sum(nil)
		tsum := tgtSha256.Sum(nil)
		allgood = bytes.Equal(ssum, tsum)
		if !allgood {
			fmt.Printf("sha256 sum mismatch: %s -> Expected(%x), Found(%x)\n", srcCtnt.Key, ssum, tsum)
		}
	}

	return allgood
}