From 21058c34d045d147416a3b6cdab3a7b7f94eb068 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 22 Jun 2020 17:27:54 -0700 Subject: [PATCH] add some description of xl.meta (#9901) --- cmd/xl-storage-format-v2.go | 4 +- docs/bucket/versioning/DESIGN.md | 65 ++++++++--------------- docs/bucket/versioning/README.md | 4 +- docs/bucket/versioning/xl-meta-to-json.go | 33 +++++++++++- 4 files changed, 58 insertions(+), 48 deletions(-) diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 8eac30b86..883148e7c 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -40,11 +40,11 @@ func checkXL2V1(buf []byte) error { } if !bytes.Equal(buf[:4], xlHeader[:]) { - return fmt.Errorf("xlMeta: unknown XLv2 header %s", xlHeader) + return fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4]) } if !bytes.Equal(buf[4:8], xlVersionV1[:]) { - return fmt.Errorf("xlMeta: unknown XLv2 version %s", xlVersionV1) + return fmt.Errorf("xlMeta: unknown XLv2 version, expected %v, got %v", xlVersionV1[:4], buf[4:8]) } return nil diff --git a/docs/bucket/versioning/DESIGN.md b/docs/bucket/versioning/DESIGN.md index 4593df048..4639cae34 100644 --- a/docs/bucket/versioning/DESIGN.md +++ b/docs/bucket/versioning/DESIGN.md @@ -1,53 +1,32 @@ # Bucket Versioning Design Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) -Example of a version enabled bucket `engineering` -``` -/mnt/data02/engineering/backup.tar.gz -├─ 0379f361-695c-4509-b0b8-a4842d414579 -│ └─ part.1 -├─ 804fea2c-c21e-490b-98ff-cdb647047cb6 -│ └─ part.1 -├─ e063d138-4d6e-4e68-8576-12d1b4509cc3 -│ └─ part.1 -├─ e675c1f6-476d-4b46-be31-281c887aea7b -│ └─ part.1 -└─ xl.meta +## Description of `xl.meta` -/mnt/data03/engineering/backup.tar.gz -├─ 0379f361-695c-4509-b0b8-a4842d414579 -│ └─ part.1 -├─ 804fea2c-c21e-490b-98ff-cdb647047cb6 -│ └─ part.1 -├─ e063d138-4d6e-4e68-8576-12d1b4509cc3 -│ └─ part.1 -├─ e675c1f6-476d-4b46-be31-281c887aea7b -│ └─ part.1 -└─ xl.meta +`xl.meta` is a new self describing backend format used by MinIO to support AWS S3 compatible versioning. This file is the source of truth for each `version` at rest. `xl.meta` is a msgpack file serialized from a well defined data structure. To understand `xl.meta` here are the few things to start with -/mnt/data04/engineering/backup.tar.gz -├─ 0379f361-695c-4509-b0b8-a4842d414579 -│ └─ part.1 -├─ 804fea2c-c21e-490b-98ff-cdb647047cb6 -│ └─ part.1 -├─ e063d138-4d6e-4e68-8576-12d1b4509cc3 -│ └─ part.1 -├─ e675c1f6-476d-4b46-be31-281c887aea7b -│ └─ part.1 -└─ xl.meta +`xl.meta` carries first 8 bytes an XL header which describes the current format and the format version, allowing the unmarshaller's to automatically use the right data structures to parse the subsequent content in the stream. -/mnt/data05/engineering/backup.tar.gz -├─ 0379f361-695c-4509-b0b8-a4842d414579 -│ └─ part.1 -├─ 804fea2c-c21e-490b-98ff-cdb647047cb6 -│ └─ part.1 -├─ e063d138-4d6e-4e68-8576-12d1b4509cc3 -│ └─ part.1 -├─ e675c1f6-476d-4b46-be31-281c887aea7b -│ └─ part.1 -└─ xl.meta +These are the current entries +```go +var ( + // XL header specifies the format + // one extra byte left for future use + xlHeader = [4]byte{'X', 'L', '2', ' '} + + // XLv2 version 1 specifies the current + // version of the XLv2 format, 3 extra bytes + // left for future use. + xlVersionV1 = [4]byte{'1', ' ', ' ', ' '} +) ``` -`xl.meta` is a msgpack file with following data structure, this is converted from binary format to JSON for convenience. +Once the header is validated, we proceed to the actual data structure of the `xl.meta` format. `xl.meta` carries three types of object entries which designate the type of version object stored. + +- ObjectType (default) +- LegacyObjectType (preserves existing deployments and older xl.json format) +- DeleteMarker (a versionId to capture the DELETE sequences implemented primarily for AWS spec compatibility) + +A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta-to-json.go][./xl-meta-to-json.go] program. ```json { "Versions": [ diff --git a/docs/bucket/versioning/README.md b/docs/bucket/versioning/README.md index 40b6d2229..772ef3e63 100644 --- a/docs/bucket/versioning/README.md +++ b/docs/bucket/versioning/README.md @@ -30,8 +30,8 @@ To permanently delete an object you need to specify the version you want to dele ![delete_version_id](versioning_DELETE_versionEnabled_id.png) -## Features -- All Buckets on MinIO are always in one of the following states: unversioned (the default), versioning-enabled, or versioning-suspended. +## Concepts +- All Buckets on MinIO are always in one of the following states: unversioned (the default) and all other existing deployments, versioning-enabled, or versioning-suspended. - Versioning state applies to all of the objects in the versioning enabled bucket. The first time you enable a bucket for versioning, objects in the bucket are thereafter always versioned and given a unique version ID. - Existing or newer buckets can be created with versioning enabled and eventually can be suspended as well. Existing versions of objects stay as is and can still be accessed using the version ID. - All versions, including delete-markers should be deleted before deleting a bucket. diff --git a/docs/bucket/versioning/xl-meta-to-json.go b/docs/bucket/versioning/xl-meta-to-json.go index 1d7544d70..6c4c187d5 100644 --- a/docs/bucket/versioning/xl-meta-to-json.go +++ b/docs/bucket/versioning/xl-meta-to-json.go @@ -1,3 +1,19 @@ +/* + * MinIO Cloud Storage, (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package main import ( @@ -18,7 +34,18 @@ func main() { app := cli.NewApp() app.Copyright = "MinIO, Inc." app.Usage = "xl.meta to JSON" - app.Version = "0.0.1" + app.HideVersion = true + app.CustomAppHelpTemplate = `NAME: + {{.Name}} - {{.Usage}} + +USAGE: + {{.Name}} {{if .VisibleFlags}}[FLAGS]{{end}} METAFILES... +{{if .VisibleFlags}} +GLOBAL FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +` + app.HideHelpCommand = true app.Flags = []cli.Flag{ @@ -29,6 +56,10 @@ func main() { } app.Action = func(c *cli.Context) error { + if !c.Args().Present() { + cli.ShowAppHelp(c) + return nil + } for _, file := range c.Args() { var r io.Reader switch file {