diff --git a/pkg/encoding/erasure/cauchy_test.go b/pkg/encoding/erasure/cauchy_test.go index d577d97d3..37f28c74c 100644 --- a/pkg/encoding/erasure/cauchy_test.go +++ b/pkg/encoding/erasure/cauchy_test.go @@ -34,19 +34,33 @@ const ( m = 5 ) -func (s *MySuite) TestCauchyDecode(c *C) { - ep, _ := ParseEncoderParams(k, m, Cauchy) +func (s *MySuite) TestCauchyEncodeDecodeFailure(c *C) { + ep, _ := ValidateParams(k, m, Cauchy) data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") - e := NewEncoder(ep) - chunks, _ := e.Encode(data) + e := NewErasure(ep) + chunks, err := e.Encode(data) + c.Assert(err, IsNil) - chunks[0] = nil - chunks[3] = nil - chunks[5] = nil - chunks[9] = nil - chunks[13] = nil + errorIndex := []int{0, 3, 5, 9, 11, 13} + chunks = corruptChunks(chunks, errorIndex) + + _, err = e.Decode(chunks, len(data)) + c.Assert(err, Not(IsNil)) +} + +func (s *MySuite) TestCauchyEncodeDecodeSuccess(c *C) { + ep, _ := ValidateParams(k, m, Cauchy) + + data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") + + e := NewErasure(ep) + chunks, err := e.Encode(data) + c.Assert(err, IsNil) + + errorIndex := []int{0, 3, 5, 9, 13} + chunks = corruptChunks(chunks, errorIndex) recoveredData, err := e.Decode(chunks, len(data)) c.Assert(err, IsNil) diff --git a/pkg/encoding/erasure/doc.go b/pkg/encoding/erasure/doc.go index 8e8e3c9db..ce47fb64d 100644 --- a/pkg/encoding/erasure/doc.go +++ b/pkg/encoding/erasure/doc.go @@ -28,8 +28,8 @@ // 2. Create a new encoder // 3. Decode data // -// Encoder parameters contain three configurable elements: -// ParseEncoderParams(k, m, technique int) (EncoderParams, error) +// Erasure parameters contain three configurable elements: +// ValidateParams(k, m, technique int) (ErasureParams, error) // k - Number of rows in matrix // m - Number of colums in matrix // technique - Matrix type, can be either Cauchy (recommended) or Vandermonde @@ -53,15 +53,15 @@ // // Creating and using an encoder // var bytes []byte -// params := erasure.ParseEncoderParams(10, 5, erasure.Cauchy) -// encoder := erasure.NewEncoder(params) +// params := erasure.ValidateParams(10, 5, erasure.Cauchy) +// encoder := erasure.NewErasure(params) // encodedData, length := encoder.Encode(bytes) // // Creating and using a decoder // var encodedData [][]byte // var length int -// params := erasure.ParseEncoderParams(10, 5, erasure.Cauchy) -// encoder := erasure.NewEncoder(params) +// params := erasure.ValidateParams(10, 5, erasure.Cauchy) +// encoder := erasure.NewErasure(params) // originalData, err := encoder.Decode(encodedData, length) // package erasure diff --git a/pkg/encoding/erasure/erasure_decode.go b/pkg/encoding/erasure/erasure_decode.go index 395e449e3..fa130a1de 100644 --- a/pkg/encoding/erasure/erasure_decode.go +++ b/pkg/encoding/erasure/erasure_decode.go @@ -36,7 +36,7 @@ import ( // are set to "nil". There must be at least "K" number of data|parity // blocks. // "dataLen" is the length of original source data -func (e *Encoder) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) { +func (e *Erasure) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) { var source, target **C.uint8_t k := int(e.params.K) diff --git a/pkg/encoding/erasure/erasure_encode.go b/pkg/encoding/erasure/erasure_encode.go index f6664ce5e..e2f1d81a1 100644 --- a/pkg/encoding/erasure/erasure_encode.go +++ b/pkg/encoding/erasure/erasure_encode.go @@ -47,27 +47,27 @@ const ( SIMDAlign = 32 ) -// EncoderParams is a configuration set for building an encoder. It is created using ValidateParams. -type EncoderParams struct { +// ErasureParams is a configuration set for building an encoder. It is created using ValidateParams(). +type ErasureParams struct { K uint8 M uint8 Technique Technique // cauchy or vandermonde matrix (RS) } -// Encoder is an object used to encode and decode data. -type Encoder struct { - params *EncoderParams +// Erasure is an object used to encode and decode data. +type Erasure struct { + params *ErasureParams encodeMatrix, encodeTbls *C.uint8_t decodeMatrix, decodeTbls *C.uint8_t decodeIndex *C.uint32_t } -// ParseEncoderParams creates an EncoderParams object. +// ValidateParams creates an ErasureParams object. // // k and m represent the matrix size, which corresponds to the protection level // technique is the matrix type. Valid inputs are Cauchy (recommended) or Vandermonde. // -func ParseEncoderParams(k, m uint8, technique Technique) (*EncoderParams, error) { +func ValidateParams(k, m uint8, technique Technique) (*ErasureParams, error) { if k < 1 { return nil, errors.New("k cannot be zero") } @@ -89,15 +89,15 @@ func ParseEncoderParams(k, m uint8, technique Technique) (*EncoderParams, error) return nil, errors.New("Technique can be either vandermonde or cauchy") } - return &EncoderParams{ + return &ErasureParams{ K: k, M: m, Technique: technique, }, nil } -// NewEncoder creates an encoder object with a given set of parameters. -func NewEncoder(ep *EncoderParams) *Encoder { +// NewErasure creates an encoder object with a given set of parameters. +func NewErasure(ep *ErasureParams) *Erasure { var k = C.int(ep.K) var m = C.int(ep.M) @@ -107,7 +107,7 @@ func NewEncoder(ep *EncoderParams) *Encoder { C.minio_init_encoder(C.int(ep.Technique), k, m, &encodeMatrix, &encodeTbls) - return &Encoder{ + return &Erasure{ params: ep, encodeMatrix: encodeMatrix, encodeTbls: encodeTbls, @@ -138,7 +138,7 @@ func GetEncodedBlockLen(inputLen int, k uint8) (encodedOutputLen int) { // Encode erasure codes a block of data in "k" data blocks and "m" parity blocks. // Output is [k+m][]blocks of data and parity slices. -func (e *Encoder) Encode(inputData []byte) (encodedBlocks [][]byte, err error) { +func (e *Erasure) Encode(inputData []byte) (encodedBlocks [][]byte, err error) { k := int(e.params.K) // "k" data blocks m := int(e.params.M) // "m" parity blocks n := k + m // "n" total encoded blocks diff --git a/pkg/encoding/erasure/vandermonde_test.go b/pkg/encoding/erasure/vandermonde_test.go index 853dd086f..490830557 100644 --- a/pkg/encoding/erasure/vandermonde_test.go +++ b/pkg/encoding/erasure/vandermonde_test.go @@ -22,21 +22,40 @@ import ( . "gopkg.in/check.v1" ) -func (s *MySuite) TestVanderMondeDecode(c *C) { - ep, _ := ParseEncoderParams(k, m, Vandermonde) +func corruptChunks(chunks [][]byte, errorIndex []int) [][]byte { + for _, err := range errorIndex { + chunks[err] = nil + } + return chunks +} + +func (s *MySuite) TestVanderMondeEncodeDecodeFailure(c *C) { + ep, _ := ValidateParams(k, m, Vandermonde) data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") - e := NewEncoder(ep) + e := NewErasure(ep) chunks, err := e.Encode(data) - c.Logf("chunks length: %d", len(chunks)) - c.Logf("length: %d", len(data)) + c.Assert(err, IsNil) - chunks[0] = nil - chunks[3] = nil - chunks[5] = nil - chunks[9] = nil - chunks[13] = nil + errorIndex := []int{0, 3, 5, 9, 11, 13} + chunks = corruptChunks(chunks, errorIndex) + + _, err = e.Decode(chunks, len(data)) + c.Assert(err, Not(IsNil)) +} + +func (s *MySuite) TestVanderMondeEncodeDecodeSuccess(c *C) { + ep, _ := ValidateParams(k, m, Vandermonde) + + data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") + + e := NewErasure(ep) + chunks, err := e.Encode(data) + c.Assert(err, IsNil) + + errorIndex := []int{0, 3, 5, 9, 13} + chunks = corruptChunks(chunks, errorIndex) recoveredData, err := e.Decode(chunks, len(data)) c.Assert(err, IsNil) diff --git a/pkg/storage/donut/erasure.go b/pkg/storage/donut/erasure.go index 8a5f34bb7..6ee38fff3 100644 --- a/pkg/storage/donut/erasure.go +++ b/pkg/storage/donut/erasure.go @@ -3,6 +3,7 @@ package donut import ( "bytes" "errors" + "hash" "io" "strconv" "strings" @@ -12,21 +13,19 @@ import ( "encoding/hex" "github.com/minio-io/iodine" - "github.com/minio-io/minio/pkg/encoding/erasure" + encoding "github.com/minio-io/minio/pkg/encoding/erasure" "github.com/minio-io/minio/pkg/utils/split" - "hash" - "log" ) // getErasureTechnique - convert technique string into Technique type -func getErasureTechnique(technique string) (erasure.Technique, error) { +func getErasureTechnique(technique string) (encoding.Technique, error) { switch true { case technique == "Cauchy": - return erasure.Cauchy, nil + return encoding.Cauchy, nil case technique == "Vandermonde": - return erasure.Cauchy, nil + return encoding.Cauchy, nil default: - return erasure.None, iodine.New(errors.New("Invalid erasure technique: "+technique), nil) + return encoding.None, iodine.New(errors.New("Invalid erasure technique: "+technique), nil) } } @@ -70,13 +69,12 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri return } hasher := md5.New() - params, err := erasure.ParseEncoderParams(k, m, technique) + params, err := encoding.ValidateParams(k, m, technique) if err != nil { writer.CloseWithError(iodine.New(err, donutMetadata)) } - encoder := erasure.NewEncoder(params) + encoder := encoding.NewErasure(params) for i := 0; i < totalChunks; i++ { - log.Println(i) totalLeft, err = decodeChunk(writer, readers, encoder, hasher, k, m, totalLeft, blockSize) if err != nil { errParams := map[string]string{ @@ -97,7 +95,7 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri return } -func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *erasure.Encoder, hasher hash.Hash, k, m uint8, totalLeft int64, blockSize int) (int64, error) { +func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *encoding.Erasure, hasher hash.Hash, k, m uint8, totalLeft int64, blockSize int) (int64, error) { curBlockSize := 0 if int64(blockSize) < totalLeft { curBlockSize = blockSize @@ -105,7 +103,7 @@ func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *erasur curBlockSize = int(totalLeft) // cast is safe, blockSize in if protects } - curChunkSize := erasure.GetEncodedBlockLen(curBlockSize, uint8(k)) + curChunkSize := encoding.GetEncodedBlockLen(curBlockSize, uint8(k)) encodedBytes := make([][]byte, 16) for i, reader := range readers { var bytesBuffer bytes.Buffer @@ -167,8 +165,8 @@ func newErasureWriter(writers []Writer) ObjectWriter { func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) { chunks := split.Stream(r, 10*1024*1024) - params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy) - encoder := erasure.NewEncoder(params) + params, _ := encoding.ValidateParams(8, 8, encoding.Cauchy) + encoder := encoding.NewErasure(params) chunkCount := 0 totalLength := 0 summer := md5.New()