/* Copyright 2014 SAP SE Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package protocol import ( "fmt" "io" "math" "unicode/utf8" "golang.org/x/text/transform" "github.com/SAP/go-hdb/internal/bufio" "github.com/SAP/go-hdb/internal/unicode" "github.com/SAP/go-hdb/internal/unicode/cesu8" ) const ( locatorIDSize = 8 writeLobRequestHeaderSize = 21 readLobRequestSize = 24 ) // variable (unit testing) //var lobChunkSize = 1 << 14 //TODO: check size var lobChunkSize int32 = 256 //TODO: check size //lob options type lobOptions int8 const ( loNullindicator lobOptions = 0x01 loDataincluded lobOptions = 0x02 loLastdata lobOptions = 0x04 ) var lobOptionsText = map[lobOptions]string{ loNullindicator: "null indicator", loDataincluded: "data included", loLastdata: "last data", } func (k lobOptions) String() string { t := make([]string, 0, len(lobOptionsText)) for option, text := range lobOptionsText { if (k & option) != 0 { t = append(t, text) } } return fmt.Sprintf("%v", t) } // LobReadDescr is the package internal representation of a lob field to be read from database. type LobReadDescr struct { col int fn func() error w lobWriter } // SetWriter sets the io.Writer destination for a lob field to be read from database. func (d *LobReadDescr) SetWriter(w io.Writer) error { if err := d.w.setWriter(w); err != nil { return err } if d.fn != nil { return d.fn() } return nil } // LobWriteDescr is the package internal representation of a lob field to be written to database. type LobWriteDescr struct { r io.Reader } // SetReader sets the io.Reader source for a lob field to be written to database. func (d *LobWriteDescr) SetReader(r io.Reader) { d.r = r } type locatorID uint64 // byte[locatorIdSize] // write lob reply type writeLobReply struct { ids []locatorID numArg int } func newWriteLobReply() *writeLobReply { return &writeLobReply{ ids: make([]locatorID, 0), } } func (r *writeLobReply) String() string { return fmt.Sprintf("write lob reply: %v", r.ids) } func (r *writeLobReply) kind() partKind { return pkWriteLobReply } func (r *writeLobReply) setNumArg(numArg int) { r.numArg = numArg } func (r *writeLobReply) read(rd *bufio.Reader) error { //resize ids if cap(r.ids) < r.numArg { r.ids = make([]locatorID, r.numArg) } else { r.ids = r.ids[:r.numArg] } for i := 0; i < r.numArg; i++ { if id, err := rd.ReadUint64(); err == nil { r.ids[i] = locatorID(id) } else { return err } } return nil } //write lob request type writeLobRequest struct { readers []lobReader } func newWriteLobRequest(readers []lobReader) *writeLobRequest { return &writeLobRequest{ readers: readers, } } func (r *writeLobRequest) kind() partKind { return pkWriteLobRequest } func (r *writeLobRequest) size() (int, error) { // TODO: check size limit size := 0 for _, reader := range r.readers { if reader.done() { continue } if err := reader.fill(); err != nil { return 0, err } size += writeLobRequestHeaderSize size += reader.size() } return size, nil } func (r *writeLobRequest) numArg() int { n := 0 for _, reader := range r.readers { if !reader.done() { n++ } } return n } func (r *writeLobRequest) write(wr *bufio.Writer) error { for _, reader := range r.readers { if !reader.done() { if err := wr.WriteUint64(uint64(reader.id())); err != nil { return err } opt := int8(0x02) // data included if reader.eof() { opt |= 0x04 // last data } if err := wr.WriteInt8(opt); err != nil { return err } if err := wr.WriteInt64(-1); err != nil { //offset (-1 := append) return err } if err := wr.WriteInt32(int32(reader.size())); err != nil { // size return err } if _, err := wr.Write(reader.bytes()); err != nil { return err } } } return nil } //read lob request type readLobRequest struct { writers []lobWriter } func (r *readLobRequest) numWriter() int { n := 0 for _, writer := range r.writers { if !writer.eof() { n++ } } return n } func (r *readLobRequest) kind() partKind { return pkReadLobRequest } func (r *readLobRequest) size() (int, error) { return r.numWriter() * readLobRequestSize, nil } func (r *readLobRequest) numArg() int { return r.numWriter() } func (r *readLobRequest) write(wr *bufio.Writer) error { for _, writer := range r.writers { if writer.eof() { continue } if err := wr.WriteUint64(uint64(writer.id())); err != nil { return err } readOfs, readLen := writer.readOfsLen() if err := wr.WriteInt64(readOfs + 1); err != nil { //1-based return err } if err := wr.WriteInt32(readLen); err != nil { return err } if err := wr.WriteZeroes(4); err != nil { return err } } return nil } // read lob reply // - seems like readLobreply gives only an result for one lob - even if more then one is requested type readLobReply struct { writers []lobWriter numArg int } func (r *readLobReply) kind() partKind { return pkReadLobReply } func (r *readLobReply) setNumArg(numArg int) { r.numArg = numArg } func (r *readLobReply) read(rd *bufio.Reader) error { for i := 0; i < r.numArg; i++ { id, err := rd.ReadUint64() if err != nil { return err } var writer lobWriter for _, writer = range r.writers { if writer.id() == locatorID(id) { break // writer found } } if writer == nil { return fmt.Errorf("internal error: no lob writer found for id %d", id) } opt, err := rd.ReadInt8() if err != nil { return err } chunkLen, err := rd.ReadInt32() if err != nil { return err } if err := rd.Skip(3); err != nil { return err } eof := (lobOptions(opt) & loLastdata) != 0 if err := writer.write(rd, int(chunkLen), eof); err != nil { return err } } return nil } // lobWriter reads lob chunks and writes them into lob field. type lobWriter interface { id() locatorID setWriter(w io.Writer) error write(rd *bufio.Reader, size int, eof bool) error readOfsLen() (int64, int32) eof() bool } // baseLobWriter is a reuse struct for binary and char lob writers. type baseLobWriter struct { _id locatorID charLen int64 byteLen int64 readOfs int64 _eof bool ofs int wr io.Writer _flush func() error b []byte } func (l *baseLobWriter) id() locatorID { return l._id } func (l *baseLobWriter) eof() bool { return l._eof } func (l *baseLobWriter) setWriter(wr io.Writer) error { l.wr = wr return l._flush() } func (l *baseLobWriter) write(rd *bufio.Reader, size int, eof bool) error { l._eof = eof // store eof if size == 0 { return nil } l.b = resizeBuffer(l.b, size+l.ofs) if err := rd.ReadFull(l.b[l.ofs:]); err != nil { return err } if l.wr != nil { return l._flush() } return nil } func (l *baseLobWriter) readOfsLen() (int64, int32) { readLen := l.charLen - l.readOfs if readLen > int64(math.MaxInt32) || readLen > int64(lobChunkSize) { return l.readOfs, lobChunkSize } return l.readOfs, int32(readLen) } // binaryLobWriter (byte based lobs). type binaryLobWriter struct { *baseLobWriter } func newBinaryLobWriter(id locatorID, charLen, byteLen int64) *binaryLobWriter { l := &binaryLobWriter{ baseLobWriter: &baseLobWriter{_id: id, charLen: charLen, byteLen: byteLen}, } l._flush = l.flush return l } func (l *binaryLobWriter) flush() error { if _, err := l.wr.Write(l.b); err != nil { return err } l.readOfs += int64(len(l.b)) return nil } type charLobWriter struct { *baseLobWriter } func newCharLobWriter(id locatorID, charLen, byteLen int64) *charLobWriter { l := &charLobWriter{ baseLobWriter: &baseLobWriter{_id: id, charLen: charLen, byteLen: byteLen}, } l._flush = l.flush return l } func (l *charLobWriter) flush() error { nDst, nSrc, err := unicode.Cesu8ToUtf8Transformer.Transform(l.b, l.b, true) // inline cesu8 to utf8 transformation if err != nil && err != transform.ErrShortSrc { return err } if _, err := l.wr.Write(l.b[:nDst]); err != nil { return err } l.ofs = len(l.b) - nSrc if l.ofs != 0 && l.ofs != cesu8.CESUMax/2 { // assert remaining bytes return unicode.ErrInvalidCesu8 } l.readOfs += int64(l.runeCount(l.b[:nDst])) if l.ofs != 0 { l.readOfs++ // add half encoding copy(l.b, l.b[nSrc:len(l.b)]) // move half encoding to buffer begin } return nil } // Caution: hdb counts 4 byte utf-8 encodings (cesu-8 6 bytes) as 2 (3 byte) chars func (l *charLobWriter) runeCount(b []byte) int { numChars := 0 for len(b) > 0 { _, size := utf8.DecodeRune(b) b = b[size:] numChars++ if size == utf8.UTFMax { numChars++ } } return numChars } // lobWriter reads field lob data chunks. type lobReader interface { id() locatorID fill() error size() int bytes() []byte eof() bool done() bool } // baseLobWriter is a reuse struct for binary and char lob writers. type baseLobReader struct { r io.Reader _id locatorID _size int _eof bool _done bool b []byte } func (l *baseLobReader) id() locatorID { return l._id } func (l *baseLobReader) eof() bool { return l._eof } func (l *baseLobReader) done() bool { return l._done } func (l *baseLobReader) size() int { return l._size } func (l *baseLobReader) bytes() []byte { if l._eof { l._done = true } return l.b[:l._size] } // binaryLobReader (byte based lobs). type binaryLobReader struct { *baseLobReader } func newBinaryLobReader(r io.Reader, id locatorID) *binaryLobReader { return &binaryLobReader{ baseLobReader: &baseLobReader{r: r, _id: id}, } } func (l *binaryLobReader) fill() error { if l._eof { return fmt.Errorf("locator id %d eof error", l._id) } var err error l.b = resizeBuffer(l.b, int(lobChunkSize)) l._size, err = l.r.Read(l.b) if err != nil && err != io.EOF { return err } l._eof = err == io.EOF return nil } // charLobReader (character based lobs - cesu8). type charLobReader struct { *baseLobReader c []byte ofs int } func newCharLobReader(r io.Reader, id locatorID) *charLobReader { return &charLobReader{ baseLobReader: &baseLobReader{r: r, _id: id}, } } func (l *charLobReader) fill() error { if l._eof { return fmt.Errorf("locator id %d eof error", l._id) } l.c = resizeBuffer(l.c, int(lobChunkSize)+l.ofs) n, err := l.r.Read(l.c[l.ofs:]) size := n + l.ofs if err != nil && err != io.EOF { return err } l._eof = err == io.EOF if l._eof && size == 0 { l._size = 0 return nil } l.b = resizeBuffer(l.b, cesu8.Size(l.c[:size])) // last rune might be incomplete, so size is one greater than needed nDst, nSrc, err := unicode.Utf8ToCesu8Transformer.Transform(l.b, l.c[:size], l._eof) if err != nil && err != transform.ErrShortSrc { return err } if l._eof && err == transform.ErrShortSrc { return unicode.ErrInvalidUtf8 } l._size = nDst l.ofs = size - nSrc if l.ofs > 0 { copy(l.c, l.c[nSrc:size]) // copy rest to buffer beginn } return nil } // helper func resizeBuffer(b1 []byte, size int) []byte { if b1 == nil || cap(b1) < size { b2 := make([]byte, size) copy(b2, b1) // !!! return b2 } return b1[:size] }