block_stream.go 4.85 KB
Newer Older
manish's avatar
manish committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fsblkstorage

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"

	"github.com/golang/protobuf/proto"
)

// ErrUnexpectedEndOfBlockfile error used to indicate an unexpected end of a file segment
// this can happen mainly if a crash occurs during appening a block and partial block contents
// get written towards the end of the file
var ErrUnexpectedEndOfBlockfile = errors.New("unexpected end of blockfile")

// blockfileStream reads blocks sequentially from a single file.
// It starts from the given offset and can traverse till the end of the file
type blockfileStream struct {
	file          *os.File
	reader        *bufio.Reader
	currentOffset int64
}

// blockStream reads blocks sequentially from multiple files.
// it starts from a given file offset and continues with the next
// file segment until the end of the last segment (`endFileNum`)
type blockStream struct {
	rootDir           string
	currentFileNum    int
	endFileNum        int
	currentFileStream *blockfileStream
}

///////////////////////////////////
// blockfileStream functions
////////////////////////////////////
func newBlockfileStream(filePath string, startOffset int64) (*blockfileStream, error) {
	logger.Debugf("newBlockfileStream(): filePath=[%s], startOffset=[%d]", filePath, startOffset)
	var file *os.File
	var err error
	if file, err = os.OpenFile(filePath, os.O_RDONLY, 0600); err != nil {
		return nil, err
	}
	var newPosition int64
	if newPosition, err = file.Seek(startOffset, 0); err != nil {
		// file.Seek does not raise an error - simply seeks to the new position
		return nil, err
	}
	if newPosition != startOffset {
		panic(fmt.Sprintf("Could not seek file [%s] to given startOffset [%d]. New position = [%d]",
			filePath, startOffset, newPosition))
	}
	s := &blockfileStream{file, bufio.NewReader(file), startOffset}
	return s, nil
}

func (s *blockfileStream) nextBlockBytes() ([]byte, error) {
	var lenBytes []byte
	var err error
	if lenBytes, err = s.reader.Peek(8); err != nil {
		// reader.Peek raises io.EOF error if enough bytes not available
		if err == io.EOF {
			if len(lenBytes) > 0 {
				return nil, ErrUnexpectedEndOfBlockfile
			}
			return nil, nil
		}
		return nil, err
	}
	len, n := proto.DecodeVarint(lenBytes)
	if n == 0 {
		panic(fmt.Errorf("Error in decoding varint bytes"))
	}
	if _, err = s.reader.Discard(n); err != nil {
		return nil, err
	}
	blockBytes := make([]byte, len)
	if _, err = io.ReadAtLeast(s.reader, blockBytes, int(len)); err != nil {
		// io.ReadAtLeast raises io.ErrUnexpectedEOF error if it is able to
		// read a fewer (non-zero) bytes and io.EOF is encountered
		if err == io.ErrUnexpectedEOF {
			return nil, ErrUnexpectedEndOfBlockfile
		}
		return nil, err
	}
	s.currentOffset += int64(n) + int64(len)
	return blockBytes, nil
}

func (s *blockfileStream) close() error {
	return s.file.Close()
}

///////////////////////////////////
// blockStream functions
////////////////////////////////////
func newBlockStream(rootDir string, startFileNum int, startOffset int64, endFileNum int) (*blockStream, error) {
	startFile := deriveBlockfilePath(rootDir, startFileNum)
	startFileStream, err := newBlockfileStream(startFile, startOffset)
	if err != nil {
		return nil, err
	}
	return &blockStream{rootDir, startFileNum, endFileNum, startFileStream}, nil
}

func (s *blockStream) moveToNextBlockfileStream() error {
	var err error
	if err = s.currentFileStream.close(); err != nil {
		return err
	}
	s.currentFileNum++
	nextFile := deriveBlockfilePath(s.rootDir, s.currentFileNum)
	if s.currentFileStream, err = newBlockfileStream(nextFile, 0); err != nil {
		return err
	}
	return nil
}

func (s *blockStream) nextBlockBytes() ([]byte, error) {
	var blockBytes []byte
	var err error
	if blockBytes, err = s.currentFileStream.nextBlockBytes(); err != nil {
		logger.Debugf("current file [%d]", s.currentFileNum)
		logger.Debugf("blockbytes [%d]. Err:%s", len(blockBytes), err)
		return nil, err
	}
	logger.Debugf("blockbytes [%d] read from file [%d]", len(blockBytes), s.currentFileNum)
	if blockBytes == nil && s.currentFileNum < s.endFileNum {
		logger.Debugf("current file [%d] exhausted. Moving to next file", s.currentFileNum)
		if err = s.moveToNextBlockfileStream(); err != nil {
			return nil, err
		}
		return s.nextBlockBytes()
	}
	return blockBytes, nil
}

func (s *blockStream) close() error {
	return s.currentFileStream.close()
}