1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.DataOutputStream;
22 import java.io.IOException;
23 import java.net.InetSocketAddress;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.List;
27
28 import org.apache.hadoop.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FSDataOutputStream;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.fs.permission.FsPermission;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.KeyValue.KVComparator;
37 import org.apache.hadoop.hbase.io.compress.Compression;
38 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.apache.hadoop.hbase.util.FSUtils;
41 import org.apache.hadoop.io.Writable;
42
43
44
45
46 @InterfaceAudience.Private
47 public abstract class AbstractHFileWriter implements HFile.Writer {
48
49
50 protected byte[] lastKeyBuffer = null;
51
52 protected int lastKeyOffset = -1;
53 protected int lastKeyLength = -1;
54
55
56 protected FSDataOutputStream outputStream;
57
58
59 protected final boolean closeOutputStream;
60
61
62 protected FileInfo fileInfo = new HFile.FileInfo();
63
64
65 protected final int blockSize;
66
67
68 protected long entryCount = 0;
69
70
71 protected long totalKeyLength = 0;
72
73
74 protected long totalValueLength = 0;
75
76
77 protected long totalUncompressedBytes = 0;
78
79
80 protected final KVComparator comparator;
81
82
83 protected List<byte[]> metaNames = new ArrayList<byte[]>();
84
85
86 protected List<Writable> metaData = new ArrayList<Writable>();
87
88
89 protected final Compression.Algorithm compressAlgo;
90
91
92
93
94
95 protected final HFileDataBlockEncoder blockEncoder;
96
97
98 protected byte[] firstKeyInBlock = null;
99
100
101 protected final Path path;
102
103
104
105 protected final CacheConfig cacheConf;
106
107
108
109
110
111 protected final String name;
112
113 public AbstractHFileWriter(CacheConfig cacheConf,
114 FSDataOutputStream outputStream, Path path, int blockSize,
115 Compression.Algorithm compressAlgo,
116 HFileDataBlockEncoder dataBlockEncoder,
117 KVComparator comparator) {
118 this.outputStream = outputStream;
119 this.path = path;
120 this.name = path != null ? path.getName() : outputStream.toString();
121 this.blockSize = blockSize;
122 this.compressAlgo = compressAlgo == null
123 ? HFile.DEFAULT_COMPRESSION_ALGORITHM : compressAlgo;
124 this.blockEncoder = dataBlockEncoder != null
125 ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
126 this.comparator = comparator != null ? comparator
127 : KeyValue.COMPARATOR;
128
129 closeOutputStream = path != null;
130 this.cacheConf = cacheConf;
131 }
132
133
134
135
136 protected void finishFileInfo() throws IOException {
137 if (lastKeyBuffer != null) {
138
139
140 fileInfo.append(FileInfo.LASTKEY, Arrays.copyOfRange(lastKeyBuffer,
141 lastKeyOffset, lastKeyOffset + lastKeyLength), false);
142 }
143
144
145 int avgKeyLen =
146 entryCount == 0 ? 0 : (int) (totalKeyLength / entryCount);
147 fileInfo.append(FileInfo.AVG_KEY_LEN, Bytes.toBytes(avgKeyLen), false);
148
149
150 int avgValueLen =
151 entryCount == 0 ? 0 : (int) (totalValueLength / entryCount);
152 fileInfo.append(FileInfo.AVG_VALUE_LEN, Bytes.toBytes(avgValueLen), false);
153 }
154
155
156
157
158
159
160
161
162
163 @Override
164 public void appendFileInfo(final byte[] k, final byte[] v)
165 throws IOException {
166 fileInfo.append(k, v, true);
167 }
168
169
170
171
172
173
174
175
176
177
178
179 protected final void writeFileInfo(FixedFileTrailer trailer, DataOutputStream out)
180 throws IOException {
181 trailer.setFileInfoOffset(outputStream.getPos());
182 finishFileInfo();
183 fileInfo.write(out);
184 }
185
186
187
188
189
190
191
192
193 protected boolean checkKey(final byte[] key, final int offset,
194 final int length) throws IOException {
195 boolean isDuplicateKey = false;
196
197 if (key == null || length <= 0) {
198 throw new IOException("Key cannot be null or empty");
199 }
200 if (lastKeyBuffer != null) {
201 int keyComp = comparator.compareFlatKey(lastKeyBuffer, lastKeyOffset,
202 lastKeyLength, key, offset, length);
203
204 if (keyComp > 0) {
205 throw new IOException("Added a key not lexically larger than"
206 + " previous key="
207 + Bytes.toStringBinary(key, offset, length)
208 + ", lastkey="
209 + Bytes.toStringBinary(lastKeyBuffer, lastKeyOffset,
210 lastKeyLength));
211 } else if (keyComp == 0) {
212 isDuplicateKey = true;
213 }
214 }
215 return isDuplicateKey;
216 }
217
218
219 protected void checkValue(final byte[] value, final int offset,
220 final int length) throws IOException {
221 if (value == null) {
222 throw new IOException("Value cannot be null");
223 }
224 }
225
226
227
228
229 @Override
230 public Path getPath() {
231 return path;
232 }
233
234 @Override
235 public String toString() {
236 return "writer=" + (path != null ? path.toString() : null) + ", name="
237 + name + ", compression=" + compressAlgo.getName();
238 }
239
240
241
242
243
244 protected void finishClose(FixedFileTrailer trailer) throws IOException {
245 trailer.setMetaIndexCount(metaNames.size());
246 trailer.setTotalUncompressedBytes(totalUncompressedBytes+ trailer.getTrailerSize());
247 trailer.setEntryCount(entryCount);
248 trailer.setCompressionCodec(compressAlgo);
249
250 trailer.serialize(outputStream);
251
252 if (closeOutputStream) {
253 outputStream.close();
254 outputStream = null;
255 }
256 }
257
258 public static Compression.Algorithm compressionByName(String algoName) {
259 if (algoName == null)
260 return HFile.DEFAULT_COMPRESSION_ALGORITHM;
261 return Compression.getCompressionAlgorithmByName(algoName);
262 }
263
264
265 protected static FSDataOutputStream createOutputStream(Configuration conf,
266 FileSystem fs, Path path, InetSocketAddress[] favoredNodes) throws IOException {
267 FsPermission perms = FSUtils.getFilePermissions(fs, conf,
268 HConstants.DATA_FILE_UMASK_KEY);
269 return FSUtils.create(fs, path, perms, favoredNodes);
270 }
271 }