1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.DataOutputStream;
22 import java.io.IOException;
23 import java.net.InetSocketAddress;
24 import java.util.ArrayList;
25 import java.util.List;
26
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.FSDataOutputStream;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.fs.permission.FsPermission;
32 import org.apache.hadoop.hbase.Cell;
33 import org.apache.hadoop.hbase.CellUtil;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.KeyValue.KVComparator;
37 import org.apache.hadoop.hbase.classification.InterfaceAudience;
38 import org.apache.hadoop.hbase.io.compress.Compression;
39 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.hbase.util.FSUtils;
43 import org.apache.hadoop.io.Writable;
44
45
46
47
48 @InterfaceAudience.Private
49 public abstract class AbstractHFileWriter implements HFile.Writer {
50
51
52 protected Cell lastCell = null;
53
54
55 protected FSDataOutputStream outputStream;
56
57
58 protected final boolean closeOutputStream;
59
60
61 protected FileInfo fileInfo = new HFile.FileInfo();
62
63
64 protected long entryCount = 0;
65
66
67 protected long totalKeyLength = 0;
68
69
70 protected long totalValueLength = 0;
71
72
73 protected long totalUncompressedBytes = 0;
74
75
76 protected final KVComparator comparator;
77
78
79 protected List<byte[]> metaNames = new ArrayList<byte[]>();
80
81
82 protected List<Writable> metaData = new ArrayList<Writable>();
83
84
85
86
87
88 protected Cell firstCellInBlock = null;
89
90
91 protected final Path path;
92
93
94
95 protected final CacheConfig cacheConf;
96
97
98
99
100
101 protected final String name;
102
103
104
105
106
107 protected final HFileDataBlockEncoder blockEncoder;
108
109 protected final HFileContext hFileContext;
110
111 public AbstractHFileWriter(CacheConfig cacheConf,
112 FSDataOutputStream outputStream, Path path,
113 KVComparator comparator, HFileContext fileContext) {
114 this.outputStream = outputStream;
115 this.path = path;
116 this.name = path != null ? path.getName() : outputStream.toString();
117 this.hFileContext = fileContext;
118 DataBlockEncoding encoding = hFileContext.getDataBlockEncoding();
119 if (encoding != DataBlockEncoding.NONE) {
120 this.blockEncoder = new HFileDataBlockEncoderImpl(encoding);
121 } else {
122 this.blockEncoder = NoOpDataBlockEncoder.INSTANCE;
123 }
124 this.comparator = comparator != null ? comparator
125 : KeyValue.COMPARATOR;
126
127 closeOutputStream = path != null;
128 this.cacheConf = cacheConf;
129 }
130
131
132
133
134 protected void finishFileInfo() throws IOException {
135 if (lastCell != null) {
136
137
138 byte [] lastKey = CellUtil.getCellKeySerializedAsKeyValueKey(this.lastCell);
139 fileInfo.append(FileInfo.LASTKEY, lastKey, false);
140 }
141
142
143 int avgKeyLen =
144 entryCount == 0 ? 0 : (int) (totalKeyLength / entryCount);
145 fileInfo.append(FileInfo.AVG_KEY_LEN, Bytes.toBytes(avgKeyLen), false);
146
147
148 int avgValueLen =
149 entryCount == 0 ? 0 : (int) (totalValueLength / entryCount);
150 fileInfo.append(FileInfo.AVG_VALUE_LEN, Bytes.toBytes(avgValueLen), false);
151 }
152
153
154
155
156
157
158
159
160
161 @Override
162 public void appendFileInfo(final byte[] k, final byte[] v)
163 throws IOException {
164 fileInfo.append(k, v, true);
165 }
166
167
168
169
170
171
172
173
174
175
176
177 protected final void writeFileInfo(FixedFileTrailer trailer, DataOutputStream out)
178 throws IOException {
179 trailer.setFileInfoOffset(outputStream.getPos());
180 finishFileInfo();
181 fileInfo.write(out);
182 }
183
184
185
186
187
188
189
190
191 protected boolean checkKey(final Cell cell) throws IOException {
192 boolean isDuplicateKey = false;
193
194 if (cell == null) {
195 throw new IOException("Key cannot be null or empty");
196 }
197 if (lastCell != null) {
198 int keyComp = comparator.compareOnlyKeyPortion(lastCell, cell);
199
200 if (keyComp > 0) {
201 throw new IOException("Added a key not lexically larger than"
202 + " previous. Current cell = " + cell + ", lastCell = " + lastCell);
203 } else if (keyComp == 0) {
204 isDuplicateKey = true;
205 }
206 }
207 return isDuplicateKey;
208 }
209
210
211 protected void checkValue(final byte[] value, final int offset,
212 final int length) throws IOException {
213 if (value == null) {
214 throw new IOException("Value cannot be null");
215 }
216 }
217
218
219
220
221 @Override
222 public Path getPath() {
223 return path;
224 }
225
226 @Override
227 public String toString() {
228 return "writer=" + (path != null ? path.toString() : null) + ", name="
229 + name + ", compression=" + hFileContext.getCompression().getName();
230 }
231
232
233
234
235
236 protected void finishClose(FixedFileTrailer trailer) throws IOException {
237 trailer.setMetaIndexCount(metaNames.size());
238 trailer.setTotalUncompressedBytes(totalUncompressedBytes+ trailer.getTrailerSize());
239 trailer.setEntryCount(entryCount);
240 trailer.setCompressionCodec(hFileContext.getCompression());
241
242 trailer.serialize(outputStream);
243
244 if (closeOutputStream) {
245 outputStream.close();
246 outputStream = null;
247 }
248 }
249
250 public static Compression.Algorithm compressionByName(String algoName) {
251 if (algoName == null)
252 return HFile.DEFAULT_COMPRESSION_ALGORITHM;
253 return Compression.getCompressionAlgorithmByName(algoName);
254 }
255
256
257 protected static FSDataOutputStream createOutputStream(Configuration conf,
258 FileSystem fs, Path path, InetSocketAddress[] favoredNodes) throws IOException {
259 FsPermission perms = FSUtils.getFilePermissions(fs, conf,
260 HConstants.DATA_FILE_UMASK_KEY);
261 return FSUtils.create(fs, path, perms, favoredNodes);
262 }
263 }