View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.GZ;
22  import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.NONE;
23  import static org.junit.Assert.assertEquals;
24  import static org.junit.Assert.assertTrue;
25  import static org.junit.Assert.fail;
26  
27  import java.io.ByteArrayOutputStream;
28  import java.io.DataOutputStream;
29  import java.io.IOException;
30  import java.io.OutputStream;
31  import java.nio.ByteBuffer;
32  import java.util.ArrayList;
33  import java.util.Collection;
34  import java.util.List;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.hadoop.fs.FSDataInputStream;
39  import org.apache.hadoop.fs.FSDataOutputStream;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.hbase.HBaseTestingUtility;
42  import org.apache.hadoop.hbase.HConstants;
43  import org.apache.hadoop.hbase.SmallTests;
44  import org.apache.hadoop.hbase.fs.HFileSystem;
45  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
46  import org.apache.hadoop.hbase.io.compress.Compression;
47  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
48  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
49  import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
50  import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.apache.hadoop.hbase.util.ChecksumType;
53  import org.apache.hadoop.io.compress.Compressor;
54  import org.junit.Before;
55  import org.junit.Test;
56  import org.junit.experimental.categories.Category;
57  import org.junit.runner.RunWith;
58  import org.junit.runners.Parameterized;
59  import org.junit.runners.Parameterized.Parameters;
60  
61  import com.google.common.base.Preconditions;
62  
63  /**
64   * This class has unit tests to prove that older versions of
65   * HFiles (without checksums) are compatible with current readers.
66   */
67  @Category(SmallTests.class)
68  @RunWith(Parameterized.class)
69  public class TestHFileBlockCompatibility {
70    // change this value to activate more logs
71    private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
72  
73    private static final Log LOG = LogFactory.getLog(TestHFileBlockCompatibility.class);
74  
75    private static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
76        NONE, GZ };
77  
78    // The mnior version for pre-checksum files
79    private static int MINOR_VERSION = 0;
80  
81    private static final HBaseTestingUtility TEST_UTIL =
82      new HBaseTestingUtility();
83    private HFileSystem fs;
84    private int uncompressedSizeV1;
85  
86    private final boolean includesMemstoreTS;
87  
88    public TestHFileBlockCompatibility(boolean includesMemstoreTS) {
89      this.includesMemstoreTS = includesMemstoreTS;
90    }
91  
92    @Parameters
93    public static Collection<Object[]> parameters() {
94      return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
95    }
96  
97    @Before
98    public void setUp() throws IOException {
99      fs = (HFileSystem)HFileSystem.get(TEST_UTIL.getConfiguration());
100   }
101 
102   public byte[] createTestV1Block(Compression.Algorithm algo)
103       throws IOException {
104     Compressor compressor = algo.getCompressor();
105     ByteArrayOutputStream baos = new ByteArrayOutputStream();
106     OutputStream os = algo.createCompressionStream(baos, compressor, 0);
107     DataOutputStream dos = new DataOutputStream(os);
108     BlockType.META.write(dos); // Let's make this a meta block.
109     TestHFileBlock.writeTestBlockContents(dos);
110     uncompressedSizeV1 = dos.size();
111     dos.flush();
112     algo.returnCompressor(compressor);
113     return baos.toByteArray();
114   }
115 
116   private Writer createTestV2Block(Compression.Algorithm algo)
117       throws IOException {
118     final BlockType blockType = BlockType.DATA;
119     Writer hbw = new Writer(algo, null,
120         includesMemstoreTS);
121     DataOutputStream dos = hbw.startWriting(blockType);
122     TestHFileBlock.writeTestBlockContents(dos);
123     // make sure the block is ready by calling hbw.getHeaderAndData()
124     hbw.getHeaderAndData();
125     assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
126     hbw.releaseCompressor();
127     return hbw;
128   }
129 
130  private String createTestBlockStr(Compression.Algorithm algo,
131       int correctLength) throws IOException {
132     Writer hbw = createTestV2Block(algo);
133     byte[] testV2Block = hbw.getHeaderAndData();
134     int osOffset = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM + 9;
135     if (testV2Block.length == correctLength) {
136       // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
137       // variations across operating systems.
138       // See http://www.gzip.org/zlib/rfc-gzip.html for gzip format.
139       testV2Block[osOffset] = 3;
140     }
141     return Bytes.toStringBinary(testV2Block);
142   }
143 
144   @Test
145   public void testNoCompression() throws IOException {
146     assertEquals(4000, createTestV2Block(NONE).getBlockForCaching().
147                        getUncompressedSizeWithoutHeader());
148   }
149 
150   @Test
151   public void testGzipCompression() throws IOException {
152     final String correctTestBlockStr =
153         "DATABLK*\\x00\\x00\\x00:\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
154             + "\\xFF\\xFF\\xFF\\xFF"
155             // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html
156             + "\\x1F\\x8B"  // gzip magic signature
157             + "\\x08"  // Compression method: 8 = "deflate"
158             + "\\x00"  // Flags
159             + "\\x00\\x00\\x00\\x00"  // mtime
160             + "\\x00"  // XFL (extra flags)
161             // OS (0 = FAT filesystems, 3 = Unix). However, this field
162             // sometimes gets set to 0 on Linux and Mac, so we reset it to 3.
163             + "\\x03"
164             + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
165             + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
166             + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00";
167     final int correctGzipBlockLength = 82;
168 
169     String returnedStr = createTestBlockStr(GZ, correctGzipBlockLength);
170     assertEquals(correctTestBlockStr, returnedStr);
171   }
172 
173   @Test
174   public void testReaderV2() throws IOException {
175     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
176       for (boolean pread : new boolean[] { false, true }) {
177           LOG.info("testReaderV2: Compression algorithm: " + algo +
178                    ", pread=" + pread);
179         Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
180             + algo);
181         FSDataOutputStream os = fs.create(path);
182         Writer hbw = new Writer(algo, null,
183             includesMemstoreTS);
184         long totalSize = 0;
185         for (int blockId = 0; blockId < 2; ++blockId) {
186           DataOutputStream dos = hbw.startWriting(BlockType.DATA);
187           for (int i = 0; i < 1234; ++i)
188             dos.writeInt(i);
189           hbw.writeHeaderAndData(os);
190           totalSize += hbw.getOnDiskSizeWithHeader();
191         }
192         os.close();
193 
194         FSDataInputStream is = fs.open(path);
195         HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
196             algo, totalSize, MINOR_VERSION, fs, path);
197         HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
198         is.close();
199 
200         b.sanityCheck();
201         assertEquals(4936, b.getUncompressedSizeWithoutHeader());
202         assertEquals(algo == GZ ? 2173 : 4936,
203                      b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
204         String blockStr = b.toString();
205 
206         if (algo == GZ) {
207           is = fs.open(path);
208           hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
209               algo, totalSize, MINOR_VERSION, fs, path);
210           b = hbr.readBlockData(0, 2173 + HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM +
211                                 b.totalChecksumBytes(), -1, pread);
212           assertEquals(blockStr, b.toString());
213           int wrongCompressedSize = 2172;
214           try {
215             b = hbr.readBlockData(0, wrongCompressedSize
216                 + HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM, -1, pread);
217             fail("Exception expected");
218           } catch (IOException ex) {
219             String expectedPrefix = "On-disk size without header provided is "
220                 + wrongCompressedSize + ", but block header contains "
221                 + b.getOnDiskSizeWithoutHeader() + ".";
222             assertTrue("Invalid exception message: '" + ex.getMessage()
223                 + "'.\nMessage is expected to start with: '" + expectedPrefix
224                 + "'", ex.getMessage().startsWith(expectedPrefix));
225           }
226           is.close();
227         }
228       }
229     }
230   }
231 
232   /**
233    * Test encoding/decoding data blocks.
234    * @throws IOException a bug or a problem with temporary files.
235    */
236   @Test
237   public void testDataBlockEncoding() throws IOException {
238     final int numBlocks = 5;
239     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
240       for (boolean pread : new boolean[] { false, true }) {
241         for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
242           LOG.info("testDataBlockEncoding algo " + algo +
243                    " pread = " + pread +
244                    " encoding " + encoding);
245           Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
246               + algo + "_" + encoding.toString());
247           FSDataOutputStream os = fs.create(path);
248           HFileDataBlockEncoder dataBlockEncoder =
249               new HFileDataBlockEncoderImpl(encoding);
250           TestHFileBlockCompatibility.Writer hbw =
251               new TestHFileBlockCompatibility.Writer(algo,
252                   dataBlockEncoder, includesMemstoreTS);
253           long totalSize = 0;
254           final List<Integer> encodedSizes = new ArrayList<Integer>();
255           final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
256           for (int blockId = 0; blockId < numBlocks; ++blockId) {
257             DataOutputStream dos = hbw.startWriting(BlockType.DATA);
258             TestHFileBlock.writeEncodedBlock(algo, encoding, dos, encodedSizes,
259                 encodedBlocks, blockId, includesMemstoreTS,
260                 TestHFileBlockCompatibility.Writer.DUMMY_HEADER);
261 
262             hbw.writeHeaderAndData(os);
263             totalSize += hbw.getOnDiskSizeWithHeader();
264           }
265           os.close();
266 
267           FSDataInputStream is = fs.open(path);
268           HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
269               algo, totalSize, MINOR_VERSION, fs, path);
270           hbr.setDataBlockEncoder(dataBlockEncoder);
271           hbr.setIncludesMemstoreTS(includesMemstoreTS);
272 
273           HFileBlock b;
274           int pos = 0;
275           for (int blockId = 0; blockId < numBlocks; ++blockId) {
276             b = hbr.readBlockData(pos, -1, -1, pread);
277             b.sanityCheck();
278             pos += b.getOnDiskSizeWithHeader();
279 
280             assertEquals((int) encodedSizes.get(blockId),
281                 b.getUncompressedSizeWithoutHeader());
282             ByteBuffer actualBuffer = b.getBufferWithoutHeader();
283             if (encoding != DataBlockEncoding.NONE) {
284               // We expect a two-byte big-endian encoding id.
285               assertEquals(0, actualBuffer.get(0));
286               assertEquals(encoding.getId(), actualBuffer.get(1));
287               actualBuffer.position(2);
288               actualBuffer = actualBuffer.slice();
289             }
290 
291             ByteBuffer expectedBuffer = encodedBlocks.get(blockId);
292             expectedBuffer.rewind();
293 
294             // test if content matches, produce nice message
295             TestHFileBlock.assertBuffersEqual(expectedBuffer, actualBuffer,
296               algo, encoding, pread);
297           }
298           is.close();
299         }
300       }
301     }
302   }
303 
304 
305 
306   /**
307    * This is the version of the HFileBlock.Writer that is used to
308    * create V2 blocks with minor version 0. These blocks do not
309    * have hbase-level checksums. The code is here to test
310    * backward compatibility. The reason we do not inherit from
311    * HFileBlock.Writer is because we never ever want to change the code
312    * in this class but the code in HFileBlock.Writer will continually
313    * evolve.
314    */
315   public static final class Writer {
316 
317     // These constants are as they were in minorVersion 0.
318     private static final int HEADER_SIZE = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;
319     private static final boolean DONT_FILL_HEADER = HFileBlock.DONT_FILL_HEADER;
320     private static final byte[] DUMMY_HEADER =
321       HFileBlock.DUMMY_HEADER_NO_CHECKSUM;
322 
323     private enum State {
324       INIT,
325       WRITING,
326       BLOCK_READY
327     };
328 
329     /** Writer state. Used to ensure the correct usage protocol. */
330     private State state = State.INIT;
331 
332     /** Compression algorithm for all blocks this instance writes. */
333     private final Compression.Algorithm compressAlgo;
334 
335     /** Data block encoder used for data blocks */
336     private final HFileDataBlockEncoder dataBlockEncoder;
337 
338     private HFileBlockEncodingContext dataBlockEncodingCtx;
339     /** block encoding context for non-data blocks */
340     private HFileBlockDefaultEncodingContext defaultBlockEncodingCtx;
341 
342     /**
343      * The stream we use to accumulate data in uncompressed format for each
344      * block. We reset this stream at the end of each block and reuse it. The
345      * header is written as the first {@link #HEADER_SIZE} bytes into this
346      * stream.
347      */
348     private ByteArrayOutputStream baosInMemory;
349 
350     /** Compressor, which is also reused between consecutive blocks. */
351     private Compressor compressor;
352 
353     /**
354      * Current block type. Set in {@link #startWriting(BlockType)}. Could be
355      * changed in {@link #encodeDataBlockForDisk()} from {@link BlockType#DATA}
356      * to {@link BlockType#ENCODED_DATA}.
357      */
358     private BlockType blockType;
359 
360     /**
361      * A stream that we write uncompressed bytes to, which compresses them and
362      * writes them to {@link #baosInMemory}.
363      */
364     private DataOutputStream userDataStream;
365 
366     /**
367      * Bytes to be written to the file system, including the header. Compressed
368      * if compression is turned on.
369      */
370     private byte[] onDiskBytesWithHeader;
371 
372     /**
373      * Valid in the READY state. Contains the header and the uncompressed (but
374      * potentially encoded, if this is a data block) bytes, so the length is
375      * {@link #uncompressedSizeWithoutHeader} + {@link org.apache.hadoop.hbase.HConstants#HFILEBLOCK_HEADER_SIZE}.
376      */
377     private byte[] uncompressedBytesWithHeader;
378 
379     /**
380      * Current block's start offset in the {@link HFile}. Set in
381      * {@link #writeHeaderAndData(FSDataOutputStream)}.
382      */
383     private long startOffset;
384 
385     /**
386      * Offset of previous block by block type. Updated when the next block is
387      * started.
388      */
389     private long[] prevOffsetByType;
390 
391     /** The offset of the previous block of the same type */
392     private long prevOffset;
393 
394     /** Whether we are including memstore timestamp after every key/value */
395     private boolean includesMemstoreTS;
396 
397     /**
398      * @param compressionAlgorithm compression algorithm to use
399      * @param dataBlockEncoderAlgo data block encoding algorithm to use
400      */
401     public Writer(Compression.Algorithm compressionAlgorithm,
402           HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) {
403       compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
404       this.dataBlockEncoder = dataBlockEncoder != null
405           ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
406 
407       defaultBlockEncodingCtx =
408           new HFileBlockDefaultEncodingContext(compressionAlgorithm,
409               null, DUMMY_HEADER);
410       dataBlockEncodingCtx =
411         this.dataBlockEncoder.newDataBlockEncodingContext(
412             compressionAlgorithm, DUMMY_HEADER);
413 
414       baosInMemory = new ByteArrayOutputStream();
415 
416       prevOffsetByType = new long[BlockType.values().length];
417       for (int i = 0; i < prevOffsetByType.length; ++i)
418         prevOffsetByType[i] = -1;
419 
420       this.includesMemstoreTS = includesMemstoreTS;
421     }
422 
423     /**
424      * Starts writing into the block. The previous block's data is discarded.
425      *
426      * @return the stream the user can write their data into
427      * @throws IOException
428      */
429     public DataOutputStream startWriting(BlockType newBlockType)
430         throws IOException {
431       if (state == State.BLOCK_READY && startOffset != -1) {
432         // We had a previous block that was written to a stream at a specific
433         // offset. Save that offset as the last offset of a block of that type.
434         prevOffsetByType[blockType.getId()] = startOffset;
435       }
436 
437       startOffset = -1;
438       blockType = newBlockType;
439 
440       baosInMemory.reset();
441       baosInMemory.write(DUMMY_HEADER);
442 
443       state = State.WRITING;
444 
445       // We will compress it later in finishBlock()
446       userDataStream = new DataOutputStream(baosInMemory);
447       return userDataStream;
448     }
449 
450     /**
451      * Returns the stream for the user to write to. The block writer takes care
452      * of handling compression and buffering for caching on write. Can only be
453      * called in the "writing" state.
454      *
455      * @return the data output stream for the user to write to
456      */
457     DataOutputStream getUserDataStream() {
458       expectState(State.WRITING);
459       return userDataStream;
460     }
461 
462     /**
463      * Transitions the block writer from the "writing" state to the "block
464      * ready" state.  Does nothing if a block is already finished.
465      */
466     private void ensureBlockReady() throws IOException {
467       Preconditions.checkState(state != State.INIT,
468           "Unexpected state: " + state);
469 
470       if (state == State.BLOCK_READY)
471         return;
472 
473       // This will set state to BLOCK_READY.
474       finishBlock();
475     }
476 
477     /**
478      * An internal method that flushes the compressing stream (if using
479      * compression), serializes the header, and takes care of the separate
480      * uncompressed stream for caching on write, if applicable. Sets block
481      * write state to "block ready".
482      */
483     private void finishBlock() throws IOException {
484       userDataStream.flush();
485       // This does an array copy, so it is safe to cache this byte array.
486       uncompressedBytesWithHeader = baosInMemory.toByteArray();
487       prevOffset = prevOffsetByType[blockType.getId()];
488 
489       // We need to set state before we can package the block up for
490       // cache-on-write. In a way, the block is ready, but not yet encoded or
491       // compressed.
492       state = State.BLOCK_READY;
493       if (blockType == BlockType.DATA) {
494         encodeDataBlockForDisk();
495       } else {
496         defaultBlockEncodingCtx.compressAfterEncodingWithBlockType(
497             uncompressedBytesWithHeader, blockType);
498         onDiskBytesWithHeader =
499           defaultBlockEncodingCtx.getOnDiskBytesWithHeader();
500       }
501 
502       // put the header for on disk bytes
503       putHeader(onDiskBytesWithHeader, 0,
504           onDiskBytesWithHeader.length,
505           uncompressedBytesWithHeader.length);
506       //set the header for the uncompressed bytes (for cache-on-write)
507       putHeader(uncompressedBytesWithHeader, 0,
508           onDiskBytesWithHeader.length,
509         uncompressedBytesWithHeader.length);
510     }
511 
512     /**
513      * Encodes this block if it is a data block and encoding is turned on in
514      * {@link #dataBlockEncoder}.
515      */
516     private void encodeDataBlockForDisk() throws IOException {
517       // do data block encoding, if data block encoder is set
518       ByteBuffer rawKeyValues =
519           ByteBuffer.wrap(uncompressedBytesWithHeader, HEADER_SIZE,
520               uncompressedBytesWithHeader.length - HEADER_SIZE).slice();
521 
522       //do the encoding
523       dataBlockEncoder.beforeWriteToDisk(rawKeyValues,
524               includesMemstoreTS, dataBlockEncodingCtx, blockType);
525 
526       uncompressedBytesWithHeader =
527           dataBlockEncodingCtx.getUncompressedBytesWithHeader();
528       onDiskBytesWithHeader =
529           dataBlockEncodingCtx.getOnDiskBytesWithHeader();
530       blockType = dataBlockEncodingCtx.getBlockType();
531     }
532 
533     /**
534      * Put the header into the given byte array at the given offset.
535      * @param onDiskSize size of the block on disk
536      * @param uncompressedSize size of the block after decompression (but
537      *          before optional data block decoding)
538      */
539     private void putHeader(byte[] dest, int offset, int onDiskSize,
540         int uncompressedSize) {
541       offset = blockType.put(dest, offset);
542       offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE);
543       offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE);
544       Bytes.putLong(dest, offset, prevOffset);
545     }
546 
547     /**
548      * Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records
549      * the offset of this block so that it can be referenced in the next block
550      * of the same type.
551      *
552      * @param out
553      * @throws IOException
554      */
555     public void writeHeaderAndData(FSDataOutputStream out) throws IOException {
556       long offset = out.getPos();
557       if (startOffset != -1 && offset != startOffset) {
558         throw new IOException("A " + blockType + " block written to a "
559             + "stream twice, first at offset " + startOffset + ", then at "
560             + offset);
561       }
562       startOffset = offset;
563 
564       writeHeaderAndData((DataOutputStream) out);
565     }
566 
567     /**
568      * Writes the header and the compressed data of this block (or uncompressed
569      * data when not using compression) into the given stream. Can be called in
570      * the "writing" state or in the "block ready" state. If called in the
571      * "writing" state, transitions the writer to the "block ready" state.
572      *
573      * @param out the output stream to write the
574      * @throws IOException
575      */
576     private void writeHeaderAndData(DataOutputStream out) throws IOException {
577       ensureBlockReady();
578       out.write(onDiskBytesWithHeader);
579     }
580 
581     /**
582      * Returns the header or the compressed data (or uncompressed data when not
583      * using compression) as a byte array. Can be called in the "writing" state
584      * or in the "block ready" state. If called in the "writing" state,
585      * transitions the writer to the "block ready" state.
586      *
587      * @return header and data as they would be stored on disk in a byte array
588      * @throws IOException
589      */
590     public byte[] getHeaderAndData() throws IOException {
591       ensureBlockReady();
592       return onDiskBytesWithHeader;
593     }
594 
595     /**
596      * Releases the compressor this writer uses to compress blocks into the
597      * compressor pool. Needs to be called before the writer is discarded.
598      */
599     public void releaseCompressor() {
600       if (compressor != null) {
601         compressAlgo.returnCompressor(compressor);
602         compressor = null;
603       }
604     }
605 
606     /**
607      * Returns the on-disk size of the data portion of the block. This is the
608      * compressed size if compression is enabled. Can only be called in the
609      * "block ready" state. Header is not compressed, and its size is not
610      * included in the return value.
611      *
612      * @return the on-disk size of the block, not including the header.
613      */
614     public int getOnDiskSizeWithoutHeader() {
615       expectState(State.BLOCK_READY);
616       return onDiskBytesWithHeader.length - HEADER_SIZE;
617     }
618 
619     /**
620      * Returns the on-disk size of the block. Can only be called in the
621      * "block ready" state.
622      *
623      * @return the on-disk size of the block ready to be written, including the
624      *         header size
625      */
626     public int getOnDiskSizeWithHeader() {
627       expectState(State.BLOCK_READY);
628       return onDiskBytesWithHeader.length;
629     }
630 
631     /**
632      * The uncompressed size of the block data. Does not include header size.
633      */
634     public int getUncompressedSizeWithoutHeader() {
635       expectState(State.BLOCK_READY);
636       return uncompressedBytesWithHeader.length - HEADER_SIZE;
637     }
638 
639     /**
640      * The uncompressed size of the block data, including header size.
641      */
642     public int getUncompressedSizeWithHeader() {
643       expectState(State.BLOCK_READY);
644       return uncompressedBytesWithHeader.length;
645     }
646 
647     /** @return true if a block is being written  */
648     public boolean isWriting() {
649       return state == State.WRITING;
650     }
651 
652     /**
653      * Returns the number of bytes written into the current block so far, or
654      * zero if not writing the block at the moment. Note that this will return
655      * zero in the "block ready" state as well.
656      *
657      * @return the number of bytes written
658      */
659     public int blockSizeWritten() {
660       if (state != State.WRITING)
661         return 0;
662       return userDataStream.size();
663     }
664 
665     /**
666      * Returns the header followed by the uncompressed data, even if using
667      * compression. This is needed for storing uncompressed blocks in the block
668      * cache. Can be called in the "writing" state or the "block ready" state.
669      *
670      * @return uncompressed block bytes for caching on write
671      */
672     private byte[] getUncompressedDataWithHeader() {
673       expectState(State.BLOCK_READY);
674 
675       return uncompressedBytesWithHeader;
676     }
677 
678     private void expectState(State expectedState) {
679       if (state != expectedState) {
680         throw new IllegalStateException("Expected state: " + expectedState +
681             ", actual state: " + state);
682       }
683     }
684 
685     /**
686      * Similar to {@link #getUncompressedBufferWithHeader()} but returns a byte
687      * buffer.
688      *
689      * @return uncompressed block for caching on write in the form of a buffer
690      */
691     public ByteBuffer getUncompressedBufferWithHeader() {
692       byte[] b = getUncompressedDataWithHeader();
693       return ByteBuffer.wrap(b, 0, b.length);
694     }
695 
696     /**
697      * Takes the given {@link BlockWritable} instance, creates a new block of
698      * its appropriate type, writes the writable into this block, and flushes
699      * the block into the output stream. The writer is instructed not to buffer
700      * uncompressed bytes for cache-on-write.
701      *
702      * @param bw the block-writable object to write as a block
703      * @param out the file system output stream
704      * @throws IOException
705      */
706     public void writeBlock(BlockWritable bw, FSDataOutputStream out)
707         throws IOException {
708       bw.writeToBlock(startWriting(bw.getBlockType()));
709       writeHeaderAndData(out);
710     }
711 
712     /**
713      * Creates a new HFileBlock.
714      */
715     public HFileBlock getBlockForCaching() {
716       return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(),
717           getUncompressedSizeWithoutHeader(), prevOffset,
718           getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset,
719           includesMemstoreTS, MINOR_VERSION, 0, ChecksumType.NULL.getCode(),
720           getOnDiskSizeWithoutHeader());
721     }
722   }
723 
724 }
725