1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.io.encoding;
18  
19  import static org.junit.Assert.assertEquals;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.Collections;
25  import java.util.List;
26  import java.util.Random;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HColumnDescriptor;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.LargeTests;
37  import org.apache.hadoop.hbase.client.Durability;
38  import org.apache.hadoop.hbase.client.Get;
39  import org.apache.hadoop.hbase.client.HBaseAdmin;
40  import org.apache.hadoop.hbase.client.HTable;
41  import org.apache.hadoop.hbase.client.Put;
42  import org.apache.hadoop.hbase.client.Result;
43  import org.apache.hadoop.hbase.regionserver.HRegionServer;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.Threads;
46  import org.junit.After;
47  import org.junit.AfterClass;
48  import org.junit.Before;
49  import org.junit.BeforeClass;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  
53  /**
54   * Tests changing data block encoding settings of a column family.
55   */
56  @Category(LargeTests.class)
57  public class TestChangingEncoding {
58  
59    private static final Log LOG = LogFactory.getLog(TestChangingEncoding.class);
60  
61    static final String CF = "EncodingTestCF";
62    static final byte[] CF_BYTES = Bytes.toBytes(CF);
63  
64    private static final int NUM_ROWS_PER_BATCH = 100;
65    private static final int NUM_COLS_PER_ROW = 20;
66  
67    private static final HBaseTestingUtility TEST_UTIL =
68        new HBaseTestingUtility();
69    private static final Configuration conf = TEST_UTIL.getConfiguration();
70  
71    private static final int TIMEOUT_MS = 600000;
72  
73    private HBaseAdmin admin;
74    private HColumnDescriptor hcd;
75  
76    private String tableName;
77    private static final List<DataBlockEncoding> ENCODINGS_TO_ITERATE =
78        createEncodingsToIterate();
79  
80    private static final List<DataBlockEncoding> createEncodingsToIterate() {
81      List<DataBlockEncoding> encodings = new ArrayList<DataBlockEncoding>(
82          Arrays.asList(DataBlockEncoding.values()));
83      encodings.add(DataBlockEncoding.NONE);
84      return Collections.unmodifiableList(encodings);
85    }
86  
87    /** A zero-based index of the current batch of test data being written */
88    private int numBatchesWritten;
89  
90    private void prepareTest(String testId) throws IOException {
91      tableName = "test_table_" + testId;
92      HTableDescriptor htd = new HTableDescriptor(tableName);
93      hcd = new HColumnDescriptor(CF);
94      htd.addFamily(hcd);
95      admin.createTable(htd);
96      numBatchesWritten = 0;
97    }
98  
99    @BeforeClass
100   public static void setUpBeforeClass() throws Exception {
101     // Use a small flush size to create more HFiles.
102     conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
103     TEST_UTIL.startMiniCluster();
104   }
105 
106   @AfterClass
107   public static void tearDownAfterClass() throws Exception {
108     TEST_UTIL.shutdownMiniCluster();
109   }
110 
111   @Before
112   public void setUp() throws Exception {
113     admin = new HBaseAdmin(conf);
114   }
115 
116   @After
117   public void tearDown() throws IOException {
118     admin.close();
119   }
120 
121   private static byte[] getRowKey(int batchId, int i) {
122     return Bytes.toBytes("batch" + batchId + "_row" + i);
123   }
124 
125   private static byte[] getQualifier(int j) {
126     return Bytes.toBytes("col" + j);
127   }
128 
129   private static byte[] getValue(int batchId, int i, int j) {
130     return Bytes.toBytes("value_for_" + Bytes.toString(getRowKey(batchId, i))
131         + "_col" + j);
132   }
133 
134   static void writeTestDataBatch(Configuration conf, String tableName,
135       int batchId) throws Exception {
136     LOG.debug("Writing test data batch " + batchId);
137     HTable table = new HTable(conf, tableName);
138     table.setAutoFlush(false);
139     for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) {
140       Put put = new Put(getRowKey(batchId, i));
141       for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
142         put.add(CF_BYTES, getQualifier(j),
143             getValue(batchId, i, j));
144       }
145       put.setDurability(Durability.SKIP_WAL);
146       table.put(put);
147     }
148     table.flushCommits();
149     table.close();
150   }
151 
152   static void verifyTestDataBatch(Configuration conf, String tableName,
153       int batchId) throws Exception {
154     LOG.debug("Verifying test data batch " + batchId);
155     HTable table = new HTable(conf, tableName);
156     for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) {
157       Get get = new Get(getRowKey(batchId, i));
158       Result result = table.get(get);
159       for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
160         KeyValue kv = result.getColumnLatest(CF_BYTES, getQualifier(j));
161         assertEquals(Bytes.toStringBinary(getValue(batchId, i, j)),
162             Bytes.toStringBinary(kv.getValue()));
163       }
164     }
165     table.close();
166   }
167 
168   private void writeSomeNewData() throws Exception {
169     writeTestDataBatch(conf, tableName, numBatchesWritten);
170     ++numBatchesWritten;
171   }
172 
173   private void verifyAllData() throws Exception {
174     for (int i = 0; i < numBatchesWritten; ++i) {
175       verifyTestDataBatch(conf, tableName, i);
176     }
177   }
178 
179   private void setEncodingConf(DataBlockEncoding encoding,
180       boolean encodeOnDisk) throws IOException {
181     LOG.debug("Setting CF encoding to " + encoding + " (ordinal="
182         + encoding.ordinal() + "), encodeOnDisk=" + encodeOnDisk);
183     admin.disableTable(tableName);
184     hcd.setDataBlockEncoding(encoding);
185     hcd.setEncodeOnDisk(encodeOnDisk);
186     admin.modifyColumn(tableName, hcd);
187     admin.enableTable(tableName);
188   }
189 
190   @Test(timeout=TIMEOUT_MS)
191   public void testChangingEncoding() throws Exception {
192     prepareTest("ChangingEncoding");
193     for (boolean encodeOnDisk : new boolean[]{false, true}) {
194       for (DataBlockEncoding encoding : ENCODINGS_TO_ITERATE) {
195         setEncodingConf(encoding, encodeOnDisk);
196         writeSomeNewData();
197         verifyAllData();
198       }
199     }
200   }
201 
202   @Test(timeout=TIMEOUT_MS)
203   public void testChangingEncodingWithCompaction() throws Exception {
204     prepareTest("ChangingEncodingWithCompaction");
205     for (boolean encodeOnDisk : new boolean[]{false, true}) {
206       for (DataBlockEncoding encoding : ENCODINGS_TO_ITERATE) {
207         setEncodingConf(encoding, encodeOnDisk);
208         writeSomeNewData();
209         verifyAllData();
210         compactAndWait();
211         verifyAllData();
212       }
213     }
214   }
215 
216   @Test(timeout=TIMEOUT_MS)
217   public void testFlippingEncodeOnDisk() throws Exception {
218     prepareTest("FlippingEncodeOnDisk");
219     // The focus of this test case is to flip the "encoding on disk" flag,
220     // so we only try a couple of encodings.
221     DataBlockEncoding[] encodings = new DataBlockEncoding[] {
222         DataBlockEncoding.NONE, DataBlockEncoding.FAST_DIFF };
223     for (DataBlockEncoding encoding : encodings) {
224       boolean[] flagValues;
225       if (encoding == DataBlockEncoding.NONE) {
226         // encodeOnDisk does not matter when not using encoding.
227         flagValues =
228             new boolean[] { HColumnDescriptor.DEFAULT_ENCODE_ON_DISK };
229       } else {
230         flagValues = new boolean[] { false, true, false, true };
231       }
232       for (boolean encodeOnDisk : flagValues) {
233         setEncodingConf(encoding, encodeOnDisk);
234         writeSomeNewData();
235         verifyAllData();
236         compactAndWait();
237         verifyAllData();
238       }
239     }
240   }
241 
242   private void compactAndWait() throws IOException, InterruptedException {
243     LOG.debug("Compacting table " + tableName);
244     admin.majorCompact(tableName);
245     HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
246 
247     // Waiting for the compaction to start, at least .5s.
248     final long maxWaitime = System.currentTimeMillis() + 500;
249     boolean cont;
250     do {
251       cont = rs.compactSplitThread.getCompactionQueueSize() == 0;
252       Threads.sleep(1);
253     } while (cont && System.currentTimeMillis() < maxWaitime);
254 
255     while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
256       Threads.sleep(5);
257     }
258     LOG.debug("Compaction queue size reached 0, continuing");
259   }
260 
261   @Test
262   public void testCrazyRandomChanges() throws Exception {
263     prepareTest("RandomChanges");
264     Random rand = new Random(2934298742974297L);
265     for (int i = 0; i < 20; ++i) {
266       int encodingOrdinal = rand.nextInt(DataBlockEncoding.values().length);
267       DataBlockEncoding encoding = DataBlockEncoding.values()[encodingOrdinal];
268       setEncodingConf(encoding, rand.nextBoolean());
269       writeSomeNewData();
270       verifyAllData();
271     }
272   }
273 
274 }