1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.encoding;
18
19 import static org.junit.Assert.assertEquals;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.Collections;
25 import java.util.List;
26 import java.util.Random;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.HColumnDescriptor;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.LargeTests;
37 import org.apache.hadoop.hbase.client.Durability;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.HBaseAdmin;
40 import org.apache.hadoop.hbase.client.HTable;
41 import org.apache.hadoop.hbase.client.Put;
42 import org.apache.hadoop.hbase.client.Result;
43 import org.apache.hadoop.hbase.regionserver.HRegionServer;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.Threads;
46 import org.junit.After;
47 import org.junit.AfterClass;
48 import org.junit.Before;
49 import org.junit.BeforeClass;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52
53
54
55
56 @Category(LargeTests.class)
57 public class TestChangingEncoding {
58
59 private static final Log LOG = LogFactory.getLog(TestChangingEncoding.class);
60
61 static final String CF = "EncodingTestCF";
62 static final byte[] CF_BYTES = Bytes.toBytes(CF);
63
64 private static final int NUM_ROWS_PER_BATCH = 100;
65 private static final int NUM_COLS_PER_ROW = 20;
66
67 private static final HBaseTestingUtility TEST_UTIL =
68 new HBaseTestingUtility();
69 private static final Configuration conf = TEST_UTIL.getConfiguration();
70
71 private static final int TIMEOUT_MS = 600000;
72
73 private HBaseAdmin admin;
74 private HColumnDescriptor hcd;
75
76 private String tableName;
77 private static final List<DataBlockEncoding> ENCODINGS_TO_ITERATE =
78 createEncodingsToIterate();
79
80 private static final List<DataBlockEncoding> createEncodingsToIterate() {
81 List<DataBlockEncoding> encodings = new ArrayList<DataBlockEncoding>(
82 Arrays.asList(DataBlockEncoding.values()));
83 encodings.add(DataBlockEncoding.NONE);
84 return Collections.unmodifiableList(encodings);
85 }
86
87
88 private int numBatchesWritten;
89
90 private void prepareTest(String testId) throws IOException {
91 tableName = "test_table_" + testId;
92 HTableDescriptor htd = new HTableDescriptor(tableName);
93 hcd = new HColumnDescriptor(CF);
94 htd.addFamily(hcd);
95 admin.createTable(htd);
96 numBatchesWritten = 0;
97 }
98
99 @BeforeClass
100 public static void setUpBeforeClass() throws Exception {
101
102 conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
103 TEST_UTIL.startMiniCluster();
104 }
105
106 @AfterClass
107 public static void tearDownAfterClass() throws Exception {
108 TEST_UTIL.shutdownMiniCluster();
109 }
110
111 @Before
112 public void setUp() throws Exception {
113 admin = new HBaseAdmin(conf);
114 }
115
116 @After
117 public void tearDown() throws IOException {
118 admin.close();
119 }
120
121 private static byte[] getRowKey(int batchId, int i) {
122 return Bytes.toBytes("batch" + batchId + "_row" + i);
123 }
124
125 private static byte[] getQualifier(int j) {
126 return Bytes.toBytes("col" + j);
127 }
128
129 private static byte[] getValue(int batchId, int i, int j) {
130 return Bytes.toBytes("value_for_" + Bytes.toString(getRowKey(batchId, i))
131 + "_col" + j);
132 }
133
134 static void writeTestDataBatch(Configuration conf, String tableName,
135 int batchId) throws Exception {
136 LOG.debug("Writing test data batch " + batchId);
137 HTable table = new HTable(conf, tableName);
138 table.setAutoFlush(false);
139 for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) {
140 Put put = new Put(getRowKey(batchId, i));
141 for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
142 put.add(CF_BYTES, getQualifier(j),
143 getValue(batchId, i, j));
144 }
145 put.setDurability(Durability.SKIP_WAL);
146 table.put(put);
147 }
148 table.flushCommits();
149 table.close();
150 }
151
152 static void verifyTestDataBatch(Configuration conf, String tableName,
153 int batchId) throws Exception {
154 LOG.debug("Verifying test data batch " + batchId);
155 HTable table = new HTable(conf, tableName);
156 for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) {
157 Get get = new Get(getRowKey(batchId, i));
158 Result result = table.get(get);
159 for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
160 KeyValue kv = result.getColumnLatest(CF_BYTES, getQualifier(j));
161 assertEquals(Bytes.toStringBinary(getValue(batchId, i, j)),
162 Bytes.toStringBinary(kv.getValue()));
163 }
164 }
165 table.close();
166 }
167
168 private void writeSomeNewData() throws Exception {
169 writeTestDataBatch(conf, tableName, numBatchesWritten);
170 ++numBatchesWritten;
171 }
172
173 private void verifyAllData() throws Exception {
174 for (int i = 0; i < numBatchesWritten; ++i) {
175 verifyTestDataBatch(conf, tableName, i);
176 }
177 }
178
179 private void setEncodingConf(DataBlockEncoding encoding,
180 boolean encodeOnDisk) throws IOException {
181 LOG.debug("Setting CF encoding to " + encoding + " (ordinal="
182 + encoding.ordinal() + "), encodeOnDisk=" + encodeOnDisk);
183 admin.disableTable(tableName);
184 hcd.setDataBlockEncoding(encoding);
185 hcd.setEncodeOnDisk(encodeOnDisk);
186 admin.modifyColumn(tableName, hcd);
187 admin.enableTable(tableName);
188 }
189
190 @Test(timeout=TIMEOUT_MS)
191 public void testChangingEncoding() throws Exception {
192 prepareTest("ChangingEncoding");
193 for (boolean encodeOnDisk : new boolean[]{false, true}) {
194 for (DataBlockEncoding encoding : ENCODINGS_TO_ITERATE) {
195 setEncodingConf(encoding, encodeOnDisk);
196 writeSomeNewData();
197 verifyAllData();
198 }
199 }
200 }
201
202 @Test(timeout=TIMEOUT_MS)
203 public void testChangingEncodingWithCompaction() throws Exception {
204 prepareTest("ChangingEncodingWithCompaction");
205 for (boolean encodeOnDisk : new boolean[]{false, true}) {
206 for (DataBlockEncoding encoding : ENCODINGS_TO_ITERATE) {
207 setEncodingConf(encoding, encodeOnDisk);
208 writeSomeNewData();
209 verifyAllData();
210 compactAndWait();
211 verifyAllData();
212 }
213 }
214 }
215
216 @Test(timeout=TIMEOUT_MS)
217 public void testFlippingEncodeOnDisk() throws Exception {
218 prepareTest("FlippingEncodeOnDisk");
219
220
221 DataBlockEncoding[] encodings = new DataBlockEncoding[] {
222 DataBlockEncoding.NONE, DataBlockEncoding.FAST_DIFF };
223 for (DataBlockEncoding encoding : encodings) {
224 boolean[] flagValues;
225 if (encoding == DataBlockEncoding.NONE) {
226
227 flagValues =
228 new boolean[] { HColumnDescriptor.DEFAULT_ENCODE_ON_DISK };
229 } else {
230 flagValues = new boolean[] { false, true, false, true };
231 }
232 for (boolean encodeOnDisk : flagValues) {
233 setEncodingConf(encoding, encodeOnDisk);
234 writeSomeNewData();
235 verifyAllData();
236 compactAndWait();
237 verifyAllData();
238 }
239 }
240 }
241
242 private void compactAndWait() throws IOException, InterruptedException {
243 LOG.debug("Compacting table " + tableName);
244 admin.majorCompact(tableName);
245 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
246
247
248 final long maxWaitime = System.currentTimeMillis() + 500;
249 boolean cont;
250 do {
251 cont = rs.compactSplitThread.getCompactionQueueSize() == 0;
252 Threads.sleep(1);
253 } while (cont && System.currentTimeMillis() < maxWaitime);
254
255 while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
256 Threads.sleep(5);
257 }
258 LOG.debug("Compaction queue size reached 0, continuing");
259 }
260
261 @Test
262 public void testCrazyRandomChanges() throws Exception {
263 prepareTest("RandomChanges");
264 Random rand = new Random(2934298742974297L);
265 for (int i = 0; i < 20; ++i) {
266 int encodingOrdinal = rand.nextInt(DataBlockEncoding.values().length);
267 DataBlockEncoding encoding = DataBlockEncoding.values()[encodingOrdinal];
268 setEncodingConf(encoding, rand.nextBoolean());
269 writeSomeNewData();
270 verifyAllData();
271 }
272 }
273
274 }