1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.fail;
26
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.List;
30 import java.util.Random;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HRegionLocation;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.LargeTests;
40 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.Pair;
44 import org.junit.After;
45 import org.junit.AfterClass;
46 import org.junit.Before;
47 import org.junit.BeforeClass;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51 @Category(LargeTests.class)
52 public class TestFromClientSide3 {
53 final Log LOG = LogFactory.getLog(getClass());
54 private final static HBaseTestingUtility TEST_UTIL
55 = new HBaseTestingUtility();
56 private static byte[] FAMILY = Bytes.toBytes("testFamily");
57 private static Random random = new Random();
58 private static int SLAVES = 3;
59 private static byte [] ROW = Bytes.toBytes("testRow");
60 private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
61 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
62 private static byte [] VALUE = Bytes.toBytes("testValue");
63 private final static byte[] COL_QUAL = Bytes.toBytes("f1");
64 private final static byte[] VAL_BYTES = Bytes.toBytes("v1");
65 private final static byte[] ROW_BYTES = Bytes.toBytes("r1");
66
67
68
69
70 @BeforeClass
71 public static void setUpBeforeClass() throws Exception {
72 TEST_UTIL.getConfiguration().setBoolean(
73 "hbase.online.schema.update.enable", true);
74 TEST_UTIL.startMiniCluster(SLAVES);
75 }
76
77
78
79
80 @AfterClass
81 public static void tearDownAfterClass() throws Exception {
82 TEST_UTIL.shutdownMiniCluster();
83 }
84
85
86
87
88 @Before
89 public void setUp() throws Exception {
90
91 }
92
93
94
95
96 @After
97 public void tearDown() throws Exception {
98
99 }
100
101 private void randomCFPuts(HTable table, byte[] row, byte[] family, int nPuts)
102 throws Exception {
103 Put put = new Put(row);
104 for (int i = 0; i < nPuts; i++) {
105 byte[] qualifier = Bytes.toBytes(random.nextInt());
106 byte[] value = Bytes.toBytes(random.nextInt());
107 put.add(family, qualifier, value);
108 }
109 table.put(put);
110 }
111
112 private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
113 byte[] row, byte[] family, int nFlushes, int nPuts) throws Exception {
114
115
116 HConnection conn = HConnectionManager.getConnection(TEST_UTIL
117 .getConfiguration());
118 HRegionLocation loc = table.getRegionLocation(row, true);
119 AdminProtos.AdminService.BlockingInterface server = conn.getAdmin(loc.getServerName());
120 byte[] regName = loc.getRegionInfo().getRegionName();
121
122 for (int i = 0; i < nFlushes; i++) {
123 randomCFPuts(table, row, family, nPuts);
124 List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY);
125 int sfCount = sf.size();
126
127
128 admin.flush(table.getTableName());
129
130
131 while (ProtobufUtil.getStoreFiles(
132 server, regName, FAMILY).size() == sfCount) {
133 Thread.sleep(40);
134 }
135 }
136 }
137
138
139 @Test(timeout = 60000)
140 public void testAdvancedConfigOverride() throws Exception {
141
142
143
144
145
146
147
148
149
150 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
151
152 String tableName = "testAdvancedConfigOverride";
153 TableName TABLE =
154 TableName.valueOf(tableName);
155 HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
156 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
157 HConnection connection = HConnectionManager.getConnection(TEST_UTIL
158 .getConfiguration());
159
160
161 byte[] row = Bytes.toBytes(random.nextInt());
162 performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
163
164
165 HRegionLocation loc = hTable.getRegionLocation(row, true);
166 byte[] regionName = loc.getRegionInfo().getRegionName();
167 AdminProtos.AdminService.BlockingInterface server =
168 connection.getAdmin(loc.getServerName());
169 assertTrue(ProtobufUtil.getStoreFiles(
170 server, regionName, FAMILY).size() > 1);
171
172
173 admin.compact(TABLE.getName());
174
175
176 for (int i = 0; i < 10 * 1000 / 40; ++i) {
177
178 loc = hTable.getRegionLocation(row, true);
179 if (!loc.getRegionInfo().isOffline()) {
180 regionName = loc.getRegionInfo().getRegionName();
181 server = connection.getAdmin(loc.getServerName());
182 if (ProtobufUtil.getStoreFiles(
183 server, regionName, FAMILY).size() <= 1) {
184 break;
185 }
186 }
187 Thread.sleep(40);
188 }
189
190 assertTrue(ProtobufUtil.getStoreFiles(
191 server, regionName, FAMILY).size() <= 1);
192
193
194 LOG.info("hbase.hstore.compaction.min should now be 5");
195 HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
196 htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
197 admin.modifyTable(TABLE, htd);
198 Pair<Integer, Integer> st;
199 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
200 LOG.debug(st.getFirst() + " regions left to update");
201 Thread.sleep(40);
202 }
203 LOG.info("alter status finished");
204
205
206 performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
207
208
209 admin.compact(TABLE.getName());
210
211
212 Thread.sleep(10 * 1000);
213 loc = hTable.getRegionLocation(row, true);
214 regionName = loc.getRegionInfo().getRegionName();
215 server = connection.getAdmin(loc.getServerName());
216 int sfCount = ProtobufUtil.getStoreFiles(
217 server, regionName, FAMILY).size();
218 assertTrue(sfCount > 1);
219
220
221 LOG.info("hbase.hstore.compaction.min should now be 2");
222 HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
223 hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
224 htd.addFamily(hcd);
225 admin.modifyTable(TABLE, htd);
226 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
227 LOG.debug(st.getFirst() + " regions left to update");
228 Thread.sleep(40);
229 }
230 LOG.info("alter status finished");
231
232
233 admin.compact(TABLE.getName());
234
235
236 for (int i = 0; i < 10 * 1000 / 40; ++i) {
237 loc = hTable.getRegionLocation(row, true);
238 regionName = loc.getRegionInfo().getRegionName();
239 try {
240 server = connection.getAdmin(loc.getServerName());
241 if (ProtobufUtil.getStoreFiles(
242 server, regionName, FAMILY).size() < sfCount) {
243 break;
244 }
245 } catch (Exception e) {
246 LOG.debug("Waiting for region to come online: " + regionName);
247 }
248 Thread.sleep(40);
249 }
250
251 assertTrue(ProtobufUtil.getStoreFiles(
252 server, regionName, FAMILY).size() < sfCount);
253
254
255 LOG.info("Removing CF config value");
256 LOG.info("hbase.hstore.compaction.min should now be 5");
257 hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
258 hcd.setValue("hbase.hstore.compaction.min", null);
259 htd.addFamily(hcd);
260 admin.modifyTable(TABLE, htd);
261 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
262 LOG.debug(st.getFirst() + " regions left to update");
263 Thread.sleep(40);
264 }
265 LOG.info("alter status finished");
266 assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
267 "hbase.hstore.compaction.min"));
268 }
269
270 @Test
271 public void testHTableExistsMethodSingleRegionSingleGet() throws Exception {
272
273
274
275 HTable table = TEST_UTIL.createTable(
276 Bytes.toBytes("testHTableExistsMethodSingleRegionSingleGet"), new byte[][] { FAMILY });
277
278 Put put = new Put(ROW);
279 put.add(FAMILY, QUALIFIER, VALUE);
280
281 Get get = new Get(ROW);
282
283 boolean exist = table.exists(get);
284 assertEquals(exist, false);
285
286 table.put(put);
287
288 exist = table.exists(get);
289 assertEquals(exist, true);
290 }
291
292 public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
293
294 HTable table = TEST_UTIL.createTable(
295 Bytes.toBytes("testHTableExistsMethodSingleRegionMultipleGets"), new byte[][] { FAMILY });
296
297 Put put = new Put(ROW);
298 put.add(FAMILY, QUALIFIER, VALUE);
299 table.put(put);
300
301 List<Get> gets = new ArrayList<Get>();
302 gets.add(new Get(ROW));
303 gets.add(null);
304 gets.add(new Get(ANOTHERROW));
305
306 Boolean[] results = table.exists(gets);
307 assertEquals(results[0], true);
308 assertEquals(results[1], false);
309 assertEquals(results[2], false);
310 }
311
312 @Test
313 public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception {
314
315 HTable table = TEST_UTIL.createTable(
316 Bytes.toBytes("testHTableExistsMethodMultipleRegionsSingleGet"), new byte[][] { FAMILY }, 1,
317 new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
318 Put put = new Put(ROW);
319 put.add(FAMILY, QUALIFIER, VALUE);
320
321 Get get = new Get(ROW);
322
323 boolean exist = table.exists(get);
324 assertEquals(exist, false);
325
326 table.put(put);
327
328 exist = table.exists(get);
329 assertEquals(exist, true);
330 }
331
332 @Test
333 public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception {
334 HTable table = TEST_UTIL.createTable(
335 Bytes.toBytes("testHTableExistsMethodMultipleRegionsMultipleGets"), new byte[][] { FAMILY },
336 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
337 Put put = new Put(ROW);
338 put.add(FAMILY, QUALIFIER, VALUE);
339 table.put (put);
340
341 List<Get> gets = new ArrayList<Get>();
342 gets.add(new Get(ANOTHERROW));
343 gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 })));
344 gets.add(new Get(ROW));
345 gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 })));
346
347 LOG.info("Calling exists");
348 Boolean[] results = table.exists(gets);
349 assertEquals(results[0], false);
350 assertEquals(results[1], false);
351 assertEquals(results[2], true);
352 assertEquals(results[3], false);
353
354
355 put = new Put(new byte[] { 0x00 });
356 put.add(FAMILY, QUALIFIER, VALUE);
357 table.put(put);
358
359 gets = new ArrayList<Get>();
360 gets.add(new Get(new byte[] { 0x00 }));
361 gets.add(new Get(new byte[] { 0x00, 0x00 }));
362 results = table.exists(gets);
363 assertEquals(results[0], true);
364 assertEquals(results[1], false);
365
366
367 put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
368 put.add(FAMILY, QUALIFIER, VALUE);
369 table.put(put);
370
371 gets = new ArrayList<Get>();
372 gets.add(new Get(new byte[] { (byte) 0xff }));
373 gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
374 gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
375 results = table.exists(gets);
376 assertEquals(results[0], false);
377 assertEquals(results[1], true);
378 assertEquals(results[2], false);
379 }
380
381 @Test
382 public void testGetEmptyRow() throws Exception {
383
384 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
385 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("test")));
386 desc.addFamily(new HColumnDescriptor(FAMILY));
387 admin.createTable(desc);
388 HTable table = new HTable(TEST_UTIL.getConfiguration(), "test");
389
390 Put put = new Put(ROW_BYTES);
391 put.add(FAMILY, COL_QUAL, VAL_BYTES);
392 table.put(put);
393 table.flushCommits();
394
395
396 Result res = null;
397 try {
398 res = table.get(new Get(new byte[0]));
399 fail();
400 } catch (IllegalArgumentException e) {
401
402 }
403 assertTrue(res == null);
404 res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
405 assertTrue(res.isEmpty() == true);
406 res = table.get(new Get(ROW_BYTES));
407 assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
408 table.close();
409 }
410 }