1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.assertFalse;
26 import static org.junit.Assert.fail;
27
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.List;
31 import java.util.Random;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HRegionLocation;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.testclassification.LargeTests;
41 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
42 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.Pair;
45 import org.junit.After;
46 import org.junit.AfterClass;
47 import org.junit.Before;
48 import org.junit.BeforeClass;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51
52 @Category(LargeTests.class)
53 public class TestFromClientSide3 {
54 final Log LOG = LogFactory.getLog(getClass());
55 private final static HBaseTestingUtility TEST_UTIL
56 = new HBaseTestingUtility();
57 private static byte[] FAMILY = Bytes.toBytes("testFamily");
58 private static Random random = new Random();
59 private static int SLAVES = 3;
60 private static byte [] ROW = Bytes.toBytes("testRow");
61 private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
62 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
63 private static byte [] VALUE = Bytes.toBytes("testValue");
64 private final static byte[] COL_QUAL = Bytes.toBytes("f1");
65 private final static byte[] VAL_BYTES = Bytes.toBytes("v1");
66 private final static byte[] ROW_BYTES = Bytes.toBytes("r1");
67
68
69
70
71 @BeforeClass
72 public static void setUpBeforeClass() throws Exception {
73 TEST_UTIL.getConfiguration().setBoolean(
74 "hbase.online.schema.update.enable", true);
75 TEST_UTIL.startMiniCluster(SLAVES);
76 }
77
78
79
80
81 @AfterClass
82 public static void tearDownAfterClass() throws Exception {
83 TEST_UTIL.shutdownMiniCluster();
84 }
85
86
87
88
89 @Before
90 public void setUp() throws Exception {
91
92 }
93
94
95
96
97 @After
98 public void tearDown() throws Exception {
99 for (HTableDescriptor htd: TEST_UTIL.getHBaseAdmin().listTables()) {
100 LOG.info("Tear down, remove table=" + htd.getTableName());
101 TEST_UTIL.deleteTable(htd.getTableName());
102 }
103 }
104
105 private void randomCFPuts(HTable table, byte[] row, byte[] family, int nPuts)
106 throws Exception {
107 Put put = new Put(row);
108 for (int i = 0; i < nPuts; i++) {
109 byte[] qualifier = Bytes.toBytes(random.nextInt());
110 byte[] value = Bytes.toBytes(random.nextInt());
111 put.add(family, qualifier, value);
112 }
113 table.put(put);
114 }
115
116 private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
117 byte[] row, byte[] family, int nFlushes, int nPuts) throws Exception {
118
119
120 HConnection conn = HConnectionManager.getConnection(TEST_UTIL
121 .getConfiguration());
122 HRegionLocation loc = table.getRegionLocation(row, true);
123 AdminProtos.AdminService.BlockingInterface server = conn.getAdmin(loc.getServerName());
124 byte[] regName = loc.getRegionInfo().getRegionName();
125
126 for (int i = 0; i < nFlushes; i++) {
127 randomCFPuts(table, row, family, nPuts);
128 List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY);
129 int sfCount = sf.size();
130
131
132 admin.flush(table.getTableName());
133
134
135 while (ProtobufUtil.getStoreFiles(
136 server, regName, FAMILY).size() == sfCount) {
137 Thread.sleep(40);
138 }
139 }
140 }
141
142
143 @Test(timeout = 60000)
144 public void testAdvancedConfigOverride() throws Exception {
145
146
147
148
149
150
151
152
153
154 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
155
156 String tableName = "testAdvancedConfigOverride";
157 TableName TABLE =
158 TableName.valueOf(tableName);
159 HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
160 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
161 HConnection connection = HConnectionManager.getConnection(TEST_UTIL
162 .getConfiguration());
163
164
165 byte[] row = Bytes.toBytes(random.nextInt());
166 performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
167
168
169 HRegionLocation loc = hTable.getRegionLocation(row, true);
170 byte[] regionName = loc.getRegionInfo().getRegionName();
171 AdminProtos.AdminService.BlockingInterface server =
172 connection.getAdmin(loc.getServerName());
173 assertTrue(ProtobufUtil.getStoreFiles(
174 server, regionName, FAMILY).size() > 1);
175
176
177 admin.compact(TABLE.getName());
178
179
180 for (int i = 0; i < 10 * 1000 / 40; ++i) {
181
182 loc = hTable.getRegionLocation(row, true);
183 if (!loc.getRegionInfo().isOffline()) {
184 regionName = loc.getRegionInfo().getRegionName();
185 server = connection.getAdmin(loc.getServerName());
186 if (ProtobufUtil.getStoreFiles(
187 server, regionName, FAMILY).size() <= 1) {
188 break;
189 }
190 }
191 Thread.sleep(40);
192 }
193
194 assertTrue(ProtobufUtil.getStoreFiles(
195 server, regionName, FAMILY).size() <= 1);
196
197
198 LOG.info("hbase.hstore.compaction.min should now be 5");
199 HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
200 htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
201 admin.modifyTable(TABLE, htd);
202 Pair<Integer, Integer> st;
203 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
204 LOG.debug(st.getFirst() + " regions left to update");
205 Thread.sleep(40);
206 }
207 LOG.info("alter status finished");
208
209
210 performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
211
212
213 admin.compact(TABLE.getName());
214
215
216 Thread.sleep(10 * 1000);
217 loc = hTable.getRegionLocation(row, true);
218 regionName = loc.getRegionInfo().getRegionName();
219 server = connection.getAdmin(loc.getServerName());
220 int sfCount = ProtobufUtil.getStoreFiles(
221 server, regionName, FAMILY).size();
222 assertTrue(sfCount > 1);
223
224
225 LOG.info("hbase.hstore.compaction.min should now be 2");
226 HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
227 hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
228 htd.addFamily(hcd);
229 admin.modifyTable(TABLE, htd);
230 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
231 LOG.debug(st.getFirst() + " regions left to update");
232 Thread.sleep(40);
233 }
234 LOG.info("alter status finished");
235
236
237 admin.compact(TABLE.getName());
238
239
240 for (int i = 0; i < 10 * 1000 / 40; ++i) {
241 loc = hTable.getRegionLocation(row, true);
242 regionName = loc.getRegionInfo().getRegionName();
243 try {
244 server = connection.getAdmin(loc.getServerName());
245 if (ProtobufUtil.getStoreFiles(
246 server, regionName, FAMILY).size() < sfCount) {
247 break;
248 }
249 } catch (Exception e) {
250 LOG.debug("Waiting for region to come online: " + regionName);
251 }
252 Thread.sleep(40);
253 }
254
255 assertTrue(ProtobufUtil.getStoreFiles(
256 server, regionName, FAMILY).size() < sfCount);
257
258
259 LOG.info("Removing CF config value");
260 LOG.info("hbase.hstore.compaction.min should now be 5");
261 hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
262 hcd.setValue("hbase.hstore.compaction.min", null);
263 htd.addFamily(hcd);
264 admin.modifyTable(TABLE, htd);
265 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
266 LOG.debug(st.getFirst() + " regions left to update");
267 Thread.sleep(40);
268 }
269 LOG.info("alter status finished");
270 assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
271 "hbase.hstore.compaction.min"));
272 }
273
274 @Test
275 public void testHTableBatchWithEmptyPut() throws Exception {
276 HTable table = TEST_UTIL.createTable(
277 Bytes.toBytes("testHTableBatchWithEmptyPut"), new byte[][] { FAMILY });
278 try {
279 List actions = (List) new ArrayList();
280 Object[] results = new Object[2];
281
282 Put put1 = new Put(ROW);
283 actions.add(put1);
284
285 Put put2 = new Put(ANOTHERROW);
286 put2.add(FAMILY, QUALIFIER, VALUE);
287 actions.add(put2);
288
289 table.batch(actions, results);
290 fail("Empty Put should have failed the batch call");
291 } catch (IllegalArgumentException iae) {
292
293 } finally {
294 table.close();
295 }
296 }
297
298 @Test
299 public void testHTableExistsMethodSingleRegionSingleGet() throws Exception {
300
301
302
303 HTable table = TEST_UTIL.createTable(
304 Bytes.toBytes("testHTableExistsMethodSingleRegionSingleGet"), new byte[][] { FAMILY });
305
306 Put put = new Put(ROW);
307 put.add(FAMILY, QUALIFIER, VALUE);
308
309 Get get = new Get(ROW);
310
311 boolean exist = table.exists(get);
312 assertEquals(exist, false);
313
314 table.put(put);
315
316 exist = table.exists(get);
317 assertEquals(exist, true);
318 }
319
320 public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
321
322 HTable table = TEST_UTIL.createTable(
323 Bytes.toBytes("testHTableExistsMethodSingleRegionMultipleGets"), new byte[][] { FAMILY });
324
325 Put put = new Put(ROW);
326 put.add(FAMILY, QUALIFIER, VALUE);
327 table.put(put);
328
329 List<Get> gets = new ArrayList<Get>();
330 gets.add(new Get(ROW));
331 gets.add(null);
332 gets.add(new Get(ANOTHERROW));
333
334 Boolean[] results = table.exists(gets);
335 assertEquals(results[0], true);
336 assertEquals(results[1], false);
337 assertEquals(results[2], false);
338 }
339
340 @Test
341 public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception {
342
343 HTable table = TEST_UTIL.createTable(
344 Bytes.toBytes("testHTableExistsMethodMultipleRegionsSingleGet"), new byte[][] { FAMILY }, 1,
345 new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
346 Put put = new Put(ROW);
347 put.add(FAMILY, QUALIFIER, VALUE);
348
349 Get get = new Get(ROW);
350
351 boolean exist = table.exists(get);
352 assertEquals(exist, false);
353
354 table.put(put);
355
356 exist = table.exists(get);
357 assertEquals(exist, true);
358 }
359
360 @Test
361 public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception {
362 HTable table = TEST_UTIL.createTable(
363 Bytes.toBytes("testHTableExistsMethodMultipleRegionsMultipleGets"), new byte[][] { FAMILY },
364 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
365 Put put = new Put(ROW);
366 put.add(FAMILY, QUALIFIER, VALUE);
367 table.put (put);
368
369 List<Get> gets = new ArrayList<Get>();
370 gets.add(new Get(ANOTHERROW));
371 gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 })));
372 gets.add(new Get(ROW));
373 gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 })));
374
375 LOG.info("Calling exists");
376 Boolean[] results = table.exists(gets);
377 assertEquals(results[0], false);
378 assertEquals(results[1], false);
379 assertEquals(results[2], true);
380 assertEquals(results[3], false);
381
382
383 put = new Put(new byte[] { 0x00 });
384 put.add(FAMILY, QUALIFIER, VALUE);
385 table.put(put);
386
387 gets = new ArrayList<Get>();
388 gets.add(new Get(new byte[] { 0x00 }));
389 gets.add(new Get(new byte[] { 0x00, 0x00 }));
390 results = table.exists(gets);
391 assertEquals(results[0], true);
392 assertEquals(results[1], false);
393
394
395 put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
396 put.add(FAMILY, QUALIFIER, VALUE);
397 table.put(put);
398
399 gets = new ArrayList<Get>();
400 gets.add(new Get(new byte[] { (byte) 0xff }));
401 gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
402 gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
403 results = table.exists(gets);
404 assertEquals(results[0], false);
405 assertEquals(results[1], true);
406 assertEquals(results[2], false);
407 }
408
409 @Test
410 public void testGetEmptyRow() throws Exception {
411
412 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
413 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("test")));
414 desc.addFamily(new HColumnDescriptor(FAMILY));
415 admin.createTable(desc);
416 HTable table = new HTable(TEST_UTIL.getConfiguration(), "test");
417
418 Put put = new Put(ROW_BYTES);
419 put.add(FAMILY, COL_QUAL, VAL_BYTES);
420 table.put(put);
421 table.flushCommits();
422
423
424 Result res = null;
425 try {
426 res = table.get(new Get(new byte[0]));
427 fail();
428 } catch (IllegalArgumentException e) {
429
430 }
431 assertTrue(res == null);
432 res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
433 assertTrue(res.isEmpty() == true);
434 res = table.get(new Get(ROW_BYTES));
435 assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
436 table.close();
437 }
438
439 @Test
440 public void testHTableExistsBeforeGet() throws Exception {
441 HTable table = TEST_UTIL.createTable(
442 Bytes.toBytes("testHTableExistsBeforeGet"), new byte[][] { FAMILY });
443 try {
444 Put put = new Put(ROW);
445 put.add(FAMILY, QUALIFIER, VALUE);
446 table.put(put);
447
448 Get get = new Get(ROW);
449
450 boolean exist = table.exists(get);
451 assertEquals(true, exist);
452
453 Result result = table.get(get);
454 assertEquals(false, result.isEmpty());
455 assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER)));
456 } finally {
457 table.close();
458 }
459 }
460
461 @Test
462 public void testHTableExistsAllBeforeGet() throws Exception {
463 final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2"));
464 HTable table = TEST_UTIL.createTable(
465 Bytes.toBytes("testHTableExistsAllBeforeGet"), new byte[][] { FAMILY });
466 try {
467 Put put = new Put(ROW);
468 put.add(FAMILY, QUALIFIER, VALUE);
469 table.put(put);
470 put = new Put(ROW2);
471 put.add(FAMILY, QUALIFIER, VALUE);
472 table.put(put);
473
474 Get get = new Get(ROW);
475 Get get2 = new Get(ROW2);
476 ArrayList<Get> getList = new ArrayList(2);
477 getList.add(get);
478 getList.add(get2);
479
480 Boolean[] exists = table.exists(getList);
481 assertEquals(true, exists[0]);
482 assertEquals(true, exists[1]);
483
484 Result[] result = table.get(getList);
485 assertEquals(false, result[0].isEmpty());
486 assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER)));
487 assertEquals(false, result[1].isEmpty());
488 assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER)));
489 } finally {
490 table.close();
491 }
492 }
493
494 @Test
495 public void testLeaseRenewal() throws Exception {
496 HTable table = TEST_UTIL.createTable(
497 Bytes.toBytes("testLeaseRenewal"), FAMILY);
498 Put p = new Put(ROW_BYTES);
499 p.add(FAMILY, COL_QUAL, VAL_BYTES);
500 table.put(p);
501 p = new Put(ANOTHERROW);
502 p.add(FAMILY, COL_QUAL, VAL_BYTES);
503 table.put(p);
504 Scan s = new Scan();
505 s.setCaching(1);
506 ResultScanner rs = table.getScanner(s);
507
508 assertTrue(((AbstractClientScanner)rs).renewLease());
509 assertTrue(Arrays.equals(rs.next().getRow(), ANOTHERROW));
510 assertTrue(((AbstractClientScanner)rs).renewLease());
511 assertTrue(Arrays.equals(rs.next().getRow(), ROW_BYTES));
512 assertTrue(((AbstractClientScanner)rs).renewLease());
513 assertNull(rs.next());
514 assertFalse(((AbstractClientScanner)rs).renewLease());
515 rs.close();
516 table.close();
517 }
518 }