View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  import static org.junit.Assert.assertFalse;
26  import static org.junit.Assert.fail;
27  
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.List;
31  import java.util.Random;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HRegionLocation;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.testclassification.LargeTests;
41  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
42  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.Pair;
45  import org.junit.After;
46  import org.junit.AfterClass;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  @Category(LargeTests.class)
53  public class TestFromClientSide3 {
54    final Log LOG = LogFactory.getLog(getClass());
55    private final static HBaseTestingUtility TEST_UTIL
56      = new HBaseTestingUtility();
57    private static byte[] FAMILY = Bytes.toBytes("testFamily");
58    private static Random random = new Random();
59    private static int SLAVES = 3;
60    private static byte [] ROW = Bytes.toBytes("testRow");
61    private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
62    private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
63    private static byte [] VALUE = Bytes.toBytes("testValue");
64    private final static byte[] COL_QUAL = Bytes.toBytes("f1");
65    private final static byte[] VAL_BYTES = Bytes.toBytes("v1");
66    private final static byte[] ROW_BYTES = Bytes.toBytes("r1");
67  
68    /**
69     * @throws java.lang.Exception
70     */
71    @BeforeClass
72    public static void setUpBeforeClass() throws Exception {
73      TEST_UTIL.getConfiguration().setBoolean(
74          "hbase.online.schema.update.enable", true);
75      TEST_UTIL.startMiniCluster(SLAVES);
76    }
77  
78    /**
79     * @throws java.lang.Exception
80     */
81    @AfterClass
82    public static void tearDownAfterClass() throws Exception {
83      TEST_UTIL.shutdownMiniCluster();
84    }
85  
86    /**
87     * @throws java.lang.Exception
88     */
89    @Before
90    public void setUp() throws Exception {
91      // Nothing to do.
92    }
93  
94    /**
95     * @throws java.lang.Exception
96     */
97    @After
98    public void tearDown() throws Exception {
99      for (HTableDescriptor htd: TEST_UTIL.getHBaseAdmin().listTables()) {
100       LOG.info("Tear down, remove table=" + htd.getTableName());
101       TEST_UTIL.deleteTable(htd.getTableName());
102     }
103   }
104 
105   private void randomCFPuts(HTable table, byte[] row, byte[] family, int nPuts)
106       throws Exception {
107     Put put = new Put(row);
108     for (int i = 0; i < nPuts; i++) {
109       byte[] qualifier = Bytes.toBytes(random.nextInt());
110       byte[] value = Bytes.toBytes(random.nextInt());
111       put.add(family, qualifier, value);
112     }
113     table.put(put);
114   }
115 
116   private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
117       byte[] row, byte[] family, int nFlushes, int nPuts) throws Exception {
118 
119     // connection needed for poll-wait
120     HConnection conn = HConnectionManager.getConnection(TEST_UTIL
121         .getConfiguration());
122     HRegionLocation loc = table.getRegionLocation(row, true);
123     AdminProtos.AdminService.BlockingInterface server = conn.getAdmin(loc.getServerName());
124     byte[] regName = loc.getRegionInfo().getRegionName();
125 
126     for (int i = 0; i < nFlushes; i++) {
127       randomCFPuts(table, row, family, nPuts);
128       List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY);
129       int sfCount = sf.size();
130 
131       // TODO: replace this api with a synchronous flush after HBASE-2949
132       admin.flush(table.getTableName());
133 
134       // synchronously poll wait for a new storefile to appear (flush happened)
135       while (ProtobufUtil.getStoreFiles(
136           server, regName, FAMILY).size() == sfCount) {
137         Thread.sleep(40);
138       }
139     }
140   }
141 
142   // override the config settings at the CF level and ensure priority
143   @Test(timeout = 60000)
144   public void testAdvancedConfigOverride() throws Exception {
145     /*
146      * Overall idea: (1) create 3 store files and issue a compaction. config's
147      * compaction.min == 3, so should work. (2) Increase the compaction.min
148      * toggle in the HTD to 5 and modify table. If we use the HTD value instead
149      * of the default config value, adding 3 files and issuing a compaction
150      * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2
151      * and modify table. The CF schema should override the Table schema and now
152      * cause a minor compaction.
153      */
154     TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
155 
156     String tableName = "testAdvancedConfigOverride";
157     TableName TABLE =
158         TableName.valueOf(tableName);
159     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
160     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
161     HConnection connection = HConnectionManager.getConnection(TEST_UTIL
162         .getConfiguration());
163 
164     // Create 3 store files.
165     byte[] row = Bytes.toBytes(random.nextInt());
166     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
167 
168     // Verify we have multiple store files.
169     HRegionLocation loc = hTable.getRegionLocation(row, true);
170     byte[] regionName = loc.getRegionInfo().getRegionName();
171     AdminProtos.AdminService.BlockingInterface server =
172       connection.getAdmin(loc.getServerName());
173     assertTrue(ProtobufUtil.getStoreFiles(
174       server, regionName, FAMILY).size() > 1);
175 
176     // Issue a compaction request
177     admin.compact(TABLE.getName());
178 
179     // poll wait for the compactions to happen
180     for (int i = 0; i < 10 * 1000 / 40; ++i) {
181       // The number of store files after compaction should be lesser.
182       loc = hTable.getRegionLocation(row, true);
183       if (!loc.getRegionInfo().isOffline()) {
184         regionName = loc.getRegionInfo().getRegionName();
185         server = connection.getAdmin(loc.getServerName());
186         if (ProtobufUtil.getStoreFiles(
187             server, regionName, FAMILY).size() <= 1) {
188           break;
189         }
190       }
191       Thread.sleep(40);
192     }
193     // verify the compactions took place and that we didn't just time out
194     assertTrue(ProtobufUtil.getStoreFiles(
195       server, regionName, FAMILY).size() <= 1);
196 
197     // change the compaction.min config option for this table to 5
198     LOG.info("hbase.hstore.compaction.min should now be 5");
199     HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
200     htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
201     admin.modifyTable(TABLE, htd);
202     Pair<Integer, Integer> st;
203     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
204       LOG.debug(st.getFirst() + " regions left to update");
205       Thread.sleep(40);
206     }
207     LOG.info("alter status finished");
208 
209     // Create 3 more store files.
210     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
211 
212     // Issue a compaction request
213     admin.compact(TABLE.getName());
214 
215     // This time, the compaction request should not happen
216     Thread.sleep(10 * 1000);
217     loc = hTable.getRegionLocation(row, true);
218     regionName = loc.getRegionInfo().getRegionName();
219     server = connection.getAdmin(loc.getServerName());
220     int sfCount = ProtobufUtil.getStoreFiles(
221       server, regionName, FAMILY).size();
222     assertTrue(sfCount > 1);
223 
224     // change an individual CF's config option to 2 & online schema update
225     LOG.info("hbase.hstore.compaction.min should now be 2");
226     HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
227     hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
228     htd.addFamily(hcd);
229     admin.modifyTable(TABLE, htd);
230     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
231       LOG.debug(st.getFirst() + " regions left to update");
232       Thread.sleep(40);
233     }
234     LOG.info("alter status finished");
235 
236     // Issue a compaction request
237     admin.compact(TABLE.getName());
238 
239     // poll wait for the compactions to happen
240     for (int i = 0; i < 10 * 1000 / 40; ++i) {
241       loc = hTable.getRegionLocation(row, true);
242       regionName = loc.getRegionInfo().getRegionName();
243       try {
244         server = connection.getAdmin(loc.getServerName());
245         if (ProtobufUtil.getStoreFiles(
246             server, regionName, FAMILY).size() < sfCount) {
247           break;
248         }
249       } catch (Exception e) {
250         LOG.debug("Waiting for region to come online: " + regionName);
251       }
252       Thread.sleep(40);
253     }
254     // verify the compaction took place and that we didn't just time out
255     assertTrue(ProtobufUtil.getStoreFiles(
256       server, regionName, FAMILY).size() < sfCount);
257 
258     // Finally, ensure that we can remove a custom config value after we made it
259     LOG.info("Removing CF config value");
260     LOG.info("hbase.hstore.compaction.min should now be 5");
261     hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
262     hcd.setValue("hbase.hstore.compaction.min", null);
263     htd.addFamily(hcd);
264     admin.modifyTable(TABLE, htd);
265     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
266       LOG.debug(st.getFirst() + " regions left to update");
267       Thread.sleep(40);
268     }
269     LOG.info("alter status finished");
270     assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
271         "hbase.hstore.compaction.min"));
272   }
273 
274   @Test
275   public void testHTableBatchWithEmptyPut() throws Exception {
276     HTable table = TEST_UTIL.createTable(
277       Bytes.toBytes("testHTableBatchWithEmptyPut"), new byte[][] { FAMILY });
278     try {
279       List actions = (List) new ArrayList();
280       Object[] results = new Object[2];
281       // create an empty Put
282       Put put1 = new Put(ROW);
283       actions.add(put1);
284 
285       Put put2 = new Put(ANOTHERROW);
286       put2.add(FAMILY, QUALIFIER, VALUE);
287       actions.add(put2);
288 
289       table.batch(actions, results);
290       fail("Empty Put should have failed the batch call");
291     } catch (IllegalArgumentException iae) {
292 
293     } finally {
294       table.close();
295     }
296   }
297 
298   @Test
299   public void testHTableExistsMethodSingleRegionSingleGet() throws Exception {
300 
301     // Test with a single region table.
302 
303     HTable table = TEST_UTIL.createTable(
304       Bytes.toBytes("testHTableExistsMethodSingleRegionSingleGet"), new byte[][] { FAMILY });
305 
306     Put put = new Put(ROW);
307     put.add(FAMILY, QUALIFIER, VALUE);
308 
309     Get get = new Get(ROW);
310 
311     boolean exist = table.exists(get);
312     assertEquals(exist, false);
313 
314     table.put(put);
315 
316     exist = table.exists(get);
317     assertEquals(exist, true);
318   }
319 
320   public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
321 
322     HTable table = TEST_UTIL.createTable(
323       Bytes.toBytes("testHTableExistsMethodSingleRegionMultipleGets"), new byte[][] { FAMILY });
324 
325     Put put = new Put(ROW);
326     put.add(FAMILY, QUALIFIER, VALUE);
327     table.put(put);
328 
329     List<Get> gets = new ArrayList<Get>();
330     gets.add(new Get(ROW));
331     gets.add(null);
332     gets.add(new Get(ANOTHERROW));
333 
334     Boolean[] results = table.exists(gets);
335     assertEquals(results[0], true);
336     assertEquals(results[1], false);
337     assertEquals(results[2], false);
338   }
339 
340   @Test
341   public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception {
342 
343     HTable table = TEST_UTIL.createTable(
344       Bytes.toBytes("testHTableExistsMethodMultipleRegionsSingleGet"), new byte[][] { FAMILY }, 1,
345       new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
346     Put put = new Put(ROW);
347     put.add(FAMILY, QUALIFIER, VALUE);
348 
349     Get get = new Get(ROW);
350 
351     boolean exist = table.exists(get);
352     assertEquals(exist, false);
353 
354     table.put(put);
355 
356     exist = table.exists(get);
357     assertEquals(exist, true);
358   }
359 
360   @Test
361   public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception {
362     HTable table = TEST_UTIL.createTable(
363       Bytes.toBytes("testHTableExistsMethodMultipleRegionsMultipleGets"), new byte[][] { FAMILY },
364       1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
365     Put put = new Put(ROW);
366     put.add(FAMILY, QUALIFIER, VALUE);
367     table.put (put);
368 
369     List<Get> gets = new ArrayList<Get>();
370     gets.add(new Get(ANOTHERROW));
371     gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 })));
372     gets.add(new Get(ROW));
373     gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 })));
374 
375     LOG.info("Calling exists");
376     Boolean[] results = table.exists(gets);
377     assertEquals(results[0], false);
378     assertEquals(results[1], false);
379     assertEquals(results[2], true);
380     assertEquals(results[3], false);
381 
382     // Test with the first region.
383     put = new Put(new byte[] { 0x00 });
384     put.add(FAMILY, QUALIFIER, VALUE);
385     table.put(put);
386 
387     gets = new ArrayList<Get>();
388     gets.add(new Get(new byte[] { 0x00 }));
389     gets.add(new Get(new byte[] { 0x00, 0x00 }));
390     results = table.exists(gets);
391     assertEquals(results[0], true);
392     assertEquals(results[1], false);
393 
394     // Test with the last region
395     put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
396     put.add(FAMILY, QUALIFIER, VALUE);
397     table.put(put);
398 
399     gets = new ArrayList<Get>();
400     gets.add(new Get(new byte[] { (byte) 0xff }));
401     gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
402     gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
403     results = table.exists(gets);
404     assertEquals(results[0], false);
405     assertEquals(results[1], true);
406     assertEquals(results[2], false);
407   }
408 
409   @Test
410   public void testGetEmptyRow() throws Exception {
411     //Create a table and put in 1 row
412     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
413     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("test")));
414     desc.addFamily(new HColumnDescriptor(FAMILY));
415     admin.createTable(desc);
416     HTable table = new HTable(TEST_UTIL.getConfiguration(), "test");
417 
418     Put put = new Put(ROW_BYTES);
419     put.add(FAMILY, COL_QUAL, VAL_BYTES);
420     table.put(put);
421     table.flushCommits();
422 
423     //Try getting the row with an empty row key
424     Result res = null;
425     try {
426       res = table.get(new Get(new byte[0]));
427       fail();
428     } catch (IllegalArgumentException e) {
429       // Expected.
430     }
431     assertTrue(res == null);
432     res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
433     assertTrue(res.isEmpty() == true);
434     res = table.get(new Get(ROW_BYTES));
435     assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
436     table.close();
437   }
438 
439   @Test
440   public void testHTableExistsBeforeGet() throws Exception {
441     HTable table = TEST_UTIL.createTable(
442       Bytes.toBytes("testHTableExistsBeforeGet"), new byte[][] { FAMILY });
443     try {
444       Put put = new Put(ROW);
445       put.add(FAMILY, QUALIFIER, VALUE);
446       table.put(put);
447 
448       Get get = new Get(ROW);
449 
450       boolean exist = table.exists(get);
451       assertEquals(true, exist);
452 
453       Result result = table.get(get);
454       assertEquals(false, result.isEmpty());
455       assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER)));
456     } finally {
457       table.close();
458     }
459   }
460 
461   @Test
462   public void testHTableExistsAllBeforeGet() throws Exception {
463     final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2"));
464     HTable table = TEST_UTIL.createTable(
465       Bytes.toBytes("testHTableExistsAllBeforeGet"), new byte[][] { FAMILY });
466     try {
467       Put put = new Put(ROW);
468       put.add(FAMILY, QUALIFIER, VALUE);
469       table.put(put);
470       put = new Put(ROW2);
471       put.add(FAMILY, QUALIFIER, VALUE);
472       table.put(put);
473 
474       Get get = new Get(ROW);
475       Get get2 = new Get(ROW2);
476       ArrayList<Get> getList = new ArrayList(2);
477       getList.add(get);
478       getList.add(get2);
479 
480       Boolean[] exists = table.exists(getList);
481       assertEquals(true, exists[0]);
482       assertEquals(true, exists[1]);
483 
484       Result[] result = table.get(getList);
485       assertEquals(false, result[0].isEmpty());
486       assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER)));
487       assertEquals(false, result[1].isEmpty());
488       assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER)));
489     } finally {
490       table.close();
491     }
492   }
493 }