1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.util.ArrayList;
26  import java.util.Random;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HRegionLocation;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.LargeTests;
35  import org.apache.hadoop.hbase.ipc.HRegionInterface;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.hbase.util.Pair;
38  import org.junit.After;
39  import org.junit.AfterClass;
40  import org.junit.Before;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  
45  import com.google.common.collect.Lists;
46  
47  @Category(LargeTests.class)
48  public class TestFromClientSide3 {
49    final Log LOG = LogFactory.getLog(getClass());
50    private final static HBaseTestingUtility TEST_UTIL
51      = new HBaseTestingUtility();
52    private static byte[] ROW = Bytes.toBytes("testRow");
53    private static byte[] FAMILY = Bytes.toBytes("testFamily");
54    private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
55    private static byte[] VALUE = Bytes.toBytes("testValue");
56    private static Random random = new Random();
57    private static int SLAVES = 3;
58  
59    /**
60     * @throws java.lang.Exception
61     */
62    @BeforeClass
63    public static void setUpBeforeClass() throws Exception {
64      TEST_UTIL.getConfiguration().setBoolean(
65          "hbase.online.schema.update.enable", true);
66      TEST_UTIL.startMiniCluster(SLAVES);
67    }
68  
69    /**
70     * @throws java.lang.Exception
71     */
72    @AfterClass
73    public static void tearDownAfterClass() throws Exception {
74      TEST_UTIL.shutdownMiniCluster();
75    }
76  
77    /**
78     * @throws java.lang.Exception
79     */
80    @Before
81    public void setUp() throws Exception {
82      // Nothing to do.
83    }
84  
85    /**
86     * @throws java.lang.Exception
87     */
88    @After
89    public void tearDown() throws Exception {
90      // Nothing to do.
91    }
92  
93    private void randomCFPuts(HTable table, byte[] row, byte[] family, int nPuts)
94        throws Exception {
95      Put put = new Put(row);
96      for (int i = 0; i < nPuts; i++) {
97        byte[] qualifier = Bytes.toBytes(random.nextInt());
98        byte[] value = Bytes.toBytes(random.nextInt());
99        put.add(family, qualifier, value);
100     }
101     table.put(put);
102   }
103 
104   private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
105       byte[] row, byte[] family, int nFlushes, int nPuts) throws Exception {
106 
107     // connection needed for poll-wait
108     HConnection conn = HConnectionManager.getConnection(TEST_UTIL
109         .getConfiguration());
110     HRegionLocation loc = table.getRegionLocation(row, true);
111     HRegionInterface server = conn.getHRegionConnection(loc.getHostname(), loc
112         .getPort());
113     byte[] regName = loc.getRegionInfo().getRegionName();
114 
115     for (int i = 0; i < nFlushes; i++) {
116       randomCFPuts(table, row, family, nPuts);
117       int sfCount = server.getStoreFileList(regName, FAMILY).size();
118 
119       // TODO: replace this api with a synchronous flush after HBASE-2949
120       admin.flush(table.getTableName());
121 
122       // synchronously poll wait for a new storefile to appear (flush happened)
123       while (server.getStoreFileList(regName, FAMILY).size() == sfCount) {
124         Thread.sleep(40);
125       }
126     }
127   }
128 
129   // override the config settings at the CF level and ensure priority
130   @Test(timeout = 60000)
131   public void testAdvancedConfigOverride() throws Exception {
132     /*
133      * Overall idea: (1) create 3 store files and issue a compaction. config's
134      * compaction.min == 3, so should work. (2) Increase the compaction.min
135      * toggle in the HTD to 5 and modify table. If we use the HTD value instead
136      * of the default config value, adding 3 files and issuing a compaction
137      * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2
138      * and modify table. The CF schema should override the Table schema and now
139      * cause a minor compaction.
140      */
141     TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
142 
143     String tableName = "testAdvancedConfigOverride";
144     byte[] TABLE = Bytes.toBytes(tableName);
145     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
146     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
147     HConnection connection = HConnectionManager.getConnection(TEST_UTIL
148         .getConfiguration());
149 
150     // Create 3 store files.
151     byte[] row = Bytes.toBytes(random.nextInt());
152     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
153 
154     // Verify we have multiple store files.
155     HRegionLocation loc = hTable.getRegionLocation(row, true);
156     byte[] regionName = loc.getRegionInfo().getRegionName();
157     HRegionInterface server = connection.getHRegionConnection(
158         loc.getHostname(), loc.getPort());
159     assertTrue(server.getStoreFileList(regionName, FAMILY).size() > 1);
160 
161     // Issue a compaction request
162     admin.compact(TABLE);
163 
164     // poll wait for the compactions to happen
165     for (int i = 0; i < 10 * 1000 / 40; ++i) {
166       // The number of store files after compaction should be lesser.
167       loc = hTable.getRegionLocation(row, true);
168       if (!loc.getRegionInfo().isOffline()) {
169         regionName = loc.getRegionInfo().getRegionName();
170         server = connection.getHRegionConnection(loc.getHostname(), loc
171             .getPort());
172         if (server.getStoreFileList(regionName, FAMILY).size() <= 1) {
173           break;
174         }
175       }
176       Thread.sleep(40);
177     }
178     // verify the compactions took place and that we didn't just time out
179     assertTrue(server.getStoreFileList(regionName, FAMILY).size() <= 1);
180 
181     // change the compaction.min config option for this table to 5
182     LOG.info("hbase.hstore.compaction.min should now be 5");
183     HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
184     htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
185     admin.modifyTable(TABLE, htd);
186     Pair<Integer, Integer> st;
187     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
188       LOG.debug(st.getFirst() + " regions left to update");
189       Thread.sleep(40);
190     }
191     LOG.info("alter status finished");
192 
193     // Create 3 more store files.
194     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
195 
196     // Issue a compaction request
197     admin.compact(TABLE);
198 
199     // This time, the compaction request should not happen
200     Thread.sleep(10 * 1000);
201     int sfCount = 0;
202     loc = hTable.getRegionLocation(row, true);
203     regionName = loc.getRegionInfo().getRegionName();
204     server = connection.getHRegionConnection(loc.getHostname(), loc.getPort());
205     sfCount = server.getStoreFileList(regionName, FAMILY).size();
206     assertTrue(sfCount > 1);
207 
208     // change an individual CF's config option to 2 & online schema update
209     LOG.info("hbase.hstore.compaction.min should now be 2");
210     HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
211     hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
212     htd.addFamily(hcd);
213     admin.modifyTable(TABLE, htd);
214     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
215       LOG.debug(st.getFirst() + " regions left to update");
216       Thread.sleep(40);
217     }
218     LOG.info("alter status finished");
219 
220     // Issue a compaction request
221     admin.compact(TABLE);
222 
223     // poll wait for the compactions to happen
224     for (int i = 0; i < 10 * 1000 / 40; ++i) {
225       loc = hTable.getRegionLocation(row, true);
226       regionName = loc.getRegionInfo().getRegionName();
227       try {
228         server = connection.getHRegionConnection(loc.getHostname(), loc
229             .getPort());
230         if (server.getStoreFileList(regionName, FAMILY).size() < sfCount) {
231           break;
232         }
233       } catch (Exception e) {
234         LOG.debug("Waiting for region to come online: " + regionName);
235       }
236       Thread.sleep(40);
237     }
238     // verify the compaction took place and that we didn't just time out
239     assertTrue(server.getStoreFileList(regionName, FAMILY).size() < sfCount);
240 
241     // Finally, ensure that we can remove a custom config value after we made it
242     LOG.info("Removing CF config value");
243     LOG.info("hbase.hstore.compaction.min should now be 5");
244     hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
245     hcd.setValue("hbase.hstore.compaction.min", null);
246     htd.addFamily(hcd);
247     admin.modifyTable(TABLE, htd);
248     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
249       LOG.debug(st.getFirst() + " regions left to update");
250       Thread.sleep(40);
251     }
252     LOG.info("alter status finished");
253     assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
254         "hbase.hstore.compaction.min"));
255   }
256 
257   @org.junit.Rule
258   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
259    new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
260 }