1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.TreeMap;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HBaseConfiguration;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.SmallTests;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.junit.Before;
38  import org.junit.Test;
39  import org.junit.experimental.categories.Category;
40  import org.mockito.Mockito;
41  
42  @Category(SmallTests.class)
43  public class TestRegionSplitPolicy {
44  
45    private Configuration conf;
46    private HTableDescriptor htd;
47    private HRegion mockRegion;
48    private TreeMap<byte[], Store> stores;
49    private static final byte [] TABLENAME = new byte [] {'t'};
50  
51    @Before
52    public void setupMocks() {
53      conf = HBaseConfiguration.create();
54      HRegionInfo hri = new HRegionInfo(TABLENAME);
55      htd = new HTableDescriptor(TABLENAME);
56      mockRegion = Mockito.mock(HRegion.class);
57      Mockito.doReturn(htd).when(mockRegion).getTableDesc();
58      Mockito.doReturn(hri).when(mockRegion).getRegionInfo();
59  
60      stores = new TreeMap<byte[], Store>(Bytes.BYTES_COMPARATOR);
61      Mockito.doReturn(stores).when(mockRegion).getStores();
62    }
63  
64    @Test
65    public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
66      // Configure IncreasingToUpperBoundRegionSplitPolicy as our split policy
67      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
68        IncreasingToUpperBoundRegionSplitPolicy.class.getName());
69      // Now make it so the mock region has a RegionServerService that will
70      // return 'online regions'.
71      RegionServerServices rss = Mockito.mock(RegionServerServices.class);
72      final List<HRegion> regions = new ArrayList<HRegion>();
73      Mockito.when(rss.getOnlineRegions(TABLENAME)).thenReturn(regions);
74      Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
75      // Set max size for this 'table'.
76      long maxSplitSize = 1024L;
77      htd.setMaxFileSize(maxSplitSize);
78      // Set flush size to 1/8.  IncreasingToUpperBoundRegionSplitPolicy
79      // grows by the cube of the number of regions times flushsize each time.
80      long flushSize = maxSplitSize/8;
81      conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
82      htd.setMemStoreFlushSize(flushSize);
83      // If RegionServerService with no regions in it -- 'online regions' == 0 --
84      // then IncreasingToUpperBoundRegionSplitPolicy should act like a
85      // ConstantSizePolicy
86      IncreasingToUpperBoundRegionSplitPolicy policy =
87        (IncreasingToUpperBoundRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
88      doConstantSizePolicyTests(policy);
89  
90      // Add a store in excess of split size.  Because there are "no regions"
91      // on this server -- rss.getOnlineRegions is 0 -- then we should split
92      // like a constantsizeregionsplitpolicy would
93      Store mockStore = Mockito.mock(Store.class);
94      Mockito.doReturn(2000L).when(mockStore).getSize();
95      Mockito.doReturn(true).when(mockStore).canSplit();
96      stores.put(new byte[]{1}, mockStore);
97      // It should split
98      assertTrue(policy.shouldSplit());
99  
100     // Now test that we increase our split size as online regions for a table
101     // grows. With one region, split size should be flushsize.
102     regions.add(mockRegion);
103     Mockito.doReturn(flushSize).when(mockStore).getSize();
104     // Should not split since store is flush size.
105     assertFalse(policy.shouldSplit());
106     // Set size of store to be > 2*flush size and we should split
107     Mockito.doReturn(flushSize*2 + 1).when(mockStore).getSize();
108     assertTrue(policy.shouldSplit());
109     // Add another region to the 'online regions' on this server and we should
110     // now be no longer be splittable since split size has gone up.
111     regions.add(mockRegion);
112     assertFalse(policy.shouldSplit());
113     // Quadruple (2 squared) the store size and make sure its just over; verify it'll split
114     Mockito.doReturn((flushSize * 2 * 2 * 2) + 1).when(mockStore).getSize();
115     assertTrue(policy.shouldSplit());
116 
117     // Finally assert that even if loads of regions, we'll split at max size
118     assertEquals(maxSplitSize, policy.getSizeToCheck(1000));
119     // Assert same is true if count of regions is zero.
120     assertEquals(maxSplitSize, policy.getSizeToCheck(0));
121   }
122 
123   @Test
124   public void testCreateDefault() throws IOException {
125     conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L);
126 
127     // Using a default HTD, should pick up the file size from
128     // configuration.
129     ConstantSizeRegionSplitPolicy policy =
130         (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
131             mockRegion, conf);
132     assertEquals(1234L, policy.getDesiredMaxFileSize());
133 
134     // If specified in HTD, should use that
135     htd.setMaxFileSize(9999L);
136     policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
137         mockRegion, conf);
138     assertEquals(9999L, policy.getDesiredMaxFileSize());
139   }
140 
141   /**
142    * Test setting up a customized split policy
143    */
144   @Test
145   public void testCustomPolicy() throws IOException {
146     HTableDescriptor myHtd = new HTableDescriptor();
147     myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
148         KeyPrefixRegionSplitPolicy.class.getName());
149     myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2));
150 
151     HRegion myMockRegion = Mockito.mock(HRegion.class);
152     Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
153     Mockito.doReturn(stores).when(myMockRegion).getStores();
154 
155     Store mockStore = Mockito.mock(Store.class);
156     Mockito.doReturn(2000L).when(mockStore).getSize();
157     Mockito.doReturn(true).when(mockStore).canSplit();
158     Mockito.doReturn(Bytes.toBytes("abcd")).when(mockStore).getSplitPoint();
159     stores.put(new byte[] { 1 }, mockStore);
160 
161     KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
162         .create(myMockRegion, conf);
163 
164     assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
165 
166     Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
167     Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion)
168         .getExplicitSplitPoint();
169 
170     policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
171         .create(myMockRegion, conf);
172 
173     assertEquals("ef", Bytes.toString(policy.getSplitPoint()));
174   }
175 
176   @Test
177   public void testConstantSizePolicy() throws IOException {
178     htd.setMaxFileSize(1024L);
179     ConstantSizeRegionSplitPolicy policy =
180       (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
181     doConstantSizePolicyTests(policy);
182   }
183 
184   /**
185    * Run through tests for a ConstantSizeRegionSplitPolicy
186    * @param policy
187    */
188   private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) {
189     // For no stores, should not split
190     assertFalse(policy.shouldSplit());
191 
192     // Add a store above the requisite size. Should split.
193     Store mockStore = Mockito.mock(Store.class);
194     Mockito.doReturn(2000L).when(mockStore).getSize();
195     Mockito.doReturn(true).when(mockStore).canSplit();
196     stores.put(new byte[]{1}, mockStore);
197 
198     assertTrue(policy.shouldSplit());
199 
200     // Act as if there's a reference file or some other reason it can't split.
201     // This should prevent splitting even though it's big enough.
202     Mockito.doReturn(false).when(mockStore).canSplit();
203     assertFalse(policy.shouldSplit());
204 
205     // Reset splittability after above
206     Mockito.doReturn(true).when(mockStore).canSplit();
207 
208     // Set to a small size but turn on forceSplit. Should result in a split.
209     Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
210     Mockito.doReturn(100L).when(mockStore).getSize();
211     assertTrue(policy.shouldSplit());
212 
213     // Turn off forceSplit, should not split
214     Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
215     assertFalse(policy.shouldSplit());
216 
217     // Clear families we added above
218     stores.clear();
219   }
220 
221   @Test
222   public void testGetSplitPoint() throws IOException {
223     ConstantSizeRegionSplitPolicy policy =
224       (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
225 
226     // For no stores, should not split
227     assertFalse(policy.shouldSplit());
228     assertNull(policy.getSplitPoint());
229 
230     // Add a store above the requisite size. Should split.
231     Store mockStore = Mockito.mock(Store.class);
232     Mockito.doReturn(2000L).when(mockStore).getSize();
233     Mockito.doReturn(true).when(mockStore).canSplit();
234     Mockito.doReturn(Bytes.toBytes("store 1 split"))
235       .when(mockStore).getSplitPoint();
236     stores.put(new byte[]{1}, mockStore);
237 
238     assertEquals("store 1 split",
239         Bytes.toString(policy.getSplitPoint()));
240 
241     // Add a bigger store. The split point should come from that one
242     Store mockStore2 = Mockito.mock(Store.class);
243     Mockito.doReturn(4000L).when(mockStore2).getSize();
244     Mockito.doReturn(true).when(mockStore2).canSplit();
245     Mockito.doReturn(Bytes.toBytes("store 2 split"))
246       .when(mockStore2).getSplitPoint();
247     stores.put(new byte[]{2}, mockStore2);
248 
249     assertEquals("store 2 split",
250         Bytes.toString(policy.getSplitPoint()));
251   }
252 
253   @Test
254   public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
255     HTableDescriptor myHtd = new HTableDescriptor();
256     myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
257         DelimitedKeyPrefixRegionSplitPolicy.class.getName());
258     myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",");
259 
260     HRegion myMockRegion = Mockito.mock(HRegion.class);
261     Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
262     Mockito.doReturn(stores).when(myMockRegion).getStores();
263 
264     Store mockStore = Mockito.mock(Store.class);
265     Mockito.doReturn(2000L).when(mockStore).getSize();
266     Mockito.doReturn(true).when(mockStore).canSplit();
267     Mockito.doReturn(Bytes.toBytes("ab,cd")).when(mockStore).getSplitPoint();
268     stores.put(new byte[] { 1 }, mockStore);
269 
270     DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
271         .create(myMockRegion, conf);
272 
273     assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
274 
275     Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
276     Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion)
277         .getExplicitSplitPoint();
278 
279     policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
280         .create(myMockRegion, conf);
281 
282     assertEquals("efg", Bytes.toString(policy.getSplitPoint()));
283 
284     Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion)
285     .getExplicitSplitPoint();
286     assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
287   }
288 
289   @org.junit.Rule
290   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
291     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
292 }