View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.TreeMap;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.HBaseConfiguration;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.testclassification.SmallTests;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.junit.Before;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  import org.mockito.Mockito;
42  
43  @Category(SmallTests.class)
44  public class TestRegionSplitPolicy {
45  
46    private Configuration conf;
47    private HTableDescriptor htd;
48    private HRegion mockRegion;
49    private TreeMap<byte[], HStore> stores;
50    private static final TableName TABLENAME = TableName.valueOf("t");
51  
52    @Before
53    public void setupMocks() {
54      conf = HBaseConfiguration.create();
55      HRegionInfo hri = new HRegionInfo(TABLENAME);
56      htd = new HTableDescriptor(TABLENAME);
57      mockRegion = Mockito.mock(HRegion.class);
58      Mockito.doReturn(htd).when(mockRegion).getTableDesc();
59      Mockito.doReturn(hri).when(mockRegion).getRegionInfo();
60  
61      stores = new TreeMap<byte[], HStore>(Bytes.BYTES_COMPARATOR);
62      Mockito.doReturn(stores).when(mockRegion).getStores();
63    }
64  
65    @Test
66    public void testForceSplitRegionWithReference() throws IOException {
67      htd.setMaxFileSize(1024L);
68      // Add a store above the requisite size. Should split.
69      HStore mockStore = Mockito.mock(HStore.class);
70      Mockito.doReturn(2000L).when(mockStore).getSize();
71      // Act as if there's a reference file or some other reason it can't split.
72      // This should prevent splitting even though it's big enough.
73      Mockito.doReturn(false).when(mockStore).canSplit();
74      stores.put(new byte[]{1}, mockStore);
75  
76      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
77        ConstantSizeRegionSplitPolicy.class.getName());
78      ConstantSizeRegionSplitPolicy policy =
79          (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
80      assertFalse(policy.shouldSplit());
81      Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
82      assertFalse(policy.shouldSplit());
83  
84      Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
85      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
86        IncreasingToUpperBoundRegionSplitPolicy.class.getName());
87      policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
88      assertFalse(policy.shouldSplit());
89      Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
90      assertFalse(policy.shouldSplit());
91    }
92  
93    @Test
94    public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
95      // Configure IncreasingToUpperBoundRegionSplitPolicy as our split policy
96      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
97        IncreasingToUpperBoundRegionSplitPolicy.class.getName());
98      conf.setFloat("hbase.hregion.max.filesize.jitter", 0.25f);
99  
100     // Now make it so the mock region has a RegionServerService that will
101     // return 'online regions'.
102     RegionServerServices rss = Mockito.mock(RegionServerServices.class);
103     final List<HRegion> regions = new ArrayList<HRegion>();
104     Mockito.when(rss.getOnlineRegions(TABLENAME)).thenReturn(regions);
105     Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
106     // Set max size for this 'table'.
107     long maxSplitSize = 1024L;
108     htd.setMaxFileSize(maxSplitSize);
109     // Set flush size to 1/8.  IncreasingToUpperBoundRegionSplitPolicy
110     // grows by the cube of the number of regions times flushsize each time.
111     long flushSize = maxSplitSize/8;
112     conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
113     htd.setMemStoreFlushSize(flushSize);
114     // If RegionServerService with no regions in it -- 'online regions' == 0 --
115     // then IncreasingToUpperBoundRegionSplitPolicy should act like a
116     // ConstantSizePolicy
117     IncreasingToUpperBoundRegionSplitPolicy policy =
118       (IncreasingToUpperBoundRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
119     doConstantSizePolicyTests(policy);
120 
121     // Add a store in excess of split size.  Because there are "no regions"
122     // on this server -- rss.getOnlineRegions is 0 -- then we should split
123     // like a constantsizeregionsplitpolicy would
124     HStore mockStore = Mockito.mock(HStore.class);
125     Mockito.doReturn(2000L).when(mockStore).getSize();
126     Mockito.doReturn(true).when(mockStore).canSplit();
127     stores.put(new byte[]{1}, mockStore);
128     // It should split
129     assertTrue(policy.shouldSplit());
130 
131     // Now test that we increase our split size as online regions for a table
132     // grows. With one region, split size should be flushsize.
133     regions.add(mockRegion);
134     Mockito.doReturn(flushSize).when(mockStore).getSize();
135     // Should not split since store is flush size.
136     assertFalse(policy.shouldSplit());
137     // Set size of store to be > 2*flush size and we should split
138     Mockito.doReturn(flushSize*2 + 1).when(mockStore).getSize();
139     assertTrue(policy.shouldSplit());
140     // Add another region to the 'online regions' on this server and we should
141     // now be no longer be splittable since split size has gone up.
142     regions.add(mockRegion);
143     assertFalse(policy.shouldSplit());
144     // make sure its just over; verify it'll split
145     Mockito.doReturn((long)(maxSplitSize * 1.25 + 1)).when(mockStore).getSize();
146     assertTrue(policy.shouldSplit());
147 
148     // Finally assert that even if loads of regions, we'll split at max size
149     assertWithinJitter(maxSplitSize, policy.getSizeToCheck(1000));
150     // Assert same is true if count of regions is zero.
151     assertWithinJitter(maxSplitSize, policy.getSizeToCheck(0));
152   }
153 
154   private void assertWithinJitter(long maxSplitSize, long sizeToCheck) {
155     assertTrue("Size greater than lower bound of jitter",
156         (long)(maxSplitSize * 0.75) <= sizeToCheck);
157     assertTrue("Size less than upper bound of jitter",
158         (long)(maxSplitSize * 1.25) >= sizeToCheck);
159   }
160 
161   @Test
162   public void testCreateDefault() throws IOException {
163     conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L);
164     conf.setFloat("hbase.hregion.max.filesize.jitter", 0.25f);
165 
166     // Using a default HTD, should pick up the file size from
167     // configuration.
168     ConstantSizeRegionSplitPolicy policy =
169         (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
170             mockRegion, conf);
171     assertWithinJitter(1234L, policy.getDesiredMaxFileSize());
172 
173     // If specified in HTD, should use that
174     htd.setMaxFileSize(9999L);
175     policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
176         mockRegion, conf);
177     assertWithinJitter(9999L, policy.getDesiredMaxFileSize());
178   }
179 
180   /**
181    * Test setting up a customized split policy
182    */
183   @Test
184   public void testCustomPolicy() throws IOException {
185     HTableDescriptor myHtd = new HTableDescriptor();
186     myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
187         KeyPrefixRegionSplitPolicy.class.getName());
188     myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2));
189 
190     HRegion myMockRegion = Mockito.mock(HRegion.class);
191     Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
192     Mockito.doReturn(stores).when(myMockRegion).getStores();
193 
194     HStore mockStore = Mockito.mock(HStore.class);
195     Mockito.doReturn(2000L).when(mockStore).getSize();
196     Mockito.doReturn(true).when(mockStore).canSplit();
197     Mockito.doReturn(Bytes.toBytes("abcd")).when(mockStore).getSplitPoint();
198     stores.put(new byte[] { 1 }, mockStore);
199 
200     KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
201         .create(myMockRegion, conf);
202 
203     assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
204 
205     Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
206     Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion)
207         .getExplicitSplitPoint();
208 
209     policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
210         .create(myMockRegion, conf);
211 
212     assertEquals("ef", Bytes.toString(policy.getSplitPoint()));
213   }
214 
215   @Test
216   public void testConstantSizePolicy() throws IOException {
217     htd.setMaxFileSize(1024L);
218     ConstantSizeRegionSplitPolicy policy =
219       (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
220     doConstantSizePolicyTests(policy);
221   }
222 
223   /**
224    * Run through tests for a ConstantSizeRegionSplitPolicy
225    * @param policy
226    */
227   private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) {
228     // For no stores, should not split
229     assertFalse(policy.shouldSplit());
230 
231     // Add a store above the requisite size. Should split.
232     HStore mockStore = Mockito.mock(HStore.class);
233     Mockito.doReturn(2000L).when(mockStore).getSize();
234     Mockito.doReturn(true).when(mockStore).canSplit();
235     stores.put(new byte[]{1}, mockStore);
236 
237     assertTrue(policy.shouldSplit());
238 
239     // Act as if there's a reference file or some other reason it can't split.
240     // This should prevent splitting even though it's big enough.
241     Mockito.doReturn(false).when(mockStore).canSplit();
242     assertFalse(policy.shouldSplit());
243 
244     // Reset splittability after above
245     Mockito.doReturn(true).when(mockStore).canSplit();
246 
247     // Set to a small size but turn on forceSplit. Should result in a split.
248     Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
249     Mockito.doReturn(100L).when(mockStore).getSize();
250     assertTrue(policy.shouldSplit());
251 
252     // Turn off forceSplit, should not split
253     Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
254     assertFalse(policy.shouldSplit());
255 
256     // Clear families we added above
257     stores.clear();
258   }
259 
260   @Test
261   public void testGetSplitPoint() throws IOException {
262     ConstantSizeRegionSplitPolicy policy =
263       (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
264 
265     // For no stores, should not split
266     assertFalse(policy.shouldSplit());
267     assertNull(policy.getSplitPoint());
268 
269     // Add a store above the requisite size. Should split.
270     HStore mockStore = Mockito.mock(HStore.class);
271     Mockito.doReturn(2000L).when(mockStore).getSize();
272     Mockito.doReturn(true).when(mockStore).canSplit();
273     Mockito.doReturn(Bytes.toBytes("store 1 split"))
274       .when(mockStore).getSplitPoint();
275     stores.put(new byte[]{1}, mockStore);
276 
277     assertEquals("store 1 split",
278         Bytes.toString(policy.getSplitPoint()));
279 
280     // Add a bigger store. The split point should come from that one
281     HStore mockStore2 = Mockito.mock(HStore.class);
282     Mockito.doReturn(4000L).when(mockStore2).getSize();
283     Mockito.doReturn(true).when(mockStore2).canSplit();
284     Mockito.doReturn(Bytes.toBytes("store 2 split"))
285       .when(mockStore2).getSplitPoint();
286     stores.put(new byte[]{2}, mockStore2);
287 
288     assertEquals("store 2 split",
289         Bytes.toString(policy.getSplitPoint()));
290   }
291 
292   @Test
293   public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
294     HTableDescriptor myHtd = new HTableDescriptor();
295     myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
296         DelimitedKeyPrefixRegionSplitPolicy.class.getName());
297     myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",");
298 
299     HRegion myMockRegion = Mockito.mock(HRegion.class);
300     Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
301     Mockito.doReturn(stores).when(myMockRegion).getStores();
302 
303     HStore mockStore = Mockito.mock(HStore.class);
304     Mockito.doReturn(2000L).when(mockStore).getSize();
305     Mockito.doReturn(true).when(mockStore).canSplit();
306     Mockito.doReturn(Bytes.toBytes("ab,cd")).when(mockStore).getSplitPoint();
307     stores.put(new byte[] { 1 }, mockStore);
308 
309     DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
310         .create(myMockRegion, conf);
311 
312     assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
313 
314     Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
315     Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion)
316         .getExplicitSplitPoint();
317 
318     policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
319         .create(myMockRegion, conf);
320 
321     assertEquals("efg", Bytes.toString(policy.getSplitPoint()));
322 
323     Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion)
324     .getExplicitSplitPoint();
325     assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
326   }
327 
328 }