View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.TreeMap;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.HBaseConfiguration;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.testclassification.SmallTests;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.junit.Before;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  import org.mockito.Mockito;
42  
43  @Category(SmallTests.class)
44  public class TestRegionSplitPolicy {
45  
46    private Configuration conf;
47    private HTableDescriptor htd;
48    private HRegion mockRegion;
49    private TreeMap<byte[], HStore> stores;
50    private static final TableName TABLENAME = TableName.valueOf("t");
51  
52    @Before
53    public void setupMocks() {
54      conf = HBaseConfiguration.create();
55      HRegionInfo hri = new HRegionInfo(TABLENAME);
56      htd = new HTableDescriptor(TABLENAME);
57      mockRegion = Mockito.mock(HRegion.class);
58      Mockito.doReturn(htd).when(mockRegion).getTableDesc();
59      Mockito.doReturn(hri).when(mockRegion).getRegionInfo();
60  
61      stores = new TreeMap<byte[], HStore>(Bytes.BYTES_COMPARATOR);
62      Mockito.doReturn(stores).when(mockRegion).getStores();
63    }
64  
65    @Test
66    public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
67      // Configure IncreasingToUpperBoundRegionSplitPolicy as our split policy
68      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
69        IncreasingToUpperBoundRegionSplitPolicy.class.getName());
70      conf.setFloat("hbase.hregion.max.filesize.jitter", 0.25f);
71  
72      // Now make it so the mock region has a RegionServerService that will
73      // return 'online regions'.
74      RegionServerServices rss = Mockito.mock(RegionServerServices.class);
75      final List<HRegion> regions = new ArrayList<HRegion>();
76      Mockito.when(rss.getOnlineRegions(TABLENAME)).thenReturn(regions);
77      Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
78      // Set max size for this 'table'.
79      long maxSplitSize = 1024L;
80      htd.setMaxFileSize(maxSplitSize);
81      // Set flush size to 1/8.  IncreasingToUpperBoundRegionSplitPolicy
82      // grows by the cube of the number of regions times flushsize each time.
83      long flushSize = maxSplitSize/8;
84      conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
85      htd.setMemStoreFlushSize(flushSize);
86      // If RegionServerService with no regions in it -- 'online regions' == 0 --
87      // then IncreasingToUpperBoundRegionSplitPolicy should act like a
88      // ConstantSizePolicy
89      IncreasingToUpperBoundRegionSplitPolicy policy =
90        (IncreasingToUpperBoundRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
91      doConstantSizePolicyTests(policy);
92  
93      // Add a store in excess of split size.  Because there are "no regions"
94      // on this server -- rss.getOnlineRegions is 0 -- then we should split
95      // like a constantsizeregionsplitpolicy would
96      HStore mockStore = Mockito.mock(HStore.class);
97      Mockito.doReturn(2000L).when(mockStore).getSize();
98      Mockito.doReturn(true).when(mockStore).canSplit();
99      stores.put(new byte[]{1}, mockStore);
100     // It should split
101     assertTrue(policy.shouldSplit());
102 
103     // Now test that we increase our split size as online regions for a table
104     // grows. With one region, split size should be flushsize.
105     regions.add(mockRegion);
106     Mockito.doReturn(flushSize).when(mockStore).getSize();
107     // Should not split since store is flush size.
108     assertFalse(policy.shouldSplit());
109     // Set size of store to be > 2*flush size and we should split
110     Mockito.doReturn(flushSize*2 + 1).when(mockStore).getSize();
111     assertTrue(policy.shouldSplit());
112     // Add another region to the 'online regions' on this server and we should
113     // now be no longer be splittable since split size has gone up.
114     regions.add(mockRegion);
115     assertFalse(policy.shouldSplit());
116     // make sure its just over; verify it'll split
117     Mockito.doReturn((long)(maxSplitSize * 1.25 + 1)).when(mockStore).getSize();
118     assertTrue(policy.shouldSplit());
119 
120     // Finally assert that even if loads of regions, we'll split at max size
121     assertWithinJitter(maxSplitSize, policy.getSizeToCheck(1000));
122     // Assert same is true if count of regions is zero.
123     assertWithinJitter(maxSplitSize, policy.getSizeToCheck(0));
124   }
125 
126   private void assertWithinJitter(long maxSplitSize, long sizeToCheck) {
127     assertTrue("Size greater than lower bound of jitter",
128         (long)(maxSplitSize * 0.75) <= sizeToCheck);
129     assertTrue("Size less than upper bound of jitter",
130         (long)(maxSplitSize * 1.25) >= sizeToCheck);
131   }
132 
133   @Test
134   public void testCreateDefault() throws IOException {
135     conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L);
136     conf.setFloat("hbase.hregion.max.filesize.jitter", 0.25f);
137 
138     // Using a default HTD, should pick up the file size from
139     // configuration.
140     ConstantSizeRegionSplitPolicy policy =
141         (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
142             mockRegion, conf);
143     assertWithinJitter(1234L, policy.getDesiredMaxFileSize());
144 
145     // If specified in HTD, should use that
146     htd.setMaxFileSize(9999L);
147     policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
148         mockRegion, conf);
149     assertWithinJitter(9999L, policy.getDesiredMaxFileSize());
150   }
151 
152   /**
153    * Test setting up a customized split policy
154    */
155   @Test
156   public void testCustomPolicy() throws IOException {
157     HTableDescriptor myHtd = new HTableDescriptor();
158     myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
159         KeyPrefixRegionSplitPolicy.class.getName());
160     myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2));
161 
162     HRegion myMockRegion = Mockito.mock(HRegion.class);
163     Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
164     Mockito.doReturn(stores).when(myMockRegion).getStores();
165 
166     HStore mockStore = Mockito.mock(HStore.class);
167     Mockito.doReturn(2000L).when(mockStore).getSize();
168     Mockito.doReturn(true).when(mockStore).canSplit();
169     Mockito.doReturn(Bytes.toBytes("abcd")).when(mockStore).getSplitPoint();
170     stores.put(new byte[] { 1 }, mockStore);
171 
172     KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
173         .create(myMockRegion, conf);
174 
175     assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
176 
177     Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
178     Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion)
179         .getExplicitSplitPoint();
180 
181     policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
182         .create(myMockRegion, conf);
183 
184     assertEquals("ef", Bytes.toString(policy.getSplitPoint()));
185   }
186 
187   @Test
188   public void testConstantSizePolicy() throws IOException {
189     htd.setMaxFileSize(1024L);
190     ConstantSizeRegionSplitPolicy policy =
191       (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
192     doConstantSizePolicyTests(policy);
193   }
194 
195   /**
196    * Run through tests for a ConstantSizeRegionSplitPolicy
197    * @param policy
198    */
199   private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) {
200     // For no stores, should not split
201     assertFalse(policy.shouldSplit());
202 
203     // Add a store above the requisite size. Should split.
204     HStore mockStore = Mockito.mock(HStore.class);
205     Mockito.doReturn(2000L).when(mockStore).getSize();
206     Mockito.doReturn(true).when(mockStore).canSplit();
207     stores.put(new byte[]{1}, mockStore);
208 
209     assertTrue(policy.shouldSplit());
210 
211     // Act as if there's a reference file or some other reason it can't split.
212     // This should prevent splitting even though it's big enough.
213     Mockito.doReturn(false).when(mockStore).canSplit();
214     assertFalse(policy.shouldSplit());
215 
216     // Reset splittability after above
217     Mockito.doReturn(true).when(mockStore).canSplit();
218 
219     // Set to a small size but turn on forceSplit. Should result in a split.
220     Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
221     Mockito.doReturn(100L).when(mockStore).getSize();
222     assertTrue(policy.shouldSplit());
223 
224     // Turn off forceSplit, should not split
225     Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
226     assertFalse(policy.shouldSplit());
227 
228     // Clear families we added above
229     stores.clear();
230   }
231 
232   @Test
233   public void testGetSplitPoint() throws IOException {
234     ConstantSizeRegionSplitPolicy policy =
235       (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
236 
237     // For no stores, should not split
238     assertFalse(policy.shouldSplit());
239     assertNull(policy.getSplitPoint());
240 
241     // Add a store above the requisite size. Should split.
242     HStore mockStore = Mockito.mock(HStore.class);
243     Mockito.doReturn(2000L).when(mockStore).getSize();
244     Mockito.doReturn(true).when(mockStore).canSplit();
245     Mockito.doReturn(Bytes.toBytes("store 1 split"))
246       .when(mockStore).getSplitPoint();
247     stores.put(new byte[]{1}, mockStore);
248 
249     assertEquals("store 1 split",
250         Bytes.toString(policy.getSplitPoint()));
251 
252     // Add a bigger store. The split point should come from that one
253     HStore mockStore2 = Mockito.mock(HStore.class);
254     Mockito.doReturn(4000L).when(mockStore2).getSize();
255     Mockito.doReturn(true).when(mockStore2).canSplit();
256     Mockito.doReturn(Bytes.toBytes("store 2 split"))
257       .when(mockStore2).getSplitPoint();
258     stores.put(new byte[]{2}, mockStore2);
259 
260     assertEquals("store 2 split",
261         Bytes.toString(policy.getSplitPoint()));
262   }
263 
264   @Test
265   public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
266     HTableDescriptor myHtd = new HTableDescriptor();
267     myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
268         DelimitedKeyPrefixRegionSplitPolicy.class.getName());
269     myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",");
270 
271     HRegion myMockRegion = Mockito.mock(HRegion.class);
272     Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
273     Mockito.doReturn(stores).when(myMockRegion).getStores();
274 
275     HStore mockStore = Mockito.mock(HStore.class);
276     Mockito.doReturn(2000L).when(mockStore).getSize();
277     Mockito.doReturn(true).when(mockStore).canSplit();
278     Mockito.doReturn(Bytes.toBytes("ab,cd")).when(mockStore).getSplitPoint();
279     stores.put(new byte[] { 1 }, mockStore);
280 
281     DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
282         .create(myMockRegion, conf);
283 
284     assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
285 
286     Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
287     Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion)
288         .getExplicitSplitPoint();
289 
290     policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
291         .create(myMockRegion, conf);
292 
293     assertEquals("efg", Bytes.toString(policy.getSplitPoint()));
294 
295     Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion)
296     .getExplicitSplitPoint();
297     assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
298   }
299 
300 }