1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.util;
21  
22  import static org.junit.Assert.*;
23  
24  import java.io.IOException;
25  import java.math.BigInteger;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.List;
29  import java.util.Map;
30  
31  import org.apache.commons.lang.ArrayUtils;
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.*;
36  import org.apache.hadoop.hbase.client.HTable;
37  import org.apache.hadoop.hbase.client.Put;
38  import org.apache.hadoop.hbase.util.RegionSplitter.HexStringSplit;
39  import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm;
40  import org.apache.hadoop.hbase.util.RegionSplitter.UniformSplit;
41  import org.junit.AfterClass;
42  import org.junit.BeforeClass;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  
46  /**
47   * Tests for {@link RegionSplitter}, which can create a pre-split table or do a
48   * rolling split of an existing table.
49   */
50  @Category(MediumTests.class)
51  public class TestRegionSplitter {
52      private final static Log LOG = LogFactory.getLog(TestRegionSplitter.class);
53      private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
54      private final static String CF_NAME = "SPLIT_TEST_CF";
55      private final static byte xFF = (byte) 0xff;
56  
57      @BeforeClass
58      public static void setup() throws Exception {
59          UTIL.startMiniCluster();
60      }
61  
62      @AfterClass
63      public static void teardown() throws Exception {
64          UTIL.shutdownMiniCluster();
65      }
66  
67      /**
68       * Test creating a pre-split table using the HexStringSplit algorithm.
69       */
70      @Test
71      public void testCreatePresplitTableHex() throws Exception {
72        final List<byte[]> expectedBounds = new ArrayList<byte[]>();
73        expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
74        expectedBounds.add("10000000".getBytes());
75        expectedBounds.add("20000000".getBytes());
76        expectedBounds.add("30000000".getBytes());
77        expectedBounds.add("40000000".getBytes());
78        expectedBounds.add("50000000".getBytes());
79        expectedBounds.add("60000000".getBytes());
80        expectedBounds.add("70000000".getBytes());
81        expectedBounds.add("80000000".getBytes());
82        expectedBounds.add("90000000".getBytes());
83        expectedBounds.add("a0000000".getBytes());
84        expectedBounds.add("b0000000".getBytes());
85        expectedBounds.add("c0000000".getBytes());
86        expectedBounds.add("d0000000".getBytes());
87        expectedBounds.add("e0000000".getBytes());
88        expectedBounds.add("f0000000".getBytes());
89            expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
90  
91            // Do table creation/pre-splitting and verification of region boundaries
92        preSplitTableAndVerify(expectedBounds,
93            HexStringSplit.class.getSimpleName(), "NewHexPresplitTable");
94      }
95  
96      /**
97       * Test creating a pre-split table using the UniformSplit algorithm.
98       */
99      @Test
100     public void testCreatePresplitTableUniform() throws Exception {
101       List<byte[]> expectedBounds = new ArrayList<byte[]>();
102       expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
103       expectedBounds.add(new byte[] {      0x10, 0, 0, 0, 0, 0, 0, 0});
104       expectedBounds.add(new byte[] {      0x20, 0, 0, 0, 0, 0, 0, 0});
105       expectedBounds.add(new byte[] {      0x30, 0, 0, 0, 0, 0, 0, 0});
106       expectedBounds.add(new byte[] {      0x40, 0, 0, 0, 0, 0, 0, 0});
107       expectedBounds.add(new byte[] {      0x50, 0, 0, 0, 0, 0, 0, 0});
108       expectedBounds.add(new byte[] {      0x60, 0, 0, 0, 0, 0, 0, 0});
109       expectedBounds.add(new byte[] {      0x70, 0, 0, 0, 0, 0, 0, 0});
110       expectedBounds.add(new byte[] {(byte)0x80, 0, 0, 0, 0, 0, 0, 0});
111       expectedBounds.add(new byte[] {(byte)0x90, 0, 0, 0, 0, 0, 0, 0});
112       expectedBounds.add(new byte[] {(byte)0xa0, 0, 0, 0, 0, 0, 0, 0});
113       expectedBounds.add(new byte[] {(byte)0xb0, 0, 0, 0, 0, 0, 0, 0});
114       expectedBounds.add(new byte[] {(byte)0xc0, 0, 0, 0, 0, 0, 0, 0});
115       expectedBounds.add(new byte[] {(byte)0xd0, 0, 0, 0, 0, 0, 0, 0});
116       expectedBounds.add(new byte[] {(byte)0xe0, 0, 0, 0, 0, 0, 0, 0});
117       expectedBounds.add(new byte[] {(byte)0xf0, 0, 0, 0, 0, 0, 0, 0});
118       expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
119 
120       // Do table creation/pre-splitting and verification of region boundaries
121       preSplitTableAndVerify(expectedBounds, UniformSplit.class.getSimpleName(),
122         "NewUniformPresplitTable");
123     }
124 
125     /**
126      * Unit tests for the HexStringSplit algorithm. Makes sure it divides up the
127      * space of keys in the way that we expect.
128      */
129     @Test
130     public void unitTestHexStringSplit() {
131         HexStringSplit splitter = new HexStringSplit();
132         // Check splitting while starting from scratch
133 
134         byte[][] twoRegionsSplits = splitter.split(2);
135         assertEquals(1, twoRegionsSplits.length);
136     assertArrayEquals(twoRegionsSplits[0], "80000000".getBytes());
137 
138         byte[][] threeRegionsSplits = splitter.split(3);
139         assertEquals(2, threeRegionsSplits.length);
140         byte[] expectedSplit0 = "55555555".getBytes();
141         assertArrayEquals(expectedSplit0, threeRegionsSplits[0]);
142         byte[] expectedSplit1 = "aaaaaaaa".getBytes();
143         assertArrayEquals(expectedSplit1, threeRegionsSplits[1]);
144 
145         // Check splitting existing regions that have start and end points
146         byte[] splitPoint = splitter.split("10000000".getBytes(), "30000000".getBytes());
147         assertArrayEquals("20000000".getBytes(), splitPoint);
148 
149         byte[] lastRow = "ffffffff".getBytes();
150         assertArrayEquals(lastRow, splitter.lastRow());
151         byte[] firstRow = "00000000".getBytes();
152         assertArrayEquals(firstRow, splitter.firstRow());
153 
154         // Halfway between 00... and 20... should be 10...
155         splitPoint = splitter.split(firstRow, "20000000".getBytes());
156         assertArrayEquals(splitPoint, "10000000".getBytes());
157 
158         // Halfway between df... and ff... should be ef....
159         splitPoint = splitter.split("dfffffff".getBytes(), lastRow);
160         assertArrayEquals(splitPoint,"efffffff".getBytes());
161     }
162 
163     /**
164      * Unit tests for the UniformSplit algorithm. Makes sure it divides up the space of
165      * keys in the way that we expect.
166      */
167     @Test
168     public void unitTestUniformSplit() {
169         UniformSplit splitter = new UniformSplit();
170 
171         // Check splitting while starting from scratch
172         try {
173             splitter.split(1);
174             throw new AssertionError("Splitting into <2 regions should have thrown exception");
175         } catch (IllegalArgumentException e) { }
176 
177         byte[][] twoRegionsSplits = splitter.split(2);
178         assertEquals(1, twoRegionsSplits.length);
179         assertArrayEquals(twoRegionsSplits[0],
180             new byte[] { (byte) 0x80, 0, 0, 0, 0, 0, 0, 0 });
181 
182         byte[][] threeRegionsSplits = splitter.split(3);
183         assertEquals(2, threeRegionsSplits.length);
184         byte[] expectedSplit0 = new byte[] {0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55};
185         assertArrayEquals(expectedSplit0, threeRegionsSplits[0]);
186         byte[] expectedSplit1 = new byte[] {(byte)0xAA, (byte)0xAA, (byte)0xAA, (byte)0xAA,
187                 (byte)0xAA, (byte)0xAA, (byte)0xAA, (byte)0xAA};
188         assertArrayEquals(expectedSplit1, threeRegionsSplits[1]);
189 
190         // Check splitting existing regions that have start and end points
191         byte[] splitPoint = splitter.split(new byte[] {0x10}, new byte[] {0x30});
192         assertArrayEquals(new byte[] {0x20}, splitPoint);
193 
194         byte[] lastRow = new byte[] {xFF, xFF, xFF, xFF, xFF, xFF, xFF, xFF};
195         assertArrayEquals(lastRow, splitter.lastRow());
196         byte[] firstRow = ArrayUtils.EMPTY_BYTE_ARRAY;
197         assertArrayEquals(firstRow, splitter.firstRow());
198 
199         splitPoint = splitter.split(firstRow, new byte[] {0x20});
200         assertArrayEquals(splitPoint, new byte[] {0x10});
201 
202         splitPoint = splitter.split(new byte[] {(byte)0xdf, xFF, xFF, xFF, xFF,
203                 xFF, xFF, xFF}, lastRow);
204         assertArrayEquals(splitPoint,
205                 new byte[] {(byte)0xef, xFF, xFF, xFF, xFF, xFF, xFF, xFF});
206     }
207 
208   @Test
209   public void testUserInput() {
210     SplitAlgorithm algo = new HexStringSplit();
211     assertFalse(splitFailsPrecondition(algo)); // default settings are fine
212     assertFalse(splitFailsPrecondition(algo, "00", "AA")); // custom is fine
213     assertTrue(splitFailsPrecondition(algo, "AA", "00")); // range error
214     assertTrue(splitFailsPrecondition(algo, "AA", "AA")); // range error
215     assertFalse(splitFailsPrecondition(algo, "0", "2", 3)); // should be fine
216     assertFalse(splitFailsPrecondition(algo, "0", "A", 11)); // should be fine
217     assertTrue(splitFailsPrecondition(algo, "0", "A", 12)); // too granular
218 
219     algo = new UniformSplit();
220     assertFalse(splitFailsPrecondition(algo)); // default settings are fine
221     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\xAA")); // custom is fine
222     assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\x00")); // range error
223     assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\xAA")); // range error
224     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x02", 3)); // should be fine
225     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x0A", 11)); // should be fine
226     assertTrue(splitFailsPrecondition(algo, "\\x00", "\\x0A", 12)); // too granular
227   }
228 
229   private boolean splitFailsPrecondition(SplitAlgorithm algo) {
230     return splitFailsPrecondition(algo, 100);
231   }
232 
233   private boolean splitFailsPrecondition(SplitAlgorithm algo, String firstRow,
234       String lastRow) {
235     return splitFailsPrecondition(algo, firstRow, lastRow, 100);
236   }
237 
238   private boolean splitFailsPrecondition(SplitAlgorithm algo, String firstRow,
239       String lastRow, int numRegions) {
240     algo.setFirstRow(firstRow);
241     algo.setLastRow(lastRow);
242     return splitFailsPrecondition(algo, numRegions);
243   }
244 
245   private boolean splitFailsPrecondition(SplitAlgorithm algo, int numRegions) {
246     try {
247       byte[][] s = algo.split(numRegions);
248       LOG.debug("split algo = " + algo);
249       if (s != null) {
250         StringBuilder sb = new StringBuilder();
251         for (byte[] b : s) {
252           sb.append(Bytes.toStringBinary(b) + "  ");
253         }
254         LOG.debug(sb.toString());
255       }
256       return false;
257     } catch (IllegalArgumentException e) {
258       return true;
259     } catch (IllegalStateException e) {
260       return true;
261     } catch (IndexOutOfBoundsException e) {
262       return true;
263     }
264   }
265 
266     /**
267      * Creates a pre-split table with expectedBounds.size()+1 regions, then
268      * verifies that the region boundaries are the same as the expected
269      * region boundaries in expectedBounds.
270      * @throws Various junit assertions
271      */
272     private void preSplitTableAndVerify(List<byte[]> expectedBounds,
273             String splitClass, String tableName) throws Exception {
274         final int numRegions = expectedBounds.size()-1;
275         final Configuration conf = UTIL.getConfiguration();
276         conf.setInt("split.count", numRegions);
277         SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass);
278         RegionSplitter.createPresplitTable(tableName, splitAlgo,
279                 new String[] {CF_NAME}, conf);
280         verifyBounds(expectedBounds, tableName);
281     }
282 
283     private void rollingSplitAndVerify(String tableName, String splitClass,
284             List<byte[]> expectedBounds)  throws Exception {
285         final Configuration conf = UTIL.getConfiguration();
286 
287         // Set this larger than the number of splits so RegionSplitter won't block
288         conf.setInt("split.outstanding", 5);
289         SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass);
290         RegionSplitter.rollingSplit(tableName, splitAlgo, conf);
291         verifyBounds(expectedBounds, tableName);
292     }
293 
294     private void verifyBounds(List<byte[]> expectedBounds, String tableName)
295             throws Exception {
296         // Get region boundaries from the cluster and verify their endpoints
297         final Configuration conf = UTIL.getConfiguration();
298         final int numRegions = expectedBounds.size()-1;
299         final HTable hTable = new HTable(conf, tableName.getBytes());
300         final Map<HRegionInfo, HServerAddress> regionInfoMap =
301                 hTable.getRegionsInfo();
302         assertEquals(numRegions, regionInfoMap.size());
303         for (Map.Entry<HRegionInfo, HServerAddress> entry:
304            regionInfoMap.entrySet()) {
305             final HRegionInfo regionInfo = entry.getKey();
306             byte[] regionStart = regionInfo.getStartKey();
307             byte[] regionEnd = regionInfo.getEndKey();
308 
309             // This region's start key should be one of the region boundaries
310             int startBoundaryIndex = indexOfBytes(expectedBounds, regionStart);
311             assertNotSame(-1, startBoundaryIndex);
312 
313             // This region's end key should be the region boundary that comes
314             // after the starting boundary.
315             byte[] expectedRegionEnd = expectedBounds.get(
316                     startBoundaryIndex+1);
317             assertEquals(0, Bytes.compareTo(regionEnd, expectedRegionEnd));
318         }
319     }
320 
321     /**
322      * List.indexOf() doesn't really work for a List<byte[]>, because byte[]
323      * doesn't override equals(). This method checks whether a list contains
324      * a given element by checking each element using the byte array
325      * comparator.
326      * @return the index of the first element that equals compareTo, or -1
327      * if no elements are equal.
328      */
329     static private int indexOfBytes(List<byte[]> list,  byte[] compareTo) {
330         int listIndex = 0;
331         for(byte[] elem: list) {
332             if(Bytes.BYTES_COMPARATOR.compare(elem, compareTo) == 0) {
333                 return listIndex;
334             }
335             listIndex++;
336         }
337         return -1;
338     }
339 
340   @org.junit.Rule
341   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
342     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
343 }
344