1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.mapreduce;
21
22 import static org.junit.Assert.assertArrayEquals;
23 import static org.junit.Assert.assertEquals;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.IOException;
27 import java.util.TreeMap;
28
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.*;
33 import org.apache.hadoop.hbase.client.HBaseAdmin;
34 import org.apache.hadoop.hbase.client.HTable;
35 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
36 import org.apache.hadoop.hbase.io.hfile.Compression;
37 import org.apache.hadoop.hbase.io.hfile.HFile;
38 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
39 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.junit.*;
42 import org.junit.experimental.categories.Category;
43
44
45
46
47
48
49 @Category(LargeTests.class)
50 public class TestLoadIncrementalHFiles {
51 private static final byte[] QUALIFIER = Bytes.toBytes("myqual");
52 private static final byte[] FAMILY = Bytes.toBytes("myfam");
53
54 private static final byte[][] SPLIT_KEYS = new byte[][] {
55 Bytes.toBytes("ddd"),
56 Bytes.toBytes("ppp")
57 };
58
59 public static int BLOCKSIZE = 64*1024;
60 public static String COMPRESSION =
61 Compression.Algorithm.NONE.getName();
62
63 static HBaseTestingUtility util = new HBaseTestingUtility();
64
65 static boolean useSecure = false;
66
67 @BeforeClass
68 public static void setUpBeforeClass() throws Exception {
69 util.startMiniCluster();
70 }
71
72 @AfterClass
73 public static void tearDownAfterClass() throws Exception {
74 util.shutdownMiniCluster();
75 }
76
77
78
79
80
81 @Test
82 public void testSimpleLoad() throws Exception {
83 runTest("testSimpleLoad", BloomType.NONE,
84 new byte[][][] {
85 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("cccc") },
86 new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") },
87 });
88 }
89
90
91
92
93
94 @Test
95 public void testRegionCrossingLoad() throws Exception {
96 runTest("testRegionCrossingLoad", BloomType.NONE,
97 new byte[][][] {
98 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
99 new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
100 });
101 }
102
103
104
105
106 @Test
107 public void testRegionCrossingRowBloom() throws Exception {
108 runTest("testRegionCrossingLoadRowBloom", BloomType.ROW,
109 new byte[][][] {
110 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
111 new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
112 });
113 }
114
115
116
117
118 @Test
119 public void testRegionCrossingRowColBloom() throws Exception {
120 runTest("testRegionCrossingLoadRowColBloom", BloomType.ROWCOL,
121 new byte[][][] {
122 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
123 new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
124 });
125 }
126
127 private void runTest(String testName, BloomType bloomType,
128 byte[][][] hfileRanges) throws Exception {
129 Path dir = util.getDataTestDir(testName);
130 FileSystem fs = util.getTestFileSystem();
131 dir = dir.makeQualified(fs);
132 Path familyDir = new Path(dir, Bytes.toString(FAMILY));
133
134 int hfileIdx = 0;
135 for (byte[][] range : hfileRanges) {
136 byte[] from = range[0];
137 byte[] to = range[1];
138 createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
139 + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000);
140 }
141 int expectedRows = hfileIdx * 1000;
142
143 final byte[] TABLE = Bytes.toBytes("mytable_"+testName);
144
145 HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
146 HTableDescriptor htd = new HTableDescriptor(TABLE);
147 HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
148 familyDesc.setBloomFilterType(bloomType);
149 htd.addFamily(familyDesc);
150 admin.createTable(htd, SPLIT_KEYS);
151
152 HTable table = new HTable(util.getConfiguration(), TABLE);
153 util.waitTableAvailable(TABLE, 30000);
154 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration(), useSecure);
155 loader.doBulkLoad(dir, table);
156
157 assertEquals(expectedRows, util.countRows(table));
158 }
159
160 @Test
161 public void testSplitStoreFile() throws IOException {
162 Path dir = util.getDataTestDir("testSplitHFile");
163 FileSystem fs = util.getTestFileSystem();
164 Path testIn = new Path(dir, "testhfile");
165 HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
166 createHFile(util.getConfiguration(), fs, testIn, FAMILY, QUALIFIER,
167 Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000);
168
169 Path bottomOut = new Path(dir, "bottom.out");
170 Path topOut = new Path(dir, "top.out");
171
172 LoadIncrementalHFiles.splitStoreFile(
173 util.getConfiguration(), testIn,
174 familyDesc, Bytes.toBytes("ggg"),
175 bottomOut,
176 topOut);
177
178 int rowCount = verifyHFile(bottomOut);
179 rowCount += verifyHFile(topOut);
180 assertEquals(1000, rowCount);
181 }
182
183 private int verifyHFile(Path p) throws IOException {
184 Configuration conf = util.getConfiguration();
185 HFile.Reader reader = HFile.createReader(
186 p.getFileSystem(conf), p, new CacheConfig(conf));
187 reader.loadFileInfo();
188 HFileScanner scanner = reader.getScanner(false, false);
189 scanner.seekTo();
190 int count = 0;
191 do {
192 count++;
193 } while (scanner.next());
194 assertTrue(count > 0);
195 reader.close();
196 return count;
197 }
198
199
200
201
202
203
204
205 static void createHFile(
206 Configuration conf,
207 FileSystem fs, Path path,
208 byte[] family, byte[] qualifier,
209 byte[] startKey, byte[] endKey, int numRows) throws IOException
210 {
211 HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf))
212 .withPath(fs, path)
213 .withBlockSize(BLOCKSIZE)
214 .withCompression(COMPRESSION)
215 .withComparator(KeyValue.KEY_COMPARATOR)
216 .create();
217 long now = System.currentTimeMillis();
218 try {
219
220 for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows-2)) {
221 KeyValue kv = new KeyValue(key, family, qualifier, now, key);
222 writer.append(kv);
223 }
224 } finally {
225 writer.close();
226 }
227 }
228
229 private void addStartEndKeysForTest(TreeMap<byte[], Integer> map, byte[] first, byte[] last) {
230 Integer value = map.containsKey(first)?(Integer)map.get(first):0;
231 map.put(first, value+1);
232
233 value = map.containsKey(last)?(Integer)map.get(last):0;
234 map.put(last, value-1);
235 }
236
237 @Test
238 public void testInferBoundaries() {
239 TreeMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
240
241
242
243
244
245
246
247
248
249
250
251 String first;
252 String last;
253
254 first = "a"; last = "e";
255 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
256
257 first = "r"; last = "s";
258 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
259
260 first = "o"; last = "p";
261 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
262
263 first = "g"; last = "k";
264 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
265
266 first = "v"; last = "x";
267 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
268
269 first = "c"; last = "i";
270 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
271
272 first = "m"; last = "q";
273 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
274
275 first = "s"; last = "t";
276 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
277
278 first = "u"; last = "w";
279 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
280
281 byte[][] keysArray = LoadIncrementalHFiles.inferBoundaries(map);
282 byte[][] compare = new byte[3][];
283 compare[0] = "m".getBytes();
284 compare[1] = "r".getBytes();
285 compare[2] = "u".getBytes();
286
287 assertEquals(keysArray.length, 3);
288
289 for (int row = 0; row<keysArray.length; row++){
290 assertArrayEquals(keysArray[row], compare[row]);
291 }
292 }
293
294
295 @org.junit.Rule
296 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
297 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
298 }
299