1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.Collection;
25 import java.util.Collections;
26 import java.util.Iterator;
27 import java.util.List;
28
29 import static org.junit.Assert.*;
30 import static org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.OPEN_KEY;
31
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseConfiguration;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.KeyValue.KVComparator;
40 import org.apache.hadoop.hbase.SmallTests;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.junit.After;
43 import org.junit.Before;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46 import org.mockito.Mockito;
47
48
49 @Category(SmallTests.class)
50 public class TestStripeStoreFileManager {
51 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
52 private static final Path BASEDIR =
53 TEST_UTIL.getDataTestDir(TestStripeStoreFileManager.class.getSimpleName());
54 private static final Path CFDIR = HStore.getStoreHomedir(BASEDIR, "region", Bytes.toBytes("cf"));
55
56 private static final byte[] KEY_A = Bytes.toBytes("aaa");
57 private static final byte[] KEY_B = Bytes.toBytes("bbb");
58 private static final byte[] KEY_C = Bytes.toBytes("ccc");
59 private static final byte[] KEY_D = Bytes.toBytes("ddd");
60
61 private static final KeyValue KV_A = new KeyValue(KEY_A, 0L);
62 private static final KeyValue KV_B = new KeyValue(KEY_B, 0L);
63 private static final KeyValue KV_C = new KeyValue(KEY_C, 0L);
64 private static final KeyValue KV_D = new KeyValue(KEY_D, 0L);
65
66 @Before
67 public void setUp() throws Exception {
68 FileSystem fs = TEST_UTIL.getTestFileSystem();
69 if (!fs.mkdirs(CFDIR)) {
70 throw new IOException("Cannot create test directory " + CFDIR);
71 }
72 }
73
74 @After
75 public void tearDown() throws Exception {
76 FileSystem fs = TEST_UTIL.getTestFileSystem();
77 if (fs.exists(CFDIR) && !fs.delete(CFDIR, true)) {
78 throw new IOException("Cannot delete test directory " + CFDIR);
79 }
80 }
81
82 @Test
83 public void testInsertFilesIntoL0() throws Exception {
84 StripeStoreFileManager manager = createManager();
85 MockStoreFile sf = createFile();
86 manager.insertNewFiles(al(sf));
87 assertEquals(1, manager.getStorefileCount());
88 Collection<StoreFile> filesForGet = manager.getFilesForScanOrGet(true, KEY_A, KEY_A);
89 assertEquals(1, filesForGet.size());
90 assertTrue(filesForGet.contains(sf));
91
92
93 manager.addCompactionResults(al(), al(createFile(OPEN_KEY, KEY_B),
94 createFile(KEY_B, OPEN_KEY)));
95 assertTrue(manager.getFilesForScanOrGet(true, KEY_A, KEY_A).contains(sf));
96 assertTrue(manager.getFilesForScanOrGet(true, KEY_C, KEY_C).contains(sf));
97 }
98
99 @Test
100 public void testClearFiles() throws Exception {
101 StripeStoreFileManager manager = createManager();
102 manager.insertNewFiles(al(createFile()));
103 manager.insertNewFiles(al(createFile()));
104 manager.addCompactionResults(al(), al(createFile(OPEN_KEY, KEY_B),
105 createFile(KEY_B, OPEN_KEY)));
106 assertEquals(4, manager.getStorefileCount());
107 Collection<StoreFile> allFiles = manager.clearFiles();
108 assertEquals(4, allFiles.size());
109 assertEquals(0, manager.getStorefileCount());
110 assertEquals(0, manager.getStorefiles().size());
111 }
112
113 private static ArrayList<StoreFile> dumpIterator(Iterator<StoreFile> iter) {
114 ArrayList<StoreFile> result = new ArrayList<StoreFile>();
115 for (; iter.hasNext(); result.add(iter.next()));
116 return result;
117 }
118
119 @Test
120 public void testRowKeyBefore() throws Exception {
121 StripeStoreFileManager manager = createManager();
122 StoreFile l0File = createFile(), l0File2 = createFile();
123 manager.insertNewFiles(al(l0File));
124 manager.insertNewFiles(al(l0File2));
125
126 Iterator<StoreFile> sfs = manager.getCandidateFilesForRowKeyBefore(KV_B);
127 sfs.next();
128 sfs.remove();
129
130 sfs = manager.updateCandidateFilesForRowKeyBefore(sfs, KV_B, KV_A);
131 assertTrue(sfs.hasNext());
132
133 MockStoreFile stripe0a = createFile(0, 100, OPEN_KEY, KEY_B),
134 stripe1 = createFile(KEY_B, OPEN_KEY);
135 manager.addCompactionResults(al(l0File), al(stripe0a, stripe1));
136
137 ArrayList<StoreFile> sfsDump = dumpIterator(manager.getCandidateFilesForRowKeyBefore(KV_A));
138 assertEquals(2, sfsDump.size());
139 assertTrue(sfsDump.contains(stripe0a));
140 assertFalse(sfsDump.contains(stripe1));
141
142 sfsDump = dumpIterator(manager.getCandidateFilesForRowKeyBefore(KV_B));
143 assertEquals(3, sfsDump.size());
144 assertTrue(sfsDump.contains(stripe1));
145
146 sfsDump = dumpIterator(manager.getCandidateFilesForRowKeyBefore(KV_D));
147 assertEquals(3, sfsDump.size());
148
149
150 sfs = manager.getCandidateFilesForRowKeyBefore(KV_D);
151 sfs.next();
152 sfs.remove();
153 sfs = manager.updateCandidateFilesForRowKeyBefore(sfs, KV_D, KV_C);
154 assertEquals(stripe1, sfs.next());
155 assertFalse(sfs.hasNext());
156
157
158
159 StoreFile stripe0b = createFile(0, 101, OPEN_KEY, KEY_B);
160 manager.addCompactionResults(al(l0File2), al(stripe0b));
161 sfs = manager.getCandidateFilesForRowKeyBefore(KV_A);
162 assertEquals(stripe0b, sfs.next());
163 sfs.remove();
164 sfs = manager.updateCandidateFilesForRowKeyBefore(sfs, KV_A, KV_A);
165 assertEquals(stripe0a, sfs.next());
166 }
167
168 @Test
169 public void testGetSplitPointEdgeCases() throws Exception {
170 StripeStoreFileManager manager = createManager();
171
172 assertNull(manager.getSplitPoint());
173
174
175 MockStoreFile sf5 = createFile(5, 0);
176 sf5.splitPoint = new byte[1];
177 manager.insertNewFiles(al(sf5));
178 manager.insertNewFiles(al(createFile(1, 0)));
179 assertEquals(sf5.splitPoint, manager.getSplitPoint());
180
181
182 manager.addCompactionResults(al(), al(createFile(2, 0, OPEN_KEY, OPEN_KEY)));
183 assertEquals(sf5.splitPoint, manager.getSplitPoint());
184
185
186 MockStoreFile sf6 = createFile(6, 0, OPEN_KEY, OPEN_KEY);
187 sf6.splitPoint = new byte[1];
188 manager.addCompactionResults(al(), al(sf6));
189 assertEquals(sf6.splitPoint, manager.getSplitPoint());
190 }
191
192 @Test
193 public void testGetStripeBoundarySplits() throws Exception {
194
195 verifySplitPointScenario(5, false, 0f, 2, 1, 1, 1, 1, 1, 10);
196 verifySplitPointScenario(0, false, 0f, 6, 3, 1, 1, 2);
197 verifySplitPointScenario(2, false, 0f, 1, 1, 1, 1, 2);
198 verifySplitPointScenario(0, false, 0f, 5, 4);
199 verifySplitPointScenario(2, false, 0f, 5, 2, 5, 5, 5);
200 }
201
202 @Test
203 public void testGetUnbalancedSplits() throws Exception {
204
205 verifySplitPointScenario(0, false, 2.1f, 4, 4, 4);
206 verifySplitPointScenario(1, true, 1.5f, 4, 4, 4);
207 verifySplitPointScenario(1, false, 1.1f, 3, 4, 1, 1, 2, 2);
208 verifySplitPointScenario(1, false, 1.1f, 3, 6, 1, 1, 2, 2);
209 verifySplitPointScenario(1, true, 1.1f, 3, 8, 1, 1, 2, 2);
210 verifySplitPointScenario(3, false, 1.1f, 2, 2, 1, 1, 4, 3);
211 verifySplitPointScenario(4, true, 1.1f, 2, 2, 1, 1, 8, 3);
212 verifySplitPointScenario(0, true, 1.5f, 10, 4);
213 verifySplitPointScenario(0, false, 1.4f, 6, 4);
214 verifySplitPointScenario(1, true, 1.5f, 4, 10);
215 verifySplitPointScenario(0, false, 1.4f, 4, 6);
216 }
217
218
219
220
221
222
223
224
225
226
227 private void verifySplitPointScenario(int splitPointAfter, boolean shouldSplitStripe,
228 float splitRatioToVerify, int... sizes) throws Exception {
229 assertTrue(sizes.length > 1);
230 ArrayList<StoreFile> sfs = new ArrayList<StoreFile>();
231 for (int sizeIx = 0; sizeIx < sizes.length; ++sizeIx) {
232 byte[] startKey = (sizeIx == 0) ? OPEN_KEY : Bytes.toBytes(sizeIx - 1);
233 byte[] endKey = (sizeIx == sizes.length - 1) ? OPEN_KEY : Bytes.toBytes(sizeIx);
234 MockStoreFile sf = createFile(sizes[sizeIx], 0, startKey, endKey);
235 sf.splitPoint = Bytes.toBytes(-sizeIx);
236 sfs.add(sf);
237 }
238
239 Configuration conf = HBaseConfiguration.create();
240 if (splitRatioToVerify != 0) {
241 conf.setFloat(StripeStoreConfig.MAX_REGION_SPLIT_IMBALANCE_KEY, splitRatioToVerify);
242 }
243 StripeStoreFileManager manager = createManager(al(), conf);
244 manager.addCompactionResults(al(), sfs);
245 int result = Bytes.toInt(manager.getSplitPoint());
246
247 assertEquals(splitPointAfter * (shouldSplitStripe ? -1 : 1), result);
248 }
249
250 private static byte[] keyAfter(byte[] key) {
251 return Arrays.copyOf(key, key.length + 1);
252 }
253
254 @Test
255 public void testGetFilesForGetAndScan() throws Exception {
256 StripeStoreFileManager manager = createManager();
257 verifyGetAndScanScenario(manager, null, null);
258 verifyGetAndScanScenario(manager, KEY_B, KEY_C);
259
260
261 MockStoreFile sf0 = createFile();
262 manager.insertNewFiles(al(sf0));
263 verifyGetAndScanScenario(manager, null, null, sf0);
264 verifyGetAndScanScenario(manager, null, KEY_C, sf0);
265 verifyGetAndScanScenario(manager, KEY_B, null, sf0);
266 verifyGetAndScanScenario(manager, KEY_B, KEY_C, sf0);
267
268
269 MockStoreFile sfA = createFile(OPEN_KEY, KEY_A);
270 MockStoreFile sfB = createFile(KEY_A, KEY_B);
271 MockStoreFile sfC = createFile(KEY_B, KEY_C);
272 MockStoreFile sfD = createFile(KEY_C, KEY_D);
273 MockStoreFile sfE = createFile(KEY_D, OPEN_KEY);
274 manager.addCompactionResults(al(), al(sfA, sfB, sfC, sfD, sfE));
275
276 verifyGetAndScanScenario(manager, null, null, sf0, sfA, sfB, sfC, sfD, sfE);
277 verifyGetAndScanScenario(manager, keyAfter(KEY_A), null, sf0, sfB, sfC, sfD, sfE);
278 verifyGetAndScanScenario(manager, null, keyAfter(KEY_C), sf0, sfA, sfB, sfC, sfD);
279 verifyGetAndScanScenario(manager, KEY_B, null, sf0, sfC, sfD, sfE);
280 verifyGetAndScanScenario(manager, null, KEY_C, sf0, sfA, sfB, sfC, sfD);
281 verifyGetAndScanScenario(manager, KEY_B, keyAfter(KEY_B), sf0, sfC);
282 verifyGetAndScanScenario(manager, keyAfter(KEY_A), KEY_B, sf0, sfB, sfC);
283 verifyGetAndScanScenario(manager, KEY_D, KEY_D, sf0, sfE);
284 verifyGetAndScanScenario(manager, keyAfter(KEY_B), keyAfter(KEY_C), sf0, sfC, sfD);
285 }
286
287 private void verifyGetAndScanScenario(StripeStoreFileManager manager,
288 byte[] start, byte[] end, StoreFile... results) throws Exception {
289 verifyGetOrScanScenario(manager, true, start, end, results);
290 verifyGetOrScanScenario(manager, false, start, end, results);
291 }
292
293 @Test
294 @SuppressWarnings("unchecked")
295 public void testLoadFilesWithRecoverableBadFiles() throws Exception {
296
297
298
299 ArrayList<StoreFile> validStripeFiles = al(createFile(OPEN_KEY, KEY_B),
300 createFile(KEY_B, KEY_C), createFile(KEY_C, OPEN_KEY),
301 createFile(KEY_C, OPEN_KEY));
302 ArrayList<StoreFile> filesToGoToL0 = al(createFile(), createFile(null, KEY_A),
303 createFile(KEY_D, null), createFile(KEY_D, KEY_A), createFile(keyAfter(KEY_A), KEY_C),
304 createFile(OPEN_KEY, KEY_D), createFile(KEY_D, keyAfter(KEY_D)));
305 ArrayList<StoreFile> allFilesToGo = flattenLists(validStripeFiles, filesToGoToL0);
306 Collections.shuffle(allFilesToGo);
307 StripeStoreFileManager manager = createManager(allFilesToGo);
308 List<StoreFile> l0Files = manager.getLevel0Files();
309 assertEquals(filesToGoToL0.size(), l0Files.size());
310 for (StoreFile sf : filesToGoToL0) {
311 assertTrue(l0Files.contains(sf));
312 }
313 verifyAllFiles(manager, allFilesToGo);
314 }
315
316 @Test
317 public void testLoadFilesWithBadStripe() throws Exception {
318
319
320 ArrayList<StoreFile> allFilesToGo = al(createFile(OPEN_KEY, KEY_B),
321 createFile(KEY_B, KEY_C), createFile(KEY_C, OPEN_KEY),
322 createFile(KEY_B, keyAfter(KEY_B)));
323 Collections.shuffle(allFilesToGo);
324 StripeStoreFileManager manager = createManager(allFilesToGo);
325 assertEquals(allFilesToGo.size(), manager.getLevel0Files().size());
326 }
327
328 @Test
329 public void testLoadFilesWithGaps() throws Exception {
330
331 StripeStoreFileManager manager =
332 createManager(al(createFile(OPEN_KEY, KEY_B), createFile(KEY_C, OPEN_KEY)));
333 assertEquals(2, manager.getLevel0Files().size());
334
335 manager = createManager(al(createFile(OPEN_KEY, OPEN_KEY)));
336 assertEquals(0, manager.getLevel0Files().size());
337 assertEquals(1, manager.getStorefileCount());
338 }
339
340 @Test
341 public void testLoadFilesAfterSplit() throws Exception {
342
343 MockStoreFile sf = createFile(KEY_B, KEY_C);
344 StripeStoreFileManager manager = createManager(al(createFile(OPEN_KEY, KEY_B), sf));
345 assertEquals(0, manager.getLevel0Files().size());
346
347 verifyInvalidCompactionScenario(manager, al(sf), al(createFile(KEY_B, KEY_C)));
348 manager.addCompactionResults(al(sf), al(createFile(KEY_B, OPEN_KEY)));
349
350 manager = createManager(al(sf, createFile(KEY_C, OPEN_KEY)));
351 verifyInvalidCompactionScenario(manager, al(sf), al(createFile(KEY_B, KEY_C)));
352 manager.addCompactionResults(al(sf), al(createFile(OPEN_KEY, KEY_C)));
353 manager = createManager(al(sf));
354 verifyInvalidCompactionScenario(manager, al(sf), al(createFile(KEY_B, KEY_C)));
355 manager.addCompactionResults(al(sf), al(createFile(OPEN_KEY, OPEN_KEY)));
356 }
357
358 @Test
359 public void testAddingCompactionResults() throws Exception {
360 StripeStoreFileManager manager = createManager();
361
362 StoreFile sf_L0_0a = createFile(), sf_L0_0b = createFile();
363 manager.insertNewFiles(al(sf_L0_0a, sf_L0_0b));
364
365
366 verifyInvalidCompactionScenario(manager, al(sf_L0_0a), al(createFile(OPEN_KEY, KEY_B)));
367 verifyInvalidCompactionScenario(manager, al(sf_L0_0a), al(createFile(OPEN_KEY, KEY_B),
368 createFile(KEY_C, OPEN_KEY)));
369 verifyInvalidCompactionScenario(manager, al(sf_L0_0a), al(createFile(OPEN_KEY, KEY_B),
370 createFile(KEY_B, OPEN_KEY), createFile(KEY_A, KEY_D)));
371 verifyInvalidCompactionScenario(manager, al(sf_L0_0a), al(createFile(OPEN_KEY, KEY_B),
372 createFile(KEY_A, KEY_B), createFile(KEY_B, OPEN_KEY)));
373
374 StoreFile sf_i2B_0 = createFile(OPEN_KEY, KEY_B);
375 StoreFile sf_B2C_0 = createFile(KEY_B, KEY_C);
376 StoreFile sf_C2i_0 = createFile(KEY_C, OPEN_KEY);
377 manager.addCompactionResults(al(sf_L0_0a), al(sf_i2B_0, sf_B2C_0, sf_C2i_0));
378 verifyAllFiles(manager, al(sf_L0_0b, sf_i2B_0, sf_B2C_0, sf_C2i_0));
379
380
381 StoreFile sf_L0_1 = createFile();
382 StoreFile sf_i2B_1 = createFile(OPEN_KEY, KEY_B);
383 StoreFile sf_B2C_1 = createFile(KEY_B, KEY_C);
384 manager.insertNewFiles(al(sf_L0_1));
385 manager.addCompactionResults(al(sf_L0_0b, sf_L0_1), al(sf_i2B_1, sf_B2C_1));
386 verifyAllFiles(manager, al(sf_i2B_0, sf_B2C_0, sf_C2i_0, sf_i2B_1, sf_B2C_1));
387
388
389 StoreFile sf_L0_2 = createFile(null, null);
390 manager.addCompactionResults(al(), al(sf_L0_2));
391 verifyAllFiles(manager, al(sf_i2B_0, sf_B2C_0, sf_C2i_0, sf_i2B_1, sf_B2C_1, sf_L0_2));
392
393 manager.addCompactionResults(al(sf_L0_2), al());
394
395
396 StoreFile sf_i2B_3 = createFile(OPEN_KEY, KEY_B);
397 manager.addCompactionResults(al(sf_i2B_0, sf_i2B_1), al(sf_i2B_3));
398 verifyAllFiles(manager, al(sf_B2C_0, sf_C2i_0, sf_B2C_1, sf_i2B_3));
399
400
401 StoreFile sf_B2D_4 = createFile(KEY_B, KEY_D);
402 StoreFile sf_D2i_4 = createFile(KEY_D, OPEN_KEY);
403 manager.addCompactionResults(al(sf_B2C_0, sf_C2i_0, sf_B2C_1), al(sf_B2D_4, sf_D2i_4));
404 verifyAllFiles(manager, al(sf_i2B_3, sf_B2D_4, sf_D2i_4));
405
406
407 StoreFile sf_i2A_5 = createFile(OPEN_KEY, KEY_A);
408 StoreFile sf_A2B_5 = createFile(KEY_A, KEY_B);
409 manager.addCompactionResults(al(sf_i2B_3), al(sf_i2A_5, sf_A2B_5));
410 verifyAllFiles(manager, al(sf_B2D_4, sf_D2i_4, sf_i2A_5, sf_A2B_5));
411
412
413 StoreFile sf_B2C_6 = createFile(KEY_B, KEY_C);
414 StoreFile sf_C2D_6 = createFile(KEY_C, KEY_D);
415 manager.addCompactionResults(al(sf_B2D_4), al(sf_B2C_6, sf_C2D_6));
416 verifyAllFiles(manager, al(sf_D2i_4, sf_i2A_5, sf_A2B_5, sf_B2C_6, sf_C2D_6));
417
418
419 StoreFile sf_A2C_7 = createFile(KEY_A, KEY_C);
420 manager.addCompactionResults(al(sf_A2B_5, sf_B2C_6), al(sf_A2C_7));
421 verifyAllFiles(manager, al(sf_D2i_4, sf_i2A_5, sf_C2D_6, sf_A2C_7));
422
423
424 StoreFile sf_i2C_8 = createFile(OPEN_KEY, KEY_C);
425 manager.addCompactionResults(al(sf_i2A_5, sf_A2C_7), al(sf_i2C_8));
426 verifyAllFiles(manager, al(sf_D2i_4, sf_C2D_6, sf_i2C_8));
427
428
429 StoreFile sf_i2i_9 = createFile(OPEN_KEY, OPEN_KEY);
430 manager.addCompactionResults(al(sf_D2i_4, sf_C2D_6, sf_i2C_8), al(sf_i2i_9));
431 verifyAllFiles(manager, al(sf_i2i_9));
432 }
433
434 @Test
435 public void testCompactionAndFlushConflict() throws Exception {
436
437 StripeStoreFileManager sfm = createManager();
438 assertEquals(0, sfm.getStripeCount());
439 StoreFile sf_i2c = createFile(OPEN_KEY, KEY_C), sf_c2i = createFile(KEY_C, OPEN_KEY);
440 sfm.insertNewFiles(al(sf_i2c, sf_c2i));
441 assertEquals(2, sfm.getStripeCount());
442
443 StoreFile sf_i2d = createFile(OPEN_KEY, KEY_D), sf_d2i = createFile(KEY_D, OPEN_KEY);
444 sfm.insertNewFiles(al(sf_i2d, sf_d2i));
445 assertEquals(2, sfm.getStripeCount());
446 assertEquals(2, sfm.getLevel0Files().size());
447 verifyGetAndScanScenario(sfm, KEY_C, KEY_C, sf_i2d, sf_d2i, sf_c2i);
448
449 sfm.addCompactionResults(al(sf_i2d, sf_d2i), al());
450 assertEquals(0, sfm.getLevel0Files().size());
451
452
453 StoreFile sf_i2c_2 = createFile(OPEN_KEY, KEY_C);
454 sfm.insertNewFiles(al(sf_i2c_2));
455 sfm.addCompactionResults(al(sf_i2c, sf_c2i), al(sf_i2d, sf_d2i));
456 assertEquals(1, sfm.getLevel0Files().size());
457 verifyGetAndScanScenario(sfm, KEY_C, KEY_C, sf_i2d, sf_i2c_2);
458 }
459
460 @Test
461 public void testEmptyResultsForStripes() throws Exception {
462
463 StripeStoreFileManager manager = createManager();
464 StoreFile sf0a = createFile();
465 StoreFile sf0b = createFile();
466 manager.insertNewFiles(al(sf0a));
467 manager.insertNewFiles(al(sf0b));
468 ArrayList<StoreFile> compacted = al(createFile(OPEN_KEY, KEY_B),
469 createFile(KEY_B, KEY_C), createFile(KEY_C, OPEN_KEY));
470 manager.addCompactionResults(al(sf0a), compacted);
471
472 ArrayList<StoreFile> compacted2 = al(createFile(OPEN_KEY, KEY_B), createFile(KEY_C, OPEN_KEY));
473 manager.addCompactionResults(al(sf0b), compacted2);
474 compacted.addAll(compacted2);
475 verifyAllFiles(manager, compacted);
476 }
477
478 @Test
479 public void testPriority() throws Exception {
480
481 testPriorityScenario(5, 5, 0, 0, 0);
482 testPriorityScenario(2, 5, 0, 0, 3);
483 testPriorityScenario(4, 25, 5, 1, 0);
484 testPriorityScenario(3, 25, 5, 1, 1);
485 testPriorityScenario(3, 25, 5, 2, 0);
486 testPriorityScenario(2, 25, 5, 4, 0);
487 testPriorityScenario(2, 25, 5, 4, 4);
488 testPriorityScenario(2, 25, 5, 1, 10);
489 testPriorityScenario(0, 25, 5, 4, 5);
490 testPriorityScenario(-5, 25, 5, 6, 0);
491 testPriorityScenario(-1, 25, 0, 0, 26);
492 }
493
494 private void testPriorityScenario(int expectedPriority,
495 int limit, int stripes, int filesInStripe, int l0Files) throws Exception
496 {
497 final byte[][] keys = { KEY_A, KEY_B, KEY_C, KEY_D };
498 assertTrue(stripes <= keys.length + 1);
499 Configuration conf = TEST_UTIL.getConfiguration();
500 conf.setInt("hbase.hstore.blockingStoreFiles", limit);
501 StripeStoreFileManager sfm = createManager(al(), conf);
502 for (int i = 0; i < l0Files; ++i) {
503 sfm.insertNewFiles(al(createFile()));
504 }
505 for (int i = 0; i < filesInStripe; ++i) {
506 ArrayList<StoreFile> stripe = new ArrayList<StoreFile>();
507 for (int j = 0; j < stripes; ++j) {
508 stripe.add(createFile(
509 (j == 0) ? OPEN_KEY : keys[j - 1], (j == stripes - 1) ? OPEN_KEY : keys[j]));
510 }
511 sfm.addCompactionResults(al(), stripe);
512 }
513 assertEquals(expectedPriority, sfm.getStoreCompactionPriority());
514 }
515
516 private void verifyInvalidCompactionScenario(StripeStoreFileManager manager,
517 ArrayList<StoreFile> filesToCompact, ArrayList<StoreFile> filesToInsert) throws Exception {
518 Collection<StoreFile> allFiles = manager.getStorefiles();
519 try {
520 manager.addCompactionResults(filesToCompact, filesToInsert);
521 fail("Should have thrown");
522 } catch (IOException ex) {
523
524 }
525 verifyAllFiles(manager, allFiles);
526 }
527
528 private void verifyGetOrScanScenario(StripeStoreFileManager manager, boolean isGet,
529 byte[] start, byte[] end, StoreFile... results) throws Exception {
530 verifyGetOrScanScenario(manager, isGet, start, end, Arrays.asList(results));
531 }
532
533 private void verifyGetOrScanScenario(StripeStoreFileManager manager, boolean isGet,
534 byte[] start, byte[] end, Collection<StoreFile> results) throws Exception {
535 start = start != null ? start : HConstants.EMPTY_START_ROW;
536 end = end != null ? end : HConstants.EMPTY_END_ROW;
537 Collection<StoreFile> sfs = manager.getFilesForScanOrGet(isGet, start, end);
538 assertEquals(results.size(), sfs.size());
539 for (StoreFile result : results) {
540 assertTrue(sfs.contains(result));
541 }
542 }
543
544 private void verifyAllFiles(
545 StripeStoreFileManager manager, Collection<StoreFile> results) throws Exception {
546 verifyGetOrScanScenario(manager, false, null, null, results);
547 }
548
549
550 private static MockStoreFile createFile(
551 long size, long seqNum, byte[] startKey, byte[] endKey) throws Exception {
552 FileSystem fs = TEST_UTIL.getTestFileSystem();
553 Path testFilePath = StoreFile.getUniqueFile(fs, CFDIR);
554 fs.create(testFilePath);
555 MockStoreFile sf = new MockStoreFile(TEST_UTIL, testFilePath, size, 0, false, seqNum);
556 if (startKey != null) {
557 sf.setMetadataValue(StripeStoreFileManager.STRIPE_START_KEY, startKey);
558 }
559 if (endKey != null) {
560 sf.setMetadataValue(StripeStoreFileManager.STRIPE_END_KEY, endKey);
561 }
562 return sf;
563 }
564
565 private static MockStoreFile createFile(long size, long seqNum) throws Exception {
566 return createFile(size, seqNum, null, null);
567 }
568
569 private static MockStoreFile createFile(byte[] startKey, byte[] endKey) throws Exception {
570 return createFile(0, 0, startKey, endKey);
571 }
572
573 private static MockStoreFile createFile() throws Exception {
574 return createFile(null, null);
575 }
576
577 private static StripeStoreFileManager createManager() throws Exception {
578 return createManager(new ArrayList<StoreFile>());
579 }
580
581 private static StripeStoreFileManager createManager(ArrayList<StoreFile> sfs) throws Exception {
582 return createManager(sfs, TEST_UTIL.getConfiguration());
583 }
584
585 private static StripeStoreFileManager createManager(
586 ArrayList<StoreFile> sfs, Configuration conf) throws Exception {
587 StripeStoreConfig config = new StripeStoreConfig(
588 conf, Mockito.mock(StoreConfigInformation.class));
589 StripeStoreFileManager result = new StripeStoreFileManager(new KVComparator(), conf, config);
590 result.loadFiles(sfs);
591 return result;
592 }
593
594 private static ArrayList<StoreFile> al(StoreFile... sfs) {
595 return new ArrayList<StoreFile>(Arrays.asList(sfs));
596 }
597
598 private static ArrayList<StoreFile> flattenLists(ArrayList<StoreFile>... sfls) {
599 ArrayList<StoreFile> result = new ArrayList<StoreFile>();
600 for (ArrayList<StoreFile> sfl : sfls) {
601 result.addAll(sfl);
602 }
603 return result;
604 }
605 }