1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.directory.mavibot.btree;
22
23
24 import static org.junit.Assert.assertEquals;
25 import static org.junit.Assert.assertFalse;
26 import static org.junit.Assert.assertNotNull;
27 import static org.junit.Assert.assertTrue;
28
29 import java.io.File;
30 import java.io.IOException;
31 import java.util.HashMap;
32 import java.util.Iterator;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.Random;
36 import java.util.Set;
37 import java.util.TreeSet;
38
39 import org.apache.directory.mavibot.btree.BulkLoader.LevelEnum;
40 import org.apache.directory.mavibot.btree.exception.BTreeAlreadyManagedException;
41 import org.apache.directory.mavibot.btree.exception.EndOfFileExceededException;
42 import org.apache.directory.mavibot.btree.exception.KeyNotFoundException;
43 import org.apache.directory.mavibot.btree.serializer.LongSerializer;
44 import org.apache.directory.mavibot.btree.serializer.StringSerializer;
45 import org.junit.Ignore;
46 import org.junit.Test;
47
48
49
50
51
52
53
54 public class BulkLoaderTest
55 {
56 private void checkBtree( BTree<Long, String> oldBtree, BTree<Long, String> newBtree )
57 throws EndOfFileExceededException, IOException, KeyNotFoundException
58 {
59 assertEquals( oldBtree.getNbElems(), newBtree.getNbElems() );
60
61 TupleCursor<Long, String> cursorOld = oldBtree.browse();
62 TupleCursor<Long, String> cursorNew = newBtree.browse();
63
64 while ( cursorOld.hasNext() && cursorNew.hasNext() )
65 {
66 Tuple<Long, String> tupleOld = cursorOld.next();
67 Tuple<Long, String> tupleNew = cursorNew.next();
68
69 assertEquals( tupleOld.getKey(), tupleNew.getKey() );
70 assertEquals( tupleOld.getValue(), tupleNew.getValue() );
71 }
72
73 assertEquals( cursorOld.hasNext(), cursorNew.hasNext() );
74 }
75
76
77
78
79
80 @Test
81 public void testInMemoryBulkLoadNoElement() throws IOException, KeyNotFoundException
82 {
83 BTree<Long, String> btree = BTreeFactory.createInMemoryBTree( "test", LongSerializer.INSTANCE,
84 StringSerializer.INSTANCE );
85 btree.setPageSize( 4 );
86
87 BTree<Long, String> newBtree = ( BTree<Long, String> ) BulkLoader.compact( btree );
88
89 checkBtree( btree, newBtree );
90 TupleCursor<Long, String> cursorOld = btree.browse();
91 TupleCursor<Long, String> cursorNew = btree.browse();
92
93 assertFalse( cursorOld.hasNext() );
94 assertFalse( cursorNew.hasNext() );
95 }
96
97
98
99
100
101 @Ignore
102 @Test
103 public void testInMemoryBulkLoad3Elements() throws IOException, KeyNotFoundException
104 {
105 BTree<Long, String> btree = BTreeFactory.createInMemoryBTree( "test", LongSerializer.INSTANCE,
106 StringSerializer.INSTANCE );
107 btree.setPageSize( 4 );
108
109 for ( Long i = 0L; i < 3L; i++ )
110 {
111 String value = "V" + i;
112 btree.insert( i, value );
113 }
114
115 BTree<Long, String> newBtree = ( BTree<Long, String> ) BulkLoader.compact( btree );
116
117 checkBtree( btree, newBtree );
118 }
119
120
121
122
123
124 @Ignore
125 @Test
126 public void testInMemoryBulkLoad8Elements() throws IOException, KeyNotFoundException
127 {
128 BTree<Long, String> btree = BTreeFactory.createInMemoryBTree( "test", LongSerializer.INSTANCE,
129 StringSerializer.INSTANCE );
130 btree.setPageSize( 4 );
131
132 for ( Long i = 0L; i < 8L; i++ )
133 {
134 String value = "V" + i;
135 btree.insert( i, value );
136 }
137
138 BTree<Long, String> newBtree = ( BTree<Long, String> ) BulkLoader.compact( btree );
139
140 checkBtree( btree, newBtree );
141 }
142
143
144
145
146
147
148 @Test
149 public void testPersistedBulkLoad1000Elements() throws IOException, KeyNotFoundException,
150 BTreeAlreadyManagedException
151 {
152 for ( int i = 0; i < 1001; i++ )
153 {
154 Random random = new Random( System.currentTimeMillis() );
155 File file = File.createTempFile( "managedbtreebuilder", ".data" );
156 file.deleteOnExit();
157
158 try
159 {
160 RecordManager rm = new RecordManager( file.getAbsolutePath() );
161 PersistedBTree<Long, String> btree = ( PersistedBTree<Long, String> ) rm.addBTree( "test",
162 LongSerializer.INSTANCE, StringSerializer.INSTANCE, false );
163
164 BulkLoader<Long, String> bulkLoader = new BulkLoader<Long, String>();
165 int nbElems = i;
166 int addedElems = 0;
167
168 final Tuple<Long, String>[] elems = new Tuple[nbElems];
169 Map<Long, Tuple<Long, Set<String>>> expected = new HashMap<Long, Tuple<Long, Set<String>>>();
170
171 long t00 = System.currentTimeMillis();
172
173 while ( addedElems < nbElems )
174 {
175 long key = random.nextLong() % 3333333L;
176
177 if ( expected.containsKey( key ) )
178 {
179 continue;
180 }
181
182 long w = random.nextLong() % 3333333L;
183 String value = "V" + w;
184 elems[addedElems] = new Tuple<Long, String>( key, value );
185
186 Tuple<Long, Set<String>> expectedTuple = expected.get( key );
187
188 if ( expectedTuple == null )
189 {
190 expectedTuple = new Tuple<Long, Set<String>>( key, new TreeSet<String>() );
191 }
192
193 expectedTuple.value.add( value );
194 expected.put( key, expectedTuple );
195 addedElems++;
196
197 if ( addedElems % 100 == 0 )
198 {
199
200 }
201 }
202 long t01 = System.currentTimeMillis();
203
204
205
206 Iterator<Tuple<Long, String>> tupleIterator = new Iterator<Tuple<Long, String>>()
207 {
208 private int pos = 0;
209
210
211 @Override
212 public Tuple<Long, String> next()
213 {
214 return elems[pos++];
215 }
216
217
218 @Override
219 public boolean hasNext()
220 {
221 return pos < elems.length;
222 }
223
224
225 @Override
226 public void remove()
227 {
228 }
229 };
230
231 long t0 = System.currentTimeMillis();
232 BTree<Long, String> result = bulkLoader.load( btree, tupleIterator, 128 );
233 long t1 = System.currentTimeMillis();
234
235 if ( i % 100 == 0 )
236 {
237 System.out.println( "== Btree #" + i + ", Time to bulkoad the " + nbElems + " elements "
238 + ( t1 - t0 ) + "ms" );
239 }
240
241 TupleCursor<Long, String> cursor = result.browse();
242 int nbFetched = 0;
243
244 long t2 = System.currentTimeMillis();
245
246 while ( cursor.hasNext() )
247 {
248 Tuple<Long, String> elem = cursor.next();
249
250 assertTrue( expected.containsKey( elem.key ) );
251 Tuple<Long, Set<String>> tuple = expected.get( elem.key );
252 assertNotNull( tuple );
253 nbFetched++;
254 }
255
256 long t3 = System.currentTimeMillis();
257
258
259 assertEquals( nbElems, nbFetched );
260
261 checkBtree( btree, result );
262 }
263 finally
264 {
265 file.delete();
266 }
267 }
268 }
269
270
271
272
273
274 @Test
275 public void testInMemoryBulkLoad20Elements() throws IOException, KeyNotFoundException
276 {
277 BTree<Long, String> btree = BTreeFactory.createInMemoryBTree( "test", LongSerializer.INSTANCE,
278 StringSerializer.INSTANCE );
279 btree.setPageSize( 4 );
280
281 for ( Long i = 0L; i < 20L; i++ )
282 {
283 String value = "V" + i;
284 btree.insert( i, value );
285 }
286
287 BTree<Long, String> newBtree = ( BTree<Long, String> ) BulkLoader.compact( btree );
288
289 checkBtree( btree, newBtree );
290 }
291
292
293
294
295
296
297 @Ignore
298 @Test
299 public void testInMemoryBulkLoad40Elements() throws IOException, KeyNotFoundException
300 {
301 BTree<Long, String> btree = BTreeFactory.createInMemoryBTree( "test", LongSerializer.INSTANCE,
302 StringSerializer.INSTANCE );
303 btree.setPageSize( 4 );
304
305 for ( Long i = 0L; i < 40L; i++ )
306 {
307 String value = "V" + i;
308 btree.insert( i, value );
309 }
310
311 BTree<Long, String> newBtree = ( BTree<Long, String> ) BulkLoader.compact( btree );
312
313 checkBtree( btree, newBtree );
314 }
315
316
317
318
319
320
321 @Test
322 public void testInMemoryBulkLoad100Elements() throws IOException, KeyNotFoundException
323 {
324 BTree<Long, String> btree = BTreeFactory.createInMemoryBTree( "test", LongSerializer.INSTANCE,
325 StringSerializer.INSTANCE );
326 btree.setPageSize( 4 );
327
328 for ( Long i = 0L; i < 100L; i++ )
329 {
330 String value = "V" + i;
331 btree.insert( i, value );
332 }
333
334 BTree<Long, String> newBtree = ( BTree<Long, String> ) BulkLoader.compact( btree );
335
336 checkBtree( btree, newBtree );
337 }
338
339
340 @Ignore
341 @Test
342 public void testInMemoryBulkLoadN() throws IOException, KeyNotFoundException
343 {
344 Random random = new Random( System.nanoTime() );
345 long t0 = System.currentTimeMillis();
346
347 for ( long n = 0L; n < 2500L; n++ )
348 {
349 BTree<Long, String> btree = BTreeFactory.createInMemoryBTree( "test", LongSerializer.INSTANCE,
350 StringSerializer.INSTANCE );
351 btree.setPageSize( 4 );
352
353 for ( Long i = 0L; i < n; i++ )
354 {
355 String value = "V" + i;
356 btree.insert( i, value );
357 }
358
359
360
361
362
363
364
365 BTree<Long, String> newBtree = ( BTree<Long, String> ) BulkLoader.compact( btree );
366
367
368
369
370
371 System.out.println( "Checking for N = " + n );
372 checkBtree( btree, newBtree );
373 }
374 }
375
376
377 @Ignore
378 @Test
379 public void testInMemoryBulkLoad21() throws IOException, KeyNotFoundException
380 {
381 Random random = new Random( System.nanoTime() );
382 long t0 = System.currentTimeMillis();
383
384 BTree<Long, String> btree = BTreeFactory.createInMemoryBTree( "test", LongSerializer.INSTANCE,
385 StringSerializer.INSTANCE );
386 btree.setPageSize( 4 );
387
388 for ( Long i = 0L; i < 21; i++ )
389 {
390 String value = "V" + i;
391 btree.insert( i, value );
392 }
393
394
395
396
397
398
399
400 BTree<Long, String> newBtree = ( BTree<Long, String> ) BulkLoader.compact( btree );
401
402
403
404
405
406 System.out.println( "Checking for N = " + 21 );
407 checkBtree( btree, newBtree );
408 }
409
410
411
412
413
414 @Test
415 public void testPersistedBulkLoadComputeLeafLevel() throws IOException, KeyNotFoundException,
416 BTreeAlreadyManagedException
417 {
418 Random random = new Random( System.currentTimeMillis() );
419 File file = File.createTempFile( "managedbtreebuilder", ".data" );
420 file.deleteOnExit();
421
422 try
423 {
424 RecordManager rm = new RecordManager( file.getAbsolutePath() );
425 PersistedBTree<Long, String> btree = ( PersistedBTree<Long, String> ) rm.addBTree( "test",
426 LongSerializer.INSTANCE, StringSerializer.INSTANCE, false );
427
428 BulkLoader<Long, String> bulkLoader = new BulkLoader<Long, String>();
429
430 int[] expectedNbPages = new int[]
431 {
432 0,
433 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
434 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
435 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3
436 };
437
438 int[] expectedLimit = new int[]
439 {
440 0,
441 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
442 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 32,
443 16, 16, 16, 16, 16, 16, 16, 32, 32, 32, 32, 32, 32, 32, 32, 48
444 };
445
446 int[] expectedKeys = new int[]
447 {
448 0,
449 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
450 9, 10, 11, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16,
451 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16
452 };
453
454 for ( int i = 0; i < 49; i++ )
455 {
456 System.out.println( "=======================================" );
457 System.out.println( "== Iteration n#" + i );
458 System.out.println( "=======================================" );
459
460 BulkLoader<Long, String>.LevelInfo leafInfo = bulkLoader.computeLevel( btree, i, LevelEnum.LEAF );
461
462 assertEquals( expectedNbPages[i], leafInfo.nbPages );
463 assertEquals( expectedLimit[i], leafInfo.nbElemsLimit );
464 assertEquals( expectedKeys[i], leafInfo.currentPage.getNbElems() );
465 }
466 }
467 finally
468 {
469 file.delete();
470 }
471 }
472
473
474
475
476
477 @Test
478 public void testPersistedBulkLoadComputeNodeLevel() throws IOException, KeyNotFoundException,
479 BTreeAlreadyManagedException
480 {
481 Random random = new Random( System.currentTimeMillis() );
482 File file = File.createTempFile( "managedbtreebuilder", ".data" );
483 file.deleteOnExit();
484
485 try
486 {
487 RecordManager rm = new RecordManager( file.getAbsolutePath() );
488 PersistedBTree<Long, String> btree = ( PersistedBTree<Long, String> ) rm.addBTree( "test",
489 LongSerializer.INSTANCE, StringSerializer.INSTANCE, false );
490
491 BulkLoader<Long, String> bulkLoader = new BulkLoader<Long, String>();
492
493 int[] expectedNbPages = new int[]
494 {
495 -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
496 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
497 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3
498 };
499
500 int[] expectedLimit = new int[]
501 {
502 -1,
503 -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
504 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 17, 34,
505 17, 17, 17, 17, 17, 17, 17, 17, 34, 34, 34, 34, 34, 34, 34, 34, 51
506 };
507
508 int[] expectedKeys = new int[]
509 {
510 -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
511 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16,
512 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16
513 };
514
515 for ( int i = 2; i < 52; i++ )
516 {
517 System.out.println( "=======================================" );
518 System.out.println( "== Iteration n#" + i );
519 System.out.println( "=======================================" );
520
521 BulkLoader<Long, String>.LevelInfo nodeInfo = bulkLoader.computeLevel( btree, i, LevelEnum.NODE );
522
523 assertEquals( expectedNbPages[i], nodeInfo.nbPages );
524 assertEquals( expectedLimit[i], nodeInfo.nbElemsLimit );
525 assertEquals( expectedKeys[i], nodeInfo.currentPage.getNbElems() );
526 }
527 }
528 finally
529 {
530 file.delete();
531 }
532 }
533
534
535
536
537
538 @Test
539 public void testPersistedBulkLoadComputeLevels() throws IOException, KeyNotFoundException,
540 BTreeAlreadyManagedException
541 {
542 Random random = new Random( System.currentTimeMillis() );
543 File file = File.createTempFile( "managedbtreebuilder", ".data" );
544 file.deleteOnExit();
545
546 try
547 {
548 RecordManager rm = new RecordManager( file.getAbsolutePath() );
549 PersistedBTree<Long, String> btree = ( PersistedBTree<Long, String> ) rm.addBTree( "test",
550 LongSerializer.INSTANCE, StringSerializer.INSTANCE, false );
551
552 BulkLoader<Long, String> bulkLoader = new BulkLoader<Long, String>();
553
554 int[] expectedNbPages = new int[]
555 {
556 -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
557 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
558 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3
559 };
560
561 int[] expectedLimit = new int[]
562 {
563 -1,
564 -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
565 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 17, 34,
566 17, 17, 17, 17, 17, 17, 17, 17, 34, 34, 34, 34, 34, 34, 34, 34, 51
567 };
568
569 int[] expectedKeys = new int[]
570 {
571 -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
572 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16,
573 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16
574 };
575
576 for ( int i = 2599; i <= 2599; i++ )
577 {
578 System.out.println( "=======================================" );
579 System.out.println( "== Iteration #" + i );
580 System.out.println( "=======================================" );
581
582 List<BulkLoader<Long, String>.LevelInfo> levels = bulkLoader.computeLevels( btree, i );
583
584 for ( BulkLoader<Long, String>.LevelInfo level : levels )
585 {
586 System.out.println( level );
587 }
588 }
589 }
590 finally
591 {
592 file.delete();
593 }
594 }
595
596
597
598
599
600
601 @Test
602 public void testPersistedBulkLoad1000ElementsMultipleValues() throws IOException, KeyNotFoundException,
603 BTreeAlreadyManagedException
604 {
605 for ( int i = 1000; i < 1001; i++ )
606 {
607 Random random = new Random( System.currentTimeMillis() );
608 File file = File.createTempFile( "managedbtreebuilder", ".data" );
609 file.deleteOnExit();
610
611 try
612 {
613 RecordManager rm = new RecordManager( file.getAbsolutePath() );
614 PersistedBTree<Long, String> btree = ( PersistedBTree<Long, String> ) rm.addBTree( "test",
615 LongSerializer.INSTANCE, StringSerializer.INSTANCE, false );
616
617 BulkLoader<Long, String> bulkLoader = new BulkLoader<Long, String>();
618 int nbElems = i;
619 int addedElems = 0;
620
621 final Tuple<Long, String>[] elems = new Tuple[nbElems];
622 Map<Long, Tuple<Long, Set<String>>> expected = new HashMap<Long, Tuple<Long, Set<String>>>();
623 long valueNumber = 0;
624
625 long t00 = System.currentTimeMillis();
626
627 while ( addedElems < nbElems )
628 {
629 long key = random.nextLong() % 33L;
630 String value = "V" + valueNumber++;
631
632 elems[addedElems] = new Tuple<Long, String>( key, value );
633
634 Tuple<Long, Set<String>> expectedTuple = expected.get( key );
635
636 if ( expectedTuple == null )
637 {
638 expectedTuple = new Tuple<Long, Set<String>>( key, new TreeSet<String>() );
639 }
640
641 expectedTuple.value.add( value );
642 expected.put( key, expectedTuple );
643 addedElems++;
644
645 if ( addedElems % 100 == 0 )
646 {
647
648 }
649 }
650
651 long t01 = System.currentTimeMillis();
652
653
654
655 Iterator<Tuple<Long, String>> tupleIterator = new Iterator<Tuple<Long, String>>()
656 {
657 private int pos = 0;
658
659
660 @Override
661 public Tuple<Long, String> next()
662 {
663 return elems[pos++];
664 }
665
666
667 @Override
668 public boolean hasNext()
669 {
670 return pos < elems.length;
671 }
672
673
674 @Override
675 public void remove()
676 {
677 }
678 };
679
680 long t0 = System.currentTimeMillis();
681 BTree<Long, String> result = bulkLoader.load( btree, tupleIterator, 128 );
682 long t1 = System.currentTimeMillis();
683
684 System.out.println( "== Btree #" + i + ", Time to bulkoad the " + nbElems + " elements "
685 + ( t1 - t0 ) + "ms" );
686
687 TupleCursor<Long, String> cursor = result.browse();
688 int nbFetched = 0;
689
690 long t2 = System.currentTimeMillis();
691
692 while ( cursor.hasNext() )
693 {
694 Tuple<Long, String> elem = cursor.next();
695
696 assertTrue( expected.containsKey( elem.key ) );
697 Tuple<Long, Set<String>> tuple = expected.get( elem.key );
698 assertNotNull( tuple );
699 nbFetched++;
700 }
701
702 long t3 = System.currentTimeMillis();
703
704
705 assertEquals( nbElems, nbFetched );
706
707 checkBtree( btree, result );
708 }
709 finally
710 {
711 file.delete();
712 }
713 }
714 }
715 }