1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.directory.mavibot.btree;
21
22
23 import java.io.Closeable;
24 import java.io.EOFException;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.RandomAccessFile;
29 import java.nio.ByteBuffer;
30 import java.nio.channels.FileChannel;
31 import java.util.concurrent.ConcurrentLinkedQueue;
32
33 import org.apache.directory.mavibot.btree.exception.InitializationException;
34 import org.apache.directory.mavibot.btree.exception.KeyNotFoundException;
35 import org.apache.directory.mavibot.btree.exception.MissingSerializerException;
36 import org.apache.directory.mavibot.btree.serializer.BufferHandler;
37 import org.apache.directory.mavibot.btree.serializer.LongSerializer;
38 import org.slf4j.Logger;
39 import org.slf4j.LoggerFactory;
40
41
42
43
44
45
46
47
48
49
50 class InMemoryBTree<K, V> extends AbstractBTree<K, V> implements Closeable
51 {
52
53 protected static final Logger LOG = LoggerFactory.getLogger( InMemoryBTree.class );
54
55
56 public static final String DEFAULT_JOURNAL = "mavibot.log";
57
58
59 public static final String DATA_SUFFIX = ".db";
60
61
62 public static final String JOURNAL_SUFFIX = ".log";
63
64
65
66 private File file;
67
68
69 private boolean withJournal;
70
71
72 private File journal;
73
74
75 private File envDir;
76
77
78 private FileChannel journalChannel = null;
79
80
81
82
83 InMemoryBTree()
84 {
85 super();
86 setType( BTreeTypeEnum.IN_MEMORY );
87 }
88
89
90
91
92
93
94
95
96 InMemoryBTree( InMemoryBTreeConfiguration<K, V> configuration )
97 {
98 super();
99 String btreeName = configuration.getName();
100
101 if ( btreeName == null )
102 {
103 throw new IllegalArgumentException( "BTree name cannot be null" );
104 }
105
106 String filePath = configuration.getFilePath();
107
108 if ( filePath != null )
109 {
110 envDir = new File( filePath );
111 }
112
113
114 setName( btreeName );
115 setPageSize( configuration.getPageSize() );
116 setKeySerializer( configuration.getKeySerializer() );
117 setValueSerializer( configuration.getValueSerializer() );
118 setAllowDuplicates( configuration.isAllowDuplicates() );
119 setType( configuration.getType() );
120
121 readTimeOut = configuration.getReadTimeOut();
122 writeBufferSize = configuration.getWriteBufferSize();
123
124 if ( keySerializer.getComparator() == null )
125 {
126 throw new IllegalArgumentException( "Comparator should not be null" );
127 }
128
129
130 BTreeHeader<K, V> newBtreeHeader = new BTreeHeader<K, V>();
131
132
133
134 newBtreeHeader.setBTreeHeaderOffset( 0L );
135 newBtreeHeader.setRevision( 0L );
136 newBtreeHeader.setNbElems( 0L );
137 newBtreeHeader.setRootPage( new InMemoryLeaf<K, V>( this ) );
138 newBtreeHeader.setRootPageOffset( 0L );
139
140 btreeRevisions.put( 0L, newBtreeHeader );
141 currentBtreeHeader = newBtreeHeader;
142
143
144 try
145 {
146 init();
147 }
148 catch ( IOException ioe )
149 {
150 throw new InitializationException( ioe.getMessage() );
151 }
152 }
153
154
155
156
157
158
159
160 private void init() throws IOException
161 {
162
163 if ( envDir != null )
164 {
165 if ( !envDir.exists() )
166 {
167 boolean created = envDir.mkdirs();
168
169 if ( !created )
170 {
171 throw new IllegalStateException( "Could not create the directory " + envDir + " for storing data" );
172 }
173 }
174
175 this.file = new File( envDir, getName() + DATA_SUFFIX );
176
177 this.journal = new File( envDir, file.getName() + JOURNAL_SUFFIX );
178 setType( BTreeTypeEnum.BACKED_ON_DISK );
179 }
180
181
182 readTransactions = new ConcurrentLinkedQueue<ReadTransaction<K, V>>();
183
184
185 transactionManager = new InMemoryTransactionManager();
186
187
188
189 if ( getType() == BTreeTypeEnum.BACKED_ON_DISK )
190 {
191 if ( file.length() > 0 )
192 {
193
194 load( file );
195 }
196
197 withJournal = true;
198
199 FileOutputStream stream = new FileOutputStream( journal );
200 journalChannel = stream.getChannel();
201
202
203
204 if ( journal.length() > 0 )
205 {
206 applyJournal();
207 }
208 }
209 else
210 {
211 setType( BTreeTypeEnum.IN_MEMORY );
212
213
214 BTreeHeader<K, V> btreeHeader = new BTreeHeader<K, V>();
215 btreeHeader.setRootPage( new InMemoryLeaf<K, V>( this ) );
216 btreeHeader.setBtree( this );
217 storeRevision( btreeHeader );
218 }
219
220
221
222
223 }
224
225
226
227
228
229 protected ReadTransaction<K, V> beginReadTransaction()
230 {
231 BTreeHeader<K, V> btreeHeader = getBtreeHeader();
232
233 ReadTransaction<K, V> readTransaction = new ReadTransaction<K, V>( btreeHeader, readTransactions );
234
235 readTransactions.add( readTransaction );
236
237 return readTransaction;
238 }
239
240
241
242
243
244 protected ReadTransaction<K, V> beginReadTransaction( long revision )
245 {
246 BTreeHeader<K, V> btreeHeader = getBtreeHeader( revision );
247
248 if ( btreeHeader != null )
249 {
250 ReadTransaction<K, V> readTransaction = new ReadTransaction<K, V>( btreeHeader, readTransactions );
251
252 readTransactions.add( readTransaction );
253
254 return readTransaction;
255 }
256 else
257 {
258 return null;
259 }
260 }
261
262
263
264
265
266 public void close() throws IOException
267 {
268
269
270
271
272 if ( getType() == BTreeTypeEnum.BACKED_ON_DISK )
273 {
274
275 flush();
276 journalChannel.close();
277 }
278 }
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293 protected Tuple<K, V> delete( K key, V value, long revision ) throws IOException
294 {
295 if ( revision == -1L )
296 {
297 revision = currentRevision.get() + 1;
298 }
299
300 BTreeHeader<K, V> oldBtreeHeader = getBtreeHeader();
301 BTreeHeader<K, V> newBtreeHeader = createNewBtreeHeader( oldBtreeHeader, revision );
302 newBtreeHeader.setBtree( this );
303
304
305
306 Tuple<K, V> tuple = null;
307
308
309
310 DeleteResult<K, V> result = getRootPage().delete( key, value, revision );
311
312 if ( result instanceof NotPresentResult )
313 {
314
315 return null;
316 }
317
318
319
320
321 if ( result instanceof RemoveResult )
322 {
323
324 RemoveResult<K, V> removeResult = ( RemoveResult<K, V> ) result;
325
326 Page<K, V> modifiedPage = removeResult.getModifiedPage();
327
328
329 newBtreeHeader.setRootPage( modifiedPage );
330 tuple = removeResult.getRemovedElement();
331 }
332
333 if ( withJournal )
334 {
335
336 writeToJournal( new Deletion<K, V>( key ) );
337 }
338
339
340 if ( tuple != null )
341 {
342 newBtreeHeader.decrementNbElems();
343 }
344
345 storeRevision( newBtreeHeader );
346
347
348 if ( oldBtreeHeader.getNbUsers() == 0 )
349 {
350 btreeRevisions.remove( oldBtreeHeader.getRevision() );
351 }
352
353 return tuple;
354 }
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370 InsertResult<K, V> insert( K key, V value, long revision ) throws IOException
371 {
372
373
374 if ( revision == -1L )
375 {
376 revision = currentRevision.get() + 1;
377 }
378
379 BTreeHeader<K, V> oldBtreeHeader = getBtreeHeader();
380 BTreeHeader<K, V> newBtreeHeader = createNewBtreeHeader( oldBtreeHeader, revision );
381 newBtreeHeader.setBtree( this );
382
383
384
385 V modifiedValue = null;
386
387
388
389
390 InsertResult<K, V> result = newBtreeHeader.getRootPage().insert( key, value, revision );
391
392 if ( result instanceof ModifyResult )
393 {
394 ModifyResult<K, V> modifyResult = ( ( ModifyResult<K, V> ) result );
395
396 Page<K, V> modifiedPage = modifyResult.getModifiedPage();
397
398
399
400 newBtreeHeader.setRootPage( modifiedPage );
401
402 modifiedValue = modifyResult.getModifiedValue();
403 }
404 else
405 {
406
407
408 SplitResult<K, V> splitResult = ( ( SplitResult<K, V> ) result );
409
410 K pivot = splitResult.getPivot();
411 Page<K, V> leftPage = splitResult.getLeftPage();
412 Page<K, V> rightPage = splitResult.getRightPage();
413
414
415 newBtreeHeader.setRootPage( new InMemoryNode<K, V>( this, revision, pivot, leftPage, rightPage ) );
416 }
417
418
419 if ( withJournal )
420 {
421 writeToJournal( new Addition<K, V>( key, value ) );
422 }
423
424
425
426 if ( modifiedValue == null )
427 {
428 newBtreeHeader.incrementNbElems();
429 }
430
431 storeRevision( newBtreeHeader );
432
433 if ( oldBtreeHeader.getNbUsers() == 0 )
434 {
435 long oldRevision = oldBtreeHeader.getRevision();
436
437 if ( oldRevision < newBtreeHeader.getRevision() )
438 {
439 btreeRevisions.remove( oldBtreeHeader.getRevision() );
440 }
441 }
442
443
444 return result;
445 }
446
447
448
449
450
451
452
453
454
455
456 private void writeBuffer( FileChannel channel, ByteBuffer bb, byte[] buffer ) throws IOException
457 {
458 int size = buffer.length;
459 int pos = 0;
460
461
462 do
463 {
464 if ( bb.remaining() >= size )
465 {
466
467 bb.put( buffer, pos, size );
468 size = 0;
469 }
470 else
471 {
472
473 int len = bb.remaining();
474 size -= len;
475 bb.put( buffer, pos, len );
476 pos += len;
477
478 bb.flip();
479
480 channel.write( bb );
481
482 bb.clear();
483 }
484 }
485 while ( size > 0 );
486 }
487
488
489
490
491
492
493 public void flush( File file ) throws IOException
494 {
495 File parentFile = file.getParentFile();
496 File baseDirectory = null;
497
498 if ( parentFile != null )
499 {
500 baseDirectory = new File( file.getParentFile().getAbsolutePath() );
501 }
502 else
503 {
504 baseDirectory = new File( "." );
505 }
506
507
508 File tmpFileFD = File.createTempFile( "mavibot", null, baseDirectory );
509 FileOutputStream stream = new FileOutputStream( tmpFileFD );
510 FileChannel ch = stream.getChannel();
511
512
513 ByteBuffer bb = ByteBuffer.allocateDirect( writeBufferSize );
514
515 try
516 {
517 TupleCursor<K, V> cursor = browse();
518
519 if ( keySerializer == null )
520 {
521 throw new MissingSerializerException( "Cannot flush the btree without a Key serializer" );
522 }
523
524 if ( valueSerializer == null )
525 {
526 throw new MissingSerializerException( "Cannot flush the btree without a Value serializer" );
527 }
528
529
530 bb.putLong( getBtreeHeader().getNbElems() );
531
532 while ( cursor.hasNext() )
533 {
534 Tuple<K, V> tuple = cursor.next();
535
536 byte[] keyBuffer = keySerializer.serialize( tuple.getKey() );
537
538 writeBuffer( ch, bb, keyBuffer );
539
540 byte[] valueBuffer = valueSerializer.serialize( tuple.getValue() );
541
542 writeBuffer( ch, bb, valueBuffer );
543 }
544
545
546 if ( bb.position() > 0 )
547 {
548 bb.flip();
549 ch.write( bb );
550 }
551
552
553 ch.force( true );
554 ch.close();
555 }
556 catch ( KeyNotFoundException knfe )
557 {
558 knfe.printStackTrace();
559 throw new IOException( knfe.getMessage() );
560 }
561
562
563 File backupFile = File.createTempFile( "mavibot", null, baseDirectory );
564 file.renameTo( backupFile );
565
566
567 tmpFileFD.renameTo( file );
568
569
570 backupFile.delete();
571 }
572
573
574
575
576
577
578
579 private void applyJournal() throws IOException
580 {
581 if ( !journal.exists() )
582 {
583 throw new IOException( "The journal does not exist" );
584 }
585
586 FileChannel channel =
587 new RandomAccessFile( journal, "rw" ).getChannel();
588 ByteBuffer buffer = ByteBuffer.allocate( 65536 );
589
590 BufferHandler bufferHandler = new BufferHandler( channel, buffer );
591
592
593 try
594 {
595 while ( true )
596 {
597
598 byte[] type = bufferHandler.read( 1 );
599
600 if ( type[0] == Modification.ADDITION )
601 {
602
603 K key = keySerializer.deserialize( bufferHandler );
604
605
606
607
608 V value = valueSerializer.deserialize( bufferHandler );
609
610
611
612
613 insert( key, value, getBtreeHeader().getRevision() );
614 }
615 else
616 {
617
618 K key = keySerializer.deserialize( bufferHandler );
619
620
621 delete( key, getBtreeHeader().getRevision() );
622 }
623 }
624 }
625 catch ( EOFException eofe )
626 {
627 eofe.printStackTrace();
628
629 journalChannel.truncate( 0 );
630 }
631 }
632
633
634
635
636
637
638
639
640
641 public void load( File file ) throws IOException
642 {
643 if ( !file.exists() )
644 {
645 throw new IOException( "The file does not exist" );
646 }
647
648 FileChannel channel =
649 new RandomAccessFile( file, "rw" ).getChannel();
650 ByteBuffer buffer = ByteBuffer.allocate( 65536 );
651
652 BufferHandler bufferHandler = new BufferHandler( channel, buffer );
653
654 long nbElems = LongSerializer.deserialize( bufferHandler.read( 8 ) );
655
656
657 boolean isJournalActivated = withJournal;
658
659 withJournal = false;
660
661
662 for ( long i = 0; i < nbElems; i++ )
663 {
664
665 K key = keySerializer.deserialize( bufferHandler );
666
667
668 V value = valueSerializer.deserialize( bufferHandler );
669
670
671 insert( key, value, getBtreeHeader().getRevision() );
672 }
673
674
675 withJournal = isJournalActivated;
676
677
678
679 }
680
681
682
683
684
685
686
687
688
689
690 public Page<K, V> getRootPage( long revision ) throws IOException, KeyNotFoundException
691 {
692
693 return getBtreeHeader().getRootPage();
694 }
695
696
697
698
699
700
701
702 public Page<K, V> getRootPage()
703 {
704 return getBtreeHeader().getRootPage();
705 }
706
707
708 void setRootPage( Page<K, V> root )
709 {
710 getBtreeHeader().setRootPage( root );
711 }
712
713
714
715
716
717
718 public void flush() throws IOException
719 {
720 if ( getType() == BTreeTypeEnum.BACKED_ON_DISK )
721 {
722
723 flush( file );
724 journalChannel.truncate( 0 );
725 }
726 }
727
728
729
730
731
732 public File getFile()
733 {
734 return file;
735 }
736
737
738
739
740
741 public File getJournal()
742 {
743 return journal;
744 }
745
746
747
748
749
750 public boolean isInMemory()
751 {
752 return getType() == BTreeTypeEnum.IN_MEMORY;
753 }
754
755
756
757
758
759 public boolean isPersistent()
760 {
761 return getType() == BTreeTypeEnum.IN_MEMORY;
762 }
763
764
765 private void writeToJournal( Modification<K, V> modification )
766 throws IOException
767 {
768 if ( modification instanceof Addition )
769 {
770 byte[] keyBuffer = keySerializer.serialize( modification.getKey() );
771 ByteBuffer bb = ByteBuffer.allocateDirect( keyBuffer.length + 1 );
772 bb.put( Modification.ADDITION );
773 bb.put( keyBuffer );
774 bb.flip();
775
776 journalChannel.write( bb );
777
778 byte[] valueBuffer = valueSerializer.serialize( modification.getValue() );
779 bb = ByteBuffer.allocateDirect( valueBuffer.length );
780 bb.put( valueBuffer );
781 bb.flip();
782
783 journalChannel.write( bb );
784 }
785 else if ( modification instanceof Deletion )
786 {
787 byte[] keyBuffer = keySerializer.serialize( modification.getKey() );
788 ByteBuffer bb = ByteBuffer.allocateDirect( keyBuffer.length + 1 );
789 bb.put( Modification.DELETION );
790 bb.put( keyBuffer );
791 bb.flip();
792
793 journalChannel.write( bb );
794 }
795
796
797 journalChannel.force( true );
798 }
799
800
801
802
803
804
805 private BTreeHeader<K, V> createNewBtreeHeader( BTreeHeader<K, V> btreeHeader, long revision )
806 {
807 BTreeHeader<K, V> newBtreeHeader = new BTreeHeader<K, V>();
808
809 newBtreeHeader.setBTreeHeaderOffset( btreeHeader.getBTreeHeaderOffset() );
810 newBtreeHeader.setRevision( revision );
811 newBtreeHeader.setNbElems( btreeHeader.getNbElems() );
812 newBtreeHeader.setRootPage( btreeHeader.getRootPage() );
813
814 return newBtreeHeader;
815 }
816
817
818
819
820
821 public String toString()
822 {
823 StringBuilder sb = new StringBuilder();
824
825 switch ( getType() )
826 {
827 case IN_MEMORY:
828 sb.append( "In-memory " );
829 break;
830
831 case BACKED_ON_DISK:
832 sb.append( "Persistent " );
833 break;
834
835 default :
836 sb.append( "Wrong type... " );
837 break;
838 }
839
840 sb.append( "BTree" );
841 sb.append( "[" ).append( getName() ).append( "]" );
842 sb.append( "( pageSize:" ).append( getPageSize() );
843
844 if ( getBtreeHeader().getRootPage() != null )
845 {
846 sb.append( ", nbEntries:" ).append( getBtreeHeader().getNbElems() );
847 }
848 else
849 {
850 sb.append( ", nbEntries:" ).append( 0 );
851 }
852
853 sb.append( ", comparator:" );
854
855 if ( keySerializer.getComparator() == null )
856 {
857 sb.append( "null" );
858 }
859 else
860 {
861 sb.append( keySerializer.getComparator().getClass().getSimpleName() );
862 }
863
864 sb.append( ", DuplicatesAllowed: " ).append( isAllowDuplicates() );
865
866 if ( getType() == BTreeTypeEnum.BACKED_ON_DISK )
867 {
868 try
869 {
870 sb.append( ", file : " );
871
872 if ( file != null )
873 {
874 sb.append( file.getCanonicalPath() );
875 }
876 else
877 {
878 sb.append( "Unknown" );
879 }
880
881 sb.append( ", journal : " );
882
883 if ( journal != null )
884 {
885 sb.append( journal.getCanonicalPath() );
886 }
887 else
888 {
889 sb.append( "Unkown" );
890 }
891 }
892 catch ( IOException ioe )
893 {
894
895 }
896 }
897
898 sb.append( ") : \n" );
899 sb.append( getRootPage().dumpPage( "" ) );
900
901 return sb.toString();
902 }
903 }