1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.directory.mavibot.btree;
21
22
23 import java.io.File;
24 import java.io.IOException;
25 import java.io.RandomAccessFile;
26 import java.nio.ByteBuffer;
27 import java.nio.channels.FileChannel;
28 import java.util.ArrayList;
29 import java.util.HashSet;
30 import java.util.LinkedHashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Set;
34 import java.util.concurrent.atomic.AtomicLong;
35
36 import org.apache.directory.mavibot.btree.exception.BTreeAlreadyManagedException;
37 import org.apache.directory.mavibot.btree.exception.EndOfFileExceededException;
38 import org.apache.directory.mavibot.btree.exception.KeyNotFoundException;
39 import org.apache.directory.mavibot.btree.serializer.ElementSerializer;
40 import org.apache.directory.mavibot.btree.serializer.IntSerializer;
41 import org.apache.directory.mavibot.btree.serializer.LongArraySerializer;
42 import org.apache.directory.mavibot.btree.serializer.LongSerializer;
43 import org.apache.directory.mavibot.btree.util.Strings;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46
47
48
49
50
51
52
53
54
55
56
57 public class RecordManager
58 {
59
60 protected static final Logger LOG = LoggerFactory.getLogger( RecordManager.class );
61
62
63 protected static final Logger LOG_CHECK = LoggerFactory.getLogger( "RM_CHECK" );
64
65
66 private File file;
67
68
69 private FileChannel fileChannel;
70
71
72 private int nbBtree;
73
74
75 private long firstFreePage;
76 private long lastFreePage;
77
78
79 List<PageIO> freePages = new ArrayList<PageIO>();
80
81
82 public AtomicLong nbFreedPages = new AtomicLong( 0 );
83 public AtomicLong nbCreatedPages = new AtomicLong( 0 );
84 public AtomicLong nbReusedPages = new AtomicLong( 0 );
85
86
87 private long endOfFileOffset;
88
89
90
91
92
93 private BTree<RevisionName, long[]> copiedPageBTree;
94
95
96 private BTree<RevisionName, Long> revisionBTree;
97
98
99 private static final long NO_PAGE = -1L;
100
101
102 private static final int NB_TREE_SIZE = 4;
103
104
105 private static final int PAGE_SIZE = 4;
106
107
108 private static final int DATA_SIZE = 4;
109
110
111 private static final int LINK_SIZE = 8;
112
113
114 private static final int BYTE_SIZE = 1;
115 private static final int INT_SIZE = 4;
116 private static final int LONG_SIZE = 8;
117
118
119 private static final int FIRST_FREE_PAGE_SIZE = 8;
120 private static final int LAST_FREE_PAGE_SIZE = 8;
121
122
123 private static final int DEFAULT_PAGE_SIZE = 512;
124
125
126 private static int HEADER_SIZE = DEFAULT_PAGE_SIZE;
127
128
129 private static ByteBuffer HEADER_BUFFER;
130
131
132 private int pageSize = DEFAULT_PAGE_SIZE;
133
134
135 private Map<String, BTree<?, ?>> managedBTrees;
136
137
138 private long lastAddedBTreeOffset = NO_PAGE;
139
140
141 private static final String DEFAULT_FILE_NAME = "mavibot.db";
142
143
144 private static final LongSerializer OFFSET_SERIALIZER = new LongSerializer();
145
146 private static final String REVISION_BTREE_NAME = "_revisionBTree_";
147
148 private static final String COPIED_PAGE_BTREE_NAME = "_copiedPageBTree_";
149
150
151 private boolean keepRevisions;
152
153
154
155
156
157
158
159
160
161 public RecordManager( String fileName )
162 {
163 this( fileName, DEFAULT_PAGE_SIZE );
164 }
165
166
167
168
169
170
171
172
173
174
175 public RecordManager( String fileName, int pageSize )
176 {
177 managedBTrees = new LinkedHashMap<String, BTree<?, ?>>();
178
179 HEADER_BUFFER = ByteBuffer.allocate( pageSize );
180 HEADER_SIZE = pageSize;
181
182
183 File tmpFile = new File( fileName );
184 boolean isNewFile = false;
185
186 if ( tmpFile.isDirectory() )
187 {
188
189 File mavibotFile = new File( tmpFile, DEFAULT_FILE_NAME );
190
191 if ( !mavibotFile.exists() )
192 {
193
194 try
195 {
196 mavibotFile.createNewFile();
197 isNewFile = true;
198 }
199 catch ( IOException ioe )
200 {
201 LOG.error( "Cannot create the file {}", mavibotFile.getName() );
202 return;
203 }
204 }
205
206 file = mavibotFile;
207 }
208 else
209 {
210
211 if ( !tmpFile.exists() || ( tmpFile.length() == 0 ) )
212 {
213 isNewFile = true;
214
215 try
216 {
217 tmpFile.createNewFile();
218 }
219 catch ( IOException ioe )
220 {
221 LOG.error( "Cannot create the file {}", tmpFile.getName() );
222 return;
223 }
224 }
225
226 file = tmpFile;
227 }
228
229 try
230 {
231 RandomAccessFile randomFile = new RandomAccessFile( file, "rw" );
232 fileChannel = randomFile.getChannel();
233
234
235 endOfFileOffset = fileChannel.size();
236
237 if ( isNewFile )
238 {
239 this.pageSize = pageSize;
240 initRecordManager();
241 }
242 else
243 {
244 loadRecordManager();
245 }
246 }
247 catch ( Exception e )
248 {
249 LOG.error( "Error while initializing the RecordManager : {}", e.getMessage() );
250 LOG.error( "", e );
251 throw new RuntimeException( e );
252 }
253 }
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279 private void initRecordManager() throws IOException
280 {
281
282 nbBtree = 0;
283 firstFreePage = NO_PAGE;
284 lastFreePage = NO_PAGE;
285 updateRecordManagerHeader();
286
287
288 endOfFileOffset = fileChannel.size();
289
290
291 copiedPageBTree = new BTree<RevisionName, long[]>( COPIED_PAGE_BTREE_NAME, new RevisionNameSerializer(),
292 new LongArraySerializer() );
293
294
295 revisionBTree = new BTree<RevisionName, Long>( REVISION_BTREE_NAME, new RevisionNameSerializer(),
296 new LongSerializer() );
297
298
299 try
300 {
301 manage( copiedPageBTree );
302 manage( revisionBTree );
303 }
304 catch ( BTreeAlreadyManagedException btame )
305 {
306
307 }
308
309
310 }
311
312
313
314
315
316
317
318
319
320 private void loadRecordManager() throws IOException, ClassNotFoundException, IllegalAccessException,
321 InstantiationException
322 {
323 if ( fileChannel.size() != 0 )
324 {
325 ByteBuffer header = ByteBuffer.allocate( HEADER_SIZE );
326
327
328 fileChannel.read( header );
329
330 header.rewind();
331
332
333 pageSize = header.getInt();
334
335
336 nbBtree = header.getInt();
337
338
339 firstFreePage = header.getLong();
340 lastFreePage = header.getLong();
341
342
343
344
345
346 long btreeOffset = HEADER_SIZE;
347
348 PageIO[] pageIos = readPageIOs( HEADER_SIZE, Long.MAX_VALUE );
349
350
351 copiedPageBTree = BTreeFactory.createBTree();
352 copiedPageBTree.setBtreeOffset( btreeOffset );
353
354 loadBTree( pageIos, copiedPageBTree );
355 long nextBtreeOffset = copiedPageBTree.getNextBTreeOffset();
356
357
358 pageIos = readPageIOs( nextBtreeOffset, Long.MAX_VALUE );
359
360 revisionBTree = BTreeFactory.createBTree();
361 revisionBTree.setBtreeOffset( nextBtreeOffset );
362
363 loadBTree( pageIos, revisionBTree );
364 nextBtreeOffset = revisionBTree.getNextBTreeOffset();
365
366
367 for ( int i = 2; i < nbBtree; i++ )
368 {
369
370 BTree<?, ?> btree = BTreeFactory.createBTree();
371 btree.setBtreeOffset( nextBtreeOffset );
372 lastAddedBTreeOffset = nextBtreeOffset;
373
374
375 pageIos = readPageIOs( nextBtreeOffset, Long.MAX_VALUE );
376
377
378 loadBTree( pageIos, btree );
379 nextBtreeOffset = btree.getNextBTreeOffset();
380
381
382 managedBTrees.put( btree.getName(), btree );
383 }
384
385
386 endOfFileOffset = fileChannel.size();
387 }
388 }
389
390
391
392
393
394
395
396
397
398 private PageIO[] readPageIOs( long position, long limit ) throws IOException, EndOfFileExceededException
399 {
400 LOG.debug( "Read PageIOs at position {}", position );
401
402 if ( limit <= 0 )
403 {
404 limit = Long.MAX_VALUE;
405 }
406
407 PageIO firstPage = fetchPage( position );
408 firstPage.setSize();
409 List<PageIO> listPages = new ArrayList<PageIO>();
410 listPages.add( firstPage );
411 long dataRead = pageSize - LONG_SIZE - INT_SIZE;
412
413
414 long nextPage = firstPage.getNextPage();
415
416 if ( ( dataRead < limit ) && ( nextPage != NO_PAGE ) )
417 {
418 while ( dataRead < limit )
419 {
420 PageIO page = fetchPage( nextPage );
421 listPages.add( page );
422 nextPage = page.getNextPage();
423 dataRead += pageSize - LONG_SIZE;
424
425 if ( nextPage == NO_PAGE )
426 {
427 page.setNextPage( NO_PAGE );
428 break;
429 }
430 }
431 }
432
433 LOG.debug( "Nb of PageIOs read : {}", listPages.size() );
434
435
436 return listPages.toArray( new PageIO[]
437 {} );
438 }
439
440
441
442
443
444
445
446
447
448
449
450 private void loadBTree( PageIO[] pageIos, BTree<?, ?> btree ) throws EndOfFileExceededException,
451 IOException, ClassNotFoundException, IllegalAccessException, InstantiationException
452 {
453 long dataPos = 0L;
454
455
456 long revision = readLong( pageIos, dataPos );
457 BTreeFactory.setRevision( btree, revision );
458 dataPos += LONG_SIZE;
459
460
461 long nbElems = readLong( pageIos, dataPos );
462 BTreeFactory.setNbElems( btree, nbElems );
463 dataPos += LONG_SIZE;
464
465
466 long rootPageOffset = readLong( pageIos, dataPos );
467 BTreeFactory.setRootPageOffset( btree, rootPageOffset );
468 dataPos += LONG_SIZE;
469
470
471 long nextBTreeOffset = readLong( pageIos, dataPos );
472 BTreeFactory.setNextBTreeOffset( btree, nextBTreeOffset );
473 dataPos += LONG_SIZE;
474
475
476 int btreePageSize = readInt( pageIos, dataPos );
477 BTreeFactory.setPageSize( btree, btreePageSize );
478 dataPos += INT_SIZE;
479
480
481 byte[] btreeNameBytes = readBytes( pageIos, dataPos );
482 dataPos += INT_SIZE + btreeNameBytes.length;
483 String btreeName = Strings.utf8ToString( btreeNameBytes );
484 BTreeFactory.setName( btree, btreeName );
485
486
487 byte[] keySerializerBytes = readBytes( pageIos, dataPos );
488 dataPos += INT_SIZE + keySerializerBytes.length;
489
490 String keySerializerFqcn = null;
491
492 if ( keySerializerBytes != null )
493 {
494 keySerializerFqcn = Strings.utf8ToString( keySerializerBytes );
495 }
496 else
497 {
498 keySerializerFqcn = "";
499 }
500
501 BTreeFactory.setKeySerializer( btree, keySerializerFqcn );
502
503
504 byte[] valueSerializerBytes = readBytes( pageIos, dataPos );
505
506 String valueSerializerFqcn = null;
507 dataPos += INT_SIZE + valueSerializerBytes.length;
508
509 if ( valueSerializerBytes != null )
510 {
511 valueSerializerFqcn = Strings.utf8ToString( valueSerializerBytes );
512 }
513 else
514 {
515 valueSerializerFqcn = "";
516 }
517
518 BTreeFactory.setValueSerializer( btree, valueSerializerFqcn );
519
520
521 int allowDuplicates = readInt( pageIos, dataPos );
522 btree.setAllowDuplicates( allowDuplicates != 0 );
523 dataPos += INT_SIZE;
524
525
526 btree.init();
527
528
529
530
531
532
533 PageIO[] rootPageIos = readPageIOs( rootPageOffset, Long.MAX_VALUE );
534
535 Page btreeRoot = readPage( btree, rootPageIos );
536 BTreeFactory.setRecordManager( btree, this );
537
538 BTreeFactory.setRoot( btree, btreeRoot );
539 }
540
541
542 private Page readNode( BTree btree, long offset, long revision, int nbElems ) throws IOException
543 {
544 Page node = BTreeFactory.createNode( btree, revision, nbElems );
545
546
547 PageIO[] pageIos = readPageIOs( offset, Long.MAX_VALUE );
548
549 return node;
550 }
551
552
553 public Page deserialize( BTree btree, long offset ) throws EndOfFileExceededException, IOException
554 {
555 PageIO[] rootPageIos = readPageIOs( offset, Long.MAX_VALUE );
556
557 Page page = readPage( btree, rootPageIos );
558
559 return page;
560 }
561
562
563 private Page readPage( BTree btree, PageIO[] pageIos ) throws IOException
564 {
565
566 long position = 0L;
567
568
569 long revision = readLong( pageIos, position );
570 position += LONG_SIZE;
571
572
573 int nbElems = readInt( pageIos, position );
574 position += INT_SIZE;
575
576
577 Page page = null;
578 ByteBuffer byteBuffer = null;
579
580
581 byte[] data = readBytes( pageIos, position );
582
583 if ( data != null )
584 {
585 byteBuffer = ByteBuffer.allocate( data.length );
586 byteBuffer.put( data );
587 byteBuffer.rewind();
588 }
589
590 if ( nbElems >= 0 )
591 {
592
593 page = BTreeFactory.createLeaf( btree, revision, nbElems );
594
595 ( ( AbstractPage ) page ).setOffset( pageIos[0].getOffset() );
596 ( ( AbstractPage ) page ).setLastOffset( pageIos[pageIos.length - 1].getOffset() );
597
598
599 for ( int i = 0; i < nbElems; i++ )
600 {
601 ElementHolder valueHolder;
602
603 if ( btree.isAllowDuplicates() )
604 {
605 long value = OFFSET_SERIALIZER.deserialize( byteBuffer );
606
607 BTree dupValueContainer = loadDupsBTree(value);
608
609 valueHolder = new DuplicateKeyMemoryHolder( btree, dupValueContainer );
610 }
611 else
612 {
613 Object value = btree.getValueSerializer().deserialize( byteBuffer );
614
615 valueHolder = new MemoryHolder( btree, value );
616 }
617
618 BTreeFactory.setValue( ( ( Leaf ) page ), i, valueHolder );
619
620 Object key = btree.getKeySerializer().deserialize( byteBuffer );
621
622 BTreeFactory.setKey( page, i, key );
623 }
624 }
625 else
626 {
627
628 int nodeNbElems = -nbElems;
629
630 page = BTreeFactory.createNode( btree, revision, nodeNbElems );
631
632
633 for ( int i = 0; i < nodeNbElems; i++ )
634 {
635
636 long offset = OFFSET_SERIALIZER.deserialize( byteBuffer );
637 long lastOffset = OFFSET_SERIALIZER.deserialize( byteBuffer );
638
639 ElementHolder valueHolder = new ReferenceHolder( btree, null, offset, lastOffset );
640 ( ( Node ) page ).setValue( i, valueHolder );
641
642 Object key = btree.getKeySerializer().deserialize( byteBuffer );
643 BTreeFactory.setKey( page, i, key );
644 }
645
646
647 long offset = OFFSET_SERIALIZER.deserialize( byteBuffer );
648 long lastOffset = OFFSET_SERIALIZER.deserialize( byteBuffer );
649
650 ElementHolder valueHolder = new ReferenceHolder( btree, null, offset, lastOffset );
651 ( ( Node ) page ).setValue( nodeNbElems, valueHolder );
652 }
653
654 return page;
655 }
656
657
658
659
660
661
662
663
664 private byte[] readBytes( PageIO[] pageIos, long position )
665 {
666
667 int length = readInt( pageIos, position );
668 position += INT_SIZE;
669
670
671
672 int pageNb = computePageNb( position );
673
674
675 int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
676
677 ByteBuffer pageData = pageIos[pageNb].getData();
678 int remaining = pageData.capacity() - pagePos;
679
680 if ( length == 0 )
681 {
682
683 return null;
684 }
685 else
686 {
687 byte[] bytes = new byte[length];
688 int bytesPos = 0;
689
690 while ( length > 0 )
691 {
692 if ( length <= remaining )
693 {
694 pageData.mark();
695 pageData.position( pagePos );
696 pageData.get( bytes, bytesPos, length );
697 pageData.reset();
698
699 return bytes;
700 }
701
702 pageData.mark();
703 pageData.position( pagePos );
704 pageData.get( bytes, bytesPos, remaining );
705 pageData.reset();
706 pageNb++;
707 pagePos = LINK_SIZE;
708 bytesPos += remaining;
709 pageData = pageIos[pageNb].getData();
710 length -= remaining;
711 remaining = pageData.capacity() - pagePos;
712 }
713
714 return bytes;
715 }
716 }
717
718
719
720
721
722
723
724
725 private int readInt( PageIO[] pageIos, long position )
726 {
727
728
729 int pageNb = computePageNb( position );
730
731
732 int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
733
734 ByteBuffer pageData = pageIos[pageNb].getData();
735 int remaining = pageData.capacity() - pagePos;
736 int value = 0;
737
738 if ( remaining >= INT_SIZE )
739 {
740 value = pageData.getInt( pagePos );
741 }
742 else
743 {
744 value = 0;
745
746 switch ( remaining )
747 {
748 case 3:
749 value += ( ( pageData.get( pagePos + 2 ) & 0x00FF ) << 8 );
750
751
752 case 2:
753 value += ( ( pageData.get( pagePos + 1 ) & 0x00FF ) << 16 );
754
755
756 case 1:
757 value += ( pageData.get( pagePos ) << 24 );
758 break;
759 }
760
761
762 pageData = pageIos[pageNb + 1].getData();
763 pagePos = LINK_SIZE;
764
765 switch ( remaining )
766 {
767 case 1:
768 value += ( pageData.get( pagePos ) & 0x00FF ) << 16;
769
770
771 case 2:
772 value += ( pageData.get( pagePos + 2 - remaining ) & 0x00FF ) << 8;
773
774
775 case 3:
776 value += ( pageData.get( pagePos + 3 - remaining ) & 0x00FF );
777 break;
778 }
779 }
780
781 return value;
782 }
783
784
785
786
787
788
789
790
791 private byte readByte( PageIO[] pageIos, long position )
792 {
793
794
795 int pageNb = computePageNb( position );
796
797
798 int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
799
800 ByteBuffer pageData = pageIos[pageNb].getData();
801 byte value = 0;
802
803 value = pageData.get( pagePos );
804
805 return value;
806 }
807
808
809
810
811
812
813
814
815 private long readLong( PageIO[] pageIos, long position )
816 {
817
818
819 int pageNb = computePageNb( position );
820
821
822 int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
823
824 ByteBuffer pageData = pageIos[pageNb].getData();
825 int remaining = pageData.capacity() - pagePos;
826 long value = 0L;
827
828 if ( remaining >= LONG_SIZE )
829 {
830 value = pageData.getLong( pagePos );
831 }
832 else
833 {
834 switch ( remaining )
835 {
836 case 7:
837 value += ( ( ( long ) pageData.get( pagePos + 6 ) & 0x00FF ) << 8 );
838
839
840 case 6:
841 value += ( ( ( long ) pageData.get( pagePos + 5 ) & 0x00FF ) << 16 );
842
843
844 case 5:
845 value += ( ( ( long ) pageData.get( pagePos + 4 ) & 0x00FF ) << 24 );
846
847
848 case 4:
849 value += ( ( ( long ) pageData.get( pagePos + 3 ) & 0x00FF ) << 32 );
850
851
852 case 3:
853 value += ( ( ( long ) pageData.get( pagePos + 2 ) & 0x00FF ) << 40 );
854
855
856 case 2:
857 value += ( ( ( long ) pageData.get( pagePos + 1 ) & 0x00FF ) << 48 );
858
859
860 case 1:
861 value += ( ( long ) pageData.get( pagePos ) << 56 );
862 break;
863 }
864
865
866 pageData = pageIos[pageNb + 1].getData();
867 pagePos = LINK_SIZE;
868
869 switch ( remaining )
870 {
871 case 1:
872 value += ( ( long ) pageData.get( pagePos ) & 0x00FF ) << 48;
873
874
875 case 2:
876 value += ( ( long ) pageData.get( pagePos + 2 - remaining ) & 0x00FF ) << 40;
877
878
879 case 3:
880 value += ( ( long ) pageData.get( pagePos + 3 - remaining ) & 0x00FF ) << 32;
881
882
883 case 4:
884 value += ( ( long ) pageData.get( pagePos + 4 - remaining ) & 0x00FF ) << 24;
885
886
887 case 5:
888 value += ( ( long ) pageData.get( pagePos + 5 - remaining ) & 0x00FF ) << 16;
889
890
891 case 6:
892 value += ( ( long ) pageData.get( pagePos + 6 - remaining ) & 0x00FF ) << 8;
893
894
895 case 7:
896 value += ( ( long ) pageData.get( pagePos + 7 - remaining ) & 0x00FF );
897 break;
898 }
899 }
900
901 return value;
902 }
903
904
905
906
907
908
909
910
911 public synchronized void manage( BTree<?, ?> btree ) throws BTreeAlreadyManagedException, IOException
912 {
913 manage( btree, false );
914 }
915
916
917
918
919
920
921
922
923
924
925
926
927 public synchronized void manage( BTree<?, ?> btree, boolean internalTree ) throws BTreeAlreadyManagedException,
928 IOException
929 {
930 LOG.debug( "Managing the btree {} which is an internam tree : {}", btree.getName(), internalTree );
931 BTreeFactory.setRecordManager( btree, this );
932
933 String name = btree.getName();
934
935 if ( managedBTrees.containsKey( name ) )
936 {
937
938 LOG.error( "There is already a BTree named '{}' managed by this recordManager", name );
939 throw new BTreeAlreadyManagedException( name );
940 }
941
942 managedBTrees.put( name, btree );
943
944
945 byte[] btreeNameBytes = Strings.getBytesUtf8( name );
946 byte[] keySerializerBytes = Strings.getBytesUtf8( btree.getKeySerializerFQCN() );
947 byte[] valueSerializerBytes = Strings.getBytesUtf8( btree.getValueSerializerFQCN() );
948
949 int bufferSize =
950 INT_SIZE +
951 btreeNameBytes.length +
952 INT_SIZE +
953 keySerializerBytes.length +
954 INT_SIZE +
955 valueSerializerBytes.length +
956 INT_SIZE +
957 LONG_SIZE +
958 LONG_SIZE +
959 LONG_SIZE +
960 LONG_SIZE +
961 INT_SIZE;
962
963
964 PageIO[] pageIos = getFreePageIOs( bufferSize );
965
966
967 long btreeOffset = pageIos[0].getOffset();
968 btree.setBtreeOffset( btreeOffset );
969
970
971
972
973
974
975
976
977
978
979
980
981 long position = 0L;
982
983
984 position = store( position, btree.getRevision(), pageIos );
985
986
987 position = store( position, btree.getNbElems(), pageIos );
988
989
990 Page rootPage = BTreeFactory.getRoot( btree );
991
992 PageIO[] rootPageIos = serializePage( btree, btree.getRevision(), rootPage );
993
994
995 PageIO rootPageIo = rootPageIos[0];
996
997
998 position = store( position, rootPageIo.getOffset(), pageIos );
999 btree.setRootPageOffset( rootPageIo.getOffset() );
1000 ( ( Leaf ) rootPage ).setOffset( rootPageIo.getOffset() );
1001
1002
1003 position = store( position, NO_PAGE, pageIos );
1004
1005
1006 position = store( position, btree.getPageSize(), pageIos );
1007
1008
1009 position = store( position, btreeNameBytes, pageIos );
1010
1011
1012 position = store( position, keySerializerBytes, pageIos );
1013
1014
1015 position = store( position, valueSerializerBytes, pageIos );
1016
1017
1018 position = store( position, ( btree.isAllowDuplicates() ? 1 : 0 ), pageIos );
1019
1020
1021 LOG.debug( "Flushing the newly managed '{}' btree header", btree.getName() );
1022 flushPages( pageIos );
1023 LOG.debug( "Flushing the newly managed '{}' btree rootpage", btree.getName() );
1024 flushPages( rootPageIos );
1025
1026
1027
1028 if ( !internalTree )
1029 {
1030 nbBtree++;
1031
1032 if ( lastAddedBTreeOffset != NO_PAGE )
1033 {
1034
1035 pageIos = readPageIOs( lastAddedBTreeOffset, LONG_SIZE + LONG_SIZE + LONG_SIZE + LONG_SIZE );
1036 store( LONG_SIZE + LONG_SIZE + LONG_SIZE, btreeOffset, pageIos );
1037
1038
1039 LOG.debug( "Updated the previous btree pointer on the added BTree {}", btree.getName() );
1040 flushPages( pageIos );
1041 }
1042
1043 lastAddedBTreeOffset = btreeOffset;
1044
1045
1046 updateRecordManagerHeader();
1047 }
1048
1049 if ( LOG_CHECK.isDebugEnabled() )
1050 {
1051 check();
1052 }
1053 }
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074 private PageIO[] serializePage( BTree btree, long revision, Page page ) throws IOException
1075 {
1076 int nbElems = page.getNbElems();
1077
1078 if ( nbElems == 0 )
1079 {
1080
1081 PageIO[] pageIos = new PageIO[1];
1082
1083
1084 PageIO newPage = fetchNewPage();
1085
1086
1087
1088
1089 long position = 0L;
1090
1091 position = store( position, revision, newPage );
1092 position = store( position, nbElems, newPage );
1093
1094
1095 newPage.setSize( ( int ) position );
1096
1097
1098 pageIos[0] = newPage;
1099
1100 return pageIos;
1101 }
1102 else
1103 {
1104
1105 int nbBuffers = 1 + 1 + 1 + nbElems * 2;
1106 int dataSize = 0;
1107 int serializedSize = 0;
1108
1109 if ( page instanceof Node )
1110 {
1111
1112 nbBuffers++;
1113 }
1114
1115
1116 List<byte[]> serializedData = new ArrayList<byte[]>( nbBuffers );
1117
1118
1119 byte[] buffer = LongSerializer.serialize( revision );
1120 serializedData.add( buffer );
1121 serializedSize += buffer.length;
1122
1123
1124
1125 int pageNbElems = nbElems;
1126
1127 if ( page instanceof Node )
1128 {
1129 pageNbElems = -nbElems;
1130 }
1131
1132 buffer = IntSerializer.serialize( pageNbElems );
1133 serializedData.add( buffer );
1134 serializedSize += buffer.length;
1135
1136
1137 for ( int pos = 0; pos < nbElems; pos++ )
1138 {
1139
1140 if ( page instanceof Node )
1141 {
1142 Page child = ( ( Node ) page ).getReference( pos );
1143
1144
1145 buffer = LongSerializer.serialize( child.getOffset() );
1146 serializedData.add( buffer );
1147 dataSize += buffer.length;
1148
1149
1150 buffer = LongSerializer.serialize( child.getLastOffset() );
1151 serializedData.add( buffer );
1152 dataSize += buffer.length;
1153 }
1154 else
1155 {
1156 if ( btree.isAllowDuplicates() )
1157 {
1158 DuplicateKeyMemoryHolder value = ( DuplicateKeyMemoryHolder ) ( ( Leaf ) page ).getValue( pos );
1159 long duplicateContainerOffset = ( ( BTree ) value.getValue( btree ) ).getBtreeOffset();
1160 buffer = OFFSET_SERIALIZER.serialize( duplicateContainerOffset );
1161 }
1162 else
1163 {
1164 ElementHolder value = ( ( Leaf ) page ).getValue( pos );
1165 buffer = btree.getValueSerializer().serialize( value.getValue( btree ) );
1166 }
1167
1168 serializedData.add( buffer );
1169 dataSize += buffer.length;
1170 }
1171
1172
1173 buffer = btree.getKeySerializer().serialize( page.getKey( pos ) );
1174 serializedData.add( buffer );
1175 dataSize += buffer.length;
1176 }
1177
1178
1179 if ( page instanceof Node )
1180 {
1181 Page child = ( ( Node ) page ).getReference( nbElems );
1182
1183
1184 buffer = LongSerializer.serialize( child.getOffset() );
1185 serializedData.add( buffer );
1186 dataSize += buffer.length;
1187
1188
1189 buffer = LongSerializer.serialize( child.getLastOffset() );
1190 serializedData.add( buffer );
1191 dataSize += buffer.length;
1192 }
1193
1194
1195 buffer = IntSerializer.serialize( dataSize );
1196 serializedData.add( 2, buffer );
1197 serializedSize += buffer.length;
1198
1199 serializedSize += dataSize;
1200
1201
1202 PageIO[] pageIos = getFreePageIOs( serializedSize );
1203
1204
1205 long position = 0L;
1206
1207 for ( byte[] bytes : serializedData )
1208 {
1209 position = storeRaw( position, bytes, pageIos );
1210 }
1211
1212 return pageIos;
1213 }
1214 }
1215
1216
1217
1218
1219
1220 private void updateRecordManagerHeader() throws IOException
1221 {
1222 HEADER_BUFFER.clear();
1223
1224
1225 HEADER_BUFFER.putInt( pageSize );
1226
1227
1228 HEADER_BUFFER.putInt( nbBtree );
1229
1230
1231 HEADER_BUFFER.putLong( firstFreePage );
1232
1233
1234 HEADER_BUFFER.putLong( lastFreePage );
1235
1236
1237 HEADER_BUFFER.limit( pageSize );
1238
1239
1240 HEADER_BUFFER.rewind();
1241
1242 LOG.debug( "Update RM header, FF : {}, LF : {}", firstFreePage, lastFreePage );
1243 fileChannel.write( HEADER_BUFFER, 0 );
1244 }
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259 IOException
1260 {
1261
1262 long offset = btree.getBtreeOffset();
1263 long headerSize = LONG_SIZE + LONG_SIZE + LONG_SIZE;
1264
1265 PageIO[] pageIos = readPageIOs( offset, headerSize );
1266
1267
1268 long position = 0;
1269
1270 position = store( position, btree.getRevision(), pageIos );
1271 position = store( position, btree.getNbElems(), pageIos );
1272 position = store( position, rootPageOffset, pageIos );
1273
1274
1275 if ( LOG.isDebugEnabled() )
1276 {
1277 LOG.debug( "-----> Flushing the '{}' BTreeHeader", btree.getName() );
1278 LOG.debug( " revision : " + btree.getRevision() + ", NbElems : " + btree.getNbElems() + ", root offset : "
1279 + rootPageOffset );
1280 }
1281
1282 flushPages( pageIos );
1283
1284 if ( LOG_CHECK.isDebugEnabled() )
1285 {
1286 check();
1287 }
1288 }
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298 private void flushPages( PageIO... pageIos ) throws IOException
1299 {
1300 for ( PageIO pageIo : pageIos )
1301 {
1302 pageIo.getData().rewind();
1303
1304 if ( fileChannel.size() < ( pageIo.getOffset() + pageSize ) )
1305 {
1306 LOG.debug( "Adding a page at the end of the file" );
1307
1308 fileChannel.write( pageIo.getData(), fileChannel.size() );
1309
1310 }
1311 else
1312 {
1313 LOG.debug( "Writing a page at position {}", pageIo.getOffset() );
1314 fileChannel.write( pageIo.getData(), pageIo.getOffset() );
1315
1316 }
1317
1318 pageIo.getData().rewind();
1319 }
1320 }
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330 private int computePageNb( long offset )
1331 {
1332 long pageNb = 0;
1333
1334 offset -= pageSize - LINK_SIZE - PAGE_SIZE;
1335
1336 if ( offset < 0 )
1337 {
1338 return ( int ) pageNb;
1339 }
1340
1341 pageNb = 1 + offset / ( pageSize - LINK_SIZE );
1342
1343 return ( int ) pageNb;
1344 }
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356 private long store( long position, byte[] bytes, PageIO... pageIos )
1357 {
1358 if ( bytes != null )
1359 {
1360
1361 position = store( position, bytes.length, pageIos );
1362
1363
1364
1365 int pageNb = computePageNb( position );
1366
1367
1368 ByteBuffer pageData = pageIos[pageNb].getData();
1369
1370
1371 int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1372
1373
1374 int remaining = pageData.capacity() - pagePos;
1375 int nbStored = bytes.length;
1376
1377
1378 while ( nbStored > 0 )
1379 {
1380 if ( remaining > nbStored )
1381 {
1382 pageData.mark();
1383 pageData.position( pagePos );
1384 pageData.put( bytes, bytes.length - nbStored, nbStored );
1385 pageData.reset();
1386 nbStored = 0;
1387 }
1388 else
1389 {
1390 pageData.mark();
1391 pageData.position( pagePos );
1392 pageData.put( bytes, bytes.length - nbStored, remaining );
1393 pageData.reset();
1394 pageNb++;
1395 pageData = pageIos[pageNb].getData();
1396 pagePos = LINK_SIZE;
1397 nbStored -= remaining;
1398 remaining = pageData.capacity() - pagePos;
1399 }
1400 }
1401
1402
1403 position += bytes.length;
1404 }
1405 else
1406 {
1407
1408 position = store( position, 0, pageIos );
1409 }
1410
1411 return position;
1412 }
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425 private long storeRaw( long position, byte[] bytes, PageIO... pageIos )
1426 {
1427 if ( bytes != null )
1428 {
1429
1430
1431 int pageNb = computePageNb( position );
1432
1433
1434 ByteBuffer pageData = pageIos[pageNb].getData();
1435
1436
1437 int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1438
1439
1440 int remaining = pageData.capacity() - pagePos;
1441 int nbStored = bytes.length;
1442
1443
1444 while ( nbStored > 0 )
1445 {
1446 if ( remaining > nbStored )
1447 {
1448 pageData.mark();
1449 pageData.position( pagePos );
1450 pageData.put( bytes, bytes.length - nbStored, nbStored );
1451 pageData.reset();
1452 nbStored = 0;
1453 }
1454 else
1455 {
1456 pageData.mark();
1457 pageData.position( pagePos );
1458 pageData.put( bytes, bytes.length - nbStored, remaining );
1459 pageData.reset();
1460 pageNb++;
1461
1462 if ( pageNb == pageIos.length )
1463 {
1464
1465 break;
1466 }
1467
1468 pageData = pageIos[pageNb].getData();
1469 pagePos = LINK_SIZE;
1470 nbStored -= remaining;
1471 remaining = pageData.capacity() - pagePos;
1472 }
1473 }
1474
1475
1476 position += bytes.length;
1477 }
1478 else
1479 {
1480
1481 position = store( position, 0, pageIos );
1482 }
1483
1484 return position;
1485 }
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497 private long store( long position, int value, PageIO... pageIos )
1498 {
1499
1500
1501 int pageNb = computePageNb( position );
1502
1503
1504 int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1505
1506
1507 ByteBuffer pageData = pageIos[pageNb].getData();
1508
1509
1510 int remaining = pageData.capacity() - pagePos;
1511
1512 if ( remaining < INT_SIZE )
1513 {
1514
1515
1516 switch ( remaining )
1517 {
1518 case 3:
1519 pageData.put( pagePos + 2, ( byte ) ( value >>> 8 ) );
1520
1521
1522 case 2:
1523 pageData.put( pagePos + 1, ( byte ) ( value >>> 16 ) );
1524
1525
1526 case 1:
1527 pageData.put( pagePos, ( byte ) ( value >>> 24 ) );
1528 break;
1529 }
1530
1531
1532 pageData = pageIos[pageNb + 1].getData();
1533 pagePos = LINK_SIZE;
1534
1535 switch ( remaining )
1536 {
1537 case 1:
1538 pageData.put( pagePos, ( byte ) ( value >>> 16 ) );
1539
1540
1541 case 2:
1542 pageData.put( pagePos + 2 - remaining, ( byte ) ( value >>> 8 ) );
1543
1544
1545 case 3:
1546 pageData.put( pagePos + 3 - remaining, ( byte ) ( value ) );
1547 break;
1548 }
1549 }
1550 else
1551 {
1552
1553 pageData.putInt( pagePos, value );
1554 }
1555
1556
1557 position += INT_SIZE;
1558
1559 return position;
1560 }
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572 private long store( long position, long value, PageIO... pageIos )
1573 {
1574
1575
1576 int pageNb = computePageNb( position );
1577
1578
1579 int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1580
1581
1582 ByteBuffer pageData = pageIos[pageNb].getData();
1583
1584
1585 int remaining = pageData.capacity() - pagePos;
1586
1587 if ( remaining < LONG_SIZE )
1588 {
1589
1590
1591 switch ( remaining )
1592 {
1593 case 7:
1594 pageData.put( pagePos + 6, ( byte ) ( value >>> 8 ) );
1595
1596
1597 case 6:
1598 pageData.put( pagePos + 5, ( byte ) ( value >>> 16 ) );
1599
1600
1601 case 5:
1602 pageData.put( pagePos + 4, ( byte ) ( value >>> 24 ) );
1603
1604
1605 case 4:
1606 pageData.put( pagePos + 3, ( byte ) ( value >>> 32 ) );
1607
1608
1609 case 3:
1610 pageData.put( pagePos + 2, ( byte ) ( value >>> 40 ) );
1611
1612
1613 case 2:
1614 pageData.put( pagePos + 1, ( byte ) ( value >>> 48 ) );
1615
1616
1617 case 1:
1618 pageData.put( pagePos, ( byte ) ( value >>> 56 ) );
1619 break;
1620 }
1621
1622
1623 pageData = pageIos[pageNb + 1].getData();
1624 pagePos = LINK_SIZE;
1625
1626 switch ( remaining )
1627 {
1628 case 1:
1629 pageData.put( pagePos, ( byte ) ( value >>> 48 ) );
1630
1631
1632 case 2:
1633 pageData.put( pagePos + 2 - remaining, ( byte ) ( value >>> 40 ) );
1634
1635
1636 case 3:
1637 pageData.put( pagePos + 3 - remaining, ( byte ) ( value >>> 32 ) );
1638
1639
1640 case 4:
1641 pageData.put( pagePos + 4 - remaining, ( byte ) ( value >>> 24 ) );
1642
1643
1644 case 5:
1645 pageData.put( pagePos + 5 - remaining, ( byte ) ( value >>> 16 ) );
1646
1647
1648 case 6:
1649 pageData.put( pagePos + 6 - remaining, ( byte ) ( value >>> 8 ) );
1650
1651
1652 case 7:
1653 pageData.put( pagePos + 7 - remaining, ( byte ) ( value ) );
1654 break;
1655 }
1656 }
1657 else
1658 {
1659
1660 pageData.putLong( pagePos, value );
1661 }
1662
1663
1664 position += LONG_SIZE;
1665
1666 return position;
1667 }
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682 throws IOException
1683 {
1684
1685 PageIO[] pageIos = serializePage( btree, newRevision, newPage );
1686
1687 LOG.debug( "Write data for '{}' btree ", btree.getName() );
1688
1689 flushPages( pageIos );
1690
1691
1692 long offset = pageIos[0].getOffset();
1693 long lastOffset = pageIos[pageIos.length - 1].getOffset();
1694 ElementHolder valueHolder = new ReferenceHolder( btree, newPage, offset, lastOffset );
1695
1696 if ( LOG_CHECK.isDebugEnabled() )
1697 {
1698 check();
1699 }
1700
1701 return valueHolder;
1702 }
1703
1704
1705
1706
1707
1708
1709
1710
1711 private int computeNbPages( int dataSize )
1712 {
1713 if ( dataSize <= 0 )
1714 {
1715 return 0;
1716 }
1717
1718
1719
1720
1721
1722
1723
1724
1725 int availableSize = ( pageSize - LONG_SIZE );
1726 int nbNeededPages = 1;
1727
1728
1729 if ( dataSize > availableSize - INT_SIZE )
1730 {
1731 int remainingSize = dataSize - ( availableSize - INT_SIZE );
1732 nbNeededPages += remainingSize / availableSize;
1733 int remain = remainingSize % availableSize;
1734
1735 if ( remain > 0 )
1736 {
1737 nbNeededPages++;
1738 }
1739 }
1740
1741 return nbNeededPages;
1742 }
1743
1744
1745
1746
1747
1748
1749
1750
1751 private PageIO[] getFreePageIOs( int dataSize ) throws IOException
1752 {
1753 if ( dataSize == 0 )
1754 {
1755 return new PageIO[]
1756 {};
1757 }
1758
1759 int nbNeededPages = computeNbPages( dataSize );
1760
1761 PageIO[] pageIOs = new PageIO[nbNeededPages];
1762
1763
1764 pageIOs[0] = fetchNewPage();
1765 pageIOs[0].setSize( dataSize );
1766
1767 for ( int i = 1; i < nbNeededPages; i++ )
1768 {
1769 pageIOs[i] = fetchNewPage();
1770
1771
1772 pageIOs[i - 1].setNextPage( pageIOs[i].getOffset() );
1773 }
1774
1775 return pageIOs;
1776 }
1777
1778
1779
1780
1781
1782
1783
1784
1785 private PageIO fetchNewPage() throws IOException
1786 {
1787 if ( firstFreePage == NO_PAGE )
1788 {
1789 nbCreatedPages.incrementAndGet();
1790
1791
1792
1793 PageIO newPage = new PageIO( endOfFileOffset );
1794
1795 endOfFileOffset += pageSize;
1796
1797 ByteBuffer data = ByteBuffer.allocateDirect( pageSize );
1798
1799 newPage.setData( data );
1800 newPage.setNextPage( NO_PAGE );
1801 newPage.setSize( 0 );
1802
1803 LOG.debug( "Requiring a new page at offset {}", newPage.getOffset() );
1804
1805 return newPage;
1806 }
1807 else
1808 {
1809 nbReusedPages.incrementAndGet();
1810
1811
1812 PageIO pageIo = fetchPage( firstFreePage );
1813
1814
1815 firstFreePage = pageIo.getNextPage();
1816
1817
1818 ByteBuffer data = ByteBuffer.allocateDirect( pageSize );
1819 pageIo.setData( data );
1820
1821 pageIo.setNextPage( NO_PAGE );
1822 pageIo.setSize( 0 );
1823
1824 LOG.debug( "Reused page at offset {}", pageIo.getOffset() );
1825
1826
1827 if ( firstFreePage == NO_PAGE )
1828 {
1829 lastFreePage = NO_PAGE;
1830 }
1831
1832
1833 updateRecordManagerHeader();
1834
1835 return pageIo;
1836 }
1837 }
1838
1839
1840
1841
1842
1843
1844
1845
1846 private PageIO fetchPage( long offset ) throws IOException, EndOfFileExceededException
1847 {
1848 if ( fileChannel.size() < offset + pageSize )
1849 {
1850
1851 throw new EndOfFileExceededException( "We are fetching a page on " + offset +
1852 " when the file's size is " + fileChannel.size() );
1853 }
1854 else
1855 {
1856
1857 fileChannel.position( offset );
1858
1859 ByteBuffer data = ByteBuffer.allocate( pageSize );
1860 fileChannel.read( data );
1861 data.rewind();
1862
1863 PageIO readPage = new PageIO( offset );
1864 readPage.setData( data );
1865
1866 return readPage;
1867 }
1868 }
1869
1870
1871
1872
1873
1874 public int getPageSize()
1875 {
1876 return pageSize;
1877 }
1878
1879
1880 public void setPageSize( int pageSize )
1881 {
1882 if ( this.pageSize != -1 )
1883 {
1884 }
1885 else
1886 {
1887 this.pageSize = pageSize;
1888 }
1889 }
1890
1891
1892
1893
1894
1895 public void close() throws IOException
1896 {
1897
1898
1899 for ( BTree tree : managedBTrees.values() )
1900 {
1901 tree.close();
1902 }
1903
1904 managedBTrees.clear();
1905
1906
1907 fileChannel.force( true );
1908
1909
1910 fileChannel.close();
1911 }
1912
1913
1914
1915
1916
1917
1918 public void dump() throws IOException
1919 {
1920 RandomAccessFile randomFile = new RandomAccessFile( file, "r" );
1921 FileChannel fileChannel = randomFile.getChannel();
1922
1923 ByteBuffer header = ByteBuffer.allocate( HEADER_SIZE );
1924
1925
1926 fileChannel.read( header );
1927
1928 header.rewind();
1929
1930
1931 int pageSize = header.getInt();
1932
1933
1934 int nbBTree = header.getInt();
1935
1936
1937 long firstFreePage = header.getLong();
1938 long lastFreePage = header.getLong();
1939
1940 if ( LOG.isDebugEnabled() )
1941 {
1942 LOG.debug( "RecordManager" );
1943 LOG.debug( "-------------" );
1944 LOG.debug( " Header " );
1945 LOG.debug( " '{}'", Strings.dumpBytes( header.array() ) );
1946 LOG.debug( " page size : {}", pageSize );
1947 LOG.debug( " nbTree : {}", nbBTree );
1948 LOG.debug( " firstFreePage : {}", firstFreePage );
1949 LOG.debug( " lastFreePage : {}", lastFreePage );
1950 }
1951
1952 long position = HEADER_SIZE;
1953
1954
1955 for ( int i = 0; i < nbBTree; i++ )
1956 {
1957 LOG.debug( " Btree[{}]", i );
1958 PageIO[] pageIos = readPageIOs( position, Long.MAX_VALUE );
1959
1960 for ( PageIO pageIo : pageIos )
1961 {
1962 LOG.debug( " {}", pageIo );
1963 }
1964 }
1965
1966 randomFile.close();
1967 }
1968
1969
1970
1971
1972
1973
1974
1975 public int getNbManagedTrees()
1976 {
1977 return nbBtree - 2;
1978 }
1979
1980
1981
1982
1983
1984
1985
1986 public Set<String> getManagedTrees()
1987 {
1988 Set<String> btrees = new HashSet<String>( managedBTrees.keySet() );
1989
1990 btrees.remove( COPIED_PAGE_BTREE_NAME );
1991 btrees.remove( REVISION_BTREE_NAME );
1992
1993 return btrees;
1994 }
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005 {
2006 if ( !isKeepRevisions() )
2007 {
2008 return;
2009 }
2010
2011 if ( ( btree == copiedPageBTree ) || ( btree == revisionBTree ) )
2012 {
2013 return;
2014 }
2015
2016 RevisionName revisionName = new RevisionName( rootPage.getRevision(), btree.getName() );
2017
2018 revisionBTree.insert( revisionName, rootPage.getOffset(), 0 );
2019
2020 if ( LOG_CHECK.isDebugEnabled() )
2021 {
2022 check();
2023 }
2024 }
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037 {
2038 if ( btree.getRevision() == revision )
2039 {
2040
2041 return btree.rootPage;
2042 }
2043
2044 RevisionName revisionName = new RevisionName( revision, btree.getName() );
2045 long rootPageOffset = revisionBTree.get( revisionName );
2046
2047
2048 PageIO[] rootPageIos = readPageIOs( rootPageOffset, Long.MAX_VALUE );
2049
2050 Page btreeRoot = readPage( btree, rootPageIos );
2051
2052 return btreeRoot;
2053 }
2054
2055
2056
2057
2058
2059
2060
2061 public BTree getManagedTree( String name )
2062 {
2063 return managedBTrees.get( name );
2064 }
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079 IOException
2080 {
2081 if ( ( btree == copiedPageBTree ) || ( btree == revisionBTree ) )
2082 {
2083 return;
2084 }
2085
2086 if ( ( pages == null ) || pages.isEmpty() )
2087 {
2088 return;
2089 }
2090
2091 if ( !keepRevisions )
2092 {
2093
2094
2095
2096
2097
2098 for ( Page page : pages )
2099 {
2100
2101 long firstOffset = page.getOffset();
2102
2103
2104
2105
2106 if ( firstOffset == NO_PAGE )
2107 {
2108 continue;
2109 }
2110
2111 long lastOffset = page.getLastOffset();
2112
2113
2114 if ( firstFreePage == NO_PAGE )
2115 {
2116
2117
2118 firstFreePage = firstOffset;
2119 }
2120 else
2121 {
2122
2123
2124 long offset = page.getLastOffset();
2125
2126 if ( offset == NO_PAGE )
2127 {
2128 offset = page.getOffset();
2129 }
2130
2131
2132 PageIO pageIo = fetchPage( offset );
2133
2134
2135 pageIo.setNextPage( firstFreePage );
2136
2137 LOG.debug( "Flushing the first free page" );
2138
2139
2140 flushPages( pageIo );
2141
2142
2143 firstFreePage = firstOffset;
2144 }
2145 }
2146
2147
2148 updateRecordManagerHeader();
2149 }
2150 else
2151 {
2152 LOG.debug( "We should not get there" );
2153
2154 for ( Page p : pages )
2155 {
2156 addFreePage( btree, p );
2157 }
2158 }
2159 }
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169 private void addFreePage( BTree btree, Page freePage )
2170 {
2171 try
2172 {
2173 RevisionName revision = new RevisionName( freePage.getRevision(), btree.getName() );
2174 long[] offsetArray = null;
2175
2176 if ( copiedPageBTree.hasKey( revision ) )
2177 {
2178 offsetArray = copiedPageBTree.get( revision );
2179 long[] tmp = new long[offsetArray.length + 1];
2180 System.arraycopy( offsetArray, 0, tmp, 0, offsetArray.length );
2181 offsetArray = tmp;
2182 }
2183 else
2184 {
2185 offsetArray = new long[1];
2186 }
2187
2188 offsetArray[offsetArray.length - 1] = freePage.getOffset();
2189
2190 copiedPageBTree.insert( revision, offsetArray, 0 );
2191 }
2192 catch ( Exception e )
2193 {
2194 throw new RuntimeException( e );
2195 }
2196 }
2197
2198
2199
2200
2201
2202 public boolean isKeepRevisions()
2203 {
2204 return keepRevisions;
2205 }
2206
2207
2208
2209
2210
2211 public void setKeepRevisions( boolean keepRevisions )
2212 {
2213 this.keepRevisions = keepRevisions;
2214 }
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228 @SuppressWarnings("all")
2229 public BTree addBTree( String name, ElementSerializer<?> keySerializer, ElementSerializer<?> valueSerializer,
2230 boolean allowDuplicates ) throws IOException, BTreeAlreadyManagedException
2231 {
2232 BTreeConfiguration config = new BTreeConfiguration();
2233
2234 config.setName( name );
2235 config.setKeySerializer( keySerializer );
2236 config.setValueSerializer( valueSerializer );
2237 config.setAllowDuplicates( allowDuplicates );
2238 config.setType( BTreeTypeEnum.MANAGED );
2239
2240 BTree btree = new BTree( config );
2241 manage( btree );
2242
2243 if ( LOG_CHECK.isDebugEnabled() )
2244 {
2245 check();
2246 }
2247
2248 return btree;
2249 }
2250
2251
2252 private void setCheckedPage( long[] checkedPages, long offset, int pageSize )
2253 {
2254 long pageOffset = ( offset - HEADER_SIZE ) / pageSize;
2255 int index = ( int ) ( pageOffset / 64L );
2256 long mask = ( 1L << ( pageOffset % 64L ) );
2257 long bits = checkedPages[index];
2258
2259 if ( ( bits & mask ) == 1 )
2260 {
2261 throw new RuntimeException( "The page at : " + offset + " has already been checked" );
2262 }
2263
2264 checkedPages[index] |= mask;
2265
2266 }
2267
2268
2269
2270
2271
2272
2273
2274
2275 private void checkFreePages( long[] checkedPages, int pageSize, long firstFreePage, long lastFreePage )
2276 throws IOException
2277 {
2278 if ( firstFreePage == NO_PAGE )
2279 {
2280 if ( lastFreePage == NO_PAGE )
2281 {
2282 return;
2283 }
2284 else
2285 {
2286 throw new RuntimeException( "Wrong last free page : " + lastFreePage );
2287 }
2288 }
2289
2290 if ( lastFreePage != NO_PAGE )
2291 {
2292 throw new RuntimeException( "Wrong last free page : " + lastFreePage );
2293 }
2294
2295
2296 long currentOffset = firstFreePage;
2297 long fileSize = fileChannel.size();
2298
2299 while ( currentOffset != NO_PAGE )
2300 {
2301 if ( currentOffset > fileSize )
2302 {
2303 throw new RuntimeException( "Wrong free page offset, above file size : " + currentOffset );
2304 }
2305
2306 try
2307 {
2308 PageIO pageIo = fetchPage( currentOffset );
2309
2310 if ( currentOffset != pageIo.getOffset() )
2311 {
2312 throw new RuntimeException( "PageIO offset is incorrect : " + currentOffset + "-"
2313 + pageIo.getOffset() );
2314 }
2315
2316 setCheckedPage( checkedPages, currentOffset, pageSize );
2317
2318 long newOffset = pageIo.getNextPage();
2319 currentOffset = newOffset;
2320 }
2321 catch ( IOException ioe )
2322 {
2323 throw new RuntimeException( "Cannot fetch page at : " + currentOffset );
2324 }
2325 }
2326 }
2327
2328
2329
2330
2331
2332
2333
2334 private void checkRoot( long[] checkedPages, long offset, int pageSize, long nbBTreeElems,
2335 ElementSerializer keySerializer, ElementSerializer valueSerializer, boolean allowDuplicates )
2336 throws EndOfFileExceededException, IOException
2337 {
2338
2339 PageIO[] rootPageIos = readPageIOs( offset, Long.MAX_VALUE );
2340
2341
2342 long position = 0L;
2343
2344
2345 long revision = readLong( rootPageIos, position );
2346 position += LONG_SIZE;
2347
2348
2349 int nbElems = readInt( rootPageIos, position );
2350 position += INT_SIZE;
2351
2352
2353 ByteBuffer byteBuffer = null;
2354
2355
2356 byte[] data = readBytes( rootPageIos, position );
2357
2358 if ( data != null )
2359 {
2360 byteBuffer = ByteBuffer.allocate( data.length );
2361 byteBuffer.put( data );
2362 byteBuffer.rewind();
2363 }
2364
2365 if ( nbElems >= 0 )
2366 {
2367
2368
2369
2370 long pageOffset = rootPageIos[0].getOffset();
2371
2372 if ( ( pageOffset < 0 ) || ( pageOffset > fileChannel.size() ) )
2373 {
2374 throw new RuntimeException( "The page offset is incorrect : " + pageOffset );
2375 }
2376
2377
2378 long pageLastOffset = rootPageIos[rootPageIos.length - 1].getOffset();
2379
2380 if ( ( pageLastOffset <= 0 ) || ( pageLastOffset > fileChannel.size() ) )
2381 {
2382 throw new RuntimeException( "The page last offset is incorrect : " + pageLastOffset );
2383 }
2384
2385
2386 for ( int i = 0; i < nbElems; i++ )
2387 {
2388
2389 if ( allowDuplicates )
2390 {
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409 }
2410 else
2411 {
2412 valueSerializer.deserialize( byteBuffer );
2413 }
2414
2415 keySerializer.deserialize( byteBuffer );
2416 }
2417 }
2418 else
2419 {
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444 }
2445 }
2446
2447
2448
2449
2450
2451
2452
2453
2454 private long checkBTree( long[] checkedPages, PageIO[] pageIos, int pageSize, boolean isLast )
2455 throws EndOfFileExceededException, IOException, InstantiationException, IllegalAccessException,
2456 ClassNotFoundException
2457 {
2458 long dataPos = 0L;
2459
2460
2461 long revision = readLong( pageIos, dataPos );
2462 dataPos += LONG_SIZE;
2463
2464
2465 long nbElems = readLong( pageIos, dataPos );
2466 dataPos += LONG_SIZE;
2467
2468
2469 long rootPageOffset = readLong( pageIos, dataPos );
2470
2471 if ( ( rootPageOffset < 0 ) || ( rootPageOffset > fileChannel.size() ) )
2472 {
2473 throw new RuntimeException( "The rootpage is incorrect : " + rootPageOffset );
2474 }
2475
2476 dataPos += LONG_SIZE;
2477
2478
2479 long nextBTreeOffset = readLong( pageIos, dataPos );
2480
2481 if ( ( ( rootPageOffset < 0 ) && ( !isLast ) ) || ( nextBTreeOffset > fileChannel.size() ) )
2482 {
2483 throw new RuntimeException( "The rootpage is incorrect : " + rootPageOffset );
2484 }
2485
2486 dataPos += LONG_SIZE;
2487
2488
2489 int btreePageSize = readInt( pageIos, dataPos );
2490
2491 if ( ( btreePageSize < 2 ) || ( ( btreePageSize & ( ~btreePageSize + 1 ) ) != btreePageSize ) )
2492 {
2493 throw new RuntimeException( "The BTree page size is not a power of 2 : " + btreePageSize );
2494 }
2495
2496 dataPos += INT_SIZE;
2497
2498
2499 byte[] btreeNameBytes = readBytes( pageIos, dataPos );
2500 dataPos += INT_SIZE;
2501
2502 dataPos += btreeNameBytes.length;
2503 String btreeName = Strings.utf8ToString( btreeNameBytes );
2504
2505
2506 byte[] keySerializerBytes = readBytes( pageIos, dataPos );
2507
2508 String keySerializerFqcn = null;
2509 dataPos += INT_SIZE;
2510
2511 if ( keySerializerBytes != null )
2512 {
2513 dataPos += keySerializerBytes.length;
2514 keySerializerFqcn = Strings.utf8ToString( keySerializerBytes );
2515 }
2516 else
2517 {
2518 keySerializerFqcn = "";
2519 }
2520
2521
2522 byte[] valueSerializerBytes = readBytes( pageIos, dataPos );
2523
2524 String valueSerializerFqcn = null;
2525 dataPos += INT_SIZE;
2526
2527 if ( valueSerializerBytes != null )
2528 {
2529 dataPos += valueSerializerBytes.length;
2530 valueSerializerFqcn = Strings.utf8ToString( valueSerializerBytes );
2531 }
2532 else
2533 {
2534 valueSerializerFqcn = "";
2535 }
2536
2537
2538 int allowDuplicates = readInt( pageIos, dataPos );
2539 dataPos += INT_SIZE;
2540
2541
2542
2543
2544 Class<?> valueSerializer = Class.forName( valueSerializerFqcn );
2545 Class<?> keySerializer = Class.forName( keySerializerFqcn );
2546
2547
2548
2549
2550
2551
2552
2553 return nextBTreeOffset;
2554 }
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565 private void checkBTrees( long[] checkedPages, int pageSize, int nbBTrees ) throws EndOfFileExceededException,
2566 IOException, InstantiationException, IllegalAccessException, ClassNotFoundException
2567 {
2568
2569
2570
2571 long position = HEADER_SIZE;
2572
2573 for ( int i = 0; i < nbBTrees; i++ )
2574 {
2575
2576 PageIO[] pageIos = readPageIOs( position, Long.MAX_VALUE );
2577
2578
2579 int pageNb = 0;
2580
2581 for ( PageIO currentPageIo : pageIos )
2582 {
2583
2584 long nextPageOffset = currentPageIo.getNextPage();
2585
2586 if ( pageNb == pageIos.length - 1 )
2587 {
2588 if ( nextPageOffset != NO_PAGE )
2589 {
2590 throw new RuntimeException( "The pointer to the next page is not valid, expected NO_PAGE" );
2591 }
2592 }
2593 else
2594 {
2595 if ( nextPageOffset == NO_PAGE )
2596 {
2597 throw new RuntimeException( "The pointer to the next page is not valid, NO_PAGE" );
2598 }
2599 }
2600
2601 if ( ( nextPageOffset != NO_PAGE ) && ( ( nextPageOffset - HEADER_SIZE ) % pageSize != 0 ) )
2602 {
2603 throw new RuntimeException( "The pointer to the next page is not valid" );
2604 }
2605
2606
2607 setCheckedPage( checkedPages, currentPageIo.getOffset(), pageSize );
2608 }
2609
2610
2611 long nextBTree = checkBTree( checkedPages, pageIos, pageSize, i == nbBTrees - 1 );
2612
2613 if ( ( nextBTree == NO_PAGE ) && ( i < nbBTrees - 1 ) )
2614 {
2615 throw new RuntimeException( "The pointer to the next BTree is incorrect" );
2616 }
2617
2618 position = nextBTree;
2619 }
2620 }
2621
2622
2623
2624
2625
2626 private void check()
2627 {
2628 try
2629 {
2630
2631 ByteBuffer header = ByteBuffer.allocate( HEADER_SIZE );
2632 long fileSize = fileChannel.size();
2633
2634 if ( fileSize < HEADER_SIZE )
2635 {
2636 throw new RuntimeException( "File size too small : " + fileSize );
2637 }
2638
2639
2640 fileChannel.read( header, 0L );
2641 header.flip();
2642
2643
2644 int pageSize = header.getInt();
2645
2646 if ( ( pageSize < 0 ) || ( pageSize < 32 ) || ( ( pageSize & ( ~pageSize + 1 ) ) != pageSize ) )
2647 {
2648 throw new RuntimeException( "Wrong page size : " + pageSize );
2649 }
2650
2651
2652 long nbPages = ( fileSize - HEADER_SIZE ) / pageSize;
2653
2654
2655 int nbBTrees = header.getInt();
2656
2657 if ( nbBTrees < 0 )
2658 {
2659 throw new RuntimeException( "Wrong nb trees : " + nbBTrees );
2660 }
2661
2662
2663
2664 long firstFreePage = header.getLong();
2665
2666 if ( firstFreePage > fileSize )
2667 {
2668 throw new RuntimeException( "First free page pointing after the end of the file : " + firstFreePage );
2669 }
2670
2671 if ( ( firstFreePage != NO_PAGE ) && ( ( ( firstFreePage - HEADER_SIZE ) % pageSize ) != 0 ) )
2672 {
2673 throw new RuntimeException( "First free page not pointing to a correct offset : " + firstFreePage );
2674 }
2675
2676
2677 long lastFreePage = header.getLong();
2678
2679 if ( ( ( lastFreePage != NO_PAGE ) && ( ( ( lastFreePage - HEADER_SIZE ) % pageSize ) != 0 ) ) )
2680
2681 {
2682 throw new RuntimeException( "Invalid last free page : " + lastFreePage );
2683 }
2684
2685 int nbPageBits = ( int ) ( nbPages / 64 );
2686
2687
2688
2689
2690 long[] checkedPages = new long[nbPageBits + 1];
2691
2692
2693 checkFreePages( checkedPages, pageSize, firstFreePage, lastFreePage );
2694
2695
2696 checkBTrees( checkedPages, pageSize, nbBTrees );
2697 }
2698 catch ( Exception e )
2699 {
2700
2701
2702 e.printStackTrace();
2703 throw new RuntimeException( "Error : " + e.getMessage() );
2704 }
2705 }
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716 {
2717 try
2718 {
2719 PageIO[] pageIos = readPageIOs( offset, Long.MAX_VALUE );
2720
2721 BTree dupValueContainer = BTreeFactory.createBTree();
2722 dupValueContainer.setBtreeOffset( offset );
2723
2724 loadBTree( pageIos, dupValueContainer );
2725
2726 return dupValueContainer;
2727 }
2728 catch ( Exception e )
2729 {
2730
2731 throw new RuntimeException( e );
2732 }
2733
2734 }
2735
2736
2737
2738
2739
2740 public String toString()
2741 {
2742 StringBuilder sb = new StringBuilder();
2743
2744 sb.append( "RM free pages : [" );
2745
2746 if ( firstFreePage != NO_PAGE )
2747 {
2748 long current = firstFreePage;
2749 boolean isFirst = true;
2750
2751 while ( current != NO_PAGE )
2752 {
2753 if ( isFirst )
2754 {
2755 isFirst = false;
2756 }
2757 else
2758 {
2759 sb.append( ", " );
2760 }
2761
2762 PageIO pageIo;
2763
2764 try
2765 {
2766 pageIo = fetchPage( current );
2767 sb.append( pageIo.getOffset() );
2768 current = pageIo.getNextPage();
2769 }
2770 catch ( EndOfFileExceededException e )
2771 {
2772 e.printStackTrace();
2773 }
2774 catch ( IOException e )
2775 {
2776 e.printStackTrace();
2777 }
2778
2779 }
2780 }
2781
2782 sb.append( "]" );
2783
2784 return sb.toString();
2785 }
2786 }