View Javadoc

1   /*
2    *  Licensed to the Apache Software Foundation (ASF) under one
3    *  or more contributor license agreements.  See the NOTICE file
4    *  distributed with this work for additional information
5    *  regarding copyright ownership.  The ASF licenses this file
6    *  to you under the Apache License, Version 2.0 (the
7    *  "License"); you may not use this file except in compliance
8    *  with the License.  You may obtain a copy of the License at
9    *
10   *    http://www.apache.org/licenses/LICENSE-2.0
11   *
12   *  Unless required by applicable law or agreed to in writing,
13   *  software distributed under the License is distributed on an
14   *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   *  KIND, either express or implied.  See the License for the
16   *  specific language governing permissions and limitations
17   *  under the License.
18   *
19   */
20  package org.apache.directory.mavibot.btree;
21  
22  
23  import java.io.File;
24  import java.io.IOException;
25  import java.io.RandomAccessFile;
26  import java.nio.ByteBuffer;
27  import java.nio.channels.FileChannel;
28  import java.util.ArrayList;
29  import java.util.HashSet;
30  import java.util.LinkedHashMap;
31  import java.util.List;
32  import java.util.Map;
33  import java.util.Set;
34  import java.util.concurrent.atomic.AtomicLong;
35  
36  import org.apache.directory.mavibot.btree.exception.BTreeAlreadyManagedException;
37  import org.apache.directory.mavibot.btree.exception.EndOfFileExceededException;
38  import org.apache.directory.mavibot.btree.exception.KeyNotFoundException;
39  import org.apache.directory.mavibot.btree.serializer.ElementSerializer;
40  import org.apache.directory.mavibot.btree.serializer.IntSerializer;
41  import org.apache.directory.mavibot.btree.serializer.LongArraySerializer;
42  import org.apache.directory.mavibot.btree.serializer.LongSerializer;
43  import org.apache.directory.mavibot.btree.util.Strings;
44  import org.slf4j.Logger;
45  import org.slf4j.LoggerFactory;
46  
47  
48  /**
49   * The RecordManager is used to manage the file in which we will store the BTrees. 
50   * A RecordManager will manage more than one BTree.<br/>
51   * 
52   * It stores data in fixed size pages (default size is 512 bytes), which may be linked one to 
53   * the other if the data we want to store is too big for a page.
54   *  
55   * @author <a href="mailto:labs@labs.apache.org">Mavibot labs Project</a>
56   */
57  public class RecordManager
58  {
59      /** The LoggerFactory used by this class */
60      protected static final Logger LOG = LoggerFactory.getLogger( RecordManager.class );
61  
62      /** A dedicated logger for the check */
63      protected static final Logger LOG_CHECK = LoggerFactory.getLogger( "RM_CHECK" );
64  
65      /** The associated file */
66      private File file;
67  
68      /** The channel used to read and write data */
69      private FileChannel fileChannel;
70  
71      /** The number of stored BTrees */
72      private int nbBtree;
73  
74      /** The first and last free page */
75      private long firstFreePage;
76      private long lastFreePage;
77  
78      /** The list of available free pages */
79      List<PageIO> freePages = new ArrayList<PageIO>();
80  
81      /** A counter to track the number of free pages */
82      public AtomicLong nbFreedPages = new AtomicLong( 0 );
83      public AtomicLong nbCreatedPages = new AtomicLong( 0 );
84      public AtomicLong nbReusedPages = new AtomicLong( 0 );
85  
86      /** The offset of the end of the file */
87      private long endOfFileOffset;
88  
89      /** 
90       * A Btree used to manage the page that has been copied in a new version.
91       * Those pages can be reclaimed when the associated version is dead. 
92       **/
93      private BTree<RevisionName, long[]> copiedPageBTree;
94  
95      /** A BTree used to store all the valid revisions for all the stored BTrees */
96      private BTree<RevisionName, Long> revisionBTree;
97  
98      /** A constant for an offset on a non existing page */
99      private static final long NO_PAGE = -1L;
100 
101     /** The number of stored BTrees */
102     private static final int NB_TREE_SIZE = 4;
103 
104     /** The header page size */
105     private static final int PAGE_SIZE = 4;
106 
107     /** The size of the data size in a page */
108     private static final int DATA_SIZE = 4;
109 
110     /** The size of the link to next page */
111     private static final int LINK_SIZE = 8;
112 
113     /** Some constants */
114     private static final int BYTE_SIZE = 1;
115     private static final int INT_SIZE = 4;
116     private static final int LONG_SIZE = 8;
117 
118     /** The size of the link to the first and last free page */
119     private static final int FIRST_FREE_PAGE_SIZE = 8;
120     private static final int LAST_FREE_PAGE_SIZE = 8;
121 
122     /** The default page size */
123     private static final int DEFAULT_PAGE_SIZE = 512;
124 
125     /** The header size */
126     private static int HEADER_SIZE = DEFAULT_PAGE_SIZE;
127 
128     /** A global buffer used to store the header */
129     private static ByteBuffer HEADER_BUFFER;
130 
131     /** The RecordManager underlying page size. */
132     private int pageSize = DEFAULT_PAGE_SIZE;
133 
134     /** The set of managed BTrees */
135     private Map<String, BTree<?, ?>> managedBTrees;
136 
137     /** The offset on the last added BTree */
138     private long lastAddedBTreeOffset = NO_PAGE;
139 
140     /** The default file name */
141     private static final String DEFAULT_FILE_NAME = "mavibot.db";
142 
143     /** A deserializer for Offsets */
144     private static final LongSerializer OFFSET_SERIALIZER = new LongSerializer();
145 
146     private static final String REVISION_BTREE_NAME = "_revisionBTree_";
147 
148     private static final String COPIED_PAGE_BTREE_NAME = "_copiedPageBTree_";
149 
150     /** A flag set to true if we want to keep old revisions */
151     private boolean keepRevisions;
152 
153 
154     /**
155      * Create a Record manager which will either create the underlying file
156      * or load an existing one. If a folder is provided, then we will create
157      * a file with a default name : mavibot.db
158      * 
159      * @param name The file name, or a folder name
160      */
161     public RecordManager( String fileName )
162     {
163         this( fileName, DEFAULT_PAGE_SIZE );
164     }
165 
166 
167     /**
168      * Create a Record manager which will either create the underlying file
169      * or load an existing one. If a folder is provider, then we will create
170      * a file with a default name : mavibot.db
171      * 
172      * @param name The file name, or a folder name
173      * @param pageSize the size of a page on disk
174      */
175     public RecordManager( String fileName, int pageSize )
176     {
177         managedBTrees = new LinkedHashMap<String, BTree<?, ?>>();
178 
179         HEADER_BUFFER = ByteBuffer.allocate( pageSize );
180         HEADER_SIZE = pageSize;
181 
182         // Open the file or create it
183         File tmpFile = new File( fileName );
184         boolean isNewFile = false;
185 
186         if ( tmpFile.isDirectory() )
187         {
188             // It's a directory. Check that we don't have an existing mavibot file
189             File mavibotFile = new File( tmpFile, DEFAULT_FILE_NAME );
190 
191             if ( !mavibotFile.exists() )
192             {
193                 // We have to create a new file
194                 try
195                 {
196                     mavibotFile.createNewFile();
197                     isNewFile = true;
198                 }
199                 catch ( IOException ioe )
200                 {
201                     LOG.error( "Cannot create the file {}", mavibotFile.getName() );
202                     return;
203                 }
204             }
205 
206             file = mavibotFile;
207         }
208         else
209         {
210             // It's a file. Let's see if it exists, otherwise create it
211             if ( !tmpFile.exists() || ( tmpFile.length() == 0 ) )
212             {
213                 isNewFile = true;
214 
215                 try
216                 {
217                     tmpFile.createNewFile();
218                 }
219                 catch ( IOException ioe )
220                 {
221                     LOG.error( "Cannot create the file {}", tmpFile.getName() );
222                     return;
223                 }
224             }
225 
226             file = tmpFile;
227         }
228 
229         try
230         {
231             RandomAccessFile randomFile = new RandomAccessFile( file, "rw" );
232             fileChannel = randomFile.getChannel();
233 
234             // get the current end of file offset
235             endOfFileOffset = fileChannel.size();
236 
237             if ( isNewFile )
238             {
239                 this.pageSize = pageSize;
240                 initRecordManager();
241             }
242             else
243             {
244                 loadRecordManager();
245             }
246         }
247         catch ( Exception e )
248         {
249             LOG.error( "Error while initializing the RecordManager : {}", e.getMessage() );
250             LOG.error( "", e );
251             throw new RuntimeException( e );
252         }
253     }
254 
255 
256     /**
257      * We will create a brand new RecordManager file, containing nothing, but the header, 
258      * a BTree to manage the old revisions we want to keep and
259      * a BTree used to manage pages associated with old versions.
260      * <br/>
261      * The Header contains the following details :
262      * <pre>
263      * +-----------+
264      * | PageSize  | 4 bytes : The size of a physical page (default to 4096)
265      * +-----------+
266      * |  NbTree   | 4 bytes : The number of managed BTrees (at least 1)
267      * +-----------+ 
268      * | FirstFree | 8 bytes : The offset of the first free page
269      * +-----------+
270      * | LastFree  | 8 bytes : The offset of the last free page
271      * +-----------+
272      * </pre>
273      * 
274      * We then store the BTree managing the pages that have been copied when we have added
275      * or deleted an element in the BTree. They are associated with a version.
276      * 
277      * Last, we add the bTree that keep a track on each revision we can have access to.
278      */
279     private void initRecordManager() throws IOException
280     {
281         // Create a new Header
282         nbBtree = 0;
283         firstFreePage = NO_PAGE;
284         lastFreePage = NO_PAGE;
285         updateRecordManagerHeader();
286 
287         // Set the offset of the end of the file
288         endOfFileOffset = fileChannel.size();
289 
290         // Now, initialize the Copied Page BTree
291         copiedPageBTree = new BTree<RevisionName, long[]>( COPIED_PAGE_BTREE_NAME, new RevisionNameSerializer(),
292             new LongArraySerializer() );
293 
294         // and initialize the Revision BTree
295         revisionBTree = new BTree<RevisionName, Long>( REVISION_BTREE_NAME, new RevisionNameSerializer(),
296             new LongSerializer() );
297 
298         // Inject these BTrees into the RecordManager
299         try
300         {
301             manage( copiedPageBTree );
302             manage( revisionBTree );
303         }
304         catch ( BTreeAlreadyManagedException btame )
305         {
306             // Can't happen here.
307         }
308 
309         // We are all set !
310     }
311 
312 
313     /**
314      * Load the BTrees from the disk. 
315      * 
316      * @throws InstantiationException 
317      * @throws IllegalAccessException 
318      * @throws ClassNotFoundException 
319      */
320     private void loadRecordManager() throws IOException, ClassNotFoundException, IllegalAccessException,
321         InstantiationException
322     {
323         if ( fileChannel.size() != 0 )
324         {
325             ByteBuffer header = ByteBuffer.allocate( HEADER_SIZE );
326 
327             // The file exists, we have to load the data now 
328             fileChannel.read( header );
329 
330             header.rewind();
331 
332             // The page size
333             pageSize = header.getInt();
334 
335             // The number of managed BTrees
336             nbBtree = header.getInt();
337 
338             // The first and last free page
339             firstFreePage = header.getLong();
340             lastFreePage = header.getLong();
341 
342             // Now read each BTree. The first one is the one which
343             // manage the modified pages. Once read, we can discard all
344             // the pages that are stored in it, as we have restarted 
345             // the RecordManager.
346             long btreeOffset = HEADER_SIZE;
347 
348             PageIO[] pageIos = readPageIOs( HEADER_SIZE, Long.MAX_VALUE );
349 
350             // Create the BTree
351             copiedPageBTree = BTreeFactory.createBTree();
352             copiedPageBTree.setBtreeOffset( btreeOffset );
353 
354             loadBTree( pageIos, copiedPageBTree );
355             long nextBtreeOffset = copiedPageBTree.getNextBTreeOffset();
356 
357             // And the Revision BTree
358             pageIos = readPageIOs( nextBtreeOffset, Long.MAX_VALUE );
359 
360             revisionBTree = BTreeFactory.createBTree();
361             revisionBTree.setBtreeOffset( nextBtreeOffset );
362 
363             loadBTree( pageIos, revisionBTree );
364             nextBtreeOffset = revisionBTree.getNextBTreeOffset();
365 
366             // Then process the next ones
367             for ( int i = 2; i < nbBtree; i++ )
368             {
369                 // Create the BTree
370                 BTree<?, ?> btree = BTreeFactory.createBTree();
371                 btree.setBtreeOffset( nextBtreeOffset );
372                 lastAddedBTreeOffset = nextBtreeOffset;
373 
374                 // Read the associated pages
375                 pageIos = readPageIOs( nextBtreeOffset, Long.MAX_VALUE );
376 
377                 // Load the BTree
378                 loadBTree( pageIos, btree );
379                 nextBtreeOffset = btree.getNextBTreeOffset();
380 
381                 // Store it into the managedBtrees map
382                 managedBTrees.put( btree.getName(), btree );
383             }
384 
385             // We are done ! Let's finish with the last initilization parts
386             endOfFileOffset = fileChannel.size();
387         }
388     }
389 
390 
391     /**
392      * Reads all the PageIOs that are linked to the page at the given position, including
393      * the first page.
394      * 
395      * @param position The position of the first page
396      * @return An array of pages
397      */
398     private PageIO[] readPageIOs( long position, long limit ) throws IOException, EndOfFileExceededException
399     {
400         LOG.debug( "Read PageIOs at position {}", position );
401 
402         if ( limit <= 0 )
403         {
404             limit = Long.MAX_VALUE;
405         }
406 
407         PageIO firstPage = fetchPage( position );
408         firstPage.setSize();
409         List<PageIO> listPages = new ArrayList<PageIO>();
410         listPages.add( firstPage );
411         long dataRead = pageSize - LONG_SIZE - INT_SIZE;
412 
413         // Iterate on the pages, if needed
414         long nextPage = firstPage.getNextPage();
415 
416         if ( ( dataRead < limit ) && ( nextPage != NO_PAGE ) )
417         {
418             while ( dataRead < limit )
419             {
420                 PageIO page = fetchPage( nextPage );
421                 listPages.add( page );
422                 nextPage = page.getNextPage();
423                 dataRead += pageSize - LONG_SIZE;
424 
425                 if ( nextPage == NO_PAGE )
426                 {
427                     page.setNextPage( NO_PAGE );
428                     break;
429                 }
430             }
431         }
432 
433         LOG.debug( "Nb of PageIOs read : {}", listPages.size() );
434 
435         // Return 
436         return listPages.toArray( new PageIO[]
437             {} );
438     }
439 
440 
441     /**
442      * Read a BTree from the disk. The meta-data are at the given position in the list of pages.
443      * 
444      * @param pageIos The list of pages containing the meta-data
445      * @param btree The BTree we have to initialize
446      * @throws InstantiationException 
447      * @throws IllegalAccessException 
448      * @throws ClassNotFoundException 
449      */
450     private void loadBTree( PageIO[] pageIos, BTree<?, ?> btree ) throws EndOfFileExceededException,
451         IOException, ClassNotFoundException, IllegalAccessException, InstantiationException
452     {
453         long dataPos = 0L;
454 
455         // The BTree current revision
456         long revision = readLong( pageIos, dataPos );
457         BTreeFactory.setRevision( btree, revision );
458         dataPos += LONG_SIZE;
459 
460         // The nb elems in the tree
461         long nbElems = readLong( pageIos, dataPos );
462         BTreeFactory.setNbElems( btree, nbElems );
463         dataPos += LONG_SIZE;
464 
465         // The BTree rootPage offset
466         long rootPageOffset = readLong( pageIos, dataPos );
467         BTreeFactory.setRootPageOffset( btree, rootPageOffset );
468         dataPos += LONG_SIZE;
469 
470         // The next BTree offset
471         long nextBTreeOffset = readLong( pageIos, dataPos );
472         BTreeFactory.setNextBTreeOffset( btree, nextBTreeOffset );
473         dataPos += LONG_SIZE;
474 
475         // The BTree page size
476         int btreePageSize = readInt( pageIos, dataPos );
477         BTreeFactory.setPageSize( btree, btreePageSize );
478         dataPos += INT_SIZE;
479 
480         // The tree name
481         byte[] btreeNameBytes = readBytes( pageIos, dataPos );
482         dataPos += INT_SIZE + btreeNameBytes.length;
483         String btreeName = Strings.utf8ToString( btreeNameBytes );
484         BTreeFactory.setName( btree, btreeName );
485 
486         // The keySerializer FQCN
487         byte[] keySerializerBytes = readBytes( pageIos, dataPos );
488         dataPos += INT_SIZE + keySerializerBytes.length;
489 
490         String keySerializerFqcn = null;
491 
492         if ( keySerializerBytes != null )
493         {
494             keySerializerFqcn = Strings.utf8ToString( keySerializerBytes );
495         }
496         else
497         {
498             keySerializerFqcn = "";
499         }
500 
501         BTreeFactory.setKeySerializer( btree, keySerializerFqcn );
502 
503         // The valueSerialier FQCN
504         byte[] valueSerializerBytes = readBytes( pageIos, dataPos );
505 
506         String valueSerializerFqcn = null;
507         dataPos += INT_SIZE + valueSerializerBytes.length;
508 
509         if ( valueSerializerBytes != null )
510         {
511             valueSerializerFqcn = Strings.utf8ToString( valueSerializerBytes );
512         }
513         else
514         {
515             valueSerializerFqcn = "";
516         }
517 
518         BTreeFactory.setValueSerializer( btree, valueSerializerFqcn );
519 
520         // The BTree allowDuplicates flag
521         int allowDuplicates = readInt( pageIos, dataPos );
522         btree.setAllowDuplicates( allowDuplicates != 0 );
523         dataPos += INT_SIZE;
524 
525         // Now, init the BTree
526         btree.init();
527 
528         // Now, load the rootPage, which can be a Leaf or a Node, depending 
529         // on the number of elements in the tree : if it's above the pageSize,
530         // it's a Node, otherwise it's a Leaf
531 
532         // Read the rootPage pages on disk
533         PageIO[] rootPageIos = readPageIOs( rootPageOffset, Long.MAX_VALUE );
534 
535         Page btreeRoot = readPage( btree, rootPageIos );
536         BTreeFactory.setRecordManager( btree, this );
537 
538         BTreeFactory.setRoot( btree, btreeRoot );
539     }
540 
541 
542     private Page readNode( BTree btree, long offset, long revision, int nbElems ) throws IOException
543     {
544         Page node = BTreeFactory.createNode( btree, revision, nbElems );
545 
546         // Read the rootPage pages on disk
547         PageIO[] pageIos = readPageIOs( offset, Long.MAX_VALUE );
548 
549         return node;
550     }
551 
552 
553     public Page deserialize( BTree btree, long offset ) throws EndOfFileExceededException, IOException
554     {
555         PageIO[] rootPageIos = readPageIOs( offset, Long.MAX_VALUE );
556 
557         Page page = readPage( btree, rootPageIos );
558 
559         return page;
560     }
561 
562 
563     private Page readPage( BTree btree, PageIO[] pageIos ) throws IOException
564     {
565         // Deserialize the rootPage now
566         long position = 0L;
567 
568         // The revision
569         long revision = readLong( pageIos, position );
570         position += LONG_SIZE;
571 
572         // The number of elements in the page
573         int nbElems = readInt( pageIos, position );
574         position += INT_SIZE;
575 
576         // The size of the data containing the keys and values
577         Page page = null;
578         ByteBuffer byteBuffer = null;
579 
580         // Reads the bytes containing all the keys and values, if we have some
581         byte[] data = readBytes( pageIos, position );
582 
583         if ( data != null )
584         {
585             byteBuffer = ByteBuffer.allocate( data.length );
586             byteBuffer.put( data );
587             byteBuffer.rewind();
588         }
589 
590         if ( nbElems >= 0 )
591         {
592             // Its a leaf
593             page = BTreeFactory.createLeaf( btree, revision, nbElems );
594 
595             ( ( AbstractPage ) page ).setOffset( pageIos[0].getOffset() );
596             ( ( AbstractPage ) page ).setLastOffset( pageIos[pageIos.length - 1].getOffset() );
597 
598             // Read each value and key
599             for ( int i = 0; i < nbElems; i++ )
600             {
601                 ElementHolder valueHolder;
602 
603                 if ( btree.isAllowDuplicates() )
604                 {
605                     long value = OFFSET_SERIALIZER.deserialize( byteBuffer );
606                     
607                     BTree dupValueContainer = loadDupsBTree(value);
608 
609                     valueHolder = new DuplicateKeyMemoryHolder( btree, dupValueContainer );
610                 }
611                 else
612                 {
613                     Object value = btree.getValueSerializer().deserialize( byteBuffer );
614 
615                     valueHolder = new MemoryHolder( btree, value );
616                 }
617 
618                 BTreeFactory.setValue( ( ( Leaf ) page ), i, valueHolder );
619 
620                 Object key = btree.getKeySerializer().deserialize( byteBuffer );
621 
622                 BTreeFactory.setKey( page, i, key );
623             }
624         }
625         else
626         {
627             // It's a node
628             int nodeNbElems = -nbElems;
629 
630             page = BTreeFactory.createNode( btree, revision, nodeNbElems );
631 
632             // Read each value and key
633             for ( int i = 0; i < nodeNbElems; i++ )
634             {
635                 // This is an Offset
636                 long offset = OFFSET_SERIALIZER.deserialize( byteBuffer );
637                 long lastOffset = OFFSET_SERIALIZER.deserialize( byteBuffer );
638 
639                 ElementHolder valueHolder = new ReferenceHolder( btree, null, offset, lastOffset );
640                 ( ( Node ) page ).setValue( i, valueHolder );
641 
642                 Object key = btree.getKeySerializer().deserialize( byteBuffer );
643                 BTreeFactory.setKey( page, i, key );
644             }
645 
646             // and read the last value, as it's a node
647             long offset = OFFSET_SERIALIZER.deserialize( byteBuffer );
648             long lastOffset = OFFSET_SERIALIZER.deserialize( byteBuffer );
649 
650             ElementHolder valueHolder = new ReferenceHolder( btree, null, offset, lastOffset );
651             ( ( Node ) page ).setValue( nodeNbElems, valueHolder );
652         }
653 
654         return page;
655     }
656 
657 
658     /**
659      * Read a byte[] from pages.
660      * @param pageIos The pages we want to read the byte[] from
661      * @param position The position in the data stored in those pages
662      * @return The byte[] we have read
663      */
664     private byte[] readBytes( PageIO[] pageIos, long position )
665     {
666         // Read the byte[] length first
667         int length = readInt( pageIos, position );
668         position += INT_SIZE;
669 
670         // Compute the page in which we will store the data given the 
671         // current position
672         int pageNb = computePageNb( position );
673 
674         // Compute the position in the current page
675         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
676 
677         ByteBuffer pageData = pageIos[pageNb].getData();
678         int remaining = pageData.capacity() - pagePos;
679 
680         if ( length == 0 )
681         {
682             // No bytes to read : return null;
683             return null;
684         }
685         else
686         {
687             byte[] bytes = new byte[length];
688             int bytesPos = 0;
689 
690             while ( length > 0 )
691             {
692                 if ( length <= remaining )
693                 {
694                     pageData.mark();
695                     pageData.position( pagePos );
696                     pageData.get( bytes, bytesPos, length );
697                     pageData.reset();
698 
699                     return bytes;
700                 }
701 
702                 pageData.mark();
703                 pageData.position( pagePos );
704                 pageData.get( bytes, bytesPos, remaining );
705                 pageData.reset();
706                 pageNb++;
707                 pagePos = LINK_SIZE;
708                 bytesPos += remaining;
709                 pageData = pageIos[pageNb].getData();
710                 length -= remaining;
711                 remaining = pageData.capacity() - pagePos;
712             }
713 
714             return bytes;
715         }
716     }
717 
718 
719     /**
720      * Read an int from pages 
721      * @param pageIos The pages we want to read the int from
722      * @param position The position in the data stored in those pages
723      * @return The int we have read
724      */
725     private int readInt( PageIO[] pageIos, long position )
726     {
727         // Compute the page in which we will store the data given the 
728         // current position
729         int pageNb = computePageNb( position );
730 
731         // Compute the position in the current page
732         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
733 
734         ByteBuffer pageData = pageIos[pageNb].getData();
735         int remaining = pageData.capacity() - pagePos;
736         int value = 0;
737 
738         if ( remaining >= INT_SIZE )
739         {
740             value = pageData.getInt( pagePos );
741         }
742         else
743         {
744             value = 0;
745 
746             switch ( remaining )
747             {
748                 case 3:
749                     value += ( ( pageData.get( pagePos + 2 ) & 0x00FF ) << 8 );
750                     // Fallthrough !!!
751 
752                 case 2:
753                     value += ( ( pageData.get( pagePos + 1 ) & 0x00FF ) << 16 );
754                     // Fallthrough !!!
755 
756                 case 1:
757                     value += ( pageData.get( pagePos ) << 24 );
758                     break;
759             }
760 
761             // Now deal with the next page
762             pageData = pageIos[pageNb + 1].getData();
763             pagePos = LINK_SIZE;
764 
765             switch ( remaining )
766             {
767                 case 1:
768                     value += ( pageData.get( pagePos ) & 0x00FF ) << 16;
769                     // fallthrough !!!
770 
771                 case 2:
772                     value += ( pageData.get( pagePos + 2 - remaining ) & 0x00FF ) << 8;
773                     // fallthrough !!!
774 
775                 case 3:
776                     value += ( pageData.get( pagePos + 3 - remaining ) & 0x00FF );
777                     break;
778             }
779         }
780 
781         return value;
782     }
783 
784 
785     /**
786      * Read a byte from pages 
787      * @param pageIos The pages we want to read the byte from
788      * @param position The position in the data stored in those pages
789      * @return The byte we have read
790      */
791     private byte readByte( PageIO[] pageIos, long position )
792     {
793         // Compute the page in which we will store the data given the 
794         // current position
795         int pageNb = computePageNb( position );
796 
797         // Compute the position in the current page
798         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
799 
800         ByteBuffer pageData = pageIos[pageNb].getData();
801         byte value = 0;
802 
803         value = pageData.get( pagePos );
804 
805         return value;
806     }
807 
808 
809     /**
810      * Read a long from pages 
811      * @param pageIos The pages we want to read the long from
812      * @param position The position in the data stored in those pages
813      * @return The long we have read
814      */
815     private long readLong( PageIO[] pageIos, long position )
816     {
817         // Compute the page in which we will store the data given the 
818         // current position
819         int pageNb = computePageNb( position );
820 
821         // Compute the position in the current page
822         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
823 
824         ByteBuffer pageData = pageIos[pageNb].getData();
825         int remaining = pageData.capacity() - pagePos;
826         long value = 0L;
827 
828         if ( remaining >= LONG_SIZE )
829         {
830             value = pageData.getLong( pagePos );
831         }
832         else
833         {
834             switch ( remaining )
835             {
836                 case 7:
837                     value += ( ( ( long ) pageData.get( pagePos + 6 ) & 0x00FF ) << 8 );
838                     // Fallthrough !!!
839 
840                 case 6:
841                     value += ( ( ( long ) pageData.get( pagePos + 5 ) & 0x00FF ) << 16 );
842                     // Fallthrough !!!
843 
844                 case 5:
845                     value += ( ( ( long ) pageData.get( pagePos + 4 ) & 0x00FF ) << 24 );
846                     // Fallthrough !!!
847 
848                 case 4:
849                     value += ( ( ( long ) pageData.get( pagePos + 3 ) & 0x00FF ) << 32 );
850                     // Fallthrough !!!
851 
852                 case 3:
853                     value += ( ( ( long ) pageData.get( pagePos + 2 ) & 0x00FF ) << 40 );
854                     // Fallthrough !!!
855 
856                 case 2:
857                     value += ( ( ( long ) pageData.get( pagePos + 1 ) & 0x00FF ) << 48 );
858                     // Fallthrough !!!
859 
860                 case 1:
861                     value += ( ( long ) pageData.get( pagePos ) << 56 );
862                     break;
863             }
864 
865             // Now deal with the next page
866             pageData = pageIos[pageNb + 1].getData();
867             pagePos = LINK_SIZE;
868 
869             switch ( remaining )
870             {
871                 case 1:
872                     value += ( ( long ) pageData.get( pagePos ) & 0x00FF ) << 48;
873                     // fallthrough !!!
874 
875                 case 2:
876                     value += ( ( long ) pageData.get( pagePos + 2 - remaining ) & 0x00FF ) << 40;
877                     // fallthrough !!!
878 
879                 case 3:
880                     value += ( ( long ) pageData.get( pagePos + 3 - remaining ) & 0x00FF ) << 32;
881                     // fallthrough !!!
882 
883                 case 4:
884                     value += ( ( long ) pageData.get( pagePos + 4 - remaining ) & 0x00FF ) << 24;
885                     // fallthrough !!!
886 
887                 case 5:
888                     value += ( ( long ) pageData.get( pagePos + 5 - remaining ) & 0x00FF ) << 16;
889                     // fallthrough !!!
890 
891                 case 6:
892                     value += ( ( long ) pageData.get( pagePos + 6 - remaining ) & 0x00FF ) << 8;
893                     // fallthrough !!!
894 
895                 case 7:
896                     value += ( ( long ) pageData.get( pagePos + 7 - remaining ) & 0x00FF );
897                     break;
898             }
899         }
900 
901         return value;
902     }
903 
904 
905     /**
906      * Manage a BTree. The btree will be added and managed by this RecordManager. We will create a 
907      * new RootPage for this added BTree, which will contain no data. 
908      *  
909      * @param btree The new BTree to manage.
910      */
911     public synchronized void manage( BTree<?, ?> btree ) throws BTreeAlreadyManagedException, IOException
912     {
913         manage( btree, false );
914     }
915 
916 
917     /**
918      * works the same as @see #manage(BTree) except the given tree will not be linked to top level trees that will be
919      * loaded initially if the internalTree flag is set to true
920      * 
921      * @param btree The new BTree to manage.
922      * @param internalTree flag indicating if this is an internal tree
923      * 
924      * @throws BTreeAlreadyManagedException
925      * @throws IOException
926      */
927     public synchronized void manage( BTree<?, ?> btree, boolean internalTree ) throws BTreeAlreadyManagedException,
928         IOException
929     {
930         LOG.debug( "Managing the btree {} which is an internam tree : {}", btree.getName(), internalTree );
931         BTreeFactory.setRecordManager( btree, this );
932 
933         String name = btree.getName();
934 
935         if ( managedBTrees.containsKey( name ) )
936         {
937             // There is already a BTree with this name in the recordManager...
938             LOG.error( "There is already a BTree named '{}' managed by this recordManager", name );
939             throw new BTreeAlreadyManagedException( name );
940         }
941 
942         managedBTrees.put( name, btree );
943 
944         // We will add the newly managed BTree at the end of the header.
945         byte[] btreeNameBytes = Strings.getBytesUtf8( name );
946         byte[] keySerializerBytes = Strings.getBytesUtf8( btree.getKeySerializerFQCN() );
947         byte[] valueSerializerBytes = Strings.getBytesUtf8( btree.getValueSerializerFQCN() );
948 
949         int bufferSize =
950             INT_SIZE + // The name size
951                 btreeNameBytes.length + // The name
952                 INT_SIZE + // The keySerializerBytes size 
953                 keySerializerBytes.length + // The keySerializerBytes
954                 INT_SIZE + // The valueSerializerBytes size
955                 valueSerializerBytes.length + // The valueSerializerBytes
956                 INT_SIZE + // The page size
957                 LONG_SIZE + // The revision
958                 LONG_SIZE + // the number of element
959                 LONG_SIZE + // the nextBtree offset
960                 LONG_SIZE + // The root offset
961                 INT_SIZE; // The allowDuplicates flag
962 
963         // Get the pageIOs we need to store the data. We may need more than one.
964         PageIO[] pageIos = getFreePageIOs( bufferSize );
965 
966         // Store the BTree Offset into the BTree
967         long btreeOffset = pageIos[0].getOffset();
968         btree.setBtreeOffset( btreeOffset );
969 
970         // Now store the BTree data in the pages :
971         // - the BTree revision
972         // - the BTree number of elements
973         // - The RootPage offset
974         // - The next Btree offset 
975         // - the BTree page size
976         // - the BTree name
977         // - the keySerializer FQCN
978         // - the valueSerializer FQCN
979         // - the flags that tell if the dups are allowed
980         // Starts at 0
981         long position = 0L;
982 
983         // The BTree current revision
984         position = store( position, btree.getRevision(), pageIos );
985 
986         // The nb elems in the tree
987         position = store( position, btree.getNbElems(), pageIos );
988 
989         // Serialize the BTree root page
990         Page rootPage = BTreeFactory.getRoot( btree );
991 
992         PageIO[] rootPageIos = serializePage( btree, btree.getRevision(), rootPage );
993 
994         // Get the reference on the first page
995         PageIO rootPageIo = rootPageIos[0];
996 
997         // Now, we can inject the BTree rootPage offset into the BTree header
998         position = store( position, rootPageIo.getOffset(), pageIos );
999         btree.setRootPageOffset( rootPageIo.getOffset() );
1000         ( ( Leaf ) rootPage ).setOffset( rootPageIo.getOffset() );
1001 
1002         // The next BTree Header offset (-1L, as it's a new BTree)
1003         position = store( position, NO_PAGE, pageIos );
1004 
1005         // The BTree page size
1006         position = store( position, btree.getPageSize(), pageIos );
1007 
1008         // The tree name
1009         position = store( position, btreeNameBytes, pageIos );
1010 
1011         // The keySerializer FQCN
1012         position = store( position, keySerializerBytes, pageIos );
1013 
1014         // The valueSerialier FQCN
1015         position = store( position, valueSerializerBytes, pageIos );
1016 
1017         // The allowDuplicates flag
1018         position = store( position, ( btree.isAllowDuplicates() ? 1 : 0 ), pageIos );
1019 
1020         // And flush the pages to disk now
1021         LOG.debug( "Flushing the newly managed '{}' btree header", btree.getName() );
1022         flushPages( pageIos );
1023         LOG.debug( "Flushing the newly managed '{}' btree rootpage", btree.getName() );
1024         flushPages( rootPageIos );
1025 
1026         // Now, if this added BTree is not the first BTree, we have to link it with the 
1027         // latest added BTree
1028         if ( !internalTree )
1029         {
1030             nbBtree++;
1031 
1032             if ( lastAddedBTreeOffset != NO_PAGE )
1033             {
1034                 // We have to update the nextBtreeOffset from the previous BTreeHeader
1035                 pageIos = readPageIOs( lastAddedBTreeOffset, LONG_SIZE + LONG_SIZE + LONG_SIZE + LONG_SIZE );
1036                 store( LONG_SIZE + LONG_SIZE + LONG_SIZE, btreeOffset, pageIos );
1037 
1038                 // Write the pages on disk
1039                 LOG.debug( "Updated the previous btree pointer on the added BTree {}", btree.getName() );
1040                 flushPages( pageIos );
1041             }
1042 
1043             lastAddedBTreeOffset = btreeOffset;
1044 
1045             // Last, not least, update the number of managed BTrees in the header
1046             updateRecordManagerHeader();
1047         }
1048 
1049         if ( LOG_CHECK.isDebugEnabled() )
1050         {
1051             check();
1052         }
1053     }
1054 
1055 
1056     /**
1057      * Serialize a new Page. It will contain the following data :<br/>
1058      * <ul>
1059      * <li>the revision : a long</li>
1060      * <li>the number of elements : an int (if <= 0, it's a Node, otherwise it's a Leaf)</li>
1061      * <li>the size of the values/keys when serialized
1062      * <li>the keys : an array of serialized keys</li>
1063      * <li>the values : an array of references to the children pageIO offset (stored as long)
1064      * if it's a Node, or a list of values if it's a Leaf</li>
1065      * <li></li>
1066      * </ul>
1067      * 
1068      * @param revision The node revision
1069      * @param keys The keys to serialize
1070      * @param children The references to the children
1071      * @return An array of pages containing the serialized node
1072      * @throws IOException
1073      */
1074     private PageIO[] serializePage( BTree btree, long revision, Page page ) throws IOException
1075     {
1076         int nbElems = page.getNbElems();
1077 
1078         if ( nbElems == 0 )
1079         {
1080             // We will have 1 single page if we have no elements
1081             PageIO[] pageIos = new PageIO[1];
1082 
1083             // This is either a new root page or a new page that will be filled later
1084             PageIO newPage = fetchNewPage();
1085 
1086             // We need first to create a byte[] that will contain all the data
1087             // For the root page, this is easy, as we only have to store the revision, 
1088             // and the number of elements, which is 0.
1089             long position = 0L;
1090 
1091             position = store( position, revision, newPage );
1092             position = store( position, nbElems, newPage );
1093 
1094             // Update the page size now
1095             newPage.setSize( ( int ) position );
1096 
1097             // Insert the result into the array of PageIO
1098             pageIos[0] = newPage;
1099 
1100             return pageIos;
1101         }
1102         else
1103         {
1104             // Prepare a list of byte[] that will contain the serialized page
1105             int nbBuffers = 1 + 1 + 1 + nbElems * 2;
1106             int dataSize = 0;
1107             int serializedSize = 0;
1108 
1109             if ( page instanceof Node )
1110             {
1111                 // A Node has one more value to store
1112                 nbBuffers++;
1113             }
1114 
1115             // Now, we can create the list with the right size
1116             List<byte[]> serializedData = new ArrayList<byte[]>( nbBuffers );
1117 
1118             // The revision
1119             byte[] buffer = LongSerializer.serialize( revision );
1120             serializedData.add( buffer );
1121             serializedSize += buffer.length;
1122 
1123             // The number of elements
1124             // Make it a negative value if it's a Node
1125             int pageNbElems = nbElems;
1126 
1127             if ( page instanceof Node )
1128             {
1129                 pageNbElems = -nbElems;
1130             }
1131 
1132             buffer = IntSerializer.serialize( pageNbElems );
1133             serializedData.add( buffer );
1134             serializedSize += buffer.length;
1135 
1136             // Iterate on the keys
1137             for ( int pos = 0; pos < nbElems; pos++ )
1138             {
1139                 // Start with the value
1140                 if ( page instanceof Node )
1141                 {
1142                     Page child = ( ( Node ) page ).getReference( pos );
1143 
1144                     // The first offset
1145                     buffer = LongSerializer.serialize( child.getOffset() );
1146                     serializedData.add( buffer );
1147                     dataSize += buffer.length;
1148 
1149                     // The last offset
1150                     buffer = LongSerializer.serialize( child.getLastOffset() );
1151                     serializedData.add( buffer );
1152                     dataSize += buffer.length;
1153                 }
1154                 else
1155                 {
1156                     if ( btree.isAllowDuplicates() )
1157                     {
1158                         DuplicateKeyMemoryHolder value = ( DuplicateKeyMemoryHolder ) ( ( Leaf ) page ).getValue( pos );
1159                         long duplicateContainerOffset = ( ( BTree ) value.getValue( btree ) ).getBtreeOffset();
1160                         buffer = OFFSET_SERIALIZER.serialize( duplicateContainerOffset );
1161                     }
1162                     else
1163                     {
1164                         ElementHolder value = ( ( Leaf ) page ).getValue( pos );
1165                         buffer = btree.getValueSerializer().serialize( value.getValue( btree ) );
1166                     }
1167 
1168                     serializedData.add( buffer );
1169                     dataSize += buffer.length;
1170                 }
1171 
1172                 // and the key
1173                 buffer = btree.getKeySerializer().serialize( page.getKey( pos ) );
1174                 serializedData.add( buffer );
1175                 dataSize += buffer.length;
1176             }
1177 
1178             // Nodes have one more value to serialize
1179             if ( page instanceof Node )
1180             {
1181                 Page child = ( ( Node ) page ).getReference( nbElems );
1182 
1183                 // The first offset
1184                 buffer = LongSerializer.serialize( child.getOffset() );
1185                 serializedData.add( buffer );
1186                 dataSize += buffer.length;
1187 
1188                 // The last offset
1189                 buffer = LongSerializer.serialize( child.getLastOffset() );
1190                 serializedData.add( buffer );
1191                 dataSize += buffer.length;
1192             }
1193 
1194             // Store the data size
1195             buffer = IntSerializer.serialize( dataSize );
1196             serializedData.add( 2, buffer );
1197             serializedSize += buffer.length;
1198 
1199             serializedSize += dataSize;
1200 
1201             // We are done. Allocate the pages we need to store the data
1202             PageIO[] pageIos = getFreePageIOs( serializedSize );
1203 
1204             // And store the data into those pages
1205             long position = 0L;
1206 
1207             for ( byte[] bytes : serializedData )
1208             {
1209                 position = storeRaw( position, bytes, pageIos );
1210             }
1211 
1212             return pageIos;
1213         }
1214     }
1215 
1216 
1217     /**
1218      * Update the header, injecting the nbBtree, firstFreePage and lastFreePage
1219      */
1220     private void updateRecordManagerHeader() throws IOException
1221     {
1222         HEADER_BUFFER.clear();
1223 
1224         // The page size
1225         HEADER_BUFFER.putInt( pageSize );
1226 
1227         // The number of managed BTree (currently we have only one : the discardedPage BTree
1228         HEADER_BUFFER.putInt( nbBtree );
1229 
1230         // The first free page
1231         HEADER_BUFFER.putLong( firstFreePage );
1232 
1233         // The last free page
1234         HEADER_BUFFER.putLong( lastFreePage );
1235 
1236         // Set the limit to the end of the page
1237         HEADER_BUFFER.limit( pageSize );
1238 
1239         // Write the header on disk
1240         HEADER_BUFFER.rewind();
1241 
1242         LOG.debug( "Update RM header, FF : {}, LF : {}", firstFreePage, lastFreePage );
1243         fileChannel.write( HEADER_BUFFER, 0 );
1244     }
1245 
1246 
1247     /**
1248      * Update the BTree header after a BTree modification. We update the following fields :
1249      * <ul>
1250      * <li>the revision</li>
1251      * <li>the number of elements</li>
1252      * <li>the rootPage offset</li>
1253      * </ul>
1254      * @param btree
1255      * @throws IOException 
1256      * @throws EndOfFileExceededException 
1257      */
1258     /* No qualifier*/void updateBtreeHeader( BTree btree, long rootPageOffset ) throws EndOfFileExceededException,
1259         IOException
1260     {
1261         // Read the pageIOs associated with this BTree
1262         long offset = btree.getBtreeOffset();
1263         long headerSize = LONG_SIZE + LONG_SIZE + LONG_SIZE;
1264 
1265         PageIO[] pageIos = readPageIOs( offset, headerSize );
1266 
1267         // Now, update the revision
1268         long position = 0;
1269 
1270         position = store( position, btree.getRevision(), pageIos );
1271         position = store( position, btree.getNbElems(), pageIos );
1272         position = store( position, rootPageOffset, pageIos );
1273 
1274         // Write the pages on disk
1275         if ( LOG.isDebugEnabled() )
1276         {
1277             LOG.debug( "-----> Flushing the '{}' BTreeHeader", btree.getName() );
1278             LOG.debug( "  revision : " + btree.getRevision() + ", NbElems : " + btree.getNbElems() + ", root offset : "
1279                 + rootPageOffset );
1280         }
1281 
1282         flushPages( pageIos );
1283 
1284         if ( LOG_CHECK.isDebugEnabled() )
1285         {
1286             check();
1287         }
1288     }
1289 
1290 
1291     /**
1292      * Write the pages in the disk, either at the end of the file, or at
1293      * the position they were taken from.
1294      * 
1295      * @param pageIos The list of pages to write
1296      * @throws IOException If the write failed
1297      */
1298     private void flushPages( PageIO... pageIos ) throws IOException
1299     {
1300         for ( PageIO pageIo : pageIos )
1301         {
1302             pageIo.getData().rewind();
1303 
1304             if ( fileChannel.size() < ( pageIo.getOffset() + pageSize ) )
1305             {
1306                 LOG.debug( "Adding a page at the end of the file" );
1307                 // This is a page we have to add to the file
1308                 fileChannel.write( pageIo.getData(), fileChannel.size() );
1309                 //fileChannel.force( false );
1310             }
1311             else
1312             {
1313                 LOG.debug( "Writing a page at position {}", pageIo.getOffset() );
1314                 fileChannel.write( pageIo.getData(), pageIo.getOffset() );
1315                 //fileChannel.force( false );
1316             }
1317 
1318             pageIo.getData().rewind();
1319         }
1320     }
1321 
1322 
1323     /**
1324      * Compute the page in which we will store data given an offset, when 
1325      * we have a list of pages.
1326      * 
1327      * @param offset The position in the data
1328      * @return The page number in which the offset will start
1329      */
1330     private int computePageNb( long offset )
1331     {
1332         long pageNb = 0;
1333 
1334         offset -= pageSize - LINK_SIZE - PAGE_SIZE;
1335 
1336         if ( offset < 0 )
1337         {
1338             return ( int ) pageNb;
1339         }
1340 
1341         pageNb = 1 + offset / ( pageSize - LINK_SIZE );
1342 
1343         return ( int ) pageNb;
1344     }
1345 
1346 
1347     /**
1348      * Stores a byte[] into one ore more pageIO (depending if the long is stored
1349      * across a boundary or not)
1350      * 
1351      * @param position The position in a virtual byte[] if all the pages were contiguous
1352      * @param bytes The byte[] to serialize
1353      * @param pageIos The pageIOs we have to store the data in
1354      * @return The new position
1355      */
1356     private long store( long position, byte[] bytes, PageIO... pageIos )
1357     {
1358         if ( bytes != null )
1359         {
1360             // Write the bytes length
1361             position = store( position, bytes.length, pageIos );
1362 
1363             // Compute the page in which we will store the data given the 
1364             // current position
1365             int pageNb = computePageNb( position );
1366 
1367             // Get back the buffer in this page
1368             ByteBuffer pageData = pageIos[pageNb].getData();
1369 
1370             // Compute the position in the current page
1371             int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1372 
1373             // Compute the remaining size in the page
1374             int remaining = pageData.capacity() - pagePos;
1375             int nbStored = bytes.length;
1376 
1377             // And now, write the bytes until we have none
1378             while ( nbStored > 0 )
1379             {
1380                 if ( remaining > nbStored )
1381                 {
1382                     pageData.mark();
1383                     pageData.position( pagePos );
1384                     pageData.put( bytes, bytes.length - nbStored, nbStored );
1385                     pageData.reset();
1386                     nbStored = 0;
1387                 }
1388                 else
1389                 {
1390                     pageData.mark();
1391                     pageData.position( pagePos );
1392                     pageData.put( bytes, bytes.length - nbStored, remaining );
1393                     pageData.reset();
1394                     pageNb++;
1395                     pageData = pageIos[pageNb].getData();
1396                     pagePos = LINK_SIZE;
1397                     nbStored -= remaining;
1398                     remaining = pageData.capacity() - pagePos;
1399                 }
1400             }
1401 
1402             // We are done
1403             position += bytes.length;
1404         }
1405         else
1406         {
1407             // No bytes : write 0 and return
1408             position = store( position, 0, pageIos );
1409         }
1410 
1411         return position;
1412     }
1413 
1414 
1415     /**
1416      * Stores a byte[] into one ore more pageIO (depending if the long is stored
1417      * across a boundary or not). We don't add the byte[] size, it's already present
1418      * in the received byte[].
1419      * 
1420      * @param position The position in a virtual byte[] if all the pages were contiguous
1421      * @param bytes The byte[] to serialize
1422      * @param pageIos The pageIOs we have to store the data in
1423      * @return The new position
1424      */
1425     private long storeRaw( long position, byte[] bytes, PageIO... pageIos )
1426     {
1427         if ( bytes != null )
1428         {
1429             // Compute the page in which we will store the data given the 
1430             // current position
1431             int pageNb = computePageNb( position );
1432 
1433             // Get back the buffer in this page
1434             ByteBuffer pageData = pageIos[pageNb].getData();
1435 
1436             // Compute the position in the current page
1437             int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1438 
1439             // Compute the remaining size in the page
1440             int remaining = pageData.capacity() - pagePos;
1441             int nbStored = bytes.length;
1442 
1443             // And now, write the bytes until we have none
1444             while ( nbStored > 0 )
1445             {
1446                 if ( remaining > nbStored )
1447                 {
1448                     pageData.mark();
1449                     pageData.position( pagePos );
1450                     pageData.put( bytes, bytes.length - nbStored, nbStored );
1451                     pageData.reset();
1452                     nbStored = 0;
1453                 }
1454                 else
1455                 {
1456                     pageData.mark();
1457                     pageData.position( pagePos );
1458                     pageData.put( bytes, bytes.length - nbStored, remaining );
1459                     pageData.reset();
1460                     pageNb++;
1461 
1462                     if ( pageNb == pageIos.length )
1463                     {
1464                         // We can stop here : we have reach the end of the page
1465                         break;
1466                     }
1467 
1468                     pageData = pageIos[pageNb].getData();
1469                     pagePos = LINK_SIZE;
1470                     nbStored -= remaining;
1471                     remaining = pageData.capacity() - pagePos;
1472                 }
1473             }
1474 
1475             // We are done
1476             position += bytes.length;
1477         }
1478         else
1479         {
1480             // No bytes : write 0 and return
1481             position = store( position, 0, pageIos );
1482         }
1483 
1484         return position;
1485     }
1486 
1487 
1488     /**
1489      * Stores an Integer into one ore more pageIO (depending if the int is stored
1490      * across a boundary or not)
1491      * 
1492      * @param position The position in a virtual byte[] if all the pages were contiguous
1493      * @param value The int to serialize
1494      * @param pageIos The pageIOs we have to store the data in
1495      * @return The new position
1496      */
1497     private long store( long position, int value, PageIO... pageIos )
1498     {
1499         // Compute the page in which we will store the data given the 
1500         // current position
1501         int pageNb = computePageNb( position );
1502 
1503         // Compute the position in the current page
1504         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1505 
1506         // Get back the buffer in this page
1507         ByteBuffer pageData = pageIos[pageNb].getData();
1508 
1509         // Compute the remaining size in the page
1510         int remaining = pageData.capacity() - pagePos;
1511 
1512         if ( remaining < INT_SIZE )
1513         {
1514             // We have to copy the serialized length on two pages
1515 
1516             switch ( remaining )
1517             {
1518                 case 3:
1519                     pageData.put( pagePos + 2, ( byte ) ( value >>> 8 ) );
1520                     // Fallthrough !!!
1521 
1522                 case 2:
1523                     pageData.put( pagePos + 1, ( byte ) ( value >>> 16 ) );
1524                     // Fallthrough !!!
1525 
1526                 case 1:
1527                     pageData.put( pagePos, ( byte ) ( value >>> 24 ) );
1528                     break;
1529             }
1530 
1531             // Now deal with the next page
1532             pageData = pageIos[pageNb + 1].getData();
1533             pagePos = LINK_SIZE;
1534 
1535             switch ( remaining )
1536             {
1537                 case 1:
1538                     pageData.put( pagePos, ( byte ) ( value >>> 16 ) );
1539                     // fallthrough !!!
1540 
1541                 case 2:
1542                     pageData.put( pagePos + 2 - remaining, ( byte ) ( value >>> 8 ) );
1543                     // fallthrough !!!
1544 
1545                 case 3:
1546                     pageData.put( pagePos + 3 - remaining, ( byte ) ( value ) );
1547                     break;
1548             }
1549         }
1550         else
1551         {
1552             // Store the value in the page at the selected position
1553             pageData.putInt( pagePos, value );
1554         }
1555 
1556         // Increment the position to reflect the addition of an Int (4 bytes)
1557         position += INT_SIZE;
1558 
1559         return position;
1560     }
1561 
1562 
1563     /**
1564      * Stores a Long into one ore more pageIO (depending if the long is stored
1565      * across a boundary or not)
1566      * 
1567      * @param position The position in a virtual byte[] if all the pages were contiguous
1568      * @param value The long to serialize
1569      * @param pageIos The pageIOs we have to store the data in
1570      * @return The new position
1571      */
1572     private long store( long position, long value, PageIO... pageIos )
1573     {
1574         // Compute the page in which we will store the data given the 
1575         // current position
1576         int pageNb = computePageNb( position );
1577 
1578         // Compute the position in the current page
1579         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1580 
1581         // Get back the buffer in this page
1582         ByteBuffer pageData = pageIos[pageNb].getData();
1583 
1584         // Compute the remaining size in the page
1585         int remaining = pageData.capacity() - pagePos;
1586 
1587         if ( remaining < LONG_SIZE )
1588         {
1589             // We have to copy the serialized length on two pages
1590 
1591             switch ( remaining )
1592             {
1593                 case 7:
1594                     pageData.put( pagePos + 6, ( byte ) ( value >>> 8 ) );
1595                     // Fallthrough !!!
1596 
1597                 case 6:
1598                     pageData.put( pagePos + 5, ( byte ) ( value >>> 16 ) );
1599                     // Fallthrough !!!
1600 
1601                 case 5:
1602                     pageData.put( pagePos + 4, ( byte ) ( value >>> 24 ) );
1603                     // Fallthrough !!!
1604 
1605                 case 4:
1606                     pageData.put( pagePos + 3, ( byte ) ( value >>> 32 ) );
1607                     // Fallthrough !!!
1608 
1609                 case 3:
1610                     pageData.put( pagePos + 2, ( byte ) ( value >>> 40 ) );
1611                     // Fallthrough !!!
1612 
1613                 case 2:
1614                     pageData.put( pagePos + 1, ( byte ) ( value >>> 48 ) );
1615                     // Fallthrough !!!
1616 
1617                 case 1:
1618                     pageData.put( pagePos, ( byte ) ( value >>> 56 ) );
1619                     break;
1620             }
1621 
1622             // Now deal with the next page
1623             pageData = pageIos[pageNb + 1].getData();
1624             pagePos = LINK_SIZE;
1625 
1626             switch ( remaining )
1627             {
1628                 case 1:
1629                     pageData.put( pagePos, ( byte ) ( value >>> 48 ) );
1630                     // fallthrough !!!
1631 
1632                 case 2:
1633                     pageData.put( pagePos + 2 - remaining, ( byte ) ( value >>> 40 ) );
1634                     // fallthrough !!!
1635 
1636                 case 3:
1637                     pageData.put( pagePos + 3 - remaining, ( byte ) ( value >>> 32 ) );
1638                     // fallthrough !!!
1639 
1640                 case 4:
1641                     pageData.put( pagePos + 4 - remaining, ( byte ) ( value >>> 24 ) );
1642                     // fallthrough !!!
1643 
1644                 case 5:
1645                     pageData.put( pagePos + 5 - remaining, ( byte ) ( value >>> 16 ) );
1646                     // fallthrough !!!
1647 
1648                 case 6:
1649                     pageData.put( pagePos + 6 - remaining, ( byte ) ( value >>> 8 ) );
1650                     // fallthrough !!!
1651 
1652                 case 7:
1653                     pageData.put( pagePos + 7 - remaining, ( byte ) ( value ) );
1654                     break;
1655             }
1656         }
1657         else
1658         {
1659             // Store the value in the page at the selected position
1660             pageData.putLong( pagePos, value );
1661         }
1662 
1663         // Increment the position to reflect the addition of an Long (8 bytes)
1664         position += LONG_SIZE;
1665 
1666         return position;
1667     }
1668 
1669 
1670     /**
1671      * Stores a new page on disk. We will add the modified page into the tree of copied pages.
1672      * The new page is serialized and saved on disk.
1673      * 
1674      * @param oldPage
1675      * @param oldRevision
1676      * @param newPage
1677      * @param newRevision
1678      * @return The offset of the new page
1679      * @throws IOException 
1680      */
1681     /* No qualifier*/ElementHolder writePage( BTree btree, Page newPage, long newRevision )
1682         throws IOException
1683     {
1684         // We first need to save the new page on disk
1685         PageIO[] pageIos = serializePage( btree, newRevision, newPage );
1686 
1687         LOG.debug( "Write data for '{}' btree ", btree.getName() );
1688         // Write the page on disk
1689         flushPages( pageIos );
1690 
1691         // Build the resulting reference
1692         long offset = pageIos[0].getOffset();
1693         long lastOffset = pageIos[pageIos.length - 1].getOffset();
1694         ElementHolder valueHolder = new ReferenceHolder( btree, newPage, offset, lastOffset );
1695 
1696         if ( LOG_CHECK.isDebugEnabled() )
1697         {
1698             check();
1699         }
1700 
1701         return valueHolder;
1702     }
1703 
1704 
1705     /**
1706      * Compute the number of pages needed to store some specific size of data.
1707      * 
1708      * @param dataSize The size of the data we want to store in pages
1709      * @return The number of pages needed
1710      */
1711     private int computeNbPages( int dataSize )
1712     {
1713         if ( dataSize <= 0 )
1714         {
1715             return 0;
1716         }
1717 
1718         // Compute the number of pages needed.
1719         // Considering that each page can contain PageSize bytes,
1720         // but that the first 8 bytes are used for links and we 
1721         // use 4 bytes to store the data size, the number of needed
1722         // pages is :
1723         // NbPages = ( (dataSize - (PageSize - 8 - 4 )) / (PageSize - 8) ) + 1 
1724         // NbPages += ( if (dataSize - (PageSize - 8 - 4 )) % (PageSize - 8) > 0 : 1 : 0 )
1725         int availableSize = ( pageSize - LONG_SIZE );
1726         int nbNeededPages = 1;
1727 
1728         // Compute the number of pages that will be full but the first page
1729         if ( dataSize > availableSize - INT_SIZE )
1730         {
1731             int remainingSize = dataSize - ( availableSize - INT_SIZE );
1732             nbNeededPages += remainingSize / availableSize;
1733             int remain = remainingSize % availableSize;
1734 
1735             if ( remain > 0 )
1736             {
1737                 nbNeededPages++;
1738             }
1739         }
1740 
1741         return nbNeededPages;
1742     }
1743 
1744 
1745     /**
1746      * Get as many pages as needed to store the data of the given size
1747      *  
1748      * @param dataSize The data size
1749      * @return An array of pages, enough to store the full data
1750      */
1751     private PageIO[] getFreePageIOs( int dataSize ) throws IOException
1752     {
1753         if ( dataSize == 0 )
1754         {
1755             return new PageIO[]
1756                 {};
1757         }
1758 
1759         int nbNeededPages = computeNbPages( dataSize );
1760 
1761         PageIO[] pageIOs = new PageIO[nbNeededPages];
1762 
1763         // The first page : set the size
1764         pageIOs[0] = fetchNewPage();
1765         pageIOs[0].setSize( dataSize );
1766 
1767         for ( int i = 1; i < nbNeededPages; i++ )
1768         {
1769             pageIOs[i] = fetchNewPage();
1770 
1771             // Create the link
1772             pageIOs[i - 1].setNextPage( pageIOs[i].getOffset() );
1773         }
1774 
1775         return pageIOs;
1776     }
1777 
1778 
1779     /**
1780      * Return a new Page. We take one of the existing free pages, or we create
1781      * a new page at the end of the file.
1782      * 
1783      * @return The fetched PageIO
1784      */
1785     private PageIO fetchNewPage() throws IOException
1786     {
1787         if ( firstFreePage == NO_PAGE )
1788         {
1789             nbCreatedPages.incrementAndGet();
1790 
1791             // We don't have any free page. Reclaim some new page at the end
1792             // of the file
1793             PageIO newPage = new PageIO( endOfFileOffset );
1794 
1795             endOfFileOffset += pageSize;
1796 
1797             ByteBuffer data = ByteBuffer.allocateDirect( pageSize );
1798 
1799             newPage.setData( data );
1800             newPage.setNextPage( NO_PAGE );
1801             newPage.setSize( 0 );
1802 
1803             LOG.debug( "Requiring a new page at offset {}", newPage.getOffset() );
1804 
1805             return newPage;
1806         }
1807         else
1808         {
1809             nbReusedPages.incrementAndGet();
1810 
1811             // We have some existing free page. Fetch it from disk
1812             PageIO pageIo = fetchPage( firstFreePage );
1813 
1814             // Update the firstFreePage pointer
1815             firstFreePage = pageIo.getNextPage();
1816 
1817             // overwrite the data of old page
1818             ByteBuffer data = ByteBuffer.allocateDirect( pageSize );
1819             pageIo.setData( data );
1820 
1821             pageIo.setNextPage( NO_PAGE );
1822             pageIo.setSize( 0 );
1823 
1824             LOG.debug( "Reused page at offset {}", pageIo.getOffset() );
1825 
1826             // If we don't have any more free page, update the last free page pointer too
1827             if ( firstFreePage == NO_PAGE )
1828             {
1829                 lastFreePage = NO_PAGE;
1830             }
1831 
1832             // Update the header
1833             updateRecordManagerHeader();
1834 
1835             return pageIo;
1836         }
1837     }
1838 
1839 
1840     /**
1841      * fetch a page from disk, knowing its position in the file.
1842      * 
1843      * @param offset The position in the file
1844      * @return The found page
1845      */
1846     private PageIO fetchPage( long offset ) throws IOException, EndOfFileExceededException
1847     {
1848         if ( fileChannel.size() < offset + pageSize )
1849         {
1850             // Error : we are past the end of the file
1851             throw new EndOfFileExceededException( "We are fetching a page on " + offset +
1852                 " when the file's size is " + fileChannel.size() );
1853         }
1854         else
1855         {
1856             // Read the page
1857             fileChannel.position( offset );
1858 
1859             ByteBuffer data = ByteBuffer.allocate( pageSize );
1860             fileChannel.read( data );
1861             data.rewind();
1862 
1863             PageIO readPage = new PageIO( offset );
1864             readPage.setData( data );
1865 
1866             return readPage;
1867         }
1868     }
1869 
1870 
1871     /**
1872      * @return the pageSize
1873      */
1874     public int getPageSize()
1875     {
1876         return pageSize;
1877     }
1878 
1879 
1880     public void setPageSize( int pageSize )
1881     {
1882         if ( this.pageSize != -1 )
1883         {
1884         }
1885         else
1886         {
1887             this.pageSize = pageSize;
1888         }
1889     }
1890 
1891 
1892     /**
1893      * Close the RecordManager and flush everything on disk
1894      */
1895     public void close() throws IOException
1896     {
1897         // TODO : we must wait for the last write to finish
1898 
1899         for ( BTree tree : managedBTrees.values() )
1900         {
1901             tree.close();
1902         }
1903 
1904         managedBTrees.clear();
1905 
1906         // Write the data
1907         fileChannel.force( true );
1908 
1909         // And close the channel
1910         fileChannel.close();
1911     }
1912 
1913 
1914     /**
1915      * Dump the RecordManager file
1916      * @throws IOException 
1917      */
1918     public void dump() throws IOException
1919     {
1920         RandomAccessFile randomFile = new RandomAccessFile( file, "r" );
1921         FileChannel fileChannel = randomFile.getChannel();
1922 
1923         ByteBuffer header = ByteBuffer.allocate( HEADER_SIZE );
1924 
1925         // load the header 
1926         fileChannel.read( header );
1927 
1928         header.rewind();
1929 
1930         // The page size
1931         int pageSize = header.getInt();
1932 
1933         // The number of managed BTrees
1934         int nbBTree = header.getInt();
1935 
1936         // The first and last free page
1937         long firstFreePage = header.getLong();
1938         long lastFreePage = header.getLong();
1939 
1940         if ( LOG.isDebugEnabled() )
1941         {
1942             LOG.debug( "RecordManager" );
1943             LOG.debug( "-------------" );
1944             LOG.debug( "  Header " );
1945             LOG.debug( "    '{}'", Strings.dumpBytes( header.array() ) );
1946             LOG.debug( "    page size : {}", pageSize );
1947             LOG.debug( "    nbTree : {}", nbBTree );
1948             LOG.debug( "    firstFreePage : {}", firstFreePage );
1949             LOG.debug( "    lastFreePage : {}", lastFreePage );
1950         }
1951 
1952         long position = HEADER_SIZE;
1953 
1954         // Dump the BTrees
1955         for ( int i = 0; i < nbBTree; i++ )
1956         {
1957             LOG.debug( "  Btree[{}]", i );
1958             PageIO[] pageIos = readPageIOs( position, Long.MAX_VALUE );
1959 
1960             for ( PageIO pageIo : pageIos )
1961             {
1962                 LOG.debug( "    {}", pageIo );
1963             }
1964         }
1965 
1966         randomFile.close();
1967     }
1968 
1969 
1970     /**
1971      * Get the number of managed trees. We don't count the CopiedPage BTree. and the Revsion BTree
1972      * 
1973      * @return The number of managed BTrees
1974      */
1975     public int getNbManagedTrees()
1976     {
1977         return nbBtree - 2;
1978     }
1979 
1980 
1981     /**
1982      * Get the managed trees. We don't return the CopiedPage BTree nor the Revision BTree.
1983      * 
1984      * @return The managed BTrees
1985      */
1986     public Set<String> getManagedTrees()
1987     {
1988         Set<String> btrees = new HashSet<String>( managedBTrees.keySet() );
1989 
1990         btrees.remove( COPIED_PAGE_BTREE_NAME );
1991         btrees.remove( REVISION_BTREE_NAME );
1992 
1993         return btrees;
1994     }
1995 
1996 
1997     /**
1998      * Store a reference to an old rootPage into the Revision BTree
1999      * 
2000      * @param btree The BTree we want to keep an old RootPage for
2001      * @param rootPage The old rootPage
2002      * @throws IOException If we have an issue while writing on disk
2003      */
2004     /* No qualifier */void storeRootPage( BTree btree, Page rootPage ) throws IOException
2005     {
2006         if ( !isKeepRevisions() )
2007         {
2008             return;
2009         }
2010 
2011         if ( ( btree == copiedPageBTree ) || ( btree == revisionBTree ) )
2012         {
2013             return;
2014         }
2015 
2016         RevisionName revisionName = new RevisionName( rootPage.getRevision(), btree.getName() );
2017 
2018         revisionBTree.insert( revisionName, rootPage.getOffset(), 0 );
2019 
2020         if ( LOG_CHECK.isDebugEnabled() )
2021         {
2022             check();
2023         }
2024     }
2025 
2026 
2027     /**
2028      * Fetch the rootPage of a given BTree for a given revision.
2029      * 
2030      * @param btree The BTree we are interested in
2031      * @param revision The revision we want to get back
2032      * @return The rootPage for this BTree and this revision, if any
2033      * @throws KeyNotFoundException If we can't find the rootPage for this revision and this BTree
2034      * @throws IOException If we had an ise while accessing the data on disk
2035      */
2036     /* No qualifier */Page getRootPage( BTree btree, long revision ) throws KeyNotFoundException, IOException
2037     {
2038         if ( btree.getRevision() == revision )
2039         {
2040             // We are asking for the current revision
2041             return btree.rootPage;
2042         }
2043 
2044         RevisionName revisionName = new RevisionName( revision, btree.getName() );
2045         long rootPageOffset = revisionBTree.get( revisionName );
2046 
2047         // Read the rootPage pages on disk
2048         PageIO[] rootPageIos = readPageIOs( rootPageOffset, Long.MAX_VALUE );
2049 
2050         Page btreeRoot = readPage( btree, rootPageIos );
2051 
2052         return btreeRoot;
2053     }
2054 
2055 
2056     /**
2057      * Get one managed trees, knowing its name. 
2058      * 
2059      * @return The managed BTrees
2060      */
2061     public BTree getManagedTree( String name )
2062     {
2063         return managedBTrees.get( name );
2064     }
2065 
2066 
2067     /**
2068      * Move a list of pages to the free page list. A logical page is associated with on 
2069      * or physical PageIO, which are on the disk. We have to move all those PagIO instance
2070      * to the free list, and do the same in memory (we try to keep a reference to a set of 
2071      * free pages.
2072      *  
2073      * @param btree The BTree which were owning the pages
2074      * @param pages The pages to free
2075      * @throws IOException 
2076      * @throws EndOfFileExceededException 
2077      */
2078     /* Package protected */void addFreePages( BTree btree, List<Page> pages ) throws EndOfFileExceededException,
2079         IOException
2080     {
2081         if ( ( btree == copiedPageBTree ) || ( btree == revisionBTree ) )
2082         {
2083             return;
2084         }
2085 
2086         if ( ( pages == null ) || pages.isEmpty() )
2087         {
2088             return;
2089         }
2090 
2091         if ( !keepRevisions )
2092         {
2093             // if the btree doesn't keep revisions, we can safely move
2094             // the pages to the free page list.
2095             // NOTE : potential improvement : we can update the header only when
2096             // we have processed all the logical pages.
2097 
2098             for ( Page page : pages )
2099             {
2100                 // Retrieve all the PageIO associated with this logical page
2101                 long firstOffset = page.getOffset();
2102 
2103                 // skip the page with offset 0, this is the first in-memory root page that
2104                 // was copied during first insert in a BTree.
2105                 // a Node or Leaf will *never* have 0 or -1 as its offset 
2106                 if ( firstOffset == NO_PAGE )
2107                 {
2108                     continue;
2109                 }
2110 
2111                 long lastOffset = page.getLastOffset();
2112 
2113                 // Update the pointers
2114                 if ( firstFreePage == NO_PAGE )
2115                 {
2116                     // We don't have yet any free pageIos. The
2117                     // PageIOs for this Page will be used
2118                     firstFreePage = firstOffset;
2119                 }
2120                 else
2121                 {
2122                     // We add the Page's PageIOs before the 
2123                     // existing free pages.
2124                     long offset = page.getLastOffset();
2125 
2126                     if ( offset == NO_PAGE )
2127                     {
2128                         offset = page.getOffset();
2129                     }
2130 
2131                     // Fetch the pageIO
2132                     PageIO pageIo = fetchPage( offset );
2133 
2134                     // Link it to the first free page
2135                     pageIo.setNextPage( firstFreePage );
2136 
2137                     LOG.debug( "Flushing the first free page" );
2138 
2139                     // And flush it to disk
2140                     flushPages( pageIo );
2141 
2142                     // We can update the lastFreePage offset 
2143                     firstFreePage = firstOffset;
2144                 }
2145             }
2146 
2147             // Last, not least, flush the header
2148             updateRecordManagerHeader();
2149         }
2150         else
2151         {
2152             LOG.debug( "We should not get there" );
2153 
2154             for ( Page p : pages )
2155             {
2156                 addFreePage( btree, p );
2157             }
2158         }
2159     }
2160 
2161 
2162     /**
2163      * 
2164      * TODO addFreePage.
2165      *
2166      * @param btree
2167      * @param freePage
2168      */
2169     private void addFreePage( BTree btree, Page freePage )
2170     {
2171         try
2172         {
2173             RevisionName revision = new RevisionName( freePage.getRevision(), btree.getName() );
2174             long[] offsetArray = null;
2175 
2176             if ( copiedPageBTree.hasKey( revision ) )
2177             {
2178                 offsetArray = copiedPageBTree.get( revision );
2179                 long[] tmp = new long[offsetArray.length + 1];
2180                 System.arraycopy( offsetArray, 0, tmp, 0, offsetArray.length );
2181                 offsetArray = tmp;
2182             }
2183             else
2184             {
2185                 offsetArray = new long[1];
2186             }
2187 
2188             offsetArray[offsetArray.length - 1] = freePage.getOffset();
2189 
2190             copiedPageBTree.insert( revision, offsetArray, 0 );
2191         }
2192         catch ( Exception e )
2193         {
2194             throw new RuntimeException( e );
2195         }
2196     }
2197 
2198 
2199     /**
2200      * @return the keepRevisions
2201      */
2202     public boolean isKeepRevisions()
2203     {
2204         return keepRevisions;
2205     }
2206 
2207 
2208     /**
2209      * @param keepRevisions the keepRevisions to set
2210      */
2211     public void setKeepRevisions( boolean keepRevisions )
2212     {
2213         this.keepRevisions = keepRevisions;
2214     }
2215 
2216 
2217     /**
2218      * Creates a BTree and automatically adds it to the list of managed btrees
2219      * 
2220      * @param name the name of the BTree
2221      * @param keySerializer key serializer
2222      * @param valueSerializer value serializer
2223      * @param allowDuplicates flag for allowing duplicate keys 
2224      * @return a managed BTree
2225      * @throws IOException
2226      * @throws BTreeAlreadyManagedException
2227      */
2228     @SuppressWarnings("all")
2229     public BTree addBTree( String name, ElementSerializer<?> keySerializer, ElementSerializer<?> valueSerializer,
2230         boolean allowDuplicates ) throws IOException, BTreeAlreadyManagedException
2231     {
2232         BTreeConfiguration config = new BTreeConfiguration();
2233 
2234         config.setName( name );
2235         config.setKeySerializer( keySerializer );
2236         config.setValueSerializer( valueSerializer );
2237         config.setAllowDuplicates( allowDuplicates );
2238         config.setType( BTreeTypeEnum.MANAGED );
2239 
2240         BTree btree = new BTree( config );
2241         manage( btree );
2242 
2243         if ( LOG_CHECK.isDebugEnabled() )
2244         {
2245             check();
2246         }
2247 
2248         return btree;
2249     }
2250 
2251 
2252     private void setCheckedPage( long[] checkedPages, long offset, int pageSize )
2253     {
2254         long pageOffset = ( offset - HEADER_SIZE ) / pageSize;
2255         int index = ( int ) ( pageOffset / 64L );
2256         long mask = ( 1L << ( pageOffset % 64L ) );
2257         long bits = checkedPages[index];
2258 
2259         if ( ( bits & mask ) == 1 )
2260         {
2261             throw new RuntimeException( "The page at : " + offset + " has already been checked" );
2262         }
2263 
2264         checkedPages[index] |= mask;
2265 
2266     }
2267 
2268 
2269     /**
2270      * Check the free pages
2271      * 
2272      * @param checkedPages
2273      * @throws IOException 
2274      */
2275     private void checkFreePages( long[] checkedPages, int pageSize, long firstFreePage, long lastFreePage )
2276         throws IOException
2277     {
2278         if ( firstFreePage == NO_PAGE )
2279         {
2280             if ( lastFreePage == NO_PAGE )
2281             {
2282                 return;
2283             }
2284             else
2285             {
2286                 throw new RuntimeException( "Wrong last free page : " + lastFreePage );
2287             }
2288         }
2289 
2290         if ( lastFreePage != NO_PAGE )
2291         {
2292             throw new RuntimeException( "Wrong last free page : " + lastFreePage );
2293         }
2294 
2295         // Now, read all the free pages
2296         long currentOffset = firstFreePage;
2297         long fileSize = fileChannel.size();
2298 
2299         while ( currentOffset != NO_PAGE )
2300         {
2301             if ( currentOffset > fileSize )
2302             {
2303                 throw new RuntimeException( "Wrong free page offset, above file size : " + currentOffset );
2304             }
2305 
2306             try
2307             {
2308                 PageIO pageIo = fetchPage( currentOffset );
2309 
2310                 if ( currentOffset != pageIo.getOffset() )
2311                 {
2312                     throw new RuntimeException( "PageIO offset is incorrect : " + currentOffset + "-"
2313                         + pageIo.getOffset() );
2314                 }
2315 
2316                 setCheckedPage( checkedPages, currentOffset, pageSize );
2317 
2318                 long newOffset = pageIo.getNextPage();
2319                 currentOffset = newOffset;
2320             }
2321             catch ( IOException ioe )
2322             {
2323                 throw new RuntimeException( "Cannot fetch page at : " + currentOffset );
2324             }
2325         }
2326     }
2327 
2328 
2329     /**
2330      * Check the root page for a given BTree
2331      * @throws IOException 
2332      * @throws EndOfFileExceededException 
2333      */
2334     private void checkRoot( long[] checkedPages, long offset, int pageSize, long nbBTreeElems,
2335         ElementSerializer keySerializer, ElementSerializer valueSerializer, boolean allowDuplicates )
2336         throws EndOfFileExceededException, IOException
2337     {
2338         // Read the rootPage pages on disk
2339         PageIO[] rootPageIos = readPageIOs( offset, Long.MAX_VALUE );
2340 
2341         // Deserialize the rootPage now
2342         long position = 0L;
2343 
2344         // The revision
2345         long revision = readLong( rootPageIos, position );
2346         position += LONG_SIZE;
2347 
2348         // The number of elements in the page
2349         int nbElems = readInt( rootPageIos, position );
2350         position += INT_SIZE;
2351 
2352         // The size of the data containing the keys and values
2353         ByteBuffer byteBuffer = null;
2354 
2355         // Reads the bytes containing all the keys and values, if we have some
2356         byte[] data = readBytes( rootPageIos, position );
2357 
2358         if ( data != null )
2359         {
2360             byteBuffer = ByteBuffer.allocate( data.length );
2361             byteBuffer.put( data );
2362             byteBuffer.rewind();
2363         }
2364 
2365         if ( nbElems >= 0 )
2366         {
2367             // Its a leaf
2368 
2369             // Check the page offset
2370             long pageOffset = rootPageIos[0].getOffset();
2371 
2372             if ( ( pageOffset < 0 ) || ( pageOffset > fileChannel.size() ) )
2373             {
2374                 throw new RuntimeException( "The page offset is incorrect : " + pageOffset );
2375             }
2376 
2377             // Check the page last offset
2378             long pageLastOffset = rootPageIos[rootPageIos.length - 1].getOffset();
2379 
2380             if ( ( pageLastOffset <= 0 ) || ( pageLastOffset > fileChannel.size() ) )
2381             {
2382                 throw new RuntimeException( "The page last offset is incorrect : " + pageLastOffset );
2383             }
2384 
2385             // Read each value and key
2386             for ( int i = 0; i < nbElems; i++ )
2387             {
2388                 // Just deserialize all the keys and values
2389                 if ( allowDuplicates )
2390                 {
2391                     /*
2392                     long value = OFFSET_SERIALIZER.deserialize( byteBuffer );
2393 
2394                     rootPageIos = readPageIOs( value, Long.MAX_VALUE );
2395 
2396                     BTree dupValueContainer = BTreeFactory.createBTree();
2397                     dupValueContainer.setBtreeOffset( value );
2398 
2399                     try
2400                     {
2401                         loadBTree( pageIos, dupValueContainer );
2402                     }
2403                     catch ( Exception e )
2404                     {
2405                         // should not happen
2406                         throw new RuntimeException( e );
2407                     }
2408                     */
2409                 }
2410                 else
2411                 {
2412                     valueSerializer.deserialize( byteBuffer );
2413                 }
2414 
2415                 keySerializer.deserialize( byteBuffer );
2416             }
2417         }
2418         else
2419         {
2420             /*
2421             // It's a node
2422             int nodeNbElems = -nbElems;
2423 
2424             // Read each value and key
2425             for ( int i = 0; i < nodeNbElems; i++ )
2426             {
2427                 // This is an Offset
2428                 long offset = OFFSET_SERIALIZER.deserialize( byteBuffer );
2429                 long lastOffset = OFFSET_SERIALIZER.deserialize( byteBuffer );
2430 
2431                 ElementHolder valueHolder = new ReferenceHolder( btree, null, offset, lastOffset );
2432                 ( ( Node ) page ).setValue( i, valueHolder );
2433 
2434                 Object key = btree.getKeySerializer().deserialize( byteBuffer );
2435                 BTreeFactory.setKey( page, i, key );
2436             }
2437 
2438             // and read the last value, as it's a node
2439             long offset = OFFSET_SERIALIZER.deserialize( byteBuffer );
2440             long lastOffset = OFFSET_SERIALIZER.deserialize( byteBuffer );
2441 
2442             ElementHolder valueHolder = new ReferenceHolder( btree, null, offset, lastOffset );
2443             ( ( Node ) page ).setValue( nodeNbElems, valueHolder );*/
2444         }
2445     }
2446 
2447 
2448     /**
2449      * Check a BTree
2450      * @throws IllegalAccessException 
2451      * @throws InstantiationException 
2452      * @throws ClassNotFoundException 
2453      */
2454     private long checkBTree( long[] checkedPages, PageIO[] pageIos, int pageSize, boolean isLast )
2455         throws EndOfFileExceededException, IOException, InstantiationException, IllegalAccessException,
2456         ClassNotFoundException
2457     {
2458         long dataPos = 0L;
2459 
2460         // The BTree current revision
2461         long revision = readLong( pageIos, dataPos );
2462         dataPos += LONG_SIZE;
2463 
2464         // The nb elems in the tree
2465         long nbElems = readLong( pageIos, dataPos );
2466         dataPos += LONG_SIZE;
2467 
2468         // The BTree rootPage offset
2469         long rootPageOffset = readLong( pageIos, dataPos );
2470 
2471         if ( ( rootPageOffset < 0 ) || ( rootPageOffset > fileChannel.size() ) )
2472         {
2473             throw new RuntimeException( "The rootpage is incorrect : " + rootPageOffset );
2474         }
2475 
2476         dataPos += LONG_SIZE;
2477 
2478         // The next BTree offset
2479         long nextBTreeOffset = readLong( pageIos, dataPos );
2480 
2481         if ( ( ( rootPageOffset < 0 ) && ( !isLast ) ) || ( nextBTreeOffset > fileChannel.size() ) )
2482         {
2483             throw new RuntimeException( "The rootpage is incorrect : " + rootPageOffset );
2484         }
2485 
2486         dataPos += LONG_SIZE;
2487 
2488         // The BTree page size
2489         int btreePageSize = readInt( pageIos, dataPos );
2490 
2491         if ( ( btreePageSize < 2 ) || ( ( btreePageSize & ( ~btreePageSize + 1 ) ) != btreePageSize ) )
2492         {
2493             throw new RuntimeException( "The BTree page size is not a power of 2 : " + btreePageSize );
2494         }
2495 
2496         dataPos += INT_SIZE;
2497 
2498         // The tree name
2499         byte[] btreeNameBytes = readBytes( pageIos, dataPos );
2500         dataPos += INT_SIZE;
2501 
2502         dataPos += btreeNameBytes.length;
2503         String btreeName = Strings.utf8ToString( btreeNameBytes );
2504 
2505         // The keySerializer FQCN
2506         byte[] keySerializerBytes = readBytes( pageIos, dataPos );
2507 
2508         String keySerializerFqcn = null;
2509         dataPos += INT_SIZE;
2510 
2511         if ( keySerializerBytes != null )
2512         {
2513             dataPos += keySerializerBytes.length;
2514             keySerializerFqcn = Strings.utf8ToString( keySerializerBytes );
2515         }
2516         else
2517         {
2518             keySerializerFqcn = "";
2519         }
2520 
2521         // The valueSerialier FQCN
2522         byte[] valueSerializerBytes = readBytes( pageIos, dataPos );
2523 
2524         String valueSerializerFqcn = null;
2525         dataPos += INT_SIZE;
2526 
2527         if ( valueSerializerBytes != null )
2528         {
2529             dataPos += valueSerializerBytes.length;
2530             valueSerializerFqcn = Strings.utf8ToString( valueSerializerBytes );
2531         }
2532         else
2533         {
2534             valueSerializerFqcn = "";
2535         }
2536 
2537         // The BTree allowDuplicates flag
2538         int allowDuplicates = readInt( pageIos, dataPos );
2539         dataPos += INT_SIZE;
2540 
2541         // Now, check the rootPage, which can be a Leaf or a Node, depending 
2542         // on the number of elements in the tree : if it's above the pageSize,
2543         // it's a Node, otherwise it's a Leaf
2544         Class<?> valueSerializer = Class.forName( valueSerializerFqcn );
2545         Class<?> keySerializer = Class.forName( keySerializerFqcn );
2546 
2547         /*
2548         checkRoot( checkedPages, rootPageOffset, pageSize, nbElems,
2549             ( ElementSerializer<?> ) keySerializer.newInstance(),
2550             ( ElementSerializer<?> ) valueSerializer.newInstance(), allowDuplicates != 0 );
2551         */
2552 
2553         return nextBTreeOffset;
2554     }
2555 
2556 
2557     /**
2558      * Check each BTree we manage
2559      * @throws IOException 
2560      * @throws EndOfFileExceededException 
2561      * @throws ClassNotFoundException 
2562      * @throws IllegalAccessException 
2563      * @throws InstantiationException 
2564      */
2565     private void checkBTrees( long[] checkedPages, int pageSize, int nbBTrees ) throws EndOfFileExceededException,
2566         IOException, InstantiationException, IllegalAccessException, ClassNotFoundException
2567     {
2568         // Iterate on each BTree until we have exhausted all of them. The number
2569         // of btrees is just used to check that we have the correct number
2570         // of stored BTrees, as they are all linked.
2571         long position = HEADER_SIZE;
2572 
2573         for ( int i = 0; i < nbBTrees; i++ )
2574         {
2575             // Load the pageIOs containing the BTree
2576             PageIO[] pageIos = readPageIOs( position, Long.MAX_VALUE );
2577 
2578             // Check that they are correctly linked and not already used
2579             int pageNb = 0;
2580 
2581             for ( PageIO currentPageIo : pageIos )
2582             {
2583                 // 
2584                 long nextPageOffset = currentPageIo.getNextPage();
2585 
2586                 if ( pageNb == pageIos.length - 1 )
2587                 {
2588                     if ( nextPageOffset != NO_PAGE )
2589                     {
2590                         throw new RuntimeException( "The pointer to the next page is not valid, expected NO_PAGE" );
2591                     }
2592                 }
2593                 else
2594                 {
2595                     if ( nextPageOffset == NO_PAGE )
2596                     {
2597                         throw new RuntimeException( "The pointer to the next page is not valid, NO_PAGE" );
2598                     }
2599                 }
2600 
2601                 if ( ( nextPageOffset != NO_PAGE ) && ( ( nextPageOffset - HEADER_SIZE ) % pageSize != 0 ) )
2602                 {
2603                     throw new RuntimeException( "The pointer to the next page is not valid" );
2604                 }
2605 
2606                 // Update the array of processed pages
2607                 setCheckedPage( checkedPages, currentPageIo.getOffset(), pageSize );
2608             }
2609 
2610             // Now check the BTree
2611             long nextBTree = checkBTree( checkedPages, pageIos, pageSize, i == nbBTrees - 1 );
2612 
2613             if ( ( nextBTree == NO_PAGE ) && ( i < nbBTrees - 1 ) )
2614             {
2615                 throw new RuntimeException( "The pointer to the next BTree is incorrect" );
2616             }
2617 
2618             position = nextBTree;
2619         }
2620     }
2621 
2622 
2623     /**
2624      * Check the whole file
2625      */
2626     private void check()
2627     {
2628         try
2629         {
2630             // First check the header
2631             ByteBuffer header = ByteBuffer.allocate( HEADER_SIZE );
2632             long fileSize = fileChannel.size();
2633 
2634             if ( fileSize < HEADER_SIZE )
2635             {
2636                 throw new RuntimeException( "File size too small : " + fileSize );
2637             }
2638 
2639             // Read the header
2640             fileChannel.read( header, 0L );
2641             header.flip();
2642 
2643             // The page size. It must be a power of 2, and above 16.
2644             int pageSize = header.getInt();
2645 
2646             if ( ( pageSize < 0 ) || ( pageSize < 32 ) || ( ( pageSize & ( ~pageSize + 1 ) ) != pageSize ) )
2647             {
2648                 throw new RuntimeException( "Wrong page size : " + pageSize );
2649             }
2650 
2651             // Compute the number of pages in this file
2652             long nbPages = ( fileSize - HEADER_SIZE ) / pageSize;
2653 
2654             // The number of trees. It must be at least 2 and > 0
2655             int nbBTrees = header.getInt();
2656 
2657             if ( nbBTrees < 0 )
2658             {
2659                 throw new RuntimeException( "Wrong nb trees : " + nbBTrees );
2660             }
2661 
2662             // The first free page offset. It must be either -1 or below file size
2663             // and its value must be a modulo of pageSize
2664             long firstFreePage = header.getLong();
2665 
2666             if ( firstFreePage > fileSize )
2667             {
2668                 throw new RuntimeException( "First free page pointing after the end of the file : " + firstFreePage );
2669             }
2670 
2671             if ( ( firstFreePage != NO_PAGE ) && ( ( ( firstFreePage - HEADER_SIZE ) % pageSize ) != 0 ) )
2672             {
2673                 throw new RuntimeException( "First free page not pointing to a correct offset : " + firstFreePage );
2674             }
2675 
2676             // The last free page offset. It must be -1
2677             long lastFreePage = header.getLong();
2678 
2679             if ( ( ( lastFreePage != NO_PAGE ) && ( ( ( lastFreePage - HEADER_SIZE ) % pageSize ) != 0 ) ) )
2680             //|| ( lastFreePage != 0 ) )
2681             {
2682                 throw new RuntimeException( "Invalid last free page : " + lastFreePage );
2683             }
2684 
2685             int nbPageBits = ( int ) ( nbPages / 64 );
2686 
2687             // Create an array of pages to be checked
2688             // We use one bit per page. It's 0 when the page
2689             // hasn't been checked, 1 otherwise.
2690             long[] checkedPages = new long[nbPageBits + 1];
2691 
2692             // Then the free files
2693             checkFreePages( checkedPages, pageSize, firstFreePage, lastFreePage );
2694 
2695             // The BTrees
2696             checkBTrees( checkedPages, pageSize, nbBTrees );
2697         }
2698         catch ( Exception e )
2699         {
2700             // We catch the exception and rethrow it immediately to be able to
2701             // put a breakpoint here
2702             e.printStackTrace();
2703             throw new RuntimeException( "Error : " + e.getMessage() );
2704         }
2705     }
2706 
2707 
2708     /**
2709      * Loads a BTree holding the values of a duplicate key
2710      * This tree is also called as dups tree or sub tree
2711      * 
2712      * @param offset the offset of the BTree header
2713      * @return the deserialized BTree
2714      */
2715     /* No qualifier */BTree loadDupsBTree( long offset )
2716     {
2717         try
2718         {
2719             PageIO[] pageIos = readPageIOs( offset, Long.MAX_VALUE );
2720             
2721             BTree dupValueContainer = BTreeFactory.createBTree();
2722             dupValueContainer.setBtreeOffset( offset );
2723 
2724             loadBTree( pageIos, dupValueContainer );
2725             
2726             return dupValueContainer;
2727         }
2728         catch ( Exception e )
2729         {
2730             // should not happen
2731             throw new RuntimeException( e );
2732         }
2733         
2734     }
2735     
2736     
2737     /**
2738      * @see Object#toString()
2739      */
2740     public String toString()
2741     {
2742         StringBuilder sb = new StringBuilder();
2743 
2744         sb.append( "RM free pages : [" );
2745 
2746         if ( firstFreePage != NO_PAGE )
2747         {
2748             long current = firstFreePage;
2749             boolean isFirst = true;
2750 
2751             while ( current != NO_PAGE )
2752             {
2753                 if ( isFirst )
2754                 {
2755                     isFirst = false;
2756                 }
2757                 else
2758                 {
2759                     sb.append( ", " );
2760                 }
2761 
2762                 PageIO pageIo;
2763 
2764                 try
2765                 {
2766                     pageIo = fetchPage( current );
2767                     sb.append( pageIo.getOffset() );
2768                     current = pageIo.getNextPage();
2769                 }
2770                 catch ( EndOfFileExceededException e )
2771                 {
2772                     e.printStackTrace();
2773                 }
2774                 catch ( IOException e )
2775                 {
2776                     e.printStackTrace();
2777                 }
2778 
2779             }
2780         }
2781 
2782         sb.append( "]" );
2783 
2784         return sb.toString();
2785     }
2786 }