View Javadoc

1   /*
2    *  Licensed to the Apache Software Foundation (ASF) under one
3    *  or more contributor license agreements.  See the NOTICE file
4    *  distributed with this work for additional information
5    *  regarding copyright ownership.  The ASF licenses this file
6    *  to you under the Apache License, Version 2.0 (the
7    *  "License"); you may not use this file except in compliance
8    *  with the License.  You may obtain a copy of the License at
9    *
10   *    http://www.apache.org/licenses/LICENSE-2.0
11   *
12   *  Unless required by applicable law or agreed to in writing,
13   *  software distributed under the License is distributed on an
14   *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   *  KIND, either express or implied.  See the License for the
16   *  specific language governing permissions and limitations
17   *  under the License.
18   *
19   */
20  package org.apache.directory.mavibot.btree.managed;
21  
22  
23  import java.io.File;
24  import java.io.IOException;
25  import java.io.RandomAccessFile;
26  import java.nio.ByteBuffer;
27  import java.nio.channels.FileChannel;
28  import java.util.ArrayList;
29  import java.util.HashSet;
30  import java.util.LinkedHashMap;
31  import java.util.List;
32  import java.util.Map;
33  import java.util.Set;
34  import java.util.concurrent.atomic.AtomicLong;
35  
36  import org.apache.directory.mavibot.btree.exception.BTreeAlreadyManagedException;
37  import org.apache.directory.mavibot.btree.exception.EndOfFileExceededException;
38  import org.apache.directory.mavibot.btree.exception.KeyNotFoundException;
39  import org.apache.directory.mavibot.btree.serializer.ElementSerializer;
40  import org.apache.directory.mavibot.btree.serializer.IntSerializer;
41  import org.apache.directory.mavibot.btree.serializer.LongArraySerializer;
42  import org.apache.directory.mavibot.btree.serializer.LongSerializer;
43  import org.apache.directory.mavibot.btree.util.Strings;
44  import org.slf4j.Logger;
45  import org.slf4j.LoggerFactory;
46  
47  
48  /**
49   * The RecordManager is used to manage the file in which we will store the BTrees. 
50   * A RecordManager will manage more than one BTree.<br/>
51   * 
52   * It stores data in fixed size pages (default size is 512 bytes), which may be linked one to 
53   * the other if the data we want to store is too big for a page.
54   *  
55   * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
56   */
57  public class RecordManager
58  {
59      /** The LoggerFactory used by this class */
60      protected static final Logger LOG = LoggerFactory.getLogger( RecordManager.class );
61  
62      /** A dedicated logger for the check */
63      protected static final Logger LOG_CHECK = LoggerFactory.getLogger( "RM_CHECK" );
64  
65      /** The associated file */
66      private File file;
67  
68      /** The channel used to read and write data */
69      private FileChannel fileChannel;
70  
71      /** The number of stored BTrees */
72      private int nbBtree;
73  
74      /** The first and last free page */
75      private long firstFreePage;
76      private long lastFreePage;
77  
78      /** The list of available free pages */
79      List<PageIO> freePages = new ArrayList<PageIO>();
80  
81      /** A counter to track the number of free pages */
82      public AtomicLong nbFreedPages = new AtomicLong( 0 );
83      public AtomicLong nbCreatedPages = new AtomicLong( 0 );
84      public AtomicLong nbReusedPages = new AtomicLong( 0 );
85      public AtomicLong nbUpdateRMHeader = new AtomicLong( 0 );
86      public AtomicLong nbUpdateBTreeHeader = new AtomicLong( 0 );
87      public AtomicLong nbUpdatePageIOs = new AtomicLong( 0 );
88  
89      /** The offset of the end of the file */
90      private long endOfFileOffset;
91  
92      /** 
93       * A Btree used to manage the page that has been copied in a new version.
94       * Those pages can be reclaimed when the associated version is dead. 
95       **/
96      private BTree<RevisionName, long[]> copiedPageBTree;
97  
98      /** A BTree used to store all the valid revisions for all the stored BTrees */
99      private BTree<RevisionName, Long> revisionBTree;
100 
101     /** A constant for an offset on a non existing page */
102     private static final long NO_PAGE = -1L;
103 
104     /** The number of stored BTrees */
105     private static final int NB_TREE_SIZE = 4;
106 
107     /** The header page size */
108     private static final int PAGE_SIZE = 4;
109 
110     /** The size of the data size in a page */
111     private static final int DATA_SIZE = 4;
112 
113     /** The size of the link to next page */
114     private static final int LINK_SIZE = 8;
115 
116     /** Some constants */
117     private static final int BYTE_SIZE = 1;
118     private static final int INT_SIZE = 4;
119     private static final int LONG_SIZE = 8;
120 
121     /** The size of the link to the first and last free page */
122     private static final int FIRST_FREE_PAGE_SIZE = 8;
123     private static final int LAST_FREE_PAGE_SIZE = 8;
124 
125     /** The default page size */
126     private static final int DEFAULT_PAGE_SIZE = 512;
127 
128     /** The header size */
129     private static int HEADER_SIZE = DEFAULT_PAGE_SIZE;
130 
131     /** A global buffer used to store the header */
132     private static ByteBuffer HEADER_BUFFER;
133 
134     /** A static buffer used to store the header */
135     private static byte[] HEADER_BYTES;
136 
137     /** The length of an Offset, as a nagative value */
138     private static byte[] LONG_LENGTH = new byte[]
139         { ( byte ) 0xFF, ( byte ) 0xFF, ( byte ) 0xFF, ( byte ) 0xF8 };
140 
141     /** The RecordManager underlying page size. */
142     private int pageSize = DEFAULT_PAGE_SIZE;
143 
144     /** The set of managed BTrees */
145     private Map<String, BTree<Object, Object>> managedBTrees;
146 
147     /** The offset on the last added BTree */
148     private long lastAddedBTreeOffset = NO_PAGE;
149 
150     /** The default file name */
151     private static final String DEFAULT_FILE_NAME = "mavibot.db";
152 
153     /** A deserializer for Offsets */
154     private static final LongSerializer OFFSET_SERIALIZER = new LongSerializer();
155 
156     private static final String REVISION_BTREE_NAME = "_revisionBTree_";
157 
158     private static final String COPIED_PAGE_BTREE_NAME = "_copiedPageBTree_";
159 
160     /** A flag set to true if we want to keep old revisions */
161     private boolean keepRevisions;
162 
163 
164     /**
165      * Create a Record manager which will either create the underlying file
166      * or load an existing one. If a folder is provided, then we will create
167      * a file with a default name : mavibot.db
168      * 
169      * @param name The file name, or a folder name
170      */
171     public RecordManager( String fileName )
172     {
173         this( fileName, DEFAULT_PAGE_SIZE );
174     }
175 
176 
177     /**
178      * Create a Record manager which will either create the underlying file
179      * or load an existing one. If a folder is provider, then we will create
180      * a file with a default name : mavibot.db
181      * 
182      * @param name The file name, or a folder name
183      * @param pageSize the size of a page on disk
184      */
185     public RecordManager( String fileName, int pageSize )
186     {
187         managedBTrees = new LinkedHashMap<String, BTree<Object, Object>>();
188 
189         HEADER_BUFFER = ByteBuffer.allocate( pageSize );
190         HEADER_BYTES = new byte[pageSize];
191         HEADER_SIZE = pageSize;
192 
193         // Open the file or create it
194         File tmpFile = new File( fileName );
195         boolean isNewFile = false;
196 
197         if ( tmpFile.isDirectory() )
198         {
199             // It's a directory. Check that we don't have an existing mavibot file
200             File mavibotFile = new File( tmpFile, DEFAULT_FILE_NAME );
201 
202             if ( !mavibotFile.exists() )
203             {
204                 // We have to create a new file
205                 try
206                 {
207                     mavibotFile.createNewFile();
208                     isNewFile = true;
209                 }
210                 catch ( IOException ioe )
211                 {
212                     LOG.error( "Cannot create the file {}", mavibotFile.getName() );
213                     return;
214                 }
215             }
216 
217             file = mavibotFile;
218         }
219         else
220         {
221             // It's a file. Let's see if it exists, otherwise create it
222             if ( !tmpFile.exists() || ( tmpFile.length() == 0 ) )
223             {
224                 isNewFile = true;
225 
226                 try
227                 {
228                     tmpFile.createNewFile();
229                 }
230                 catch ( IOException ioe )
231                 {
232                     LOG.error( "Cannot create the file {}", tmpFile.getName() );
233                     return;
234                 }
235             }
236 
237             file = tmpFile;
238         }
239 
240         try
241         {
242             RandomAccessFile randomFile = new RandomAccessFile( file, "rw" );
243             fileChannel = randomFile.getChannel();
244 
245             // get the current end of file offset
246             endOfFileOffset = fileChannel.size();
247 
248             if ( isNewFile )
249             {
250                 this.pageSize = pageSize;
251                 initRecordManager();
252             }
253             else
254             {
255                 loadRecordManager();
256             }
257         }
258         catch ( Exception e )
259         {
260             LOG.error( "Error while initializing the RecordManager : {}", e.getMessage() );
261             LOG.error( "", e );
262             throw new RuntimeException( e );
263         }
264     }
265 
266 
267     /**
268      * We will create a brand new RecordManager file, containing nothing, but the header, 
269      * a BTree to manage the old revisions we want to keep and
270      * a BTree used to manage pages associated with old versions.
271      * <br/>
272      * The Header contains the following details :
273      * <pre>
274      * +-----------+
275      * | PageSize  | 4 bytes : The size of a physical page (default to 4096)
276      * +-----------+
277      * |  NbTree   | 4 bytes : The number of managed BTrees (at least 1)
278      * +-----------+ 
279      * | FirstFree | 8 bytes : The offset of the first free page
280      * +-----------+
281      * | LastFree  | 8 bytes : The offset of the last free page
282      * +-----------+
283      * </pre>
284      * 
285      * We then store the BTree managing the pages that have been copied when we have added
286      * or deleted an element in the BTree. They are associated with a version.
287      * 
288      * Last, we add the bTree that keep a track on each revision we can have access to.
289      */
290     private void initRecordManager() throws IOException
291     {
292         // Create a new Header
293         nbBtree = 0;
294         firstFreePage = NO_PAGE;
295         lastFreePage = NO_PAGE;
296         updateRecordManagerHeader();
297 
298         // Set the offset of the end of the file
299         endOfFileOffset = fileChannel.size();
300 
301         // Now, initialize the Copied Page BTree
302         copiedPageBTree = new BTree<RevisionName, long[]>( COPIED_PAGE_BTREE_NAME, new RevisionNameSerializer(),
303             new LongArraySerializer() );
304 
305         // and initialize the Revision BTree
306         revisionBTree = new BTree<RevisionName, Long>( REVISION_BTREE_NAME, new RevisionNameSerializer(),
307             new LongSerializer() );
308 
309         // Inject these BTrees into the RecordManager
310         try
311         {
312             manage( copiedPageBTree );
313             manage( revisionBTree );
314         }
315         catch ( BTreeAlreadyManagedException btame )
316         {
317             // Can't happen here.
318         }
319 
320         // We are all set !
321     }
322 
323 
324     /**
325      * Load the BTrees from the disk. 
326      * 
327      * @throws InstantiationException 
328      * @throws IllegalAccessException 
329      * @throws ClassNotFoundException 
330      */
331     private void loadRecordManager() throws IOException, ClassNotFoundException, IllegalAccessException,
332         InstantiationException
333     {
334         if ( fileChannel.size() != 0 )
335         {
336             ByteBuffer header = ByteBuffer.allocate( HEADER_SIZE );
337 
338             // The file exists, we have to load the data now 
339             fileChannel.read( header );
340 
341             header.rewind();
342 
343             // The page size
344             pageSize = header.getInt();
345 
346             // The number of managed BTrees
347             nbBtree = header.getInt();
348 
349             // The first and last free page
350             firstFreePage = header.getLong();
351             lastFreePage = header.getLong();
352 
353             // Now read each BTree. The first one is the one which
354             // manage the modified pages. Once read, we can discard all
355             // the pages that are stored in it, as we have restarted 
356             // the RecordManager.
357             long btreeOffset = HEADER_SIZE;
358 
359             PageIO[] pageIos = readPageIOs( HEADER_SIZE, Long.MAX_VALUE );
360 
361             // Create the BTree
362             copiedPageBTree = BTreeFactory.<RevisionName, long[]> createBTree();
363             copiedPageBTree.setBtreeOffset( btreeOffset );
364 
365             loadBTree( pageIos, copiedPageBTree );
366             long nextBtreeOffset = copiedPageBTree.getNextBTreeOffset();
367 
368             // And the Revision BTree
369             pageIos = readPageIOs( nextBtreeOffset, Long.MAX_VALUE );
370 
371             revisionBTree = BTreeFactory.<RevisionName, Long> createBTree();
372             revisionBTree.setBtreeOffset( nextBtreeOffset );
373 
374             loadBTree( pageIos, revisionBTree );
375             nextBtreeOffset = revisionBTree.getNextBTreeOffset();
376 
377             // Then process the next ones
378             for ( int i = 2; i < nbBtree; i++ )
379             {
380                 // Create the BTree
381                 BTree<Object, Object> btree = BTreeFactory.createBTree();
382                 btree.setBtreeOffset( nextBtreeOffset );
383                 lastAddedBTreeOffset = nextBtreeOffset;
384 
385                 // Read the associated pages
386                 pageIos = readPageIOs( nextBtreeOffset, Long.MAX_VALUE );
387 
388                 // Load the BTree
389                 loadBTree( pageIos, btree );
390                 nextBtreeOffset = btree.getNextBTreeOffset();
391 
392                 // Store it into the managedBtrees map
393                 managedBTrees.put( btree.getName(), btree );
394             }
395 
396             // We are done ! Let's finish with the last initilization parts
397             endOfFileOffset = fileChannel.size();
398         }
399     }
400 
401 
402     /**
403      * Reads all the PageIOs that are linked to the page at the given position, including
404      * the first page.
405      * 
406      * @param position The position of the first page
407      * @return An array of pages
408      */
409     private PageIO[] readPageIOs( long position, long limit ) throws IOException, EndOfFileExceededException
410     {
411         LOG.debug( "Read PageIOs at position {}", position );
412 
413         if ( limit <= 0 )
414         {
415             limit = Long.MAX_VALUE;
416         }
417 
418         PageIO firstPage = fetchPage( position );
419         firstPage.setSize();
420         List<PageIO> listPages = new ArrayList<PageIO>();
421         listPages.add( firstPage );
422         long dataRead = pageSize - LONG_SIZE - INT_SIZE;
423 
424         // Iterate on the pages, if needed
425         long nextPage = firstPage.getNextPage();
426 
427         if ( ( dataRead < limit ) && ( nextPage != NO_PAGE ) )
428         {
429             while ( dataRead < limit )
430             {
431                 PageIO page = fetchPage( nextPage );
432                 listPages.add( page );
433                 nextPage = page.getNextPage();
434                 dataRead += pageSize - LONG_SIZE;
435 
436                 if ( nextPage == NO_PAGE )
437                 {
438                     page.setNextPage( NO_PAGE );
439                     break;
440                 }
441             }
442         }
443 
444         LOG.debug( "Nb of PageIOs read : {}", listPages.size() );
445 
446         // Return 
447         return listPages.toArray( new PageIO[]
448             {} );
449     }
450 
451 
452     /**
453      * Read a BTree from the disk. The meta-data are at the given position in the list of pages.
454      * 
455      * @param pageIos The list of pages containing the meta-data
456      * @param btree The BTree we have to initialize
457      * @throws InstantiationException 
458      * @throws IllegalAccessException 
459      * @throws ClassNotFoundException 
460      */
461     private <K, V> void loadBTree( PageIO[] pageIos, BTree<K, V> btree ) throws EndOfFileExceededException,
462         IOException, ClassNotFoundException, IllegalAccessException, InstantiationException
463     {
464         long dataPos = 0L;
465 
466         // The BTree current revision
467         long revision = readLong( pageIos, dataPos );
468         BTreeFactory.setRevision( btree, revision );
469         dataPos += LONG_SIZE;
470 
471         // The nb elems in the tree
472         long nbElems = readLong( pageIos, dataPos );
473         BTreeFactory.setNbElems( btree, nbElems );
474         dataPos += LONG_SIZE;
475 
476         // The BTree rootPage offset
477         long rootPageOffset = readLong( pageIos, dataPos );
478         BTreeFactory.setRootPageOffset( btree, rootPageOffset );
479         dataPos += LONG_SIZE;
480 
481         // The next BTree offset
482         long nextBTreeOffset = readLong( pageIos, dataPos );
483         BTreeFactory.setNextBTreeOffset( btree, nextBTreeOffset );
484         dataPos += LONG_SIZE;
485 
486         // The BTree page size
487         int btreePageSize = readInt( pageIos, dataPos );
488         BTreeFactory.setPageSize( btree, btreePageSize );
489         dataPos += INT_SIZE;
490 
491         // The tree name
492         ByteBuffer btreeNameBytes = readBytes( pageIos, dataPos );
493         dataPos += INT_SIZE + btreeNameBytes.limit();
494         String btreeName = Strings.utf8ToString( btreeNameBytes );
495         BTreeFactory.setName( btree, btreeName );
496 
497         // The keySerializer FQCN
498         ByteBuffer keySerializerBytes = readBytes( pageIos, dataPos );
499         dataPos += INT_SIZE + keySerializerBytes.limit();
500 
501         String keySerializerFqcn = "";
502 
503         if ( keySerializerBytes != null )
504         {
505             keySerializerFqcn = Strings.utf8ToString( keySerializerBytes );
506         }
507 
508         BTreeFactory.setKeySerializer( btree, keySerializerFqcn );
509 
510         // The valueSerialier FQCN
511         ByteBuffer valueSerializerBytes = readBytes( pageIos, dataPos );
512 
513         String valueSerializerFqcn = "";
514         dataPos += INT_SIZE + valueSerializerBytes.limit();
515 
516         if ( valueSerializerBytes != null )
517         {
518             valueSerializerFqcn = Strings.utf8ToString( valueSerializerBytes );
519         }
520 
521         BTreeFactory.setValueSerializer( btree, valueSerializerFqcn );
522 
523         // The BTree allowDuplicates flag
524         int allowDuplicates = readInt( pageIos, dataPos );
525         btree.setAllowDuplicates( allowDuplicates != 0 );
526         dataPos += INT_SIZE;
527 
528         // Now, init the BTree
529         btree.init();
530 
531         // Now, load the rootPage, which can be a Leaf or a Node, depending 
532         // on the number of elements in the tree : if it's above the pageSize,
533         // it's a Node, otherwise it's a Leaf
534 
535         // Read the rootPage pages on disk
536         PageIO[] rootPageIos = readPageIOs( rootPageOffset, Long.MAX_VALUE );
537 
538         Page<K, V> btreeRoot = readPage( btree, rootPageIos );
539         BTreeFactory.setRecordManager( btree, this );
540 
541         BTreeFactory.setRoot( btree, btreeRoot );
542     }
543 
544 
545     private <K, V> Page<K, V> readNode( BTree<K, V> btree, long offset, long revision, int nbElems ) throws IOException
546     {
547         Page<K, V> node = BTreeFactory.createNode( btree, revision, nbElems );
548 
549         // Read the rootPage pages on disk
550         PageIO[] pageIos = readPageIOs( offset, Long.MAX_VALUE );
551 
552         return node;
553     }
554 
555 
556     public <K, V> Page<K, V> deserialize( BTree<K, V> btree, long offset ) throws EndOfFileExceededException,
557         IOException
558     {
559         PageIO[] rootPageIos = readPageIOs( offset, Long.MAX_VALUE );
560 
561         Page<K, V> page = readPage( btree, rootPageIos );
562 
563         ( ( AbstractPage<K, V> ) page ).setOffset( rootPageIos[0].getOffset() );
564         ( ( AbstractPage<K, V> ) page ).setLastOffset( rootPageIos[rootPageIos.length - 1].getOffset() );
565 
566         return page;
567     }
568 
569 
570     private <K, V> Page<K, V> readPage( BTree<K, V> btree, PageIO[] pageIos ) throws IOException
571     {
572         // Deserialize the rootPage now
573         long position = 0L;
574 
575         // The revision
576         long revision = readLong( pageIos, position );
577         position += LONG_SIZE;
578 
579         // The number of elements in the page
580         int nbElems = readInt( pageIos, position );
581         position += INT_SIZE;
582 
583         // The size of the data containing the keys and values
584         Page<K, V> page = null;
585 
586         // Reads the bytes containing all the keys and values, if we have some
587         // We read  big blog of data into  ByteBuffer, then we will process
588         // this ByteBuffer
589         ByteBuffer byteBuffer = readBytes( pageIos, position );
590 
591         // Now, deserialize the data block. If the number of elements
592         // is positive, it's a Leaf, otherwise it's a Node
593         // Note that only a leaf can have 0 elements, and it's the root page then.
594         if ( nbElems >= 0 )
595         {
596             // It's a leaf
597             page = readLeaf( btree, nbElems, revision, byteBuffer, pageIos );
598         }
599         else
600         {
601             // It's a node
602             page = readNode( btree, -nbElems, revision, byteBuffer, pageIos );
603         }
604 
605         return page;
606     }
607 
608 
609     /**
610      * Deserialize a Leaf from some PageIOs
611      */
612     private <K, V> Leaf<K, V> readLeaf( BTree<K, V> btree, int nbElems, long revision, ByteBuffer byteBuffer,
613         PageIO[] pageIos )
614     {
615         // Its a leaf, create it
616         Leaf<K, V> leaf = BTreeFactory.createLeaf( btree, revision, nbElems );
617 
618         // Store the page offset on disk
619         leaf.setOffset( pageIos[0].getOffset() );
620         leaf.setLastOffset( pageIos[pageIos.length - 1].getOffset() );
621 
622         int[] keyLengths = new int[nbElems];
623         int[] valueLengths = new int[nbElems];
624 
625         // Read each key and value
626         for ( int i = 0; i < nbElems; i++ )
627         {
628             // Read the number of values
629             int nbValues = byteBuffer.getInt();
630             ValueHolder<V> valueHolder = null;
631 
632             if ( nbValues < 0 )
633             {
634                 // This is a sub-btree
635                 long btreeOffset = byteBuffer.getLong();
636 
637                 // Load the BTree now.
638                 try
639                 {
640                     PageIO[] rootPageIos = readPageIOs( btreeOffset, Long.MAX_VALUE );
641 
642                     BTree<V, V> subBtree = BTreeFactory.createBTree();
643                     subBtree.setBtreeOffset( btreeOffset );
644 
645                     try
646                     {
647                         loadBTree( rootPageIos, subBtree );
648                     }
649                     catch ( Exception e )
650                     {
651                         // should not happen
652                         throw new RuntimeException( e );
653                     }
654 
655                     valueHolder = new ValueHolder<V>( this, btree.getValueSerializer(), subBtree );
656 
657                     valueHolder.setSubBtree( subBtree );
658                 }
659                 catch ( EndOfFileExceededException e1 )
660                 {
661                     // TODO Auto-generated catch block
662                     e1.printStackTrace();
663                 }
664                 catch ( IOException e1 )
665                 {
666                     // TODO Auto-generated catch block
667                     e1.printStackTrace();
668                 }
669             }
670             else
671             {
672                 // This is an array
673                 // Read the value's array length
674                 valueLengths[i] = byteBuffer.getInt();
675 
676                 // This is an Array of values, read the byte[] associated with it
677                 byte[] valueBytes = new byte[valueLengths[i]];
678                 byteBuffer.get( valueBytes );
679                 valueHolder = new ValueHolder<V>( this, btree.getValueSerializer(), false, nbValues,
680                     valueBytes );
681             }
682 
683             BTreeFactory.setValue( leaf, i, valueHolder );
684 
685             keyLengths[i] = byteBuffer.getInt();
686             byte[] data = new byte[keyLengths[i]];
687             byteBuffer.get( data );
688             BTreeFactory.setKey( leaf, i, data );
689         }
690 
691         return leaf;
692     }
693 
694 
695     /**
696      * Deserialize a Node from some PageIos
697      */
698     private <K, V> Node<K, V> readNode( BTree<K, V> btree, int nbElems, long revision, ByteBuffer byteBuffer,
699         PageIO[] pageIos ) throws IOException
700     {
701         Node<K, V> node = BTreeFactory.createNode( btree, revision, nbElems );
702 
703         // Read each value and key
704         for ( int i = 0; i < nbElems; i++ )
705         {
706             // This is an Offset
707             long offset = OFFSET_SERIALIZER.deserialize( byteBuffer );
708             long lastOffset = OFFSET_SERIALIZER.deserialize( byteBuffer );
709 
710             PageHolder<K, V> valueHolder = new PageHolder<K, V>( btree, null, offset, lastOffset );
711             node.setValue( i, valueHolder );
712 
713             K key = btree.getKeySerializer().deserialize( byteBuffer );
714             BTreeFactory.setKey( node, i, key );
715         }
716 
717         // and read the last value, as it's a node
718         long offset = OFFSET_SERIALIZER.deserialize( byteBuffer );
719         long lastOffset = OFFSET_SERIALIZER.deserialize( byteBuffer );
720 
721         PageHolder<K, V> valueHolder = new PageHolder<K, V>( btree, null, offset, lastOffset );
722         node.setValue( nbElems, valueHolder );
723 
724         return node;
725     }
726 
727 
728     /**
729      * Read a byte[] from pages.
730      * @param pageIos The pages we want to read the byte[] from
731      * @param position The position in the data stored in those pages
732      * @return The byte[] we have read
733      */
734     private ByteBuffer readBytes( PageIO[] pageIos, long position )
735     {
736         // Read the byte[] length first
737         int length = readInt( pageIos, position );
738         position += INT_SIZE;
739 
740         // Compute the page in which we will store the data given the 
741         // current position
742         int pageNb = computePageNb( position );
743 
744         // Compute the position in the current page
745         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
746 
747         ByteBuffer pageData = pageIos[pageNb].getData();
748         int remaining = pageData.capacity() - pagePos;
749 
750         if ( length == 0 )
751         {
752             // No bytes to read : return null;
753             return null;
754         }
755         else
756         {
757             ByteBuffer bytes = ByteBuffer.allocate( length );
758             int bytesPos = 0;
759 
760             while ( length > 0 )
761             {
762                 if ( length <= remaining )
763                 {
764                     pageData.mark();
765                     pageData.position( pagePos );
766                     int oldLimit = pageData.limit();
767                     pageData.limit( pagePos + length );
768                     bytes.put( pageData );
769                     pageData.limit( oldLimit );
770                     pageData.reset();
771                     bytes.rewind();
772 
773                     return bytes;
774                 }
775 
776                 pageData.mark();
777                 pageData.position( pagePos );
778                 int oldLimit = pageData.limit();
779                 pageData.limit( pagePos + remaining );
780                 bytes.put( pageData );
781                 pageData.limit( oldLimit );
782                 pageData.reset();
783                 pageNb++;
784                 pagePos = LINK_SIZE;
785                 bytesPos += remaining;
786                 pageData = pageIos[pageNb].getData();
787                 length -= remaining;
788                 remaining = pageData.capacity() - pagePos;
789             }
790 
791             bytes.rewind();
792 
793             return bytes;
794         }
795     }
796 
797 
798     /**
799      * Read an int from pages 
800      * @param pageIos The pages we want to read the int from
801      * @param position The position in the data stored in those pages
802      * @return The int we have read
803      */
804     private int readInt( PageIO[] pageIos, long position )
805     {
806         // Compute the page in which we will store the data given the 
807         // current position
808         int pageNb = computePageNb( position );
809 
810         // Compute the position in the current page
811         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
812 
813         ByteBuffer pageData = pageIos[pageNb].getData();
814         int remaining = pageData.capacity() - pagePos;
815         int value = 0;
816 
817         if ( remaining >= INT_SIZE )
818         {
819             value = pageData.getInt( pagePos );
820         }
821         else
822         {
823             value = 0;
824 
825             switch ( remaining )
826             {
827                 case 3:
828                     value += ( ( pageData.get( pagePos + 2 ) & 0x00FF ) << 8 );
829                     // Fallthrough !!!
830 
831                 case 2:
832                     value += ( ( pageData.get( pagePos + 1 ) & 0x00FF ) << 16 );
833                     // Fallthrough !!!
834 
835                 case 1:
836                     value += ( pageData.get( pagePos ) << 24 );
837                     break;
838             }
839 
840             // Now deal with the next page
841             pageData = pageIos[pageNb + 1].getData();
842             pagePos = LINK_SIZE;
843 
844             switch ( remaining )
845             {
846                 case 1:
847                     value += ( pageData.get( pagePos ) & 0x00FF ) << 16;
848                     // fallthrough !!!
849 
850                 case 2:
851                     value += ( pageData.get( pagePos + 2 - remaining ) & 0x00FF ) << 8;
852                     // fallthrough !!!
853 
854                 case 3:
855                     value += ( pageData.get( pagePos + 3 - remaining ) & 0x00FF );
856                     break;
857             }
858         }
859 
860         return value;
861     }
862 
863 
864     /**
865      * Read a byte from pages 
866      * @param pageIos The pages we want to read the byte from
867      * @param position The position in the data stored in those pages
868      * @return The byte we have read
869      */
870     private byte readByte( PageIO[] pageIos, long position )
871     {
872         // Compute the page in which we will store the data given the 
873         // current position
874         int pageNb = computePageNb( position );
875 
876         // Compute the position in the current page
877         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
878 
879         ByteBuffer pageData = pageIos[pageNb].getData();
880         byte value = 0;
881 
882         value = pageData.get( pagePos );
883 
884         return value;
885     }
886 
887 
888     /**
889      * Read a long from pages 
890      * @param pageIos The pages we want to read the long from
891      * @param position The position in the data stored in those pages
892      * @return The long we have read
893      */
894     private long readLong( PageIO[] pageIos, long position )
895     {
896         // Compute the page in which we will store the data given the 
897         // current position
898         int pageNb = computePageNb( position );
899 
900         // Compute the position in the current page
901         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
902 
903         ByteBuffer pageData = pageIos[pageNb].getData();
904         int remaining = pageData.capacity() - pagePos;
905         long value = 0L;
906 
907         if ( remaining >= LONG_SIZE )
908         {
909             value = pageData.getLong( pagePos );
910         }
911         else
912         {
913             switch ( remaining )
914             {
915                 case 7:
916                     value += ( ( ( long ) pageData.get( pagePos + 6 ) & 0x00FF ) << 8 );
917                     // Fallthrough !!!
918 
919                 case 6:
920                     value += ( ( ( long ) pageData.get( pagePos + 5 ) & 0x00FF ) << 16 );
921                     // Fallthrough !!!
922 
923                 case 5:
924                     value += ( ( ( long ) pageData.get( pagePos + 4 ) & 0x00FF ) << 24 );
925                     // Fallthrough !!!
926 
927                 case 4:
928                     value += ( ( ( long ) pageData.get( pagePos + 3 ) & 0x00FF ) << 32 );
929                     // Fallthrough !!!
930 
931                 case 3:
932                     value += ( ( ( long ) pageData.get( pagePos + 2 ) & 0x00FF ) << 40 );
933                     // Fallthrough !!!
934 
935                 case 2:
936                     value += ( ( ( long ) pageData.get( pagePos + 1 ) & 0x00FF ) << 48 );
937                     // Fallthrough !!!
938 
939                 case 1:
940                     value += ( ( long ) pageData.get( pagePos ) << 56 );
941                     break;
942             }
943 
944             // Now deal with the next page
945             pageData = pageIos[pageNb + 1].getData();
946             pagePos = LINK_SIZE;
947 
948             switch ( remaining )
949             {
950                 case 1:
951                     value += ( ( long ) pageData.get( pagePos ) & 0x00FF ) << 48;
952                     // fallthrough !!!
953 
954                 case 2:
955                     value += ( ( long ) pageData.get( pagePos + 2 - remaining ) & 0x00FF ) << 40;
956                     // fallthrough !!!
957 
958                 case 3:
959                     value += ( ( long ) pageData.get( pagePos + 3 - remaining ) & 0x00FF ) << 32;
960                     // fallthrough !!!
961 
962                 case 4:
963                     value += ( ( long ) pageData.get( pagePos + 4 - remaining ) & 0x00FF ) << 24;
964                     // fallthrough !!!
965 
966                 case 5:
967                     value += ( ( long ) pageData.get( pagePos + 5 - remaining ) & 0x00FF ) << 16;
968                     // fallthrough !!!
969 
970                 case 6:
971                     value += ( ( long ) pageData.get( pagePos + 6 - remaining ) & 0x00FF ) << 8;
972                     // fallthrough !!!
973 
974                 case 7:
975                     value += ( ( long ) pageData.get( pagePos + 7 - remaining ) & 0x00FF );
976                     break;
977             }
978         }
979 
980         return value;
981     }
982 
983 
984     /**
985      * Manage a BTree. The btree will be added and managed by this RecordManager. We will create a 
986      * new RootPage for this added BTree, which will contain no data. 
987      *  
988      * @param btree The new BTree to manage.
989      */
990     public synchronized <K, V> void manage( BTree<K, V> btree ) throws BTreeAlreadyManagedException, IOException
991     {
992         manage( ( BTree<Object, Object> ) btree, false );
993     }
994 
995 
996     /**
997      * works the same as @see #manage(BTree) except the given tree will not be linked to top level trees that will be
998      * loaded initially if the internalTree flag is set to true
999      * 
1000      * @param btree The new BTree to manage.
1001      * @param internalTree flag indicating if this is an internal tree
1002      * 
1003      * @throws BTreeAlreadyManagedException
1004      * @throws IOException
1005      */
1006     public synchronized <K, V> void manage( BTree<K, V> btree, boolean internalTree )
1007         throws BTreeAlreadyManagedException,
1008         IOException
1009     {
1010         LOG.debug( "Managing the btree {} which is an internam tree : {}", btree.getName(), internalTree );
1011         BTreeFactory.setRecordManager( btree, this );
1012 
1013         String name = btree.getName();
1014 
1015         if ( managedBTrees.containsKey( name ) )
1016         {
1017             // There is already a BTree with this name in the recordManager...
1018             LOG.error( "There is already a BTree named '{}' managed by this recordManager", name );
1019             throw new BTreeAlreadyManagedException( name );
1020         }
1021 
1022         managedBTrees.put( name, ( BTree<Object, Object> ) btree );
1023 
1024         // We will add the newly managed BTree at the end of the header.
1025         byte[] btreeNameBytes = Strings.getBytesUtf8( name );
1026         byte[] keySerializerBytes = Strings.getBytesUtf8( btree.getKeySerializerFQCN() );
1027         byte[] valueSerializerBytes = Strings.getBytesUtf8( btree.getValueSerializerFQCN() );
1028 
1029         int bufferSize =
1030             INT_SIZE + // The name size
1031                 btreeNameBytes.length + // The name
1032                 INT_SIZE + // The keySerializerBytes size 
1033                 keySerializerBytes.length + // The keySerializerBytes
1034                 INT_SIZE + // The valueSerializerBytes size
1035                 valueSerializerBytes.length + // The valueSerializerBytes
1036                 INT_SIZE + // The page size
1037                 LONG_SIZE + // The revision
1038                 LONG_SIZE + // the number of element
1039                 LONG_SIZE + // the nextBtree offset
1040                 LONG_SIZE + // The root offset
1041                 INT_SIZE; // The allowDuplicates flag
1042 
1043         // Get the pageIOs we need to store the data. We may need more than one.
1044         PageIO[] pageIos = getFreePageIOs( bufferSize );
1045 
1046         // Store the BTree Offset into the BTree
1047         long btreeOffset = pageIos[0].getOffset();
1048         btree.setBtreeOffset( btreeOffset );
1049 
1050         // Now store the BTree data in the pages :
1051         // - the BTree revision
1052         // - the BTree number of elements
1053         // - The RootPage offset
1054         // - The next Btree offset 
1055         // - the BTree page size
1056         // - the BTree name
1057         // - the keySerializer FQCN
1058         // - the valueSerializer FQCN
1059         // - the flags that tell if the dups are allowed
1060         // Starts at 0
1061         long position = 0L;
1062 
1063         // The BTree current revision
1064         position = store( position, btree.getRevision(), pageIos );
1065 
1066         // The nb elems in the tree
1067         position = store( position, btree.getNbElems(), pageIos );
1068 
1069         // Serialize the BTree root page
1070         Page<K, V> rootPage = BTreeFactory.getRoot( btree );
1071 
1072         PageIO[] rootPageIos = serializePage( btree, btree.getRevision(), rootPage );
1073 
1074         // Get the reference on the first page
1075         PageIO rootPageIo = rootPageIos[0];
1076 
1077         // Now, we can inject the BTree rootPage offset into the BTree header
1078         position = store( position, rootPageIo.getOffset(), pageIos );
1079         btree.setRootPageOffset( rootPageIo.getOffset() );
1080         ( ( Leaf<K, V> ) rootPage ).setOffset( rootPageIo.getOffset() );
1081 
1082         // The next BTree Header offset (-1L, as it's a new BTree)
1083         position = store( position, NO_PAGE, pageIos );
1084 
1085         // The BTree page size
1086         position = store( position, btree.getPageSize(), pageIos );
1087 
1088         // The tree name
1089         position = store( position, btreeNameBytes, pageIos );
1090 
1091         // The keySerializer FQCN
1092         position = store( position, keySerializerBytes, pageIos );
1093 
1094         // The valueSerialier FQCN
1095         position = store( position, valueSerializerBytes, pageIos );
1096 
1097         // The allowDuplicates flag
1098         position = store( position, ( btree.isAllowDuplicates() ? 1 : 0 ), pageIos );
1099 
1100         // And flush the pages to disk now
1101         LOG.debug( "Flushing the newly managed '{}' btree header", btree.getName() );
1102         flushPages( pageIos );
1103         LOG.debug( "Flushing the newly managed '{}' btree rootpage", btree.getName() );
1104         flushPages( rootPageIos );
1105 
1106         // Now, if this added BTree is not the first BTree, we have to link it with the 
1107         // latest added BTree
1108         if ( !internalTree )
1109         {
1110             nbBtree++;
1111 
1112             if ( lastAddedBTreeOffset != NO_PAGE )
1113             {
1114                 // We have to update the nextBtreeOffset from the previous BTreeHeader
1115                 pageIos = readPageIOs( lastAddedBTreeOffset, LONG_SIZE + LONG_SIZE + LONG_SIZE + LONG_SIZE );
1116                 store( LONG_SIZE + LONG_SIZE + LONG_SIZE, btreeOffset, pageIos );
1117 
1118                 // Write the pages on disk
1119                 LOG.debug( "Updated the previous btree pointer on the added BTree {}", btree.getName() );
1120                 flushPages( pageIos );
1121             }
1122 
1123             lastAddedBTreeOffset = btreeOffset;
1124 
1125             // Last, not least, update the number of managed BTrees in the header
1126             updateRecordManagerHeader();
1127         }
1128 
1129         if ( LOG_CHECK.isDebugEnabled() )
1130         {
1131             check();
1132         }
1133     }
1134 
1135 
1136     /**
1137      * Serialize a new Page. It will contain the following data :<br/>
1138      * <ul>
1139      * <li>the revision : a long</li>
1140      * <li>the number of elements : an int (if <= 0, it's a Node, otherwise it's a Leaf)</li>
1141      * <li>the size of the values/keys when serialized
1142      * <li>the keys : an array of serialized keys</li>
1143      * <li>the values : an array of references to the children pageIO offset (stored as long)
1144      * if it's a Node, or a list of values if it's a Leaf</li>
1145      * <li></li>
1146      * </ul>
1147      * 
1148      * @param revision The node revision
1149      * @param keys The keys to serialize
1150      * @param children The references to the children
1151      * @return An array of pages containing the serialized node
1152      * @throws IOException
1153      */
1154     private <K, V> PageIO[] serializePage( BTree<K, V> btree, long revision, Page<K, V> page ) throws IOException
1155     {
1156         int nbElems = page.getNbElems();
1157 
1158         if ( nbElems == 0 )
1159         {
1160             return serializeRootPage( revision );
1161         }
1162         else
1163         {
1164             // Prepare a list of byte[] that will contain the serialized page
1165             int nbBuffers = 1 + 1 + 1 + nbElems * 2;
1166             int dataSize = 0;
1167             int serializedSize = 0;
1168 
1169             if ( page instanceof Node )
1170             {
1171                 // A Node has one more value to store
1172                 nbBuffers++;
1173             }
1174 
1175             // Now, we can create the list with the right size
1176             List<byte[]> serializedData = new ArrayList<byte[]>( nbBuffers );
1177 
1178             // The revision
1179             byte[] buffer = LongSerializer.serialize( revision );
1180             serializedData.add( buffer );
1181             serializedSize += buffer.length;
1182 
1183             // The number of elements
1184             // Make it a negative value if it's a Node
1185             int pageNbElems = nbElems;
1186 
1187             if ( page instanceof Node )
1188             {
1189                 pageNbElems = -nbElems;
1190             }
1191 
1192             buffer = IntSerializer.serialize( pageNbElems );
1193             serializedData.add( buffer );
1194             serializedSize += buffer.length;
1195 
1196             // Iterate on the keys and values. We first serialize the value, then the key
1197             // until we are done with all of them. If we are serializing a page, we have
1198             // to serialize one more value
1199             for ( int pos = 0; pos < nbElems; pos++ )
1200             {
1201                 // Start with the value
1202                 if ( page instanceof Node )
1203                 {
1204                     dataSize += serializeNodeValue( ( Node<K, V> ) page, pos, serializedData );
1205                     dataSize += serializeNodeKey( ( Node<K, V> ) page, pos, serializedData );
1206                 }
1207                 else
1208                 {
1209                     dataSize += serializeLeafValue( ( Leaf<K, V> ) page, pos, serializedData );
1210                     dataSize += serializeLeafKey( ( Leaf<K, V> ) page, pos, serializedData );
1211                 }
1212             }
1213 
1214             // Nodes have one more value to serialize
1215             if ( page instanceof Node )
1216             {
1217                 dataSize += serializeNodeValue( ( Node<K, V> ) page, nbElems, serializedData );
1218             }
1219 
1220             // Store the data size
1221             buffer = IntSerializer.serialize( dataSize );
1222             serializedData.add( 2, buffer );
1223             serializedSize += buffer.length;
1224 
1225             serializedSize += dataSize;
1226 
1227             // We are done. Allocate the pages we need to store the data
1228             PageIO[] pageIos = getFreePageIOs( serializedSize );
1229 
1230             // And store the data into those pages
1231             long position = 0L;
1232 
1233             for ( byte[] bytes : serializedData )
1234             {
1235                 position = storeRaw( position, bytes, pageIos );
1236             }
1237 
1238             return pageIos;
1239         }
1240     }
1241 
1242 
1243     /**
1244      * Serialize a Node's key
1245      */
1246     private <K, V> int serializeNodeKey( Node<K, V> node, int pos, List<byte[]> serializedData )
1247     {
1248         byte[] buffer = node.btree.getKeySerializer().serialize( node.getKey( pos ) );
1249         serializedData.add( buffer );
1250 
1251         return buffer.length;
1252     }
1253 
1254 
1255     /**
1256      * Serialize a Node's Value. We store the two offsets of the child page.
1257      */
1258     private <K, V> int serializeNodeValue( Node<K, V> node, int pos, List<byte[]> serializedData )
1259         throws IOException
1260     {
1261         // For a node, we just store the children's offsets
1262         Page<K, V> child = node.getReference( pos );
1263 
1264         // The first offset
1265         byte[] buffer = LongSerializer.serialize( child.getOffset() );
1266         serializedData.add( buffer );
1267         int dataSize = buffer.length;
1268 
1269         // The last offset
1270         buffer = LongSerializer.serialize( child.getLastOffset() );
1271         serializedData.add( buffer );
1272         dataSize += buffer.length;
1273 
1274         return dataSize;
1275     }
1276 
1277 
1278     /**
1279      * Serialize a Leaf's key
1280      */
1281     private <K, V> int serializeLeafKey( Leaf<K, V> leaf, int pos, List<byte[]> serializedData )
1282     {
1283         int dataSize = 0;
1284         KeyHolder<K> keyHolder = leaf.getKeyHolder( pos );
1285         byte[] keyData = keyHolder.getBuffer();
1286 
1287         if ( keyData != null )
1288         {
1289             byte[] data = new byte[keyData.length];
1290 
1291             // The key length
1292             byte[] buffer = IntSerializer.serialize( data.length );
1293             serializedData.add( buffer );
1294             dataSize += buffer.length;
1295 
1296             // The key data
1297             serializedData.add( keyData );
1298             dataSize += data.length;
1299         }
1300         else
1301         {
1302             serializedData.add( IntSerializer.serialize( 0 ) );
1303             dataSize += 4;
1304         }
1305 
1306         return dataSize;
1307     }
1308 
1309 
1310     /**
1311      * Serialize a Leaf's Value. We store 
1312      */
1313     private <K, V> int serializeLeafValue( Leaf<K, V> leaf, int pos, List<byte[]> serializedData )
1314         throws IOException
1315     {
1316         // The value can be an Array or a sub-btree, but we don't care
1317         // we just iterate on all the values
1318         ValueHolder<V> valueHolder = leaf.getValue( pos );
1319 
1320         // First take the number of values
1321         int nbValues = valueHolder.size();
1322         int dataSize = 0;
1323 
1324         if ( nbValues == 0 )
1325         {
1326             // No value. 
1327             byte[] buffer = IntSerializer.serialize( nbValues );
1328             serializedData.add( buffer );
1329 
1330             return buffer.length;
1331         }
1332 
1333         if ( valueHolder.isSubBtree() )
1334         {
1335             byte[] buffer = IntSerializer.serialize( -nbValues );
1336             serializedData.add( buffer );
1337             dataSize += buffer.length;
1338 
1339             // the BTree offset
1340             buffer = LongSerializer.serialize( valueHolder.getOffset() );
1341             serializedData.add( buffer );
1342             dataSize += buffer.length;
1343         }
1344         else
1345         {
1346             // This is an array, store the nb of values as a positive number
1347             byte[] buffer = IntSerializer.serialize( nbValues );
1348             serializedData.add( buffer );
1349             dataSize += buffer.length;
1350 
1351             // Now store each value
1352             byte[] data = valueHolder.getRaw();
1353             buffer = IntSerializer.serialize( data.length );
1354             serializedData.add( buffer );
1355             dataSize += buffer.length;
1356 
1357             if ( data.length > 0 )
1358             {
1359                 serializedData.add( data );
1360             }
1361 
1362             dataSize += data.length;
1363         }
1364 
1365         return dataSize;
1366     }
1367 
1368 
1369     /**
1370      * Write a root page with no elements in it
1371      */
1372     private PageIO[] serializeRootPage( long revision ) throws IOException
1373     {
1374         // We will have 1 single page if we have no elements
1375         PageIO[] pageIos = new PageIO[1];
1376 
1377         // This is either a new root page or a new page that will be filled later
1378         PageIO newPage = fetchNewPage();
1379 
1380         // We need first to create a byte[] that will contain all the data
1381         // For the root page, this is easy, as we only have to store the revision, 
1382         // and the number of elements, which is 0.
1383         long position = 0L;
1384 
1385         position = store( position, revision, newPage );
1386         position = store( position, 0, newPage );
1387 
1388         // Update the page size now
1389         newPage.setSize( ( int ) position );
1390 
1391         // Insert the result into the array of PageIO
1392         pageIos[0] = newPage;
1393 
1394         return pageIos;
1395     }
1396 
1397 
1398     /**
1399      * Update the header, injecting the nbBtree, firstFreePage and lastFreePage
1400      */
1401     private void updateRecordManagerHeader() throws IOException
1402     {
1403         // The page size
1404         HEADER_BYTES[0] = ( byte ) ( pageSize >>> 24 );
1405         HEADER_BYTES[1] = ( byte ) ( pageSize >>> 16 );
1406         HEADER_BYTES[2] = ( byte ) ( pageSize >>> 8 );
1407         HEADER_BYTES[3] = ( byte ) ( pageSize );
1408 
1409         // The number of managed BTree (currently we have only one : the discardedPage BTree
1410         HEADER_BYTES[4] = ( byte ) ( nbBtree >>> 24 );
1411         HEADER_BYTES[5] = ( byte ) ( nbBtree >>> 16 );
1412         HEADER_BYTES[6] = ( byte ) ( nbBtree >>> 8 );
1413         HEADER_BYTES[7] = ( byte ) ( nbBtree );
1414 
1415         // The first free page
1416         HEADER_BYTES[8] = ( byte ) ( firstFreePage >>> 56 );
1417         HEADER_BYTES[9] = ( byte ) ( firstFreePage >>> 48 );
1418         HEADER_BYTES[10] = ( byte ) ( firstFreePage >>> 40 );
1419         HEADER_BYTES[11] = ( byte ) ( firstFreePage >>> 32 );
1420         HEADER_BYTES[12] = ( byte ) ( firstFreePage >>> 24 );
1421         HEADER_BYTES[13] = ( byte ) ( firstFreePage >>> 16 );
1422         HEADER_BYTES[14] = ( byte ) ( firstFreePage >>> 8 );
1423         HEADER_BYTES[15] = ( byte ) ( firstFreePage );
1424 
1425         // The last free page
1426         HEADER_BYTES[16] = ( byte ) ( lastFreePage >>> 56 );
1427         HEADER_BYTES[17] = ( byte ) ( lastFreePage >>> 48 );
1428         HEADER_BYTES[18] = ( byte ) ( lastFreePage >>> 40 );
1429         HEADER_BYTES[19] = ( byte ) ( lastFreePage >>> 32 );
1430         HEADER_BYTES[20] = ( byte ) ( lastFreePage >>> 24 );
1431         HEADER_BYTES[21] = ( byte ) ( lastFreePage >>> 16 );
1432         HEADER_BYTES[22] = ( byte ) ( lastFreePage >>> 8 );
1433         HEADER_BYTES[23] = ( byte ) ( lastFreePage );
1434 
1435         // Write the header on disk
1436         HEADER_BUFFER.put( HEADER_BYTES );
1437         HEADER_BUFFER.flip();
1438 
1439         LOG.debug( "Update RM header, FF : {}, LF : {}", firstFreePage, lastFreePage );
1440         fileChannel.write( HEADER_BUFFER, 0 );
1441         HEADER_BUFFER.clear();
1442 
1443         nbUpdateRMHeader.incrementAndGet();
1444     }
1445 
1446 
1447     /**
1448      * Update the BTree header after a BTree modification. We update the following fields :
1449      * <ul>
1450      * <li>the revision</li>
1451      * <li>the number of elements</li>
1452      * <li>the rootPage offset</li>
1453      * </ul>
1454      * @param btree
1455      * @throws IOException 
1456      * @throws EndOfFileExceededException 
1457      */
1458     /* No qualifier*/<K, V> void updateBtreeHeader( BTree<K, V> btree, long rootPageOffset )
1459         throws EndOfFileExceededException,
1460         IOException
1461     {
1462         // Read the pageIOs associated with this BTree
1463         long offset = btree.getBtreeOffset();
1464         long headerSize = LONG_SIZE + LONG_SIZE + LONG_SIZE;
1465 
1466         PageIO[] pageIos = readPageIOs( offset, headerSize );
1467 
1468         // Now, update the revision
1469         long position = 0;
1470 
1471         position = store( position, btree.getRevision(), pageIos );
1472         position = store( position, btree.getNbElems(), pageIos );
1473         position = store( position, rootPageOffset, pageIos );
1474 
1475         // Write the pages on disk
1476         if ( LOG.isDebugEnabled() )
1477         {
1478             LOG.debug( "-----> Flushing the '{}' BTreeHeader", btree.getName() );
1479             LOG.debug( "  revision : " + btree.getRevision() + ", NbElems : " + btree.getNbElems() + ", root offset : "
1480                 + rootPageOffset );
1481         }
1482 
1483         flushPages( pageIos );
1484 
1485         nbUpdateBTreeHeader.incrementAndGet();
1486 
1487         if ( LOG_CHECK.isDebugEnabled() )
1488         {
1489             check();
1490         }
1491     }
1492 
1493 
1494     /**
1495      * Write the pages in the disk, either at the end of the file, or at
1496      * the position they were taken from.
1497      * 
1498      * @param pageIos The list of pages to write
1499      * @throws IOException If the write failed
1500      */
1501     private void flushPages( PageIO... pageIos ) throws IOException
1502     {
1503         for ( PageIO pageIo : pageIos )
1504         {
1505             pageIo.getData().rewind();
1506 
1507             if ( fileChannel.size() < ( pageIo.getOffset() + pageSize ) )
1508             {
1509                 LOG.debug( "Adding a page at the end of the file" );
1510                 // This is a page we have to add to the file
1511                 fileChannel.write( pageIo.getData(), fileChannel.size() );
1512                 //fileChannel.force( false );
1513             }
1514             else
1515             {
1516                 LOG.debug( "Writing a page at position {}", pageIo.getOffset() );
1517                 fileChannel.write( pageIo.getData(), pageIo.getOffset() );
1518                 //fileChannel.force( false );
1519             }
1520 
1521             nbUpdatePageIOs.incrementAndGet();
1522 
1523             pageIo.getData().rewind();
1524         }
1525     }
1526 
1527 
1528     /**
1529      * Compute the page in which we will store data given an offset, when 
1530      * we have a list of pages.
1531      * 
1532      * @param offset The position in the data
1533      * @return The page number in which the offset will start
1534      */
1535     private int computePageNb( long offset )
1536     {
1537         long pageNb = 0;
1538 
1539         offset -= pageSize - LINK_SIZE - PAGE_SIZE;
1540 
1541         if ( offset < 0 )
1542         {
1543             return ( int ) pageNb;
1544         }
1545 
1546         pageNb = 1 + offset / ( pageSize - LINK_SIZE );
1547 
1548         return ( int ) pageNb;
1549     }
1550 
1551 
1552     /**
1553      * Stores a byte[] into one ore more pageIO (depending if the long is stored
1554      * across a boundary or not)
1555      * 
1556      * @param position The position in a virtual byte[] if all the pages were contiguous
1557      * @param bytes The byte[] to serialize
1558      * @param pageIos The pageIOs we have to store the data in
1559      * @return The new position
1560      */
1561     private long store( long position, byte[] bytes, PageIO... pageIos )
1562     {
1563         if ( bytes != null )
1564         {
1565             // Write the bytes length
1566             position = store( position, bytes.length, pageIos );
1567 
1568             // Compute the page in which we will store the data given the 
1569             // current position
1570             int pageNb = computePageNb( position );
1571 
1572             // Get back the buffer in this page
1573             ByteBuffer pageData = pageIos[pageNb].getData();
1574 
1575             // Compute the position in the current page
1576             int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1577 
1578             // Compute the remaining size in the page
1579             int remaining = pageData.capacity() - pagePos;
1580             int nbStored = bytes.length;
1581 
1582             // And now, write the bytes until we have none
1583             while ( nbStored > 0 )
1584             {
1585                 if ( remaining > nbStored )
1586                 {
1587                     pageData.mark();
1588                     pageData.position( pagePos );
1589                     pageData.put( bytes, bytes.length - nbStored, nbStored );
1590                     pageData.reset();
1591                     nbStored = 0;
1592                 }
1593                 else
1594                 {
1595                     pageData.mark();
1596                     pageData.position( pagePos );
1597                     pageData.put( bytes, bytes.length - nbStored, remaining );
1598                     pageData.reset();
1599                     pageNb++;
1600                     pageData = pageIos[pageNb].getData();
1601                     pagePos = LINK_SIZE;
1602                     nbStored -= remaining;
1603                     remaining = pageData.capacity() - pagePos;
1604                 }
1605             }
1606 
1607             // We are done
1608             position += bytes.length;
1609         }
1610         else
1611         {
1612             // No bytes : write 0 and return
1613             position = store( position, 0, pageIos );
1614         }
1615 
1616         return position;
1617     }
1618 
1619 
1620     /**
1621      * Stores a byte[] into one ore more pageIO (depending if the long is stored
1622      * across a boundary or not). We don't add the byte[] size, it's already present
1623      * in the received byte[].
1624      * 
1625      * @param position The position in a virtual byte[] if all the pages were contiguous
1626      * @param bytes The byte[] to serialize
1627      * @param pageIos The pageIOs we have to store the data in
1628      * @return The new position
1629      */
1630     private long storeRaw( long position, byte[] bytes, PageIO... pageIos )
1631     {
1632         if ( bytes != null )
1633         {
1634             // Compute the page in which we will store the data given the 
1635             // current position
1636             int pageNb = computePageNb( position );
1637 
1638             // Get back the buffer in this page
1639             ByteBuffer pageData = pageIos[pageNb].getData();
1640 
1641             // Compute the position in the current page
1642             int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1643 
1644             // Compute the remaining size in the page
1645             int remaining = pageData.capacity() - pagePos;
1646             int nbStored = bytes.length;
1647 
1648             // And now, write the bytes until we have none
1649             while ( nbStored > 0 )
1650             {
1651                 if ( remaining > nbStored )
1652                 {
1653                     pageData.mark();
1654                     pageData.position( pagePos );
1655                     pageData.put( bytes, bytes.length - nbStored, nbStored );
1656                     pageData.reset();
1657                     nbStored = 0;
1658                 }
1659                 else
1660                 {
1661                     pageData.mark();
1662                     pageData.position( pagePos );
1663                     pageData.put( bytes, bytes.length - nbStored, remaining );
1664                     pageData.reset();
1665                     pageNb++;
1666 
1667                     if ( pageNb == pageIos.length )
1668                     {
1669                         // We can stop here : we have reach the end of the page
1670                         break;
1671                     }
1672 
1673                     pageData = pageIos[pageNb].getData();
1674                     pagePos = LINK_SIZE;
1675                     nbStored -= remaining;
1676                     remaining = pageData.capacity() - pagePos;
1677                 }
1678             }
1679 
1680             // We are done
1681             position += bytes.length;
1682         }
1683         else
1684         {
1685             // No bytes : write 0 and return
1686             position = store( position, 0, pageIos );
1687         }
1688 
1689         return position;
1690     }
1691 
1692 
1693     /**
1694      * Stores an Integer into one ore more pageIO (depending if the int is stored
1695      * across a boundary or not)
1696      * 
1697      * @param position The position in a virtual byte[] if all the pages were contiguous
1698      * @param value The int to serialize
1699      * @param pageIos The pageIOs we have to store the data in
1700      * @return The new position
1701      */
1702     private long store( long position, int value, PageIO... pageIos )
1703     {
1704         // Compute the page in which we will store the data given the 
1705         // current position
1706         int pageNb = computePageNb( position );
1707 
1708         // Compute the position in the current page
1709         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1710 
1711         // Get back the buffer in this page
1712         ByteBuffer pageData = pageIos[pageNb].getData();
1713 
1714         // Compute the remaining size in the page
1715         int remaining = pageData.capacity() - pagePos;
1716 
1717         if ( remaining < INT_SIZE )
1718         {
1719             // We have to copy the serialized length on two pages
1720 
1721             switch ( remaining )
1722             {
1723                 case 3:
1724                     pageData.put( pagePos + 2, ( byte ) ( value >>> 8 ) );
1725                     // Fallthrough !!!
1726 
1727                 case 2:
1728                     pageData.put( pagePos + 1, ( byte ) ( value >>> 16 ) );
1729                     // Fallthrough !!!
1730 
1731                 case 1:
1732                     pageData.put( pagePos, ( byte ) ( value >>> 24 ) );
1733                     break;
1734             }
1735 
1736             // Now deal with the next page
1737             pageData = pageIos[pageNb + 1].getData();
1738             pagePos = LINK_SIZE;
1739 
1740             switch ( remaining )
1741             {
1742                 case 1:
1743                     pageData.put( pagePos, ( byte ) ( value >>> 16 ) );
1744                     // fallthrough !!!
1745 
1746                 case 2:
1747                     pageData.put( pagePos + 2 - remaining, ( byte ) ( value >>> 8 ) );
1748                     // fallthrough !!!
1749 
1750                 case 3:
1751                     pageData.put( pagePos + 3 - remaining, ( byte ) ( value ) );
1752                     break;
1753             }
1754         }
1755         else
1756         {
1757             // Store the value in the page at the selected position
1758             pageData.putInt( pagePos, value );
1759         }
1760 
1761         // Increment the position to reflect the addition of an Int (4 bytes)
1762         position += INT_SIZE;
1763 
1764         return position;
1765     }
1766 
1767 
1768     /**
1769      * Stores a Long into one ore more pageIO (depending if the long is stored
1770      * across a boundary or not)
1771      * 
1772      * @param position The position in a virtual byte[] if all the pages were contiguous
1773      * @param value The long to serialize
1774      * @param pageIos The pageIOs we have to store the data in
1775      * @return The new position
1776      */
1777     private long store( long position, long value, PageIO... pageIos )
1778     {
1779         // Compute the page in which we will store the data given the 
1780         // current position
1781         int pageNb = computePageNb( position );
1782 
1783         // Compute the position in the current page
1784         int pagePos = ( int ) ( position + ( pageNb + 1 ) * LONG_SIZE + INT_SIZE ) - pageNb * pageSize;
1785 
1786         // Get back the buffer in this page
1787         ByteBuffer pageData = pageIos[pageNb].getData();
1788 
1789         // Compute the remaining size in the page
1790         int remaining = pageData.capacity() - pagePos;
1791 
1792         if ( remaining < LONG_SIZE )
1793         {
1794             // We have to copy the serialized length on two pages
1795 
1796             switch ( remaining )
1797             {
1798                 case 7:
1799                     pageData.put( pagePos + 6, ( byte ) ( value >>> 8 ) );
1800                     // Fallthrough !!!
1801 
1802                 case 6:
1803                     pageData.put( pagePos + 5, ( byte ) ( value >>> 16 ) );
1804                     // Fallthrough !!!
1805 
1806                 case 5:
1807                     pageData.put( pagePos + 4, ( byte ) ( value >>> 24 ) );
1808                     // Fallthrough !!!
1809 
1810                 case 4:
1811                     pageData.put( pagePos + 3, ( byte ) ( value >>> 32 ) );
1812                     // Fallthrough !!!
1813 
1814                 case 3:
1815                     pageData.put( pagePos + 2, ( byte ) ( value >>> 40 ) );
1816                     // Fallthrough !!!
1817 
1818                 case 2:
1819                     pageData.put( pagePos + 1, ( byte ) ( value >>> 48 ) );
1820                     // Fallthrough !!!
1821 
1822                 case 1:
1823                     pageData.put( pagePos, ( byte ) ( value >>> 56 ) );
1824                     break;
1825             }
1826 
1827             // Now deal with the next page
1828             pageData = pageIos[pageNb + 1].getData();
1829             pagePos = LINK_SIZE;
1830 
1831             switch ( remaining )
1832             {
1833                 case 1:
1834                     pageData.put( pagePos, ( byte ) ( value >>> 48 ) );
1835                     // fallthrough !!!
1836 
1837                 case 2:
1838                     pageData.put( pagePos + 2 - remaining, ( byte ) ( value >>> 40 ) );
1839                     // fallthrough !!!
1840 
1841                 case 3:
1842                     pageData.put( pagePos + 3 - remaining, ( byte ) ( value >>> 32 ) );
1843                     // fallthrough !!!
1844 
1845                 case 4:
1846                     pageData.put( pagePos + 4 - remaining, ( byte ) ( value >>> 24 ) );
1847                     // fallthrough !!!
1848 
1849                 case 5:
1850                     pageData.put( pagePos + 5 - remaining, ( byte ) ( value >>> 16 ) );
1851                     // fallthrough !!!
1852 
1853                 case 6:
1854                     pageData.put( pagePos + 6 - remaining, ( byte ) ( value >>> 8 ) );
1855                     // fallthrough !!!
1856 
1857                 case 7:
1858                     pageData.put( pagePos + 7 - remaining, ( byte ) ( value ) );
1859                     break;
1860             }
1861         }
1862         else
1863         {
1864             // Store the value in the page at the selected position
1865             pageData.putLong( pagePos, value );
1866         }
1867 
1868         // Increment the position to reflect the addition of an Long (8 bytes)
1869         position += LONG_SIZE;
1870 
1871         return position;
1872     }
1873 
1874 
1875     /**
1876      * Stores a new page on disk. We will add the modified page into the tree of copied pages.
1877      * The new page is serialized and saved on disk.
1878      * 
1879      * @param oldPage
1880      * @param oldRevision
1881      * @param newPage
1882      * @param newRevision
1883      * @return The offset of the new page
1884      * @throws IOException 
1885      */
1886     /* No qualifier*/<K, V> ElementHolder<Page<K, V>, K, V> writePage( BTree<K, V> btree, Page<K, V> newPage,
1887         long newRevision )
1888         throws IOException
1889     {
1890         // We first need to save the new page on disk
1891         PageIO[] pageIos = serializePage( btree, newRevision, newPage );
1892 
1893         LOG.debug( "Write data for '{}' btree ", btree.getName() );
1894 
1895         // Write the page on disk
1896         flushPages( pageIos );
1897 
1898         // Build the resulting reference
1899         long offset = pageIos[0].getOffset();
1900         long lastOffset = pageIos[pageIos.length - 1].getOffset();
1901         ElementHolder<Page<K, V>, K, V> valueHolder = new PageHolder<K, V>( btree, newPage, offset,
1902             lastOffset );
1903 
1904         if ( LOG_CHECK.isDebugEnabled() )
1905         {
1906             check();
1907         }
1908 
1909         return valueHolder;
1910     }
1911 
1912 
1913     /**
1914      * Compute the number of pages needed to store some specific size of data.
1915      * 
1916      * @param dataSize The size of the data we want to store in pages
1917      * @return The number of pages needed
1918      */
1919     private int computeNbPages( int dataSize )
1920     {
1921         if ( dataSize <= 0 )
1922         {
1923             return 0;
1924         }
1925 
1926         // Compute the number of pages needed.
1927         // Considering that each page can contain PageSize bytes,
1928         // but that the first 8 bytes are used for links and we 
1929         // use 4 bytes to store the data size, the number of needed
1930         // pages is :
1931         // NbPages = ( (dataSize - (PageSize - 8 - 4 )) / (PageSize - 8) ) + 1 
1932         // NbPages += ( if (dataSize - (PageSize - 8 - 4 )) % (PageSize - 8) > 0 : 1 : 0 )
1933         int availableSize = ( pageSize - LONG_SIZE );
1934         int nbNeededPages = 1;
1935 
1936         // Compute the number of pages that will be full but the first page
1937         if ( dataSize > availableSize - INT_SIZE )
1938         {
1939             int remainingSize = dataSize - ( availableSize - INT_SIZE );
1940             nbNeededPages += remainingSize / availableSize;
1941             int remain = remainingSize % availableSize;
1942 
1943             if ( remain > 0 )
1944             {
1945                 nbNeededPages++;
1946             }
1947         }
1948 
1949         return nbNeededPages;
1950     }
1951 
1952 
1953     /**
1954      * Get as many pages as needed to store the data of the given size
1955      *  
1956      * @param dataSize The data size
1957      * @return An array of pages, enough to store the full data
1958      */
1959     private PageIO[] getFreePageIOs( int dataSize ) throws IOException
1960     {
1961         if ( dataSize == 0 )
1962         {
1963             return new PageIO[]
1964                 {};
1965         }
1966 
1967         int nbNeededPages = computeNbPages( dataSize );
1968 
1969         PageIO[] pageIOs = new PageIO[nbNeededPages];
1970 
1971         // The first page : set the size
1972         pageIOs[0] = fetchNewPage();
1973         pageIOs[0].setSize( dataSize );
1974 
1975         for ( int i = 1; i < nbNeededPages; i++ )
1976         {
1977             pageIOs[i] = fetchNewPage();
1978 
1979             // Create the link
1980             pageIOs[i - 1].setNextPage( pageIOs[i].getOffset() );
1981         }
1982 
1983         return pageIOs;
1984     }
1985 
1986 
1987     /**
1988      * Return a new Page. We take one of the existing free pages, or we create
1989      * a new page at the end of the file.
1990      * 
1991      * @return The fetched PageIO
1992      */
1993     private PageIO fetchNewPage() throws IOException
1994     {
1995         if ( firstFreePage == NO_PAGE )
1996         {
1997             nbCreatedPages.incrementAndGet();
1998 
1999             // We don't have any free page. Reclaim some new page at the end
2000             // of the file
2001             PageIO newPage = new PageIO( endOfFileOffset );
2002 
2003             endOfFileOffset += pageSize;
2004 
2005             ByteBuffer data = ByteBuffer.allocateDirect( pageSize );
2006 
2007             newPage.setData( data );
2008             newPage.setNextPage( NO_PAGE );
2009             newPage.setSize( 0 );
2010 
2011             LOG.debug( "Requiring a new page at offset {}", newPage.getOffset() );
2012 
2013             return newPage;
2014         }
2015         else
2016         {
2017             nbReusedPages.incrementAndGet();
2018 
2019             // We have some existing free page. Fetch it from disk
2020             PageIO pageIo = fetchPage( firstFreePage );
2021 
2022             // Update the firstFreePage pointer
2023             firstFreePage = pageIo.getNextPage();
2024 
2025             // overwrite the data of old page
2026             ByteBuffer data = ByteBuffer.allocateDirect( pageSize );
2027             pageIo.setData( data );
2028 
2029             pageIo.setNextPage( NO_PAGE );
2030             pageIo.setSize( 0 );
2031 
2032             LOG.debug( "Reused page at offset {}", pageIo.getOffset() );
2033 
2034             // If we don't have any more free page, update the last free page pointer too
2035             if ( firstFreePage == NO_PAGE )
2036             {
2037                 lastFreePage = NO_PAGE;
2038             }
2039 
2040             // Update the header
2041             updateRecordManagerHeader();
2042 
2043             return pageIo;
2044         }
2045     }
2046 
2047 
2048     /**
2049      * fetch a page from disk, knowing its position in the file.
2050      * 
2051      * @param offset The position in the file
2052      * @return The found page
2053      */
2054     private PageIO fetchPage( long offset ) throws IOException, EndOfFileExceededException
2055     {
2056         if ( fileChannel.size() < offset + pageSize )
2057         {
2058             // Error : we are past the end of the file
2059             throw new EndOfFileExceededException( "We are fetching a page on " + offset +
2060                 " when the file's size is " + fileChannel.size() );
2061         }
2062         else
2063         {
2064             // Read the page
2065             fileChannel.position( offset );
2066 
2067             ByteBuffer data = ByteBuffer.allocate( pageSize );
2068             fileChannel.read( data );
2069             data.rewind();
2070 
2071             PageIO readPage = new PageIO( offset );
2072             readPage.setData( data );
2073 
2074             return readPage;
2075         }
2076     }
2077 
2078 
2079     /**
2080      * @return the pageSize
2081      */
2082     public int getPageSize()
2083     {
2084         return pageSize;
2085     }
2086 
2087 
2088     public void setPageSize( int pageSize )
2089     {
2090         if ( this.pageSize != -1 )
2091         {
2092         }
2093         else
2094         {
2095             this.pageSize = pageSize;
2096         }
2097     }
2098 
2099 
2100     /**
2101      * Close the RecordManager and flush everything on disk
2102      */
2103     public void close() throws IOException
2104     {
2105         // TODO : we must wait for the last write to finish
2106 
2107         for ( BTree<Object, Object> tree : managedBTrees.values() )
2108         {
2109             tree.close();
2110         }
2111 
2112         managedBTrees.clear();
2113 
2114         // Write the data
2115         fileChannel.force( true );
2116 
2117         // And close the channel
2118         fileChannel.close();
2119     }
2120 
2121 
2122     /**
2123      * Dump the RecordManager file
2124      * @throws IOException 
2125      */
2126     public void dump() throws IOException
2127     {
2128         RandomAccessFile randomFile = new RandomAccessFile( file, "r" );
2129         FileChannel fileChannel = randomFile.getChannel();
2130 
2131         ByteBuffer header = ByteBuffer.allocate( HEADER_SIZE );
2132 
2133         // load the header 
2134         fileChannel.read( header );
2135 
2136         header.rewind();
2137 
2138         // The page size
2139         int pageSize = header.getInt();
2140 
2141         // The number of managed BTrees
2142         int nbBTree = header.getInt();
2143 
2144         // The first and last free page
2145         long firstFreePage = header.getLong();
2146         long lastFreePage = header.getLong();
2147 
2148         if ( LOG.isDebugEnabled() )
2149         {
2150             LOG.debug( "RecordManager" );
2151             LOG.debug( "-------------" );
2152             LOG.debug( "  Header " );
2153             LOG.debug( "    '{}'", Strings.dumpBytes( header.array() ) );
2154             LOG.debug( "    page size : {}", pageSize );
2155             LOG.debug( "    nbTree : {}", nbBTree );
2156             LOG.debug( "    firstFreePage : {}", firstFreePage );
2157             LOG.debug( "    lastFreePage : {}", lastFreePage );
2158         }
2159 
2160         long position = HEADER_SIZE;
2161 
2162         // Dump the BTrees
2163         for ( int i = 0; i < nbBTree; i++ )
2164         {
2165             LOG.debug( "  Btree[{}]", i );
2166             PageIO[] pageIos = readPageIOs( position, Long.MAX_VALUE );
2167 
2168             for ( PageIO pageIo : pageIos )
2169             {
2170                 LOG.debug( "    {}", pageIo );
2171             }
2172         }
2173 
2174         randomFile.close();
2175     }
2176 
2177 
2178     /**
2179      * Get the number of managed trees. We don't count the CopiedPage BTree. and the Revsion BTree
2180      * 
2181      * @return The number of managed BTrees
2182      */
2183     public int getNbManagedTrees()
2184     {
2185         return nbBtree - 2;
2186     }
2187 
2188 
2189     /**
2190      * Get the managed trees. We don't return the CopiedPage BTree nor the Revision BTree.
2191      * 
2192      * @return The managed BTrees
2193      */
2194     public Set<String> getManagedTrees()
2195     {
2196         Set<String> btrees = new HashSet<String>( managedBTrees.keySet() );
2197 
2198         btrees.remove( COPIED_PAGE_BTREE_NAME );
2199         btrees.remove( REVISION_BTREE_NAME );
2200 
2201         return btrees;
2202     }
2203 
2204 
2205     /**
2206      * Store a reference to an old rootPage into the Revision BTree
2207      * 
2208      * @param btree The BTree we want to keep an old RootPage for
2209      * @param rootPage The old rootPage
2210      * @throws IOException If we have an issue while writing on disk
2211      */
2212     /* No qualifier */<K, V> void storeRootPage( BTree<K, V> btree, Page<K, V> rootPage ) throws IOException
2213     {
2214         if ( !isKeepRevisions() )
2215         {
2216             return;
2217         }
2218 
2219         if ( ( btree == copiedPageBTree ) || ( btree == revisionBTree ) )
2220         {
2221             return;
2222         }
2223 
2224         RevisionName revisionName = new RevisionName( rootPage.getRevision(), btree.getName() );
2225 
2226         revisionBTree.insert( revisionName, rootPage.getOffset(), 0 );
2227 
2228         if ( LOG_CHECK.isDebugEnabled() )
2229         {
2230             check();
2231         }
2232     }
2233 
2234 
2235     /**
2236      * Fetch the rootPage of a given BTree for a given revision.
2237      * 
2238      * @param btree The BTree we are interested in
2239      * @param revision The revision we want to get back
2240      * @return The rootPage for this BTree and this revision, if any
2241      * @throws KeyNotFoundException If we can't find the rootPage for this revision and this BTree
2242      * @throws IOException If we had an ise while accessing the data on disk
2243      */
2244     /* No qualifier */<K, V> Page<K, V> getRootPage( BTree<K, V> btree, long revision ) throws KeyNotFoundException,
2245         IOException
2246     {
2247         if ( btree.getRevision() == revision )
2248         {
2249             // We are asking for the current revision
2250             return btree.rootPage;
2251         }
2252 
2253         RevisionName revisionName = new RevisionName( revision, btree.getName() );
2254         long rootPageOffset = revisionBTree.get( revisionName );
2255 
2256         // Read the rootPage pages on disk
2257         PageIO[] rootPageIos = readPageIOs( rootPageOffset, Long.MAX_VALUE );
2258 
2259         Page<K, V> btreeRoot = readPage( btree, rootPageIos );
2260 
2261         return btreeRoot;
2262     }
2263 
2264 
2265     /**
2266      * Get one managed trees, knowing its name. 
2267      * 
2268      * @return The managed BTrees
2269      */
2270     public <K, V> BTree<K, V> getManagedTree( String name )
2271     {
2272         return ( BTree<K, V> ) managedBTrees.get( name );
2273     }
2274 
2275 
2276     /**
2277      * Move a list of pages to the free page list. A logical page is associated with on 
2278      * or physical PageIO, which are on the disk. We have to move all those PagIO instance
2279      * to the free list, and do the same in memory (we try to keep a reference to a set of 
2280      * free pages.
2281      *  
2282      * @param btree The BTree which were owning the pages
2283      * @param pages The pages to free
2284      * @throws IOException 
2285      * @throws EndOfFileExceededException 
2286      */
2287     /* Package protected */<K, V> void addFreePages( BTree<K, V> btree, List<Page<K, V>> pages )
2288         throws EndOfFileExceededException,
2289         IOException
2290     {
2291         if ( ( btree == copiedPageBTree ) || ( btree == revisionBTree ) )
2292         {
2293             return;
2294         }
2295 
2296         if ( ( pages == null ) || pages.isEmpty() )
2297         {
2298             return;
2299         }
2300 
2301         if ( !keepRevisions )
2302         {
2303             // if the btree doesn't keep revisions, we can safely move
2304             // the pages to the free page list.
2305             // NOTE : potential improvement : we can update the header only when
2306             // we have processed all the logical pages.
2307 
2308             for ( Page<K, V> page : pages )
2309             {
2310                 // Retrieve all the PageIO associated with this logical page
2311                 long firstOffset = page.getOffset();
2312 
2313                 // skip the page with offset 0, this is the first in-memory root page that
2314                 // was copied during first insert in a BTree.
2315                 // a Node or Leaf will *never* have 0 or -1 as its offset 
2316                 if ( firstOffset == NO_PAGE )
2317                 {
2318                     continue;
2319                 }
2320 
2321                 long lastOffset = page.getLastOffset();
2322 
2323                 // Update the pointers
2324                 if ( firstFreePage == NO_PAGE )
2325                 {
2326                     // We don't have yet any free pageIos. The
2327                     // PageIOs for this Page will be used
2328                     firstFreePage = firstOffset;
2329                 }
2330                 else
2331                 {
2332                     // We add the Page's PageIOs before the 
2333                     // existing free pages.
2334                     long offset = page.getLastOffset();
2335 
2336                     if ( offset == NO_PAGE )
2337                     {
2338                         offset = page.getOffset();
2339                     }
2340 
2341                     // Fetch the pageIO
2342                     PageIO pageIo = fetchPage( offset );
2343 
2344                     // Link it to the first free page
2345                     pageIo.setNextPage( firstFreePage );
2346 
2347                     LOG.debug( "Flushing the first free page" );
2348 
2349                     // And flush it to disk
2350                     flushPages( pageIo );
2351 
2352                     // We can update the lastFreePage offset 
2353                     firstFreePage = firstOffset;
2354                 }
2355             }
2356 
2357             // Last, not least, flush the header
2358             updateRecordManagerHeader();
2359         }
2360         else
2361         {
2362             LOG.debug( "We should not get there" );
2363 
2364             for ( Page<K, V> p : pages )
2365             {
2366                 addFreePage( btree, p );
2367             }
2368         }
2369     }
2370 
2371 
2372     /**
2373      * 
2374      * TODO addFreePage.
2375      *
2376      * @param btree
2377      * @param freePage
2378      */
2379     private <K, V> void addFreePage( BTree<K, V> btree, Page<K, V> freePage )
2380     {
2381         try
2382         {
2383             RevisionName revision = new RevisionName( freePage.getRevision(), btree.getName() );
2384             long[] offsetArray = null;
2385 
2386             if ( copiedPageBTree.hasKey( revision ) )
2387             {
2388                 offsetArray = copiedPageBTree.get( revision );
2389                 long[] tmp = new long[offsetArray.length + 1];
2390                 System.arraycopy( offsetArray, 0, tmp, 0, offsetArray.length );
2391                 offsetArray = tmp;
2392             }
2393             else
2394             {
2395                 offsetArray = new long[1];
2396             }
2397 
2398             offsetArray[offsetArray.length - 1] = freePage.getOffset();
2399 
2400             copiedPageBTree.insert( revision, offsetArray, 0 );
2401         }
2402         catch ( Exception e )
2403         {
2404             throw new RuntimeException( e );
2405         }
2406     }
2407 
2408 
2409     /**
2410      * @return the keepRevisions
2411      */
2412     public boolean isKeepRevisions()
2413     {
2414         return keepRevisions;
2415     }
2416 
2417 
2418     /**
2419      * @param keepRevisions the keepRevisions to set
2420      */
2421     public void setKeepRevisions( boolean keepRevisions )
2422     {
2423         this.keepRevisions = keepRevisions;
2424     }
2425 
2426 
2427     /**
2428      * Creates a BTree and automatically adds it to the list of managed btrees
2429      * 
2430      * @param name the name of the BTree
2431      * @param keySerializer key serializer
2432      * @param valueSerializer value serializer
2433      * @param allowDuplicates flag for allowing duplicate keys 
2434      * @return a managed BTree
2435      * @throws IOException
2436      * @throws BTreeAlreadyManagedException
2437      */
2438     @SuppressWarnings("all")
2439     public <K, V> BTree<K, V> addBTree( String name, ElementSerializer<K> keySerializer,
2440         ElementSerializer<V> valueSerializer,
2441         boolean allowDuplicates ) throws IOException, BTreeAlreadyManagedException
2442     {
2443         BTreeConfiguration config = new BTreeConfiguration();
2444 
2445         config.setName( name );
2446         config.setKeySerializer( keySerializer );
2447         config.setValueSerializer( valueSerializer );
2448         config.setAllowDuplicates( allowDuplicates );
2449 
2450         BTree btree = new BTree( config );
2451         manage( btree );
2452 
2453         if ( LOG_CHECK.isDebugEnabled() )
2454         {
2455             check();
2456         }
2457 
2458         return btree;
2459     }
2460 
2461 
2462     private void setCheckedPage( long[] checkedPages, long offset, int pageSize )
2463     {
2464         long pageOffset = ( offset - HEADER_SIZE ) / pageSize;
2465         int index = ( int ) ( pageOffset / 64L );
2466         long mask = ( 1L << ( pageOffset % 64L ) );
2467         long bits = checkedPages[index];
2468 
2469         if ( ( bits & mask ) == 1 )
2470         {
2471             throw new RuntimeException( "The page at : " + offset + " has already been checked" );
2472         }
2473 
2474         checkedPages[index] |= mask;
2475 
2476     }
2477 
2478 
2479     /**
2480      * Check the free pages
2481      * 
2482      * @param checkedPages
2483      * @throws IOException 
2484      */
2485     private void checkFreePages( long[] checkedPages, int pageSize, long firstFreePage, long lastFreePage )
2486         throws IOException
2487     {
2488         if ( firstFreePage == NO_PAGE )
2489         {
2490             if ( lastFreePage == NO_PAGE )
2491             {
2492                 return;
2493             }
2494             else
2495             {
2496                 throw new RuntimeException( "Wrong last free page : " + lastFreePage );
2497             }
2498         }
2499 
2500         if ( lastFreePage != NO_PAGE )
2501         {
2502             throw new RuntimeException( "Wrong last free page : " + lastFreePage );
2503         }
2504 
2505         // Now, read all the free pages
2506         long currentOffset = firstFreePage;
2507         long fileSize = fileChannel.size();
2508 
2509         while ( currentOffset != NO_PAGE )
2510         {
2511             if ( currentOffset > fileSize )
2512             {
2513                 throw new RuntimeException( "Wrong free page offset, above file size : " + currentOffset );
2514             }
2515 
2516             try
2517             {
2518                 PageIO pageIo = fetchPage( currentOffset );
2519 
2520                 if ( currentOffset != pageIo.getOffset() )
2521                 {
2522                     throw new RuntimeException( "PageIO offset is incorrect : " + currentOffset + "-"
2523                         + pageIo.getOffset() );
2524                 }
2525 
2526                 setCheckedPage( checkedPages, currentOffset, pageSize );
2527 
2528                 long newOffset = pageIo.getNextPage();
2529                 currentOffset = newOffset;
2530             }
2531             catch ( IOException ioe )
2532             {
2533                 throw new RuntimeException( "Cannot fetch page at : " + currentOffset );
2534             }
2535         }
2536     }
2537 
2538 
2539     /**
2540      * Check the root page for a given BTree
2541      * @throws IOException 
2542      * @throws EndOfFileExceededException 
2543      */
2544     private void checkRoot( long[] checkedPages, long offset, int pageSize, long nbBTreeElems,
2545         ElementSerializer keySerializer, ElementSerializer valueSerializer, boolean allowDuplicates )
2546         throws EndOfFileExceededException, IOException
2547     {
2548         // Read the rootPage pages on disk
2549         PageIO[] rootPageIos = readPageIOs( offset, Long.MAX_VALUE );
2550 
2551         // Deserialize the rootPage now
2552         long position = 0L;
2553 
2554         // The revision
2555         long revision = readLong( rootPageIos, position );
2556         position += LONG_SIZE;
2557 
2558         // The number of elements in the page
2559         int nbElems = readInt( rootPageIos, position );
2560         position += INT_SIZE;
2561 
2562         // The size of the data containing the keys and values
2563         ByteBuffer byteBuffer = null;
2564 
2565         // Reads the bytes containing all the keys and values, if we have some
2566         ByteBuffer data = readBytes( rootPageIos, position );
2567 
2568         if ( nbElems >= 0 )
2569         {
2570             // Its a leaf
2571 
2572             // Check the page offset
2573             long pageOffset = rootPageIos[0].getOffset();
2574 
2575             if ( ( pageOffset < 0 ) || ( pageOffset > fileChannel.size() ) )
2576             {
2577                 throw new RuntimeException( "The page offset is incorrect : " + pageOffset );
2578             }
2579 
2580             // Check the page last offset
2581             long pageLastOffset = rootPageIos[rootPageIos.length - 1].getOffset();
2582 
2583             if ( ( pageLastOffset <= 0 ) || ( pageLastOffset > fileChannel.size() ) )
2584             {
2585                 throw new RuntimeException( "The page last offset is incorrect : " + pageLastOffset );
2586             }
2587 
2588             // Read each value and key
2589             for ( int i = 0; i < nbElems; i++ )
2590             {
2591                 // Just deserialize all the keys and values
2592                 if ( allowDuplicates )
2593                 {
2594                     /*
2595                     long value = OFFSET_SERIALIZER.deserialize( byteBuffer );
2596 
2597                     rootPageIos = readPageIOs( value, Long.MAX_VALUE );
2598 
2599                     BTree dupValueContainer = BTreeFactory.createBTree();
2600                     dupValueContainer.setBtreeOffset( value );
2601 
2602                     try
2603                     {
2604                         loadBTree( pageIos, dupValueContainer );
2605                     }
2606                     catch ( Exception e )
2607                     {
2608                         // should not happen
2609                         throw new RuntimeException( e );
2610                     }
2611                     */
2612                 }
2613                 else
2614                 {
2615                     valueSerializer.deserialize( byteBuffer );
2616                 }
2617 
2618                 keySerializer.deserialize( byteBuffer );
2619             }
2620         }
2621         else
2622         {
2623             /*
2624             // It's a node
2625             int nodeNbElems = -nbElems;
2626 
2627             // Read each value and key
2628             for ( int i = 0; i < nodeNbElems; i++ )
2629             {
2630                 // This is an Offset
2631                 long offset = OFFSET_SERIALIZER.deserialize( byteBuffer );
2632                 long lastOffset = OFFSET_SERIALIZER.deserialize( byteBuffer );
2633 
2634                 ElementHolder valueHolder = new ReferenceHolder( btree, null, offset, lastOffset );
2635                 ( ( Node ) page ).setValue( i, valueHolder );
2636 
2637                 Object key = btree.getKeySerializer().deserialize( byteBuffer );
2638                 BTreeFactory.setKey( page, i, key );
2639             }
2640 
2641             // and read the last value, as it's a node
2642             long offset = OFFSET_SERIALIZER.deserialize( byteBuffer );
2643             long lastOffset = OFFSET_SERIALIZER.deserialize( byteBuffer );
2644 
2645             ElementHolder valueHolder = new ReferenceHolder( btree, null, offset, lastOffset );
2646             ( ( Node ) page ).setValue( nodeNbElems, valueHolder );*/
2647         }
2648     }
2649 
2650 
2651     /**
2652      * Check a BTree
2653      * @throws IllegalAccessException 
2654      * @throws InstantiationException 
2655      * @throws ClassNotFoundException 
2656      */
2657     private long checkBTree( long[] checkedPages, PageIO[] pageIos, int pageSize, boolean isLast )
2658         throws EndOfFileExceededException, IOException, InstantiationException, IllegalAccessException,
2659         ClassNotFoundException
2660     {
2661         long dataPos = 0L;
2662 
2663         // The BTree current revision
2664         long revision = readLong( pageIos, dataPos );
2665         dataPos += LONG_SIZE;
2666 
2667         // The nb elems in the tree
2668         long nbElems = readLong( pageIos, dataPos );
2669         dataPos += LONG_SIZE;
2670 
2671         // The BTree rootPage offset
2672         long rootPageOffset = readLong( pageIos, dataPos );
2673 
2674         if ( ( rootPageOffset < 0 ) || ( rootPageOffset > fileChannel.size() ) )
2675         {
2676             throw new RuntimeException( "The rootpage is incorrect : " + rootPageOffset );
2677         }
2678 
2679         dataPos += LONG_SIZE;
2680 
2681         // The next BTree offset
2682         long nextBTreeOffset = readLong( pageIos, dataPos );
2683 
2684         if ( ( ( rootPageOffset < 0 ) && ( !isLast ) ) || ( nextBTreeOffset > fileChannel.size() ) )
2685         {
2686             throw new RuntimeException( "The rootpage is incorrect : " + rootPageOffset );
2687         }
2688 
2689         dataPos += LONG_SIZE;
2690 
2691         // The BTree page size
2692         int btreePageSize = readInt( pageIos, dataPos );
2693 
2694         if ( ( btreePageSize < 2 ) || ( ( btreePageSize & ( ~btreePageSize + 1 ) ) != btreePageSize ) )
2695         {
2696             throw new RuntimeException( "The BTree page size is not a power of 2 : " + btreePageSize );
2697         }
2698 
2699         dataPos += INT_SIZE;
2700 
2701         // The tree name
2702         ByteBuffer btreeNameBytes = readBytes( pageIos, dataPos );
2703         dataPos += INT_SIZE;
2704 
2705         dataPos += btreeNameBytes.limit();
2706         String btreeName = Strings.utf8ToString( btreeNameBytes );
2707 
2708         // The keySerializer FQCN
2709         ByteBuffer keySerializerBytes = readBytes( pageIos, dataPos );
2710 
2711         String keySerializerFqcn = null;
2712         dataPos += INT_SIZE;
2713 
2714         if ( keySerializerBytes != null )
2715         {
2716             dataPos += keySerializerBytes.limit();
2717             keySerializerFqcn = Strings.utf8ToString( keySerializerBytes );
2718         }
2719         else
2720         {
2721             keySerializerFqcn = "";
2722         }
2723 
2724         // The valueSerialier FQCN
2725         ByteBuffer valueSerializerBytes = readBytes( pageIos, dataPos );
2726 
2727         String valueSerializerFqcn = null;
2728         dataPos += INT_SIZE;
2729 
2730         if ( valueSerializerBytes != null )
2731         {
2732             dataPos += valueSerializerBytes.limit();
2733             valueSerializerFqcn = Strings.utf8ToString( valueSerializerBytes );
2734         }
2735         else
2736         {
2737             valueSerializerFqcn = "";
2738         }
2739 
2740         // The BTree allowDuplicates flag
2741         int allowDuplicates = readInt( pageIos, dataPos );
2742         dataPos += INT_SIZE;
2743 
2744         // Now, check the rootPage, which can be a Leaf or a Node, depending 
2745         // on the number of elements in the tree : if it's above the pageSize,
2746         // it's a Node, otherwise it's a Leaf
2747         Class<?> valueSerializer = Class.forName( valueSerializerFqcn );
2748         Class<?> keySerializer = Class.forName( keySerializerFqcn );
2749 
2750         /*
2751         checkRoot( checkedPages, rootPageOffset, pageSize, nbElems,
2752             ( ElementSerializer<?> ) keySerializer.newInstance(),
2753             ( ElementSerializer<?> ) valueSerializer.newInstance(), allowDuplicates != 0 );
2754         */
2755 
2756         return nextBTreeOffset;
2757     }
2758 
2759 
2760     /**
2761      * Check each BTree we manage
2762      * @throws IOException 
2763      * @throws EndOfFileExceededException 
2764      * @throws ClassNotFoundException 
2765      * @throws IllegalAccessException 
2766      * @throws InstantiationException 
2767      */
2768     private void checkBTrees( long[] checkedPages, int pageSize, int nbBTrees ) throws EndOfFileExceededException,
2769         IOException, InstantiationException, IllegalAccessException, ClassNotFoundException
2770     {
2771         // Iterate on each BTree until we have exhausted all of them. The number
2772         // of btrees is just used to check that we have the correct number
2773         // of stored BTrees, as they are all linked.
2774         long position = HEADER_SIZE;
2775 
2776         for ( int i = 0; i < nbBTrees; i++ )
2777         {
2778             // Load the pageIOs containing the BTree
2779             PageIO[] pageIos = readPageIOs( position, Long.MAX_VALUE );
2780 
2781             // Check that they are correctly linked and not already used
2782             int pageNb = 0;
2783 
2784             for ( PageIO currentPageIo : pageIos )
2785             {
2786                 // 
2787                 long nextPageOffset = currentPageIo.getNextPage();
2788 
2789                 if ( pageNb == pageIos.length - 1 )
2790                 {
2791                     if ( nextPageOffset != NO_PAGE )
2792                     {
2793                         throw new RuntimeException( "The pointer to the next page is not valid, expected NO_PAGE" );
2794                     }
2795                 }
2796                 else
2797                 {
2798                     if ( nextPageOffset == NO_PAGE )
2799                     {
2800                         throw new RuntimeException( "The pointer to the next page is not valid, NO_PAGE" );
2801                     }
2802                 }
2803 
2804                 if ( ( nextPageOffset != NO_PAGE ) && ( ( nextPageOffset - HEADER_SIZE ) % pageSize != 0 ) )
2805                 {
2806                     throw new RuntimeException( "The pointer to the next page is not valid" );
2807                 }
2808 
2809                 // Update the array of processed pages
2810                 setCheckedPage( checkedPages, currentPageIo.getOffset(), pageSize );
2811             }
2812 
2813             // Now check the BTree
2814             long nextBTree = checkBTree( checkedPages, pageIos, pageSize, i == nbBTrees - 1 );
2815 
2816             if ( ( nextBTree == NO_PAGE ) && ( i < nbBTrees - 1 ) )
2817             {
2818                 throw new RuntimeException( "The pointer to the next BTree is incorrect" );
2819             }
2820 
2821             position = nextBTree;
2822         }
2823     }
2824 
2825 
2826     /**
2827      * Check the whole file
2828      */
2829     private void check()
2830     {
2831         try
2832         {
2833             // First check the header
2834             ByteBuffer header = ByteBuffer.allocate( HEADER_SIZE );
2835             long fileSize = fileChannel.size();
2836 
2837             if ( fileSize < HEADER_SIZE )
2838             {
2839                 throw new RuntimeException( "File size too small : " + fileSize );
2840             }
2841 
2842             // Read the header
2843             fileChannel.read( header, 0L );
2844             header.flip();
2845 
2846             // The page size. It must be a power of 2, and above 16.
2847             int pageSize = header.getInt();
2848 
2849             if ( ( pageSize < 0 ) || ( pageSize < 32 ) || ( ( pageSize & ( ~pageSize + 1 ) ) != pageSize ) )
2850             {
2851                 throw new RuntimeException( "Wrong page size : " + pageSize );
2852             }
2853 
2854             // Compute the number of pages in this file
2855             long nbPages = ( fileSize - HEADER_SIZE ) / pageSize;
2856 
2857             // The number of trees. It must be at least 2 and > 0
2858             int nbBTrees = header.getInt();
2859 
2860             if ( nbBTrees < 0 )
2861             {
2862                 throw new RuntimeException( "Wrong nb trees : " + nbBTrees );
2863             }
2864 
2865             // The first free page offset. It must be either -1 or below file size
2866             // and its value must be a modulo of pageSize
2867             long firstFreePage = header.getLong();
2868 
2869             if ( firstFreePage > fileSize )
2870             {
2871                 throw new RuntimeException( "First free page pointing after the end of the file : " + firstFreePage );
2872             }
2873 
2874             if ( ( firstFreePage != NO_PAGE ) && ( ( ( firstFreePage - HEADER_SIZE ) % pageSize ) != 0 ) )
2875             {
2876                 throw new RuntimeException( "First free page not pointing to a correct offset : " + firstFreePage );
2877             }
2878 
2879             // The last free page offset. It must be -1
2880             long lastFreePage = header.getLong();
2881 
2882             if ( ( ( lastFreePage != NO_PAGE ) && ( ( ( lastFreePage - HEADER_SIZE ) % pageSize ) != 0 ) ) )
2883             //|| ( lastFreePage != 0 ) )
2884             {
2885                 throw new RuntimeException( "Invalid last free page : " + lastFreePage );
2886             }
2887 
2888             int nbPageBits = ( int ) ( nbPages / 64 );
2889 
2890             // Create an array of pages to be checked
2891             // We use one bit per page. It's 0 when the page
2892             // hasn't been checked, 1 otherwise.
2893             long[] checkedPages = new long[nbPageBits + 1];
2894 
2895             // Then the free files
2896             checkFreePages( checkedPages, pageSize, firstFreePage, lastFreePage );
2897 
2898             // The BTrees
2899             checkBTrees( checkedPages, pageSize, nbBTrees );
2900         }
2901         catch ( Exception e )
2902         {
2903             // We catch the exception and rethrow it immediately to be able to
2904             // put a breakpoint here
2905             e.printStackTrace();
2906             throw new RuntimeException( "Error : " + e.getMessage() );
2907         }
2908     }
2909 
2910 
2911     /**
2912      * Loads a BTree holding the values of a duplicate key
2913      * This tree is also called as dups tree or sub tree
2914      * 
2915      * @param offset the offset of the BTree header
2916      * @return the deserialized BTree
2917      */
2918     /* No qualifier */<K, V> BTree<K, V> loadDupsBTree( long offset )
2919     {
2920         try
2921         {
2922             PageIO[] pageIos = readPageIOs( offset, Long.MAX_VALUE );
2923 
2924             BTree<K, V> dupValueContainer = BTreeFactory.createBTree();
2925             dupValueContainer.setBtreeOffset( offset );
2926 
2927             loadBTree( pageIos, dupValueContainer );
2928 
2929             return dupValueContainer;
2930         }
2931         catch ( Exception e )
2932         {
2933             // should not happen
2934             throw new RuntimeException( e );
2935         }
2936     }
2937 
2938 
2939     /**
2940      * @see Object#toString()
2941      */
2942     public String toString()
2943     {
2944         StringBuilder sb = new StringBuilder();
2945 
2946         sb.append( "RM free pages : [" );
2947 
2948         if ( firstFreePage != NO_PAGE )
2949         {
2950             long current = firstFreePage;
2951             boolean isFirst = true;
2952 
2953             while ( current != NO_PAGE )
2954             {
2955                 if ( isFirst )
2956                 {
2957                     isFirst = false;
2958                 }
2959                 else
2960                 {
2961                     sb.append( ", " );
2962                 }
2963 
2964                 PageIO pageIo;
2965 
2966                 try
2967                 {
2968                     pageIo = fetchPage( current );
2969                     sb.append( pageIo.getOffset() );
2970                     current = pageIo.getNextPage();
2971                 }
2972                 catch ( EndOfFileExceededException e )
2973                 {
2974                     e.printStackTrace();
2975                 }
2976                 catch ( IOException e )
2977                 {
2978                     e.printStackTrace();
2979                 }
2980 
2981             }
2982         }
2983 
2984         sb.append( "]" );
2985 
2986         return sb.toString();
2987     }
2988 }