View Javadoc
1   /*
2    *   Licensed to the Apache Software Foundation (ASF) under one
3    *   or more contributor license agreements.  See the NOTICE file
4    *   distributed with this work for additional information
5    *   regarding copyright ownership.  The ASF licenses this file
6    *   to you under the Apache License, Version 2.0 (the
7    *   "License"); you may not use this file except in compliance
8    *   with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   *   Unless required by applicable law or agreed to in writing,
13   *   software distributed under the License is distributed on an
14   *   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   *   KIND, either express or implied.  See the License for the
16   *   specific language governing permissions and limitations
17   *   under the License.
18   *
19   */
20  package org.apache.directory.mavibot.btree;
21  
22  
23  import java.io.BufferedReader;
24  import java.io.File;
25  import java.io.IOException;
26  import java.io.InputStreamReader;
27  import java.io.RandomAccessFile;
28  import java.nio.BufferUnderflowException;
29  import java.nio.ByteBuffer;
30  import java.nio.channels.FileChannel;
31  import java.util.HashMap;
32  import java.util.HashSet;
33  import java.util.Map;
34  import java.util.Set;
35  
36  import org.apache.directory.mavibot.btree.exception.InvalidBTreeException;
37  import org.apache.directory.mavibot.btree.serializer.ElementSerializer;
38  import org.apache.directory.mavibot.btree.serializer.LongSerializer;
39  import org.apache.directory.mavibot.btree.serializer.StringSerializer;
40  import org.apache.directory.mavibot.btree.util.Strings;
41  
42  
43  /**
44   * A class to examine a Mavibot database file.
45   *
46   * @author <a href="mailto:dev@directory.apache.org">Apache Directory Project</a>
47   */
48  public class MavibotInspector
49  {
50      // The file to be read
51      private File dbFile;
52  
53      // The recordManager
54      private RecordManager rm;
55  
56      private BufferedReader br = new BufferedReader( new InputStreamReader( System.in ) );
57  
58      // The name of the two page arrays for the global file and teh free pages
59      private static final String GLOBAL_PAGES_NAME = "__global__";
60      private static final String FREE_PAGES_NAME = "__free-pages__";
61  
62      // The set of page array we already know about
63      private static Set<String> knownPagesArrays =  new HashSet<String>();
64      
65      static
66      {
67          knownPagesArrays.add( GLOBAL_PAGES_NAME );
68          knownPagesArrays.add( FREE_PAGES_NAME );
69          knownPagesArrays.add( RecordManager.BTREE_OF_BTREES_NAME );
70          knownPagesArrays.add( RecordManager.COPIED_PAGE_BTREE_NAME );
71      }
72      
73      /**
74       * A private class to store a few informations about a btree
75       *
76      
77      private static BtreeInfo btreeInfo;
78      
79      static
80      {
81          btreeInfo = new BtreeInfo();
82      }
83  
84      /**
85       * Create an instance of MavibotInspector
86       * @param dbFile The file to read
87       */
88      public MavibotInspector( File dbFile )
89      {
90          this.dbFile = dbFile;
91      }
92  
93  
94      /**
95       * Check that the file exists
96       */
97      private boolean checkFilePresence()
98      {
99          if ( dbFile == null )
100         {
101             System.out.println( "No mavibot database file was given" );
102             return false;
103         }
104 
105         if ( !dbFile.exists() )
106         {
107             System.out.println( "Given mavibot database file " + dbFile + " doesn't exist" );
108             return false;
109         }
110 
111         return true;
112     }
113 
114 
115     /**
116      * Pretty print the file size
117      */
118     public void printFileSize() throws IOException
119     {
120         FileChannel fileChannel = new RandomAccessFile( dbFile, "r" ).getChannel();
121 
122         long l = fileChannel.size();
123 
124         fileChannel.close();
125 
126         String msg;
127 
128         if ( l < 1024 )
129         {
130             msg = l + " bytes";
131         }
132         else
133         {
134             msg = ( l / 1024 ) + " KB";
135         }
136 
137         System.out.println( msg );
138         
139         fileChannel.close();
140     }
141 
142 
143     /**
144      * Print the number of B-trees
145      */
146     public void printNumberOfBTrees()
147     {
148         int nbBtrees = 0;
149         
150         if ( rm != null )
151         {
152             nbBtrees = rm.getNbManagedTrees();
153         }
154 
155         // The number of trees. It must be at least 2 and > 0
156         System.out.println( "Total Number of BTrees: " + nbBtrees );
157     }
158 
159 
160     /**
161      * Print the B-tree's name
162      */
163     public void printBTreeNames()
164     {
165         if ( rm == null )
166         {
167             System.out.println( "Couldn't find the number of managed btrees" );
168             return;
169         }
170 
171         Set<String> trees = rm.getManagedTrees();
172         System.out.println( "\nManaged BTrees:" );
173         
174         for ( String tree : trees )
175         {
176             System.out.println( tree );
177         }
178         
179         System.out.println();
180     }
181 
182 
183     /**
184      * Check a B-tree
185      */
186     public void checkBTree()
187     {
188         if ( rm == null )
189         {
190             System.out.println( "Cannot check BTree(s)" );
191             return;
192         }
193 
194         System.out.print( "BTree Name: " );
195         String name = readLine();
196 
197         PersistedBTree<?, ?> pb = ( PersistedBTree<?, ?> ) rm.getManagedTree( name );
198 
199         if ( pb == null )
200         {
201             System.out.println( "No BTree exists with the name '" + name + "'" );
202             return;
203         }
204 
205         System.out.println( "\nBTree offset: " + String.format( "0x%1$08x", pb.getBtreeOffset() ) );
206         System.out.println( "BTree _info_ offset: " + String.format( "0x%1$08x", pb.getBtreeInfoOffset() ) );
207         System.out.println( "BTree root page offset: " + String.format( "0x%1$08x", pb.getRootPageOffset() ) );
208         System.out.println( "Number of elements present: " + pb.getNbElems() );
209         System.out.println( "BTree Page size: " + pb.getPageSize() );
210         System.out.println( "BTree revision: " + pb.getRevision() );
211         System.out.println( "Key serializer: " + pb.getKeySerializerFQCN() );
212         System.out.println( "Value serializer: " + pb.getValueSerializerFQCN() );
213         System.out.println();
214     }
215 
216 
217     /**
218      * Load the full fie into a new RecordManager
219      */
220     private boolean loadRm()
221     {
222         try
223         {
224             if( rm != null )
225             {
226                 System.out.println("Closing record manager");
227                 rm.close();
228             }
229             
230             rm = new RecordManager( dbFile.getAbsolutePath() );
231             System.out.println("Loaded record manager");
232         }
233         catch ( Exception e )
234         {
235             System.out.println( "Given database file seems to be corrupted. " + e.getMessage() );
236             return false;
237         }
238 
239         return true;
240     }
241     
242     
243     /**
244      * Check the whole file
245      */
246     /* no qualifier */ static void check( RecordManager recordManager )
247     {
248         try
249         {
250             // First check the RMheader
251             ByteBuffer recordManagerHeader = ByteBuffer.allocate( RecordManager.RECORD_MANAGER_HEADER_SIZE );
252             long fileSize = recordManager.fileChannel.size();
253 
254             if ( fileSize < RecordManager.RECORD_MANAGER_HEADER_SIZE )
255             {
256                 throw new InvalidBTreeException( "File size too small : " + fileSize );
257             }
258 
259             // Read the RMHeader
260             recordManager.fileChannel.read( recordManagerHeader, 0L );
261             recordManagerHeader.flip();
262 
263             // The page size. It must be a power of 2, and above 16.
264             int pageSize = recordManagerHeader.getInt();
265 
266             if ( ( pageSize != recordManager.pageSize ) || ( pageSize < 32 ) || ( ( pageSize & ( ~pageSize + 1 ) ) != pageSize ) )
267             {
268                 throw new InvalidBTreeException( "Wrong page size : " + pageSize );
269             }
270 
271             // Compute the number of pages in this file
272             long nbPages = ( fileSize - RecordManager.RECORD_MANAGER_HEADER_SIZE ) / pageSize;
273 
274             // The number of trees. It must be at least 2 and > 0
275             int nbBtrees = recordManagerHeader.getInt();
276 
277             if ( ( nbBtrees < 0 ) || ( nbBtrees != recordManager.nbBtree ) )
278             {
279                 throw new InvalidBTreeException( "Wrong nb trees : " + nbBtrees );
280             }
281 
282             // The first free page offset. It must be either -1 or below file size
283             // and its value must be a modulo of pageSize
284             long firstFreePage = recordManagerHeader.getLong();
285 
286             if ( firstFreePage != RecordManager.NO_PAGE )
287             {
288                 checkOffset( recordManager, firstFreePage );
289             }
290 
291             int nbPageBits = ( int ) ( nbPages / 32 );
292 
293             // Create an array of pages to be checked for each B-tree, plus
294             // two others for the free pages and the global one
295             // We use one bit per page. It's 0 when the page
296             // hasn't been checked, 1 otherwise.
297             Map<String, int[]> checkedPages = new HashMap<String, int[]>(nbBtrees + 4);
298 
299             // The global page array
300             checkedPages.put( GLOBAL_PAGES_NAME, new int[nbPageBits + 1] );
301 
302             // The freePages array
303             checkedPages.put( FREE_PAGES_NAME, new int[nbPageBits + 1] );
304             
305             // The B-tree of B-trees array
306             checkedPages.put( RecordManager.BTREE_OF_BTREES_NAME, new int[nbPageBits + 1] );
307             
308             // Last, the Copied Pages B-tree array
309             checkedPages.put( RecordManager.COPIED_PAGE_BTREE_NAME, new int[nbPageBits + 1] );
310 
311             // Check the free files
312             checkFreePages( recordManager, checkedPages );
313 
314             // The B-trees offsets
315             long currentBtreeOfBtreesOffset = recordManagerHeader.getLong();
316             long previousBtreeOfBtreesOffset = recordManagerHeader.getLong();
317             long currentCopiedPagesBtreeOffset = recordManagerHeader.getLong();
318             long previousCopiedPagesBtreeOffset = recordManagerHeader.getLong();
319 
320             // Check that the previous BOB offset is not pointing to something
321             if ( previousBtreeOfBtreesOffset != RecordManager.NO_PAGE )
322             {
323                 System.out.println( "The previous Btree of Btrees offset is not valid : "
324                     + previousBtreeOfBtreesOffset );
325                 return;
326             }
327 
328             // Check that the previous CPB offset is not pointing to something
329             if ( previousCopiedPagesBtreeOffset != RecordManager.NO_PAGE )
330             {
331                 System.out.println( "The previous Copied Pages Btree offset is not valid : "
332                     + previousCopiedPagesBtreeOffset );
333                 return;
334             }
335 
336             // Check that the current BOB offset is valid
337             checkOffset( recordManager, currentBtreeOfBtreesOffset );
338 
339             // Check that the current CPB offset is valid
340             checkOffset( recordManager, currentCopiedPagesBtreeOffset );
341 
342             // Now, check the BTree of Btrees
343             checkBtreeOfBtrees( recordManager, checkedPages );
344 
345             // And the Copied Pages BTree
346             checkBtree( recordManager, currentCopiedPagesBtreeOffset, checkedPages );
347 
348             // We can now dump the checked pages
349             dumpCheckedPages( recordManager, checkedPages );
350         }
351         catch ( Exception e )
352         {
353             // We catch the exception and rethrow it immediately to be able to
354             // put a breakpoint here
355             e.printStackTrace();
356             throw new InvalidBTreeException( "Error : " + e.getMessage() );
357         }
358     }
359 
360     
361     /**
362      * Check the Btree of Btrees
363      */
364     private static <K, V> void checkBtreeOfBtrees( RecordManager recordManager, Map<String, int[]> checkedPages ) throws Exception
365     {
366         // Read the BOB header
367         PageIO[] bobHeaderPageIos = recordManager.readPageIOs( recordManager.currentBtreeOfBtreesOffset, Long.MAX_VALUE );
368 
369         // update the checkedPages
370         updateCheckedPages( checkedPages.get( RecordManager.BTREE_OF_BTREES_NAME), recordManager.pageSize, bobHeaderPageIos );
371         updateCheckedPages( checkedPages.get( GLOBAL_PAGES_NAME ), recordManager.pageSize, bobHeaderPageIos );
372 
373         long dataPos = 0L;
374 
375         // The B-tree current revision
376         recordManager.readLong( bobHeaderPageIos, dataPos );
377         dataPos += RecordManager.LONG_SIZE;
378 
379         // The nb elems in the tree
380         recordManager.readLong( bobHeaderPageIos, dataPos );
381         dataPos += RecordManager.LONG_SIZE;
382 
383         // The B-tree rootPage offset
384         long rootPageOffset = recordManager.readLong( bobHeaderPageIos, dataPos );
385 
386         checkOffset( recordManager, rootPageOffset );
387 
388         dataPos += RecordManager.LONG_SIZE;
389 
390         // The B-tree info offset
391         long btreeInfoOffset = recordManager.readLong( bobHeaderPageIos, dataPos );
392 
393         checkOffset( recordManager, btreeInfoOffset );
394 
395         checkBtreeInfo( recordManager, checkedPages, btreeInfoOffset, -1L );
396 
397         // Check the elements in the btree itself
398         // We will read every single page
399         checkBtreeOfBtreesPage( recordManager, checkedPages, rootPageOffset );
400     }
401 
402     
403     /**
404      * Check a user's B-tree
405      */
406     private static <K, V> void checkBtree( RecordManager recordManager, long btreeOffset, Map<String, int[]> checkedPages ) throws Exception
407     {
408         // Read the B-tree header
409         PageIO[] btreeHeaderPageIos = recordManager.readPageIOs( btreeOffset, Long.MAX_VALUE );
410 
411         long dataPos = 0L;
412 
413         // The B-tree current revision
414         long btreeRevision = recordManager.readLong( btreeHeaderPageIos, dataPos );
415         dataPos += RecordManager.LONG_SIZE;
416 
417         // The nb elems in the tree
418         recordManager.readLong( btreeHeaderPageIos, dataPos );
419         dataPos += RecordManager.LONG_SIZE;
420 
421         // The B-tree rootPage offset
422         long rootPageOffset = recordManager.readLong( btreeHeaderPageIos, dataPos );
423 
424         checkOffset( recordManager, rootPageOffset );
425 
426         dataPos += RecordManager.LONG_SIZE;
427 
428         // The B-tree info offset
429         long btreeInfoOffset = recordManager.readLong( btreeHeaderPageIos, dataPos );
430 
431         checkOffset( recordManager, btreeInfoOffset );
432 
433         BtreeInfo<K, V> btreeInfo = checkBtreeInfo( recordManager, checkedPages, btreeInfoOffset, btreeRevision );
434 
435         // Update the checked pages
436         updateCheckedPages( checkedPages.get( btreeInfo.btreeName ), recordManager.pageSize, btreeHeaderPageIos );
437         updateCheckedPages( checkedPages.get( GLOBAL_PAGES_NAME ), recordManager.pageSize, btreeHeaderPageIos );
438         
439         // And now, process the rootPage
440         checkBtreePage( recordManager, btreeInfo, checkedPages, rootPageOffset );
441     }
442 
443     
444     /**
445      * Check the Btree of Btrees rootPage
446      */
447     private static <K, V> void checkBtreePage( RecordManager recordManager, BtreeInfo<K, V> btreeInfo, Map<String, int[]> checkedPages, long pageOffset ) throws Exception
448     {
449         PageIO[] pageIos = recordManager.readPageIOs( pageOffset, Long.MAX_VALUE );
450 
451         // Update the checkedPages array
452         updateCheckedPages( checkedPages.get( btreeInfo.btreeName ), recordManager.pageSize, pageIos );
453         updateCheckedPages( checkedPages.get( GLOBAL_PAGES_NAME ), recordManager.pageSize, pageIos );
454 
455         // Deserialize the page now
456         long position = 0L;
457 
458         // The revision
459         long revision = recordManager.readLong( pageIos, position );
460         position += RecordManager.LONG_SIZE;
461 
462         // The number of elements in the page
463         int nbElems = recordManager.readInt( pageIos, position );
464         position += RecordManager.INT_SIZE;
465 
466         // The size of the data containing the keys and values
467         // Reads the bytes containing all the keys and values, if we have some
468         // We read  big blob of data into  ByteBuffer, then we will process
469         // this ByteBuffer
470         ByteBuffer byteBuffer = recordManager.readBytes( pageIos, position );
471 
472         // Now, deserialize the data block. If the number of elements
473         // is positive, it's a Leaf, otherwise it's a Node
474         // Note that only a leaf can have 0 elements, and it's the root page then.
475         if ( nbElems >= 0 )
476         {
477             // It's a leaf, process it as we may have sub-btrees
478             checkBtreeLeaf( recordManager, btreeInfo, checkedPages, nbElems, revision, byteBuffer, pageIos );
479         }
480         else
481         {
482             // It's a node
483             long[] children = checkBtreeNode( recordManager, btreeInfo, checkedPages, -nbElems, revision, byteBuffer, pageIos );
484 
485             for ( int pos = 0; pos <= -nbElems; pos++ )
486             {
487                 // Recursively check the children
488                 checkBtreePage( recordManager, btreeInfo, checkedPages, children[pos] );
489             }
490         }
491     }
492 
493     
494     /**
495      * Check the Btree info page
496      * @throws ClassNotFoundException 
497      */
498     private static <K, V> BtreeInfo<K, V> checkBtreeInfo( RecordManager recordManager, Map<String, int[]> checkedPages, long btreeInfoOffset, long btreeRevision ) throws IOException
499     {
500         BtreeInfo<K, V> btreeInfo = new BtreeInfo<K, V>();
501         
502         PageIO[] btreeInfoPagesIos = recordManager.readPageIOs( btreeInfoOffset, Long.MAX_VALUE );
503 
504         long dataPos = 0L;
505 
506         // The B-tree page size
507         recordManager.readInt( btreeInfoPagesIos, dataPos );
508         dataPos += RecordManager.INT_SIZE;
509 
510         // The tree name
511         ByteBuffer btreeNameBytes = recordManager.readBytes( btreeInfoPagesIos, dataPos );
512         dataPos += RecordManager.INT_SIZE + btreeNameBytes.limit();
513         String btreeName = Strings.utf8ToString( btreeNameBytes );
514 
515         // The keySerializer FQCN
516         ByteBuffer keySerializerBytes = recordManager.readBytes( btreeInfoPagesIos, dataPos );
517 
518         if ( keySerializerBytes != null )
519         {
520             String keySerializerFqcn = Strings.utf8ToString( keySerializerBytes );
521             
522             btreeInfo.keySerializer = getSerializer(  keySerializerFqcn );
523         }
524 
525         dataPos += RecordManager.INT_SIZE + keySerializerBytes.limit();
526 
527         // The valueSerialier FQCN
528         ByteBuffer valueSerializerBytes = recordManager.readBytes( btreeInfoPagesIos, dataPos );
529 
530         if ( valueSerializerBytes != null )
531         {
532             String valueSerializerFqcn = Strings.utf8ToString( valueSerializerBytes );
533             
534             btreeInfo.valueSerializer = getSerializer( valueSerializerFqcn );
535         }
536 
537         dataPos += RecordManager.INT_SIZE + valueSerializerBytes.limit();
538 
539         // The B-tree allowDuplicates flag
540         recordManager.readInt( btreeInfoPagesIos, dataPos );
541         dataPos += RecordManager.INT_SIZE;
542 
543         // update the checkedPages
544         if ( !RecordManager.COPIED_PAGE_BTREE_NAME.equals( btreeName ) && !RecordManager.BTREE_OF_BTREES_NAME.equals( btreeName ) )
545         {
546             btreeName = btreeName + "<" + btreeRevision + ">";
547         }
548 
549         btreeInfo.btreeName = btreeName;
550         
551         // Update the checkedPages
552         int[] checkedPagesArray = checkedPages.get( btreeName );
553         
554         if ( checkedPagesArray == null )
555         {
556             // Add the new name in the checkedPage name if it's not already there
557             checkedPagesArray = createPageArray( recordManager );
558             checkedPages.put( btreeName, checkedPagesArray );
559         }
560         
561         updateCheckedPages( checkedPagesArray, recordManager.pageSize, btreeInfoPagesIos );
562         updateCheckedPages( checkedPages.get( GLOBAL_PAGES_NAME ), recordManager.pageSize, btreeInfoPagesIos );
563         
564         return btreeInfo;
565     }
566     
567 
568     /**
569      * Get back the serializer instance
570      */
571     @SuppressWarnings("unchecked")
572     private static <T> ElementSerializer<T> getSerializer( String serializerFqcn )
573     {
574         try
575         {
576             Class<?> serializerClass = Class.forName( serializerFqcn );
577             ElementSerializer<T> serializer = null;
578             
579             try
580             {
581                 serializer = ( ElementSerializer<T> ) serializerClass.getDeclaredField( "INSTANCE" ).get( null );
582             }
583             catch( NoSuchFieldException e )
584             {
585                 // ignore
586             }
587     
588             if ( serializer == null )
589             {
590                 serializer = ( ElementSerializer<T> ) serializerClass.newInstance();
591             }
592     
593             return serializer;
594         }
595         catch ( Exception e )
596         {
597             throw new InvalidBTreeException( "Error : " + e.getMessage() );
598         }
599     }
600 
601     
602     /**
603      * Check the Btree of Btrees rootPage
604      */
605     private static <K, V> void checkBtreeOfBtreesPage( RecordManager recordManager, Map<String, int[]> checkedPages, long pageOffset ) throws Exception
606     {
607         PageIO[] pageIos = recordManager.readPageIOs( pageOffset, Long.MAX_VALUE );
608 
609         // Update the checkedPages array
610         updateCheckedPages( checkedPages.get( RecordManager.BTREE_OF_BTREES_NAME), recordManager.pageSize, pageIos );
611         updateCheckedPages( checkedPages.get( GLOBAL_PAGES_NAME ), recordManager.pageSize, pageIos );
612 
613         // Deserialize the page now
614         long position = 0L;
615 
616         // The revision
617         long revision = recordManager.readLong( pageIos, position );
618         position += RecordManager.LONG_SIZE;
619 
620         // The number of elements in the page
621         int nbElems = recordManager.readInt( pageIos, position );
622         position += RecordManager.INT_SIZE;
623 
624         // The size of the data containing the keys and values
625         // Reads the bytes containing all the keys and values, if we have some
626         // We read  big blob of data into  ByteBuffer, then we will process
627         // this ByteBuffer
628         ByteBuffer byteBuffer = recordManager.readBytes( pageIos, position );
629 
630         // Now, deserialize the data block. If the number of elements
631         // is positive, it's a Leaf, otherwise it's a Node
632         // Note that only a leaf can have 0 elements, and it's the root page then.
633         if ( nbElems >= 0 )
634         {
635             // It's a leaf, process it as we may have sub-btrees
636             checkBtreeOfBtreesLeaf( recordManager, checkedPages, nbElems, revision, byteBuffer, pageIos );
637         }
638         else
639         {
640             // It's a node
641             long[] children = checkBtreeOfBtreesNode( recordManager, checkedPages, -nbElems, revision, byteBuffer, pageIos );
642 
643             for ( int pos = 0; pos <= -nbElems; pos++ )
644             {
645                 // Recursively check the children
646                 checkBtreeOfBtreesPage( recordManager, checkedPages, children[pos] );
647             }
648         }
649     }
650     
651     
652     /**
653      * Check a Btree of Btrees leaf. It contains <revision, name> -> offset.
654      */
655     private static <K, V> void checkBtreeOfBtreesLeaf( RecordManager recordManager, Map<String, int[]> checkedPages, int nbElems, long revision, ByteBuffer byteBuffer, PageIO[] pageIos ) throws Exception
656     {
657         // Read each key and value
658         for ( int i = 0; i < nbElems; i++ )
659         {
660             try
661             {
662                 // Read the number of values
663                 int nbValues = byteBuffer.getInt();
664     
665                 if ( nbValues != 1 )
666                 {
667                     throw new InvalidBTreeException( "We should have only one value for a BOB " + nbValues );
668                 }
669     
670                 // This is a normal value
671                 // First, the value, which is an offset, which length should be 12
672                 int valueLength = byteBuffer.getInt();
673                 
674                 if ( valueLength != RecordManager.LONG_SIZE + RecordManager.INT_SIZE )
675                 {
676                     throw new InvalidBTreeException( "The BOB value length is invalid " + valueLength );
677                 }
678     
679                 // Second, the offset length, which should be 8
680                 int offsetLength = byteBuffer.getInt();
681                 
682                 if ( offsetLength != RecordManager.LONG_SIZE )
683                 {
684                     throw new InvalidBTreeException( "The BOB value offset length is invalid " + offsetLength );
685                 }
686                 
687                 // Then the offset
688                 long btreeOffset = byteBuffer.getLong();
689                 
690                 checkOffset( recordManager, btreeOffset );
691                 
692                 // Now, process the key
693                 // First the key length
694                 int keyLength = byteBuffer.getInt();
695     
696                 // The length should be at least 12 bytes long
697                 if ( keyLength < RecordManager.LONG_SIZE + RecordManager.INT_SIZE )
698                 {
699                     throw new InvalidBTreeException( "The BOB key length is invalid " + keyLength );
700                 }
701                 
702                 // Read the revision
703                 long btreeRevision = byteBuffer.getLong();
704                 
705                 // read the btreeName
706                 int btreeNameLength = byteBuffer.getInt();
707     
708                 // The length should be equals to the btreeRevision + btreeNameLength + 4
709                 if ( keyLength != RecordManager.LONG_SIZE + RecordManager.INT_SIZE + btreeNameLength )
710                 {
711                     throw new InvalidBTreeException( "The BOB key length is not the expected value " + 
712                         ( RecordManager.LONG_SIZE + RecordManager.INT_SIZE + btreeNameLength ) + ", expected " + keyLength );
713                 }
714                 
715                 byte[] bytes = new byte[btreeNameLength];
716                 byteBuffer.get( bytes );
717                 String btreeName = Strings.utf8ToString( bytes );
718                 
719                 // Add the new name in the checkedPage name if it's not already there
720                 int[] btreePagesArray = createPageArray( recordManager );
721                 checkedPages.put( btreeName + "<" + btreeRevision + ">", btreePagesArray );
722                 
723                 // Now, we can check the Btree we just found
724                 checkBtree( recordManager, btreeOffset, checkedPages );
725                 
726                 //System.out.println( "read <" + btreeName + "," + btreeRevision + "> : 0x" + Long.toHexString( btreeOffset ) );
727             }
728             catch ( BufferUnderflowException bue )
729             {
730                 throw new InvalidBTreeException( "The BOB leaf byte buffer is too short : " + bue.getMessage() );
731             }
732         }
733     }
734     
735     
736     /**
737      * Check a Btree leaf.
738      */
739     private static <K, V> void checkBtreeLeaf( RecordManager recordManager, BtreeInfo<K, V> btreeInfo, Map<String, int[]> checkedPages, int nbElems, long revision, ByteBuffer byteBuffer, PageIO[] pageIos ) throws Exception
740     {
741         // Read each key and value
742         for ( int i = 0; i < nbElems; i++ )
743         {
744             try
745             {
746                 // Read the number of values
747                 int nbValues = byteBuffer.getInt();
748     
749                 if ( nbValues < 0 )
750                 {
751                     // This is a sub-btree. Read the offset
752                     long subBtreeOffset = byteBuffer.getLong();
753                     
754                     // And process the sub-btree
755                     checkBtree( recordManager, subBtreeOffset, checkedPages );
756                     
757                     // Now, process the key
758                     // The key length
759                     byteBuffer.getInt();
760                     
761                     // The key itself
762                     btreeInfo.keySerializer.deserialize( byteBuffer );
763                 }
764                 else
765                 {
766                     // just deserialize the keys and values
767                     // The value
768                     byteBuffer.getInt();
769                     btreeInfo.valueSerializer.deserialize( byteBuffer );
770                     
771                     // the key
772                     byteBuffer.getInt();
773                     
774                     btreeInfo.keySerializer.deserialize( byteBuffer );
775                 }
776             }
777             catch ( BufferUnderflowException bue )
778             {
779                 throw new InvalidBTreeException( "The BOB leaf byte buffer is too short : " + bue.getMessage() );
780             }
781         }
782     }
783     
784     /**
785      * Check a Btree of Btrees Node
786      */
787     private static <K, V> long[] checkBtreeOfBtreesNode( RecordManager recordManager, Map<String, int[]> checkedPages, int nbElems, long revision,
788         ByteBuffer byteBuffer, PageIO[] pageIos ) throws IOException
789     {
790         long[] children = new long[nbElems + 1];
791 
792         // Read each value
793         for ( int i = 0; i < nbElems; i++ )
794         {
795             // The offsets of the child
796             long firstOffset = LongSerializer.INSTANCE.deserialize( byteBuffer );
797             
798             checkOffset( recordManager, firstOffset );
799             
800             long lastOffset = LongSerializer.INSTANCE.deserialize( byteBuffer );
801 
802             checkOffset( recordManager, lastOffset );
803             
804             children[i] = firstOffset;
805 
806             // Read the key length
807             int keyLength = byteBuffer.getInt();
808             
809             // The length should be at least 12 bytes long
810             if ( keyLength < RecordManager.LONG_SIZE + RecordManager.INT_SIZE )
811             {
812                 throw new InvalidBTreeException( "The BOB key length is invalid " + keyLength );
813             }
814             
815             // Read the revision
816             byteBuffer.getLong();
817             
818             // read the btreeName
819             int btreeNameLength = byteBuffer.getInt();
820 
821             // The length should be equals to the btreeRevision + btreeNameLength + 4
822             if ( keyLength != RecordManager.LONG_SIZE + RecordManager.INT_SIZE + btreeNameLength )
823             {
824                 throw new InvalidBTreeException( "The BOB key length is not the expected value " + 
825                     ( RecordManager.LONG_SIZE + RecordManager.INT_SIZE + btreeNameLength ) + ", expected " + keyLength );
826             }
827             
828             // Read the Btree name
829             byte[] bytes = new byte[btreeNameLength];
830             byteBuffer.get( bytes );
831         }
832 
833         // And read the last child
834         // The offsets of the child
835         long firstOffset = LongSerializer.INSTANCE.deserialize( byteBuffer );
836         
837         checkOffset( recordManager, firstOffset );
838         
839         long lastOffset = LongSerializer.INSTANCE.deserialize( byteBuffer );
840 
841         checkOffset( recordManager, lastOffset );
842         
843         children[nbElems] = firstOffset;
844 
845         // and read the last value, as it's a node
846         return children;
847     }
848     
849     
850     /**
851      * Check a Btree node.
852      */
853     private static <K, V> long[] checkBtreeNode( RecordManager recordManager, BtreeInfo<K, V> btreeInfo, Map<String, int[]> checkedPages, int nbElems, long revision, ByteBuffer byteBuffer, PageIO[] pageIos ) throws Exception
854     {
855         long[] children = new long[nbElems + 1];
856 
857         // Read each key and value
858         for ( int i = 0; i < nbElems; i++ )
859         {
860             try
861             {
862                 // The offsets of the child
863                 long firstOffset = LongSerializer.INSTANCE.deserialize( byteBuffer );
864                 
865                 checkOffset( recordManager, firstOffset );
866                 
867                 long lastOffset = LongSerializer.INSTANCE.deserialize( byteBuffer );
868 
869                 checkOffset( recordManager, lastOffset );
870                 
871                 children[i] = firstOffset;
872                 
873                 // Now, read the key
874                 // The key lenth
875                 byteBuffer.getInt();
876                 
877                 // The key itself
878                 btreeInfo.keySerializer.deserialize( byteBuffer );
879             }
880             catch ( BufferUnderflowException bue )
881             {
882                 throw new InvalidBTreeException( "The BOB leaf byte buffer is too short : " + bue.getMessage() );
883             }
884         }
885         
886         // The last child
887         // The offsets of the child
888         long firstOffset = LongSerializer.INSTANCE.deserialize( byteBuffer );
889         
890         checkOffset( recordManager, firstOffset );
891         
892         long lastOffset = LongSerializer.INSTANCE.deserialize( byteBuffer );
893 
894         checkOffset( recordManager, lastOffset );
895         
896         children[nbElems] = firstOffset;
897         
898         return children;
899     }
900     
901     
902     /**
903      * Create an array of bits for pages 
904      */
905     private static int[] createPageArray( RecordManager recordManager ) throws IOException
906     {
907         long fileSize = recordManager.fileChannel.size();
908         int pageSize = recordManager.pageSize;
909         long nbPages = ( fileSize - RecordManager.RECORD_MANAGER_HEADER_SIZE ) / pageSize;
910         int nbPageBits = ( int ) ( nbPages / 32 );
911         
912         return new int[nbPageBits + 1];
913     }
914 
915 
916     /**
917      * Update the array of seen pages.
918      */
919     private static void updateCheckedPages( int[] checkedPages, int pageSize, PageIO... pageIos )
920     {
921         for ( PageIO pageIO : pageIos )
922         {
923             long offset = pageIO.getOffset();
924 
925             if ( ( offset % pageSize ) != 0 )
926             {
927                 throw new InvalidBTreeException( "Offset invalid : " + offset );
928             }
929 
930             int pageNumber = (int)(offset / pageSize);
931             int nbBitsPage = ( RecordManager.INT_SIZE << 3 );
932             int pageMask = checkedPages[ pageNumber / nbBitsPage ];
933             int mask = 1 << pageNumber % nbBitsPage;
934 
935             if ( ( pageMask & mask ) != 0 )
936             {
937                 //throw new InvalidBTreeException( "The page " + offset + " has already been referenced" );
938             }
939 
940             pageMask |= mask;
941             checkedPages[ pageNumber / nbBitsPage ] = pageMask;
942         }
943     }
944 
945 
946     /**
947      * Check the offset to be sure it's a valid one :
948      * <ul>
949      * <li>It's >= 0</li>
950      * <li>It's below the end of the file</li>
951      * <li>It's a multiple of the pageSize
952      * </ul>
953      */
954     private static void checkOffset( RecordManager recordManager, long offset ) throws IOException
955     {
956         if ( ( offset == RecordManager.NO_PAGE ) ||
957              ( ( ( offset - RecordManager.RECORD_MANAGER_HEADER_SIZE ) % recordManager.pageSize ) != 0 ) ||
958              ( offset > recordManager.fileChannel.size() ) )
959         {
960             throw new InvalidBTreeException( "Invalid Offset : " + offset );
961         }
962     }
963 
964 
965     /**
966      * Check the free pages
967      */
968     private static void checkFreePages( RecordManager recordManager, Map<String, int[]> checkedPages )
969         throws IOException
970     {
971         if ( recordManager.firstFreePage == RecordManager.NO_PAGE )
972         {
973             return;
974         }
975 
976         // Now, read all the free pages
977         long currentOffset = recordManager.firstFreePage;
978         long fileSize = recordManager.fileChannel.size();
979 
980         while ( currentOffset != RecordManager.NO_PAGE )
981         {
982             if ( currentOffset > fileSize )
983             {
984                 System.out.println( "Wrong free page offset, above file size : " + currentOffset );
985                 return;
986             }
987 
988             try
989             {
990                 PageIO pageIo = recordManager.fetchPage( currentOffset );
991 
992                 if ( currentOffset != pageIo.getOffset() )
993                 {
994                     System.out.println( "PageIO offset is incorrect : " + currentOffset + "-"
995                         + pageIo.getOffset() );
996                     return;
997                 }
998 
999                 setCheckedPage( recordManager, checkedPages.get( GLOBAL_PAGES_NAME ), currentOffset );
1000                 setCheckedPage( recordManager, checkedPages.get( FREE_PAGES_NAME ), currentOffset );
1001 
1002                 long newOffset = pageIo.getNextPage();
1003                 currentOffset = newOffset;
1004             }
1005             catch ( IOException ioe )
1006             {
1007                 throw new InvalidBTreeException( "Cannot fetch page at : " + currentOffset );
1008             }
1009         }
1010     }
1011 
1012     
1013     /**
1014      * Update the ChekcedPages array
1015      */
1016     private static void setCheckedPage( RecordManager recordManager, int[] checkedPages, long offset )
1017     {
1018         int pageNumber = ( int ) offset / recordManager.pageSize;
1019         int nbBitsPage = ( RecordManager.INT_SIZE << 3 );
1020         long pageMask = checkedPages[ pageNumber / nbBitsPage ];
1021         long mask = 1L << pageNumber % nbBitsPage;
1022         
1023         if ( ( pageMask & mask ) != 0 )
1024         {
1025             //throw new InvalidBTreeException( "The page " + offset + " has already been referenced" );
1026         }
1027 
1028         pageMask |= mask;
1029         
1030         checkedPages[ pageNumber / nbBitsPage ] |= pageMask;
1031     }
1032 
1033 
1034     /**
1035      * Output the pages that has been seen ('1') and those which has not been seen ('0'). The '.' represent non-pages
1036      * at the end of the file.
1037      */
1038     private static void dumpCheckedPages( RecordManager recordManager, Map<String, int[]> checkedPages ) throws IOException
1039     {
1040         // First dump the global array
1041         int[] globalArray = checkedPages.get( GLOBAL_PAGES_NAME );
1042         String result = dumpPageArray( recordManager, globalArray );
1043         
1044         String dump = String.format( "%1$-40s : %2$s", GLOBAL_PAGES_NAME, result );
1045         System.out.println( dump );
1046         
1047         // The free pages array
1048         int[] freePagesArray = checkedPages.get( FREE_PAGES_NAME );
1049         result = dumpPageArray( recordManager, freePagesArray );
1050         
1051         dump = String.format( "%1$-40s : %2$s", FREE_PAGES_NAME, result );
1052         System.out.println( dump );
1053         
1054         // The B-tree of B-trees pages array
1055         int[] btreeOfBtreesArray = checkedPages.get( RecordManager.BTREE_OF_BTREES_NAME );
1056         result = dumpPageArray( recordManager, btreeOfBtreesArray );
1057         
1058         dump = String.format( "%1$-40s : %2$s", RecordManager.BTREE_OF_BTREES_NAME, result );
1059         System.out.println( dump );
1060         
1061         // The Copied page B-tree pages array
1062         int[] copiedPagesArray = checkedPages.get( RecordManager.COPIED_PAGE_BTREE_NAME );
1063         result = dumpPageArray( recordManager, copiedPagesArray );
1064         
1065         dump = String.format( "%1$-40s : %2$s", RecordManager.COPIED_PAGE_BTREE_NAME, result );
1066         System.out.println( dump );
1067         
1068         // And now, all the other btree arrays
1069         for ( String btreeName : checkedPages.keySet() )
1070         {
1071             // Don't do the array we have already processed
1072             if ( knownPagesArrays.contains( btreeName ) )
1073             {
1074                 continue;
1075             }
1076 
1077             int[] btreePagesArray = checkedPages.get( btreeName );
1078             result = dumpPageArray( recordManager, btreePagesArray );
1079             
1080             dump = String.format( "%1$-40s : %2$s", btreeName, result );
1081             System.out.println( dump );
1082         }
1083     }
1084 
1085     
1086     /**
1087      * Process a page array
1088      */
1089     private static String dumpPageArray( RecordManager recordManager, int[] checkedPages ) throws IOException
1090     {
1091         StringBuilder sb = new StringBuilder();
1092         int i = -1;
1093         int nbPagesChecked = 0;
1094         long fileSize = recordManager.fileChannel.size();
1095         long nbPages = ( fileSize - RecordManager.RECORD_MANAGER_HEADER_SIZE ) / recordManager.pageSize;
1096 
1097         for ( int checkedPage : checkedPages )
1098         {
1099             if ( i == 0 )
1100             {
1101                 sb.append( " " );
1102                 i++;
1103             }
1104             else
1105             {
1106                 i = 0;
1107             }
1108 
1109             sb.append( "[" ).append( i ).append(  "] " );
1110 
1111 
1112             for ( int j = 0; j < 32; j++ )
1113             {
1114                 if ( nbPagesChecked >= nbPages + 1 )
1115                 {
1116                     sb.append( "." );
1117                 }
1118                 else
1119                 {
1120                     if ( ( checkedPage & ( 1 << j ) )  == 0 )
1121                     {
1122                         sb.append( "0" );
1123                     }
1124                     else
1125                     {
1126                         sb.append( "1" );
1127                     }
1128                 }
1129 
1130                 nbPagesChecked++;
1131             }
1132         }
1133 
1134         return sb.toString();
1135     }
1136     
1137 
1138     /**
1139      * The entry point method
1140      */
1141     public void start() throws Exception
1142     {
1143         if ( !checkFilePresence() )
1144         {
1145             return;
1146         }
1147 
1148         if ( !loadRm() )
1149         {
1150             return;
1151         }
1152 
1153         boolean stop = false;
1154 
1155         while ( !stop )
1156         {
1157             System.out.println( "Choose an option:" );
1158             System.out.println( "1. Print Number of BTrees" );
1159             System.out.println( "2. Print BTree Names" );
1160             System.out.println( "3. Inspect BTree" );
1161             System.out.println( "4. Check Free Pages" );
1162             System.out.println( "5. Get database file size" );
1163             System.out.println( "6. Dump RecordManager" );
1164             System.out.println( "7. Reload RecordManager" );
1165             System.out.println( "q. Quit" );
1166 
1167             char c = readOption();
1168 
1169             switch ( c )
1170             {
1171                 case '1':
1172                     printNumberOfBTrees();
1173                     break;
1174 
1175                 case '2':
1176                     printBTreeNames();
1177                     break;
1178 
1179                 case '3':
1180                     checkBTree();
1181                     break;
1182 
1183                 case '4':
1184                     long fileSize = rm.fileChannel.size();
1185                     long nbPages = fileSize / rm.pageSize;
1186                     int nbPageBits = ( int ) ( nbPages / RecordManager.INT_SIZE );
1187 
1188                     Map<String, int[]> checkedPages = new HashMap<String, int[]>(2);
1189 
1190                     // The global page array
1191                     checkedPages.put( GLOBAL_PAGES_NAME, new int[nbPageBits + 1] );
1192 
1193                     // The freePages array
1194                     checkedPages.put( FREE_PAGES_NAME, new int[nbPageBits + 1] );
1195 
1196                     checkFreePages( rm, checkedPages );
1197                     break;
1198 
1199                 case '5':
1200                     printFileSize();
1201                     break;
1202 
1203                 case '6':
1204                     check( rm );
1205                     break;
1206 
1207                 case '7':
1208                     loadRm();
1209                     break;
1210 
1211                 case 'q':
1212                     stop = true;
1213                     break;
1214 
1215                 default:
1216                     System.out.println( "Invalid option" );
1217                     //c = readOption( br );
1218                     break;
1219             }
1220         }
1221 
1222         try
1223         {
1224             rm.close();
1225             br.close();
1226         }
1227         catch ( Exception e )
1228         {
1229             //ignore
1230         }
1231     }
1232 
1233 
1234     /**
1235      * Read the user's interaction
1236      */
1237     private String readLine()
1238     {
1239         try
1240         {
1241             return br.readLine().trim();
1242         }
1243         catch ( Exception e )
1244         {
1245             throw new RuntimeException( e );
1246         }
1247     }
1248 
1249 
1250     /**
1251      * Process the input and get back the selected choice
1252      */
1253     private char readOption()
1254     {
1255         try
1256         {
1257             String s = br.readLine();
1258 
1259             if ( s.length() == 0 )
1260             {
1261                 return ' ';
1262             }
1263 
1264             return s.charAt( 0 );
1265         }
1266         catch ( Exception e )
1267         {
1268             throw new RuntimeException( e );
1269         }
1270     }
1271 
1272 
1273     /**
1274      * Main method
1275      */
1276     public static void main( String[] args ) throws Exception
1277     {
1278         File f = new File( "/tmp/mavibotispector.db" );
1279 
1280         RecordManager rm = new RecordManager( f.getAbsolutePath() );
1281         String name1 = "corpus";
1282         String name2 = "multiValues";
1283         /*
1284         if ( !rm.getManagedTrees().contains( name1 ) )
1285         {
1286             rm.addBTree( name1, StringSerializer.INSTANCE, StringSerializer.INSTANCE, true );
1287         }
1288         */
1289         if ( !rm.getManagedTrees().contains( name2 ) )
1290         {
1291             rm.addBTree( name2, StringSerializer.INSTANCE, StringSerializer.INSTANCE, true );
1292         }
1293 
1294         /*
1295         // Load some elements in the single value btree
1296         BTree<String, String> btree1 = rm.getManagedTree( name1 );
1297         
1298         for ( int i = 0; i < 10; i++ )
1299         {
1300             btree1.insert( Integer.toString( i ), Integer.toString( i ) );
1301         }
1302         */
1303 
1304         // Load some elements in the multi value btree
1305         BTree<String, String> btree2 = rm.getManagedTree( name2 );
1306         
1307         for ( int i = 0; i < 1; i++ )
1308         {
1309             for ( int j = 0; j < 10; j++)
1310             {
1311                 btree2.insert( Integer.toString( i ), Integer.toString( j ) );
1312             }
1313         }
1314 
1315         rm.close();
1316         
1317         MavibotInspector mi = new MavibotInspector( f );
1318         mi.start();
1319     }
1320 }
1321 
1322 
1323 /**
1324  * A class used to store some information about the Btree 
1325  */
1326 final class BtreeInfo<K, V>
1327 {
1328     // The btree name
1329     /* no qualifier */ String btreeName;
1330     
1331     // The key serializer
1332     /* no qualifier */ElementSerializer<K> keySerializer;
1333     
1334     // The value serializer
1335     /* no qualifier */ElementSerializer<V> valueSerializer;
1336     
1337     public String toString()
1338     {
1339         StringBuilder sb = new StringBuilder();
1340         
1341         sb.append( "B-tree Info :" );
1342         sb.append( "\n    name              : " ).append( btreeName );
1343         sb.append( "\n    key serializer    : " ).append( keySerializer.getClass().getName() );
1344         sb.append( "\n    value serializer  : " ).append( valueSerializer.getClass().getName() );
1345         
1346         return sb.toString();
1347     }
1348 }