View Javadoc

1   package org.apache.jcs.auxiliary.disk.block;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.File;
23  import java.io.FileNotFoundException;
24  import java.io.IOException;
25  import java.io.RandomAccessFile;
26  import java.io.Serializable;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.jcs.utils.serialization.StandardSerializer;
31  import org.apache.jcs.utils.struct.SingleLinkedList;
32  
33  /***
34   * This class manages reading an writing data to disk. When asked to write a value, it returns a
35   * block array. It can read an object from the block numbers in a byte array.
36   * <p>
37   * @author Aaron Smuts
38   */
39  public class BlockDisk
40  {
41      /*** The logger */
42      private static final Log log = LogFactory.getLog( BlockDisk.class );
43  
44      /*** The size of the header that indicates the amount of data stored in an occupied block. */
45      public static final byte HEADER_SIZE_BYTES = 4;
46  
47      /*** defaults to 4kb */
48      private static final int DEFAULT_BLOCK_SIZE_BYTES = 4 * 1024;
49  
50      /*** Size of the blocks */
51      private int blockSizeBytes = DEFAULT_BLOCK_SIZE_BYTES;
52  
53      /***
54       * the total number of blocks that have been used. If there are no free, we will use this to
55       * calculate the position of the next block.
56       */
57      private int numberOfBlocks = 0;
58  
59      /*** Empty blocks that can be reused. */
60      private SingleLinkedList emptyBlocks = new SingleLinkedList();
61  
62      /*** Handles serializing the objects */
63      private static final StandardSerializer SERIALIZER = new StandardSerializer();
64  
65      /*** Location of the spot on disk */
66      private final String filepath;
67  
68      /*** The file handle. */
69      private RandomAccessFile raf;
70  
71      /*** How many bytes have we put to disk */
72      private long putBytes = 0;
73  
74      /*** How many items have we put to disk */
75      private long putCount = 0;
76  
77      /***
78       * Constructor for the Disk object
79       * <p>
80       * @param file
81       * @exception FileNotFoundException
82       */
83      public BlockDisk( File file )
84          throws FileNotFoundException
85      {
86          this( file, DEFAULT_BLOCK_SIZE_BYTES );
87          if ( log.isInfoEnabled() )
88          {
89              log.info( "Used default block size [" + DEFAULT_BLOCK_SIZE_BYTES + "]" );
90          }
91  
92      }
93  
94      /***
95       * Creates the file and set the block size in bytes.
96       * <p>
97       * @param file
98       * @param blockSizeBytes
99       * @throws FileNotFoundException
100      */
101     public BlockDisk( File file, int blockSizeBytes )
102         throws FileNotFoundException
103     {
104         this.filepath = file.getAbsolutePath();
105         raf = new RandomAccessFile( filepath, "rw" );
106 
107         if ( log.isInfoEnabled() )
108         {
109             log.info( "Constructing BlockDisk, blockSizeBytes [" + blockSizeBytes + "]" );
110         }
111         this.blockSizeBytes = blockSizeBytes;
112     }
113 
114     /***
115      * This writes an object to disk and returns the blocks it was stored in.
116      * <p>
117      * The program flow is as follows:
118      * <ol>
119      * <li>Serialize the object.</li>
120      * <li>Detemine the number of blocks needed.</li>
121      * <li>Look for free blocks in the emptyBlock list.</li>
122      * <li>If there were not enough in the empty list. Take the nextBlock and increment it.</li>
123      * <li>If the data will not fit in one block, create sub arrays.</li>
124      * <li>Write the subarrays to disk.</li>
125      * <li>If the process fails we should decrement the block count if we took from it.</li>
126      * </ol>
127      * @param object
128      * @return the blocks we used.
129      * @throws IOException
130      */
131     protected int[] write( Serializable object )
132         throws IOException
133     {
134         // serialize the object
135         byte[] data = SERIALIZER.serialize( object );
136 
137         this.addToPutBytes( data.length );
138         this.incrementPutCount();
139 
140         // figure out how many blocks we need.
141         int numBlocksNeeded = calculateTheNumberOfBlocksNeeded( data );
142 
143         int[] blocks = new int[numBlocksNeeded];
144 
145         // get them from the empty list or take the next one
146         for ( short i = 0; i < numBlocksNeeded; i++ )
147         {
148             Integer emptyBlock = (Integer) emptyBlocks.takeFirst();
149             if ( emptyBlock != null )
150             {
151                 blocks[i] = emptyBlock.intValue();
152             }
153             else
154             {
155                 blocks[i] = takeNextBlock();
156             }
157         }
158 
159         // get the individual sub arrays.
160         byte[][] chunks = getBlockChunks( data, numBlocksNeeded );
161 
162         // write the blocks
163         for ( byte i = 0; i < numBlocksNeeded; i++ )
164         {
165             int position = calculateByteOffsetForBlock( blocks[i] );
166             write( position, chunks[i] );
167         }
168 
169         return blocks;
170     }
171 
172     /***
173      * Return the amount to put in each block. Fill them all the way, minus the header.
174      * <p>
175      * @param complete
176      * @param numBlocksNeeded
177      * @return byte[][]
178      */
179     protected byte[][] getBlockChunks( byte[] complete, int numBlocksNeeded )
180     {
181         byte[][] chunks = new byte[numBlocksNeeded][];
182 
183         if ( numBlocksNeeded == 1 )
184         {
185             chunks[0] = complete;
186         }
187         else
188         {
189             int maxChunkSize = this.blockSizeBytes - HEADER_SIZE_BYTES;
190             int totalBytes = complete.length;
191             int totalUsed = 0;
192             for ( short i = 0; i < numBlocksNeeded; i++ )
193             {
194                 // use the max that can be written to a block or whatever is left in the original
195                 // array
196                 int chunkSize = Math.min( totalUsed + maxChunkSize, totalBytes - totalUsed );
197                 byte[] chunk = new byte[chunkSize];
198                 // copy from the used position to the chunk size on the complete array to the chunk
199                 // array.
200                 System.arraycopy( complete, totalUsed, chunk, 0, chunkSize );
201                 chunks[i] = chunk;
202                 totalUsed += chunkSize;
203             }
204         }
205 
206         return chunks;
207     }
208 
209     /***
210      * Writes the given byte array to the Disk at the specified position.
211      * <p>
212      * @param position
213      * @param data
214      * @return true if we wrote successfully
215      * @throws IOException
216      */
217     private boolean write( long position, byte[] data )
218         throws IOException
219     {
220         synchronized ( this )
221         {
222             raf.seek( position );
223             raf.writeInt( data.length );
224             raf.write( data, 0, data.length );
225         }
226         return true;
227     }
228 
229     /***
230      * Reads an object that is located in the specified blocks.
231      * <p>
232      * @param blockNumbers
233      * @return Serializable
234      * @throws IOException
235      * @throws ClassNotFoundException
236      */
237     protected Serializable read( int[] blockNumbers )
238         throws IOException, ClassNotFoundException
239     {
240         byte[] data = null;
241 
242         if ( blockNumbers.length == 1 )
243         {
244             data = readBlock( blockNumbers[0] );
245         }
246         else
247         {
248             data = new byte[0];
249             // get all the blocks into data
250             for ( short i = 0; i < blockNumbers.length; i++ )
251             {
252                 byte[] chunk = readBlock( blockNumbers[i] );
253                 byte[] newTotal = new byte[data.length + chunk.length];
254                 // copy data into the new array
255                 System.arraycopy( data, 0, newTotal, 0, data.length );
256                 // copyt the chunk into the new array
257                 System.arraycopy( chunk, 0, newTotal, data.length, chunk.length );
258                 // swap the new and old.
259                 data = newTotal;
260             }
261         }
262 
263         return (Serializable) SERIALIZER.deSerialize( data );
264     }
265 
266     /***
267      * This reads the occupied data in a block.
268      * <p>
269      * The first four bytes of the record should tell us how long it is. The data is read into a
270      * byte array and then an object is constructed from the byte array.
271      * <p>
272      * @return byte[]
273      * @param block
274      * @throws IOException
275      */
276     private byte[] readBlock( int block )
277         throws IOException
278     {
279         byte[] data = null;
280         int datalen = 0;
281         synchronized ( this )
282         {
283             String message = null;
284             boolean corrupted = false;
285             long fileLength = raf.length();
286 
287             int position = calculateByteOffsetForBlock( block );
288             if ( position > fileLength )
289             {
290                 corrupted = true;
291                 message = "Record " + position + " starts past EOF.";
292             }
293             else
294             {
295                 raf.seek( position );
296                 datalen = raf.readInt();
297                 if ( position + datalen > fileLength )
298                 {
299                     corrupted = true;
300                     message = "Record " + position + " exceeds file length.";
301                 }
302             }
303 
304             if ( corrupted )
305             {
306                 log.warn( "\n The file is corrupt: " + "\n " + message );
307                 throw new IOException( "The File Is Corrupt, need to reset" );
308             }
309 
310             raf.readFully( data = new byte[datalen] );
311         }
312         return data;
313     }
314 
315     /***
316      * Add these blocks to the emptyBlock list.
317      * <p>
318      * @param blocksToFree
319      */
320     protected void freeBlocks( int[] blocksToFree )
321     {
322         if ( blocksToFree != null )
323         {
324             for ( short i = 0; i < blocksToFree.length; i++ )
325             {
326                 emptyBlocks.addLast( new Integer( blocksToFree[i] ) );
327             }
328         }
329     }
330 
331     /***
332      * Add to to total put size.
333      * <p>
334      * @param length
335      */
336     private synchronized void addToPutBytes( long length )
337     {
338         this.putBytes += length;
339     }
340 
341     /***
342      * Thread safe increment.
343      */
344     private synchronized void incrementPutCount()
345     {
346         this.putCount++;
347     }
348 
349     /***
350      * Returns the current number and adds one.
351      * <p>
352      * @return the block number to use.
353      */
354     private synchronized int takeNextBlock()
355     {
356         return this.numberOfBlocks++;
357     }
358 
359     /***
360      * Calcuates the file offset for a particular block.
361      * <p>
362      * @param block
363      * @return the offset for this block
364      */
365     protected int calculateByteOffsetForBlock( int block )
366     {
367         return block * blockSizeBytes;
368     }
369 
370     /***
371      * The number of blocks needed.
372      * <p>
373      * @param data
374      * @return the number of blocks needed to store the byte array
375      */
376     protected int calculateTheNumberOfBlocksNeeded( byte[] data )
377     {
378         int dataLength = data.length;
379 
380         int oneBlock = blockSizeBytes - HEADER_SIZE_BYTES;
381 
382         // takes care of 0 = HEADER_SIZE_BYTES + blockSizeBytes
383         if ( dataLength <= oneBlock )
384         {
385             return 1;
386         }
387 
388         int dividend = dataLength / oneBlock;
389 
390         if ( dataLength % oneBlock != 0 )
391         {
392             dividend++;
393         }
394         return dividend;
395     }
396 
397     /***
398      * Returns the raf length.
399      * <p>
400      * @return the size of the file.
401      * @exception IOException
402      */
403     protected long length()
404         throws IOException
405     {
406         synchronized ( this )
407         {
408             return raf.length();
409         }
410     }
411 
412     /***
413      * Closes the raf.
414      * <p>
415      * @exception IOException
416      */
417     protected synchronized void close()
418         throws IOException
419     {
420         raf.close();
421     }
422 
423     /***
424      * Returns the serialized form of the given object in a byte array.
425      * <p>
426      * Use the Serilizer abstraction layer.
427      * <p>
428      * @return a byte array of the serialized object.
429      * @param obj
430      * @exception IOException
431      */
432     protected static byte[] serialize( Serializable obj )
433         throws IOException
434     {
435         return SERIALIZER.serialize( obj );
436     }
437 
438     /***
439      * @return Returns the numberOfBlocks.
440      */
441     protected int getNumberOfBlocks()
442     {
443         return numberOfBlocks;
444     }
445 
446     /***
447      * @return Returns the blockSizeBytes.
448      */
449     protected int getBlockSizeBytes()
450     {
451         return blockSizeBytes;
452     }
453 
454     /***
455      * @return Returns the average size of the an element inserted.
456      */
457     protected long getAveragePutSizeBytes()
458     {
459         if ( this.putCount == 0 )
460         {
461             return 0;
462         }
463         return this.putBytes / this.putCount;
464     }
465 
466     /***
467      * @return Returns the number of empty blocks.
468      */
469     protected int getEmptyBlocks()
470     {
471         return this.emptyBlocks.size();
472     }
473 
474     /***
475      * For debugging only.
476      * <p>
477      * @return String with details.
478      */
479     public String toString()
480     {
481         StringBuffer buf = new StringBuffer();
482         buf.append( "\nBlock Disk " );
483         buf.append( "\n  Filepath [" + filepath + "]" );
484         buf.append( "\n  NumberOfBlocks [" + getNumberOfBlocks() + "]" );
485         buf.append( "\n  BlockSizeBytes [" + getBlockSizeBytes() + "]" );
486         buf.append( "\n  Put Bytes [" + this.putBytes + "]" );
487         buf.append( "\n  Put Count [" + this.putCount + "]" );
488         buf.append( "\n  Average Size [" + getAveragePutSizeBytes() + "]" );
489         buf.append( "\n  Empty Blocks [" + this.getEmptyBlocks() + "]" );
490         try
491         {
492             buf.append( "\n  Length [" + length() + "]" );
493         }
494         catch ( IOException e )
495         {
496             // swallow
497         }
498         return buf.toString();
499     }
500 }