View Javadoc

1   package org.apache.torque.util;
2   
3   /*
4    * Copyright 2001-2004 The Apache Software Foundation.
5    *
6    * Licensed under the Apache License, Version 2.0 (the "License")
7    * you may not use this file except in compliance with the License.
8    * You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  import java.sql.Connection;
20  import java.sql.SQLException;
21  
22  import java.util.Iterator;
23  import java.util.List;
24  import java.util.ArrayList;
25  import java.util.Hashtable;
26  import java.util.Set;
27  import java.io.Serializable;
28  import java.lang.reflect.Method;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  
33  import org.apache.torque.Torque;
34  import org.apache.torque.TorqueException;
35  
36  import com.workingdogs.village.QueryDataSet;
37  import com.workingdogs.village.DataSetException;
38  
39  /***
40   * This class can be used to retrieve a large result set from a database query.
41   * The query is started and then rows are returned a page at a time.  The <code>
42   * LargeSelect</code> is meant to be placed into the Session or User.Temp, so
43   * that it can be used in response to several related requests.  Note that in
44   * order to use <code>LargeSelect</code> you need to be willing to accept the
45   * fact that the result set may become inconsistent with the database if updates
46   * are processed subsequent to the queries being executed.  Specifying a memory
47   * page limit of 1 will give you a consistent view of the records but the totals
48   * may not be accurate and the performance will be terrible.  In most cases
49   * the potential for inconsistencies data should not cause any serious problems
50   * and performance should be pretty good (but read on for further warnings).
51   *
52   * <p>The idea here is that the full query result would consume too much memory
53   * and if displayed to a user the page would be too long to be useful.  Rather
54   * than loading the full result set into memory, a window of data (the memory
55   * limit) is loaded and retrieved a page at a time.  If a request occurs for
56   * data that falls outside the currently loaded window of data then a new query
57   * is executed to fetch the required data.  Performance is optimized by
58   * starting a thread to execute the database query and fetch the results.  This
59   * will perform best when paging forwards through the data, but a minor
60   * optimization where the window is moved backwards by two rather than one page
61   * is included for when a user pages past the beginning of the window.
62   *
63   * <p>As the query is performed in in steps, it is often the case that the total
64   * number of records and pages of data is unknown.  <code>LargeSelect</code>
65   * provides various methods for indicating how many records and pages it is
66   * currently aware of and for presenting this information to users.
67   *
68   * <p><code>LargeSelect</code> utilises the <code>Criteria</code> methods
69   * <code>setOffset()</code> and <code>setLimit()</code> to limit the amount of
70   * data retrieved from the database - these values are either passed through to
71   * the DBMS when supported (efficient with the caveat below) or handled by
72   * the Village API when it is not (not so efficient).  At time of writing
73   * <code>Criteria</code> will only pass the offset and limit through to MySQL
74   * and PostgreSQL (with a few changes to <code>DBOracle</code> and <code>
75   * BasePeer</code> Oracle support can be implemented by utilising the <code>
76   * rownum</code> pseudo column).
77   *
78   * <p>As <code>LargeSelect</code> must re-execute the query each time the user
79   * pages out of the window of loaded data, you should consider the impact of
80   * non-index sort orderings and other criteria that will require the DBMS to
81   * execute the entire query before filtering down to the offset and limit either
82   * internally or via Village.
83   *
84   * <p>The memory limit defaults to 5 times the page size you specify, but
85   * alternative constructors and the class method <code>setMemoryPageLimit()
86   * </code> allow you to override this for a specific instance of
87   * <code>LargeSelect</code> or future instances respectively.
88   *
89   * <p>Some of the constructors allow you to specify the name of the class to use
90   * to build the returnd rows.  This works by using reflection to find <code>
91   * addSelectColumns(Criteria)</code> and <code>populateObjects(List)</code>
92   * methods to add the necessary select columns to the criteria (only if it
93   * doesn't already contain any) and to convert query results from Village
94   * <code>Record</code> objects to a class defined within the builder class.
95   * This allows you to use any of the Torque generated Peer classes, but also
96   * makes it fairly simple to construct business object classes that can be used
97   * for this purpose (simply copy and customise the <code>addSelectColumns()
98   * </code>, <code>populateObjects()</code>, <code>row2Object()</code> and <code>
99   * populateObject()</code> methods from an existing Peer class).
100  *
101  * <p>Typically you will create a <code>LargeSelect</code> using your <code>
102  * Criteria</code> (perhaps created from the results of a search parameter
103  * page), page size, memory page limit and return class name (for which you may
104  * have defined a business object class before hand) and place this in user.Temp
105  * thus:
106  *
107  * <pre>
108  *     data.getUser().setTemp("someName", largeSelect);
109  * </pre>
110  *
111  * <p>In your template you will then use something along the lines of:
112  *
113  * <pre>
114  *    #set ($largeSelect = $data.User.getTemp("someName"))
115  *    #set ($searchop = $data.Parameters.getString("searchop"))
116  *    #if ($searchop.equals("prev"))
117  *      #set ($recs = $largeSelect.PreviousResults)
118  *    #else
119  *      #if ($searchop.equals("goto"))
120  *        #set ($recs
121  *                = $largeSelect.getPage($data.Parameters.getInt("page", 1)))
122  *      #else
123  *        #set ($recs = $largeSelect.NextResults)
124  *      #end
125  *    #end
126  * </pre>
127  *
128  * <p>...to move through the records.  <code>LargeSelect</code> implements a
129  * number of convenience methods that make it easy to add all of the necessary
130  * bells and whistles to your template.
131  *
132  * @author <a href="mailto:john.mcnally@clearink.com">John D. McNally</a>
133  * @author <a href="mailto:seade@backstagetech.com.au">Scott Eade</a>
134  * @version $Id: LargeSelect.java,v 1.16 2005/02/16 07:58:49 tfischer Exp $
135  */
136 public class LargeSelect implements Runnable, Serializable
137 {
138     /*** The number of records that a page consists of.  */
139     private int pageSize;
140     /*** The maximum number of records to maintain in memory. */
141     private int memoryLimit;
142 
143     /*** The record number of the first record in memory. */
144     private int blockBegin = 0;
145     /*** The record number of the last record in memory. */
146     private int blockEnd;
147     /*** How much of the memory block is currently occupied with result data. */
148     private volatile int currentlyFilledTo = -1;
149 
150     /*** The SQL query that this <code>LargeSelect</code> represents. */
151     private String query;
152     /*** The database name to get from Torque. */
153     private String dbName;
154     /*** Used to retrieve query results from Village. */
155     private QueryDataSet qds = null;
156 
157     /*** The memory store of records. */
158     private List results = null;
159 
160     /*** The thread that executes the query. */
161     private Thread thread = null;
162     /***
163      * A flag used to kill the thread when the currently executing query is no
164      * longer required.
165      */
166     private volatile boolean killThread = false;
167     /*** A flag that indicates whether or not the query thread is running. */
168     private volatile boolean threadRunning = false;
169     /***
170      * An indication of whether or not the current query has completed
171      * processing.
172      */
173     private volatile boolean queryCompleted = false;
174     /***
175      * An indication of whether or not the totals (records and pages) are at
176      * their final values.
177      */
178     private boolean totalsFinalized = false;
179 
180     /*** The cursor position in the result set. */
181     private int position;
182     /*** The total number of pages known to exist. */
183     private int totalPages = -1;
184     /*** The total number of records known to exist. */
185     private int totalRecords = 0;
186     /*** The number of the page that was last retrieved. */
187     private int currentPageNumber = 0;
188 
189     /*** The criteria used for the query. */
190     private Criteria criteria = null;
191     /*** The last page of results that were returned. */
192     private List lastResults;
193 
194     /***
195      * The class that is possibly used to construct the criteria and used
196      * to transform the Village Records into the desired OM or business objects.
197      */
198     private Class returnBuilderClass = null;
199     /***
200      * A reference to the method in the return builder class that will
201      * convert the Village Records to the desired class.
202      */
203     private Method populateObjectsMethod = null;
204 
205     /***
206      * The default value ("&gt;") used to indicate that the total number of
207      * records or pages is unknown. You can use <code>setMoreIndicator()</code>
208      * to change this to whatever value you like (e.g. "more than").
209      */
210     public static final String DEFAULT_MORE_INDICATOR = "&gt;";
211 
212     private static String moreIndicator = DEFAULT_MORE_INDICATOR;
213 
214     /***
215      * The default value for the maximum number of pages of data to be retained
216      * in memory - you can provide your own default value using
217      * <code>setMemoryPageLimit()</code>.
218      */
219     public static final int DEFAULT_MEMORY_LIMIT_PAGES = 5;
220 
221     private static int memoryPageLimit = DEFAULT_MEMORY_LIMIT_PAGES;
222 
223     /*** A place to store search parameters that relate to this query. */
224     private Hashtable params = null;
225 
226     /*** Logging */
227     private static Log log = LogFactory.getLog(LargeSelect.class);
228 
229     /***
230      * Creates a LargeSelect whose results are returned as a <code>List</code>
231      * containing a maximum of <code>pageSize</code> Village <code>Record</code>
232      * objects at a time, maintaining a maximum of
233      * <code>LargeSelect.memoryPageLimit</code> pages of results in memory.
234      *
235      * @param criteria object used by BasePeer to build the query.  In order to
236      * allow this class to utilise database server implemented offsets and
237      * limits (when available), the provided criteria must not have any limit or
238      * offset defined.
239      * @param pageSize number of rows to return in one block.
240      * @throws IllegalArgumentException if <code>criteria</code> uses one or
241      * both of offset and limit, or if <code>pageSize</code> is less than 1;
242      */
243     public LargeSelect(Criteria criteria, int pageSize)
244             throws IllegalArgumentException
245     {
246         this(criteria, pageSize, LargeSelect.memoryPageLimit);
247     }
248 
249     /***
250      * Creates a LargeSelect whose results are returned as a <code>List</code>
251      * containing a maximum of <code>pageSize</code> Village <code>Record</code>
252      * objects at a time, maintaining a maximum of <code>memoryPageLimit</code>
253      * pages of results in memory.
254      *
255      * @param criteria object used by BasePeer to build the query.  In order to
256      * allow this class to utilise database server implemented offsets and
257      * limits (when available), the provided criteria must not have any limit or
258      * offset defined.
259      * @param pageSize number of rows to return in one block.
260      * @param memoryPageLimit maximum number of pages worth of rows to be held
261      * in memory at one time.
262      * @throws IllegalArgumentException if <code>criteria</code> uses one or
263      * both of offset and limit, or if <code>pageSize</code> or
264      * <code>memoryLimitPages</code> are less than 1;
265      */
266     public LargeSelect(Criteria criteria, int pageSize, int memoryPageLimit)
267             throws IllegalArgumentException
268     {
269         init(criteria, pageSize, memoryPageLimit);
270     }
271 
272     /***
273      * Creates a LargeSelect whose results are returned as a <code>List</code>
274      * containing a maximum of <code>pageSize</code> objects of the type
275      * defined within the class named <code>returnBuilderClassName</code> at a
276      * time, maintaining a maximum of <code>LargeSelect.memoryPageLimit</code>
277      * pages of results in memory.
278      *
279      * @param criteria object used by BasePeer to build the query.  In order to
280      * allow this class to utilise database server implemented offsets and
281      * limits (when available), the provided criteria must not have any limit or
282      * offset defined.  If the criteria does not include the definition of any
283      * select columns the <code>addSelectColumns(Criteria)</code> method of
284      * the class named as <code>returnBuilderClassName</code> will be used to
285      * add them.
286      * @param pageSize number of rows to return in one block.
287      * @param returnBuilderClassName The name of the class that will be used to
288      * build the result records (may implement <code>addSelectColumns(Criteria)
289      * </code> and must implement <code>populateObjects(List)</code>).
290      * @throws IllegalArgumentException if <code>criteria</code> uses one or
291      * both of offset and limit, if <code>pageSize</code> is less than 1, or if
292      * problems are experienced locating and invoking either one or both of
293      * <code>addSelectColumns(Criteria)</code> and <code> populateObjects(List)
294      * </code> in the class named <code>returnBuilderClassName</code>.
295      */
296     public LargeSelect(
297             Criteria criteria,
298             int pageSize,
299             String returnBuilderClassName)
300             throws IllegalArgumentException
301     {
302         this(
303             criteria,
304             pageSize,
305             LargeSelect.memoryPageLimit,
306             returnBuilderClassName);
307     }
308 
309     /***
310      * Creates a LargeSelect whose results are returned as a <code>List</code>
311      * containing a maximum of <code>pageSize</code> objects of the type
312      * defined within the class named <code>returnBuilderClassName</code> at a
313      * time, maintaining a maximum of <code>memoryPageLimit</code> pages of
314      * results in memory.
315      *
316      * @param criteria object used by BasePeer to build the query.  In order to
317      * allow this class to utilise database server implemented offsets and
318      * limits (when available), the provided criteria must not have any limit or
319      * offset defined.  If the criteria does not include the definition of any
320      * select columns the <code>addSelectColumns(Criteria)</code> method of
321      * the class named as <code>returnBuilderClassName</code> will be used to
322      * add them.
323      * @param pageSize number of rows to return in one block.
324      * @param memoryPageLimit maximum number of pages worth of rows to be held
325      * in memory at one time.
326      * @param returnBuilderClassName The name of the class that will be used to
327      * build the result records (may implement <code>addSelectColumns(Criteria)
328      * </code> and must implement <code>populateObjects(List)</code>).
329      * @throws IllegalArgumentException if <code>criteria</code> uses one or
330      * both of offset and limit, if <code>pageSize</code> or <code>
331      * memoryLimitPages</code> are less than 1, or if problems are experienced
332      * locating and invoking either one or both of <code>
333      * addSelectColumns(Criteria)</code> and <code> populateObjects(List)</code>
334      * in the class named <code>returnBuilderClassName</code>.
335      */
336     public LargeSelect(
337             Criteria criteria,
338             int pageSize,
339             int memoryPageLimit,
340             String returnBuilderClassName)
341             throws IllegalArgumentException
342     {
343         try
344         {
345             this.returnBuilderClass = Class.forName(returnBuilderClassName);
346 
347             // Add the select columns if necessary.
348             if (criteria.getSelectColumns().size() == 0)
349             {
350                 Class[] argTypes = { Criteria.class };
351                 Method selectColumnAdder =
352                     returnBuilderClass.getMethod("addSelectColumns", argTypes);
353                 Object[] theArgs = { criteria };
354                 selectColumnAdder.invoke(returnBuilderClass.newInstance(),
355                         theArgs);
356             }
357 
358             // Locate the populateObjects() method - this will be used later
359             Class[] argTypes = { List.class };
360             populateObjectsMethod =
361                 returnBuilderClass.getMethod("populateObjects", argTypes);
362         }
363         catch (Exception e)
364         {
365             throw new IllegalArgumentException(
366                     "The class named as returnBuilderClassName does not "
367                     + "provide the necessary facilities - see javadoc.");
368         }
369 
370         init(criteria, pageSize, memoryPageLimit);
371     }
372 
373     /***
374      * Called by the constructors to start the query.
375      *
376      * @param criteria Object used by <code>BasePeer</code> to build the query.
377      * In order to allow this class to utilise database server implemented
378      * offsets and limits (when available), the provided criteria must not have
379      * any limit or offset defined.
380      * @param pageSize number of rows to return in one block.
381      * @param memoryLimitPages maximum number of pages worth of rows to be held
382      * in memory at one time.
383      * @throws IllegalArgumentException if <code>criteria</code> uses one or
384      * both of offset and limit and if <code>pageSize</code> or
385      * <code>memoryLimitPages</code> are less than 1;
386      */
387     private void init(Criteria criteria, int pageSize, int memoryLimitPages)
388             throws IllegalArgumentException
389     {
390         if (criteria.getOffset() != 0 || criteria.getLimit() != -1)
391         {
392             throw new IllegalArgumentException(
393                     "criteria must not use Offset and/or Limit.");
394         }
395 
396         if (pageSize < 1)
397         {
398             throw new IllegalArgumentException(
399                     "pageSize must be greater than zero.");
400         }
401 
402         if (memoryLimitPages < 1)
403         {
404             throw new IllegalArgumentException(
405                     "memoryPageLimit must be greater than zero.");
406         }
407 
408         this.pageSize = pageSize;
409         this.memoryLimit = pageSize * memoryLimitPages;
410         this.criteria = criteria;
411         dbName = criteria.getDbName();
412         blockEnd = blockBegin + memoryLimit - 1;
413         startQuery(pageSize);
414     }
415 
416     /***
417      * Retrieve a specific page, if it exists.
418      *
419      * @param pageNumber the number of the page to be retrieved - must be
420      * greater than zero.  An empty <code>List</code> will be returned if
421      * <code>pageNumber</code> exceeds the total number of pages that exist.
422      * @return a <code>List</code> of query results containing a maximum of
423      * <code>pageSize</code> results.
424      * @throws IllegalArgumentException when <code>pageNo</code> is not
425      * greater than zero.
426      * @throws TorqueException if invoking the <code>populateObjects()<code>
427      * method runs into problems or a sleep is unexpectedly interrupted.
428      */
429     public List getPage(int pageNumber) throws TorqueException
430     {
431         if (pageNumber < 1)
432         {
433             throw new IllegalArgumentException("pageNumber must be greater "
434                     + "than zero.");
435         }
436         currentPageNumber = pageNumber;
437         return getResults((pageNumber - 1) * pageSize);
438     }
439 
440     /***
441      * Gets the next page of rows.
442      *
443      * @return a <code>List</code> of query results containing a maximum of
444      * <code>pageSize</code> reslts.
445      * @throws TorqueException if invoking the <code>populateObjects()<code>
446      * method runs into problems or a sleep is unexpectedly interrupted.
447      */
448     public List getNextResults() throws TorqueException
449     {
450         if (!getNextResultsAvailable())
451         {
452             return getCurrentPageResults();
453         }
454         currentPageNumber++;
455         return getResults(position);
456     }
457 
458     /***
459      * Provide access to the results from the current page.
460      *
461      * @return a <code>List</code> of query results containing a maximum of
462      * <code>pageSize</code> reslts.
463      */
464     public List getCurrentPageResults()
465     {
466         return lastResults;
467     }
468 
469     /***
470      * Gets the previous page of rows.
471      *
472      * @return a <code>List</code> of query results containing a maximum of
473      * <code>pageSize</code> reslts.
474      * @throws TorqueException if invoking the <code>populateObjects()<code>
475      * method runs into problems or a sleep is unexpectedly interrupted.
476      */
477     public List getPreviousResults() throws TorqueException
478     {
479         if (!getPreviousResultsAvailable())
480         {
481             return getCurrentPageResults();
482         }
483 
484         int start;
485         if (position - 2 * pageSize < 0)
486         {
487             start = 0;
488             currentPageNumber = 1;
489         }
490         else
491         {
492             start = position - 2 * pageSize;
493             currentPageNumber--;
494         }
495         return getResults(start);
496     }
497 
498     /***
499      * Gets a page of rows starting at a specified row.
500      *
501      * @param start the starting row.
502      * @return a <code>List</code> of query results containing a maximum of
503      * <code>pageSize</code> reslts.
504      * @throws TorqueException if invoking the <code>populateObjects()<code>
505      * method runs into problems or a sleep is unexpectedly interrupted.
506      */
507     private List getResults(int start) throws TorqueException
508     {
509         return getResults(start, pageSize);
510     }
511 
512     /***
513      * Gets a block of rows starting at a specified row and containing a
514      * specified number of rows.
515      *
516      * @param start the starting row.
517      * @param size the number of rows.
518      * @return a <code>List</code> of query results containing a maximum of
519      * <code>pageSize</code> reslts.
520      * @throws IllegalArgumentException if <code>size &gt; memoryLimit</code> or
521      * <code>start</code> and <code>size</code> result in a situation that is
522      * not catered for.
523      * @throws TorqueException if invoking the <code>populateObjects()<code>
524      * method runs into problems or a sleep is unexpectedly interrupted.
525      */
526     private synchronized List getResults(int start, int size)
527             throws IllegalArgumentException, TorqueException
528     {
529         if (log.isDebugEnabled())
530         {
531             log.debug("getResults(start: " + start
532                     + ", size: " + size + ") invoked.");
533         }
534 
535         if (size > memoryLimit)
536         {
537             throw new IllegalArgumentException("size (" + size
538                     + ") exceeds memory limit (" + memoryLimit + ").");
539         }
540 
541         // Request was for a block of rows which should be in progess.
542         // If the rows have not yet been returned, wait for them to be
543         // retrieved.
544         if (start >= blockBegin && (start + size - 1) <= blockEnd)
545         {
546             if (log.isDebugEnabled())
547             {
548                 log.debug("getResults(): Sleeping until "
549                         + "start+size-1 (" + (start + size - 1)
550                         + ") > currentlyFilledTo (" + currentlyFilledTo
551                         + ") && !queryCompleted (!" + queryCompleted + ")");
552             }
553             while (((start + size - 1) > currentlyFilledTo) && !queryCompleted)
554             {
555                 try
556                 {
557                     Thread.sleep(500);
558                 }
559                 catch (InterruptedException e)
560                 {
561                     throw new TorqueException("Unexpected interruption", e);
562                 }
563             }
564         }
565 
566         // Going in reverse direction, trying to limit db hits so assume user
567         // might want at least 2 sets of data.
568         else if (start < blockBegin && start >= 0)
569         {
570             if (log.isDebugEnabled())
571             {
572                 log.debug("getResults(): Paging backwards as start (" + start
573                         + ") < blockBegin (" + blockBegin + ") && start >= 0");
574             }
575             stopQuery();
576             if (memoryLimit >= 2 * size)
577             {
578                 blockBegin = start - size;
579                 if (blockBegin < 0)
580                 {
581                     blockBegin = 0;
582                 }
583             }
584             else
585             {
586                 blockBegin = start;
587             }
588             blockEnd = blockBegin + memoryLimit - 1;
589             startQuery(size);
590             // Re-invoke getResults() to provide the wait processing.
591             return getResults(start, size);
592         }
593 
594         // Assume we are moving on, do not retrieve any records prior to start.
595         else if ((start + size - 1) > blockEnd)
596         {
597             if (log.isDebugEnabled())
598             {
599                 log.debug("getResults(): Paging past end of loaded data as "
600                         + "start+size-1 (" + (start + size - 1)
601                         + ") > blockEnd (" + blockEnd + ")");
602             }
603             stopQuery();
604             blockBegin = start;
605             blockEnd = blockBegin + memoryLimit - 1;
606             startQuery(size);
607             // Re-invoke getResults() to provide the wait processing.
608             return getResults(start, size);
609         }
610 
611         else
612         {
613             throw new IllegalArgumentException("Parameter configuration not "
614                     + "accounted for.");
615         }
616 
617         int fromIndex = start - blockBegin;
618         int toIndex = fromIndex + Math.min(size, results.size() - fromIndex);
619 
620         if (log.isDebugEnabled())
621         {
622             log.debug("getResults(): Retrieving records from results elements "
623                     + "start-blockBegin (" + fromIndex + ") through "
624                     + "fromIndex + Math.min(size, results.size() - fromIndex) ("
625                     + toIndex + ")");
626         }
627 
628         List returnResults;
629 
630         synchronized (results) 
631         {
632             returnResults = new ArrayList(results.subList(fromIndex, toIndex));
633         }
634 
635         if (null != returnBuilderClass)
636         {
637             // Invoke the populateObjects() method
638             Object[] theArgs = { returnResults };
639             try
640             {
641                 returnResults =
642                     (List) populateObjectsMethod.invoke(
643                         returnBuilderClass.newInstance(),
644                         theArgs);
645             }
646             catch (Exception e)
647             {
648                 throw new TorqueException("Unable to populate results", e);
649             }
650         }
651         position = start + size;
652         lastResults = returnResults;
653         return returnResults;
654     }
655 
656     /***
657      * A background thread that retrieves the rows.
658      */
659     public void run()
660     {
661         int size = pageSize;
662         /* The connection to the database. */
663         Connection conn = null;
664 
665         try
666         {
667             // Add 1 to memory limit to check if the query ends on a page break.
668             results = new ArrayList(memoryLimit + 1);
669 
670             // Use the criteria to limit the rows that are retrieved to the
671             // block of records that fit in the predefined memoryLimit.
672             criteria.setOffset(blockBegin);
673             // Add 1 to memory limit to check if the query ends on a page break.
674             criteria.setLimit(memoryLimit + 1);
675             query = BasePeer.createQueryString(criteria);
676 
677             // Get a connection to the db.
678             conn = Torque.getConnection(dbName);
679 
680             // Execute the query.
681             if (log.isDebugEnabled())
682             {
683                 log.debug("run(): query = " + query);
684                 log.debug("run(): memoryLimit = " + memoryLimit);
685                 log.debug("run(): blockBegin = " + blockBegin);
686                 log.debug("run(): blockEnd = " + blockEnd);
687             }
688             qds = new QueryDataSet(conn, query);
689 
690             // Continue getting rows one page at a time until the memory limit
691             // is reached, all results have been retrieved, or the rest
692             // of the results have been determined to be irrelevant.
693             while (!killThread
694                 && !qds.allRecordsRetrieved()
695                 && currentlyFilledTo + pageSize <= blockEnd)
696             {
697                 // This caters for when memoryLimit is not a multiple of
698                 //  pageSize which it never is because we always add 1 above.
699                 if ((currentlyFilledTo + pageSize) >= blockEnd)
700                 {
701                     // Add 1 to check if the query ends on a page break.
702                     size = blockEnd - currentlyFilledTo + 1;
703                 }
704 
705                 if (log.isDebugEnabled())
706                 {
707                     log.debug("run(): Invoking BasePeer.getSelectResults(qds, "
708                             + size + ", false)");
709                 }
710 
711                 List tempResults
712                         = BasePeer.getSelectResults(qds, size, false);
713 
714                 synchronized (results) 
715                 {
716                     for (int i = 0, n = tempResults.size(); i < n; i++)
717                     {
718                         results.add(tempResults.get(i));
719                     }
720                 }
721 
722                 currentlyFilledTo += tempResults.size();
723                 boolean perhapsLastPage = true;
724 
725                 // If the extra record was indeed found then we know we are not
726                 // on the last page but we must now get rid of it.
727                 if (results.size() == memoryLimit + 1)
728                 {
729                     synchronized (results)
730                     {
731                         results.remove(currentlyFilledTo--);
732                     }
733                     perhapsLastPage = false;
734                 }
735 
736                 if (results.size() > 0
737                     && blockBegin + currentlyFilledTo >= totalRecords)
738                 {
739                     // Add 1 because index starts at 0
740                     totalRecords = blockBegin + currentlyFilledTo + 1;
741                 }
742 
743                 if (qds.allRecordsRetrieved())
744                 {
745                     queryCompleted = true;
746                     // The following ugly condition ensures that the totals are
747                     // not finalized when a user does something like requesting
748                     // a page greater than what exists in the database.
749                     if (perhapsLastPage
750                         && getCurrentPageNumber() <= getTotalPages())
751                     {
752                         totalsFinalized = true;
753                     }
754                 }
755                 qds.clearRecords();
756             }
757 
758             if (log.isDebugEnabled())
759             {
760                 log.debug("run(): While loop terminated because either:");
761                 log.debug("run(): 1. qds.allRecordsRetrieved(): "
762                         + qds.allRecordsRetrieved());
763                 log.debug("run(): 2. killThread: " + killThread);
764                 log.debug("run(): 3. !(currentlyFilledTo + size <= blockEnd): !"
765                         + (currentlyFilledTo + pageSize <= blockEnd));
766                 log.debug("run(): - currentlyFilledTo: " + currentlyFilledTo);
767                 log.debug("run(): - size: " + pageSize);
768                 log.debug("run(): - blockEnd: " + blockEnd);
769                 log.debug("run(): - results.size(): " + results.size());
770             }
771         }
772         catch (TorqueException e)
773         {
774             log.error(e);
775         }
776         catch (SQLException e)
777         {
778             log.error(e);
779         }
780         catch (DataSetException e)
781         {
782             log.error(e);
783         }
784         finally
785         {
786             try
787             {
788                 if (qds != null)
789                 {
790                     qds.close();
791                 }
792                 Torque.closeConnection(conn);
793             }
794             catch (SQLException e)
795             {
796                 log.error(e);
797             }
798             catch (DataSetException e)
799             {
800                 log.error(e);
801             }
802             threadRunning = false;
803         }
804     }
805 
806     /***
807      * Starts a new thread to retrieve the result set.
808      *
809      * @param initialSize the initial size for each block.
810      */
811     private synchronized void startQuery(int initialSize)
812     {
813         if (!threadRunning)
814         {
815             pageSize = initialSize;
816             currentlyFilledTo = -1;
817             queryCompleted = false;
818             thread = new Thread(this);
819             thread.start();
820             threadRunning = true;
821         }
822     }
823 
824     /***
825      * Used to stop filling the memory with the current block of results, if it
826      * has been determined that they are no longer relevant.
827      *
828      * @throws TorqueException if a sleep is interrupted.
829      */
830     private synchronized void stopQuery() throws TorqueException
831     {
832         if (threadRunning)
833         {
834             killThread = true;
835             while (thread.isAlive())
836             {
837                 try
838                 {
839                     Thread.sleep(100);
840                 }
841                 catch (InterruptedException e)
842                 {
843                     throw new TorqueException("Unexpected interruption", e);
844                 }
845             }
846             killThread = false;
847         }
848     }
849 
850     /***
851      * Retrieve the number of the current page.
852      *
853      * @return the current page number.
854      */
855     public int getCurrentPageNumber()
856     {
857         return currentPageNumber;
858     }
859 
860     /***
861      * Retrieve the total number of search result records that are known to
862      * exist (this will be the actual value when the query has completeted (see
863      * <code>getTotalsFinalized()</code>).  The convenience method
864      * <code>getRecordProgressText()</code> may be more useful for presenting to
865      * users.
866      *
867      * @return the number of result records known to exist (not accurate until
868      * <code>getTotalsFinalized()</code> returns <code>true</code>).
869      */
870     public int getTotalRecords()
871     {
872         return totalRecords;
873     }
874 
875     /***
876      * Provide an indication of whether or not paging of results will be
877      * required.
878      *
879      * @return <code>true</code> when multiple pages of results exist.
880      */
881     public boolean getPaginated()
882     {
883         // Handle a page memory limit of 1 page.
884         if (!getTotalsFinalized())
885         {
886             return true;
887         }
888         return blockBegin + currentlyFilledTo + 1 > pageSize;
889     }
890 
891     /***
892      * Retrieve the total number of pages of search results that are known to
893      * exist (this will be the actual value when the query has completeted (see
894      * <code>getQyeryCompleted()</code>).  The convenience method
895      * <code>getPageProgressText()</code> may be more useful for presenting to
896      * users.
897      *
898      * @return the number of pages of results known to exist (not accurate until
899      * <code>getTotalsFinalized()</code> returns <code>true</code>).
900      */
901     public int getTotalPages()
902     {
903         if (totalPages > -1)
904         {
905             return totalPages;
906         }
907 
908         int tempPageCount =
909             getTotalRecords() / pageSize
910                 + (getTotalRecords() % pageSize > 0 ? 1 : 0);
911 
912         if (getTotalsFinalized())
913         {
914             totalPages = tempPageCount;
915         }
916 
917         return tempPageCount;
918     }
919 
920     /***
921      * Retrieve the page size.
922      *
923      * @return the number of records returned on each invocation of
924      * <code>getNextResults()</code>/<code>getPreviousResults()</code>.
925      */
926     public int getPageSize()
927     {
928         return pageSize;
929     }
930 
931     /***
932      * Provide access to indicator that the total values for the number of
933      * records and pages are now accurate as opposed to known upper limits.
934      *
935      * @return <code>true</code> when the totals are known to have been fully
936      * computed.
937      */
938     public boolean getTotalsFinalized()
939     {
940         return totalsFinalized;
941     }
942 
943     /***
944      * Provide a way of changing the more pages/records indicator.
945      *
946      * @param moreIndicator the indicator to use in place of the default
947      * ("&gt;").
948      */
949     public static void setMoreIndicator(String moreIndicator)
950     {
951         LargeSelect.moreIndicator = moreIndicator;
952     }
953 
954     /***
955      * Retrieve the more pages/records indicator.
956      */
957     public static String getMoreIndicator()
958     {
959         return LargeSelect.moreIndicator;
960     }
961 
962     /***
963      * Sets the multiplier that will be used to compute the memory limit when a
964      * constructor with no memory page limit is used - the memory limit will be
965      * this number multiplied by the page size.
966      *
967      * @param memoryPageLimit the maximum number of pages to be in memory
968      * at one time.
969      */
970     public static void setMemoryPageLimit(int memoryPageLimit)
971     {
972         LargeSelect.memoryPageLimit = memoryPageLimit;
973     }
974 
975     /***
976      * Retrieves the multiplier that will be used to compute the memory limit
977      * when a constructor with no memory page limit is used - the memory limit
978      * will be this number multiplied by the page size.
979      */
980     public static int getMemoryPageLimit()
981     {
982         return LargeSelect.memoryPageLimit;
983     }
984 
985     /***
986      * A convenience method that provides text showing progress through the
987      * selected rows on a page basis.
988      *
989      * @return progress text in the form of "1 of &gt; 5" where "&gt;" can be
990      * configured using <code>setMoreIndicator()</code>.
991      */
992     public String getPageProgressText()
993     {
994         StringBuffer result = new StringBuffer();
995         result.append(getCurrentPageNumber());
996         result.append(" of ");
997         if (!totalsFinalized)
998         {
999             result.append(moreIndicator);
1000             result.append(" ");
1001         }
1002         result.append(getTotalPages());
1003         return result.toString();
1004     }
1005 
1006     /***
1007      * Provides a count of the number of rows to be displayed on the current
1008      * page - for the last page this may be less than the configured page size.
1009      *
1010      * @return the number of records that are included on the current page of
1011      * results.
1012      */
1013     public int getCurrentPageSize()
1014     {
1015         if (null == lastResults)
1016         {
1017             return 0;
1018         }
1019         return lastResults.size();
1020     }
1021 
1022     /***
1023      * Provide the record number of the first row included on the current page.
1024      *
1025      * @return The record number of the first row of the current page.
1026      */
1027     public int getFirstRecordNoForPage()
1028     {
1029         if (getCurrentPageNumber() < 1)
1030         {
1031             return 0;
1032         }
1033         return getCurrentPageNumber() * getPageSize() - getPageSize() + 1;
1034     }
1035 
1036     /***
1037      * Provide the record number of the last row included on the current page.
1038      *
1039      * @return the record number of the last row of the current page.
1040      */
1041     public int getLastRecordNoForPage()
1042     {
1043         if (0 == currentPageNumber)
1044         {
1045             return 0;
1046         }
1047         return (getCurrentPageNumber() - 1) * getPageSize()
1048                 + getCurrentPageSize();
1049     }
1050 
1051     /***
1052      * A convenience method that provides text showing progress through the
1053      * selected rows on a record basis.
1054      *
1055      * @return progress text in the form of "26 - 50 of &gt; 250" where "&gt;"
1056      * can be configured using <code>setMoreIndicator()</code>.
1057      */
1058     public String getRecordProgressText()
1059     {
1060         StringBuffer result = new StringBuffer();
1061         result.append(getFirstRecordNoForPage());
1062         result.append(" - ");
1063         result.append(getLastRecordNoForPage());
1064         result.append(" of ");
1065         if (!totalsFinalized)
1066         {
1067             result.append(moreIndicator);
1068             result.append(" ");
1069         }
1070         result.append(getTotalRecords());
1071         return result.toString();
1072     }
1073 
1074     /***
1075      * Indicates if further result pages are available.
1076      *
1077      * @return <code>true</code> when further results are available.
1078      */
1079     public boolean getNextResultsAvailable()
1080     {
1081         if (!totalsFinalized || getCurrentPageNumber() < getTotalPages())
1082         {
1083             return true;
1084         }
1085         return false;
1086     }
1087 
1088     /***
1089      * Indicates if previous results pages are available.
1090      *
1091      * @return <code>true</code> when previous results are available.
1092      */
1093     public boolean getPreviousResultsAvailable()
1094     {
1095         if (getCurrentPageNumber() <= 1)
1096         {
1097             return false;
1098         }
1099         return true;
1100     }
1101 
1102     /***
1103      * Indicates if any results are available.
1104      *
1105      * @return <code>true</code> of any results are available.
1106      */
1107     public boolean hasResultsAvailable()
1108     {
1109         return getTotalRecords() > 0;
1110     }
1111 
1112     /***
1113      * Clear the query result so that the query is reexecuted when the next page
1114      * is retrieved.  You may want to invoke this method if you are returning to
1115      * a page after performing an operation on an item in the result set.
1116      *
1117      * @throws TorqueException if a sleep is interrupted.
1118      */
1119     public synchronized void invalidateResult() throws TorqueException
1120     {
1121         stopQuery();
1122         blockBegin = 0;
1123         blockEnd = 0;
1124         currentlyFilledTo = -1;
1125         qds = null;
1126         results = null;
1127         position = 0;
1128         totalPages = -1;
1129         totalRecords = 0;
1130         // todo Perhaps store the oldPageNumber and immediately restart the
1131         // query. 
1132         // oldPageNumber = currentPageNumber;
1133         currentPageNumber = 0;
1134         queryCompleted = false;
1135         totalsFinalized = false;
1136         lastResults = null;
1137     }
1138 
1139     /***
1140      * Retrieve a search parameter.  This acts as a convenient place to store
1141      * parameters that relate to the LargeSelect to make it easy to get at them
1142      * in order to repopulate search parameters on a form when the next page of
1143      * results is retrieved - they in no way effect the operation of
1144      * LargeSelect.
1145      *
1146      * @param name the search parameter key to retrieve.
1147      * @return the value of the search parameter.
1148      */
1149     public String getSearchParam(String name)
1150     {
1151         return getSearchParam(name, null);
1152     }
1153 
1154     /***
1155      * Retrieve a search parameter.  This acts as a convenient place to store
1156      * parameters that relate to the LargeSelect to make it easy to get at them
1157      * in order to repopulate search parameters on a form when the next page of
1158      * results is retrieved - they in no way effect the operation of
1159      * LargeSelect.
1160      *
1161      * @param name the search parameter key to retrieve.
1162      * @param defaultValue the default value to return if the key is not found.
1163      * @return the value of the search parameter.
1164      */
1165     public String getSearchParam(String name, String defaultValue)
1166     {
1167         if (null == params)
1168         {
1169             return defaultValue;
1170         }
1171         String value = (String) params.get(name);
1172         return null == value ? defaultValue : value;
1173     }
1174 
1175     /***
1176      * Set a search parameter.  If the value is <code>null</code> then the 
1177      * key will be removed from the parameters.
1178      *
1179      * @param name the search parameter key to set.
1180      * @param value the value of the search parameter to store.
1181      */
1182     public void setSearchParam(String name, String value)
1183     {
1184         if (null == value)
1185         {
1186             removeSearchParam(name);
1187         }
1188         else
1189         {
1190             if (null != name)
1191             {
1192                 if (null == params)
1193                 {
1194                     params = new Hashtable();
1195                 }
1196                 params.put(name, value);
1197             }
1198         }
1199     }
1200 
1201     /***
1202      * Remove a value from the search parameters.
1203      *
1204      * @param name the search parameter key to remove.
1205      */
1206     public void removeSearchParam(String name)
1207     {
1208         if (null != params)
1209         {
1210             params.remove(name);
1211         }
1212     }
1213 
1214     /***
1215      * Provide something useful for debugging purposes.
1216      * 
1217      * @return some basic information about this instance of LargeSelect.
1218      */
1219     public String toString()
1220     {
1221         StringBuffer result = new StringBuffer();
1222         result.append("LargeSelect - TotalRecords: ");
1223         result.append(getTotalRecords());
1224         result.append(" TotalsFinalised: ");
1225         result.append(getTotalsFinalized());
1226         result.append("\nParameters:");
1227         if (null == params || params.size() == 0)
1228         {
1229             result.append(" No parameters have been set.");
1230         }
1231         else
1232         {
1233             Set keys = params.keySet();
1234             for (Iterator iter = keys.iterator(); iter.hasNext();)
1235             {
1236                 String key = (String) iter.next();
1237                 String val = (String) params.get(key);
1238                 result.append("\n ").append(key).append(": ").append(val);
1239             }
1240         }
1241         return result.toString();
1242     }
1243 
1244 }