View Javadoc

1   package org.apache.torque.util;
2   
3   /*
4    * Copyright 2001-2004 The Apache Software Foundation.
5    *
6    * Licensed under the Apache License, Version 2.0 (the "License")
7    * you may not use this file except in compliance with the License.
8    * You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  import java.sql.Connection;
20  import java.sql.SQLException;
21  
22  import java.util.Iterator;
23  import java.util.List;
24  import java.util.ArrayList;
25  import java.util.Hashtable;
26  import java.util.Set;
27  import java.io.Serializable;
28  import java.lang.reflect.Method;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  
33  import org.apache.torque.Torque;
34  import org.apache.torque.TorqueException;
35  
36  import com.workingdogs.village.QueryDataSet;
37  import com.workingdogs.village.DataSetException;
38  
39  /***
40   * This class can be used to retrieve a large result set from a database query.
41   * The query is started and then rows are returned a page at a time.  The <code>
42   * LargeSelect</code> is meant to be placed into the Session or User.Temp, so
43   * that it can be used in response to several related requests.  Note that in
44   * order to use <code>LargeSelect</code> you need to be willing to accept the
45   * fact that the result set may become inconsistent with the database if updates
46   * are processed subsequent to the queries being executed.  Specifying a memory
47   * page limit of 1 will give you a consistent view of the records but the totals
48   * may not be accurate and the performance will be terrible.  In most cases
49   * the potential for inconsistencies data should not cause any serious problems
50   * and performance should be pretty good (but read on for further warnings).
51   *
52   * <p>The idea here is that the full query result would consume too much memory
53   * and if displayed to a user the page would be too long to be useful.  Rather
54   * than loading the full result set into memory, a window of data (the memory
55   * limit) is loaded and retrieved a page at a time.  If a request occurs for
56   * data that falls outside the currently loaded window of data then a new query
57   * is executed to fetch the required data.  Performance is optimized by
58   * starting a thread to execute the database query and fetch the results.  This
59   * will perform best when paging forwards through the data, but a minor
60   * optimization where the window is moved backwards by two rather than one page
61   * is included for when a user pages past the beginning of the window.
62   *
63   * <p>As the query is performed in in steps, it is often the case that the total
64   * number of records and pages of data is unknown.  <code>LargeSelect</code>
65   * provides various methods for indicating how many records and pages it is
66   * currently aware of and for presenting this information to users.
67   *
68   * <p><code>LargeSelect</code> utilises the <code>Criteria</code> methods
69   * <code>setOffset()</code> and <code>setLimit()</code> to limit the amount of
70   * data retrieved from the database - these values are either passed through to
71   * the DBMS when supported (efficient with the caveat below) or handled by
72   * the Village API when it is not (not so efficient).  At time of writing
73   * <code>Criteria</code> will only pass the offset and limit through to MySQL
74   * and PostgreSQL (with a few changes to <code>DBOracle</code> and <code>
75   * BasePeer</code> Oracle support can be implemented by utilising the <code>
76   * rownum</code> pseudo column).
77   *
78   * <p>As <code>LargeSelect</code> must re-execute the query each time the user
79   * pages out of the window of loaded data, you should consider the impact of
80   * non-index sort orderings and other criteria that will require the DBMS to
81   * execute the entire query before filtering down to the offset and limit either
82   * internally or via Village.
83   *
84   * <p>The memory limit defaults to 5 times the page size you specify, but
85   * alternative constructors and the class method <code>setMemoryPageLimit()
86   * </code> allow you to override this for a specific instance of
87   * <code>LargeSelect</code> or future instances respectively.
88   *
89   * <p>Some of the constructors allow you to specify the name of the class to use
90   * to build the returnd rows.  This works by using reflection to find <code>
91   * addSelectColumns(Criteria)</code> and <code>populateObjects(List)</code>
92   * methods to add the necessary select columns to the criteria (only if it
93   * doesn't already contain any) and to convert query results from Village
94   * <code>Record</code> objects to a class defined within the builder class.
95   * This allows you to use any of the Torque generated Peer classes, but also
96   * makes it fairly simple to construct business object classes that can be used
97   * for this purpose (simply copy and customise the <code>addSelectColumns()
98   * </code>, <code>populateObjects()</code>, <code>row2Object()</code> and <code>
99   * populateObject()</code> methods from an existing Peer class).
100  *
101  * <p>Typically you will create a <code>LargeSelect</code> using your <code>
102  * Criteria</code> (perhaps created from the results of a search parameter
103  * page), page size, memory page limit and return class name (for which you may
104  * have defined a business object class before hand) and place this in user.Temp
105  * thus:
106  *
107  * <pre>
108  *     data.getUser().setTemp("someName", largeSelect);
109  * </pre>
110  *
111  * <p>In your template you will then use something along the lines of:
112  *
113  * <pre>
114  *    #set ($largeSelect = $data.User.getTemp("someName"))
115  *    #set ($searchop = $data.Parameters.getString("searchop"))
116  *    #if ($searchop.equals("prev"))
117  *      #set ($recs = $largeSelect.PreviousResults)
118  *    #else
119  *      #if ($searchop.equals("goto"))
120  *        #set ($recs
121  *                = $largeSelect.getPage($data.Parameters.getInt("page", 1)))
122  *      #else
123  *        #set ($recs = $largeSelect.NextResults)
124  *      #end
125  *    #end
126  * </pre>
127  *
128  * <p>...to move through the records.  <code>LargeSelect</code> implements a
129  * number of convenience methods that make it easy to add all of the necessary
130  * bells and whistles to your template.
131  *
132  * @author <a href="mailto:john.mcnally@clearink.com">John D. McNally</a>
133  * @author <a href="mailto:seade@backstagetech.com.au">Scott Eade</a>
134  * @version $Id: LargeSelect.java,v 1.13.2.2 2004/05/20 04:36:06 seade Exp $
135  */
136 public class LargeSelect implements Runnable, Serializable
137 {
138     /*** The number of records that a page consists of.  */
139     private int pageSize;
140     /*** The maximum number of records to maintain in memory. */
141     private int memoryLimit;
142 
143     /*** The record number of the first record in memory. */
144     private int blockBegin = 0;
145     /*** The record number of the last record in memory. */
146     private int blockEnd;
147     /*** How much of the memory block is currently occupied with result data. */
148     private volatile int currentlyFilledTo = -1;
149 
150     /*** The SQL query that this <code>LargeSelect</code> represents. */
151     private String query;
152     /*** The database name to get from Torque. */
153     private String dbName;
154     /*** Used to retrieve query results from Village. */
155     private QueryDataSet qds = null;
156 
157     /*** The memory store of records. */
158     private List results = null;
159 
160     /*** The thread that executes the query. */
161     private Thread thread = null;
162     /***
163      * A flag used to kill the thread when the currently executing query is no
164      * longer required.
165      */
166     private volatile boolean killThread = false;
167     /*** A flag that indicates whether or not the query thread is running. */
168     private volatile boolean threadRunning = false;
169     /***
170      * An indication of whether or not the current query has completed
171      * processing.
172      */
173     private volatile boolean queryCompleted = false;
174     /***
175      * An indication of whether or not the totals (records and pages) are at
176      * their final values.
177      */
178     private boolean totalsFinalized = false;
179 
180     /*** The cursor position in the result set. */
181     private int position;
182     /*** The total number of pages known to exist. */
183     private int totalPages = -1;
184     /*** The total number of records known to exist. */
185     private int totalRecords = 0;
186     /*** The number of the page that was last retrieved. */
187     private int currentPageNumber = 0;
188 
189     /*** The criteria used for the query. */
190     private Criteria criteria = null;
191     /*** The last page of results that were returned. */
192     private List lastResults;
193 
194     /***
195      * The class that is possibly used to construct the criteria and used
196      * to transform the Village Records into the desired OM or business objects.
197      */
198     private Class returnBuilderClass = null;
199     /***
200      * A reference to the method in the return builder class that will
201      * convert the Village Records to the desired class.
202      */
203     private Method populateObjectsMethod = null;
204 
205     /***
206      * The default value ("&gt;") used to indicate that the total number of
207      * records or pages is unknown. You can use <code>setMoreIndicator()</code>
208      * to change this to whatever value you like (e.g. "more than").
209      */
210     public static final String DEFAULT_MORE_INDICATOR = "&gt;";
211 
212     private static String moreIndicator = DEFAULT_MORE_INDICATOR;
213 
214     /***
215      * The default value for the maximum number of pages of data to be retained
216      * in memory - you can provide your own default value using
217      * <code>setMemoryPageLimit()</code>.
218      */
219     public static final int DEFAULT_MEMORY_LIMIT_PAGES = 5;
220 
221     private static int memoryPageLimit = DEFAULT_MEMORY_LIMIT_PAGES;
222 
223     /*** A place to store search parameters that relate to this query. */
224     private Hashtable params = null;
225 
226     /*** Logging */
227     private static Log log = LogFactory.getLog(LargeSelect.class);
228 
229     /***
230      * Creates a LargeSelect whose results are returned as a <code>List</code>
231      * containing a maximum of <code>pageSize</code> Village <code>Record</code>
232      * objects at a time, maintaining a maximum of
233      * <code>LargeSelect.memoryPageLimit</code> pages of results in memory.
234      *
235      * @param criteria object used by BasePeer to build the query.  In order to
236      * allow this class to utilise database server implemented offsets and
237      * limits (when available), the provided criteria must not have any limit or
238      * offset defined.
239      * @param pageSize number of rows to return in one block.
240      * @throws IllegalArgumentException if <code>criteria</code> uses one or
241      * both of offset and limit, or if <code>pageSize</code> is less than 1;
242      */
243     public LargeSelect(Criteria criteria, int pageSize)
244             throws IllegalArgumentException
245     {
246         this(criteria, pageSize, LargeSelect.memoryPageLimit);
247     }
248 
249     /***
250      * Creates a LargeSelect whose results are returned as a <code>List</code>
251      * containing a maximum of <code>pageSize</code> Village <code>Record</code>
252      * objects at a time, maintaining a maximum of <code>memoryPageLimit</code>
253      * pages of results in memory.
254      *
255      * @param criteria object used by BasePeer to build the query.  In order to
256      * allow this class to utilise database server implemented offsets and
257      * limits (when available), the provided criteria must not have any limit or
258      * offset defined.
259      * @param pageSize number of rows to return in one block.
260      * @param memoryPageLimit maximum number of pages worth of rows to be held
261      * in memory at one time.
262      * @throws IllegalArgumentException if <code>criteria</code> uses one or
263      * both of offset and limit, or if <code>pageSize</code> or
264      * <code>memoryLimitPages</code> are less than 1;
265      */
266     public LargeSelect(Criteria criteria, int pageSize, int memoryPageLimit)
267             throws IllegalArgumentException
268     {
269         init(criteria, pageSize, memoryPageLimit);
270     }
271 
272     /***
273      * Creates a LargeSelect whose results are returned as a <code>List</code>
274      * containing a maximum of <code>pageSize</code> objects of the type
275      * defined within the class named <code>returnBuilderClassName</code> at a
276      * time, maintaining a maximum of <code>LargeSelect.memoryPageLimit</code>
277      * pages of results in memory.
278      *
279      * @param criteria object used by BasePeer to build the query.  In order to
280      * allow this class to utilise database server implemented offsets and
281      * limits (when available), the provided criteria must not have any limit or
282      * offset defined.  If the criteria does not include the definition of any
283      * select columns the <code>addSelectColumns(Criteria)</code> method of
284      * the class named as <code>returnBuilderClassName</code> will be used to
285      * add them.
286      * @param pageSize number of rows to return in one block.
287      * @param returnBuilderClassName The name of the class that will be used to
288      * build the result records (may implement <code>addSelectColumns(Criteria)
289      * </code> and must implement <code>populateObjects(List)</code>).
290      * @throws IllegalArgumentException if <code>criteria</code> uses one or
291      * both of offset and limit, if <code>pageSize</code> is less than 1, or if
292      * problems are experienced locating and invoking either one or both of
293      * <code>addSelectColumns(Criteria)</code> and <code> populateObjects(List)
294      * </code> in the class named <code>returnBuilderClassName</code>.
295      */
296     public LargeSelect(
297             Criteria criteria,
298             int pageSize,
299             String returnBuilderClassName)
300             throws IllegalArgumentException
301     {
302         this(
303             criteria,
304             pageSize,
305             LargeSelect.memoryPageLimit,
306             returnBuilderClassName);
307     }
308 
309     /***
310      * Creates a LargeSelect whose results are returned as a <code>List</code>
311      * containing a maximum of <code>pageSize</code> objects of the type
312      * defined within the class named <code>returnBuilderClassName</code> at a
313      * time, maintaining a maximum of <code>memoryPageLimit</code> pages of
314      * results in memory.
315      *
316      * @param criteria object used by BasePeer to build the query.  In order to
317      * allow this class to utilise database server implemented offsets and
318      * limits (when available), the provided criteria must not have any limit or
319      * offset defined.  If the criteria does not include the definition of any
320      * select columns the <code>addSelectColumns(Criteria)</code> method of
321      * the class named as <code>returnBuilderClassName</code> will be used to
322      * add them.
323      * @param pageSize number of rows to return in one block.
324      * @param memoryPageLimit maximum number of pages worth of rows to be held
325      * in memory at one time.
326      * @param returnBuilderClassName The name of the class that will be used to
327      * build the result records (may implement <code>addSelectColumns(Criteria)
328      * </code> and must implement <code>populateObjects(List)</code>).
329      * @throws IllegalArgumentException if <code>criteria</code> uses one or
330      * both of offset and limit, if <code>pageSize</code> or <code>
331      * memoryLimitPages</code> are less than 1, or if problems are experienced
332      * locating and invoking either one or both of <code>
333      * addSelectColumns(Criteria)</code> and <code> populateObjects(List)</code>
334      * in the class named <code>returnBuilderClassName</code>.
335      */
336     public LargeSelect(
337             Criteria criteria,
338             int pageSize,
339             int memoryPageLimit,
340             String returnBuilderClassName)
341             throws IllegalArgumentException
342     {
343         try
344         {
345             this.returnBuilderClass = Class.forName(returnBuilderClassName);
346 
347             // Add the select columns if necessary.
348             if (criteria.getSelectColumns().size() == 0)
349             {
350                 Class[] argTypes = { Criteria.class };
351                 Method selectColumnAdder =
352                     returnBuilderClass.getMethod("addSelectColumns", argTypes);
353                 Object[] theArgs = { criteria };
354                 selectColumnAdder.invoke(returnBuilderClass.newInstance(),
355                         theArgs);
356             }
357 
358             // Locate the populateObjects() method - this will be used later
359             Class[] argTypes = { List.class };
360             populateObjectsMethod =
361                 returnBuilderClass.getMethod("populateObjects", argTypes);
362         }
363         catch (Exception e)
364         {
365             throw new IllegalArgumentException(
366                     "The class named as returnBuilderClassName does not "
367                     + "provide the necessary facilities - see javadoc.");
368         }
369 
370         init(criteria, pageSize, memoryPageLimit);
371     }
372 
373     /***
374      * Called by the constructors to start the query.
375      *
376      * @param criteria Object used by <code>BasePeer</code> to build the query.
377      * In order to allow this class to utilise database server implemented
378      * offsets and limits (when available), the provided criteria must not have
379      * any limit or offset defined.
380      * @param pageSize number of rows to return in one block.
381      * @param memoryLimitPages maximum number of pages worth of rows to be held
382      * in memory at one time.
383      * @throws IllegalArgumentException if <code>criteria</code> uses one or
384      * both of offset and limit and if <code>pageSize</code> or
385      * <code>memoryLimitPages</code> are less than 1;
386      */
387     private void init(Criteria criteria, int pageSize, int memoryLimitPages)
388             throws IllegalArgumentException
389     {
390         if (criteria.getOffset() != 0 || criteria.getLimit() != -1)
391         {
392             throw new IllegalArgumentException(
393                     "criteria must not use Offset and/or Limit.");
394         }
395 
396         if (pageSize < 1)
397         {
398             throw new IllegalArgumentException(
399                     "pageSize must be greater than zero.");
400         }
401 
402         if (memoryLimitPages < 1)
403         {
404             throw new IllegalArgumentException(
405                     "memoryPageLimit must be greater than zero.");
406         }
407 
408         this.pageSize = pageSize;
409         this.memoryLimit = pageSize * memoryLimitPages;
410         this.criteria = criteria;
411         dbName = criteria.getDbName();
412         blockEnd = blockBegin + memoryLimit - 1;
413         startQuery(pageSize);
414     }
415 
416     /***
417      * Retrieve a specific page, if it exists.
418      *
419      * @param pageNumber the number of the page to be retrieved - must be
420      * greater than zero.  An empty <code>List</code> will be returned if
421      * <code>pageNumber</code> exceeds the total number of pages that exist.
422      * @return a <code>List</code> of query results containing a maximum of
423      * <code>pageSize</code> results.
424      * @throws IllegalArgumentException when <code>pageNo</code> is not
425      * greater than zero.
426      * @throws TorqueException if invoking the <code>populateObjects()<code>
427      * method runs into problems or a sleep is unexpectedly interrupted.
428      */
429     public List getPage(int pageNumber) throws TorqueException
430     {
431         if (pageNumber < 1)
432         {
433             throw new IllegalArgumentException("pageNumber must be greater "
434                     + "than zero.");
435         }
436         currentPageNumber = pageNumber;
437         return getResults((pageNumber - 1) * pageSize);
438     }
439 
440     /***
441      * Gets the next page of rows.
442      *
443      * @return a <code>List</code> of query results containing a maximum of
444      * <code>pageSize</code> reslts.
445      * @throws TorqueException if invoking the <code>populateObjects()<code>
446      * method runs into problems or a sleep is unexpectedly interrupted.
447      */
448     public List getNextResults() throws TorqueException
449     {
450         if (!getNextResultsAvailable())
451         {
452             return getCurrentPageResults();
453         }
454         currentPageNumber++;
455         return getResults(position);
456     }
457 
458     /***
459      * Provide access to the results from the current page.
460      *
461      * @return a <code>List</code> of query results containing a maximum of
462      * <code>pageSize</code> reslts.
463      */
464     public List getCurrentPageResults()
465     {
466         return lastResults;
467     }
468 
469     /***
470      * Gets the previous page of rows.
471      *
472      * @return a <code>List</code> of query results containing a maximum of
473      * <code>pageSize</code> reslts.
474      * @throws TorqueException if invoking the <code>populateObjects()<code>
475      * method runs into problems or a sleep is unexpectedly interrupted.
476      */
477     public List getPreviousResults() throws TorqueException
478     {
479         if (!getPreviousResultsAvailable())
480         {
481             return getCurrentPageResults();
482         }
483 
484         int start;
485         if (position - 2 * pageSize < 0)
486         {
487             start = 0;
488             currentPageNumber = 1;
489         }
490         else
491         {
492             start = position - 2 * pageSize;
493             currentPageNumber--;
494         }
495         return getResults(start);
496     }
497 
498     /***
499      * Gets a page of rows starting at a specified row.
500      *
501      * @param start the starting row.
502      * @return a <code>List</code> of query results containing a maximum of
503      * <code>pageSize</code> reslts.
504      * @throws TorqueException if invoking the <code>populateObjects()<code>
505      * method runs into problems or a sleep is unexpectedly interrupted.
506      */
507     private List getResults(int start) throws TorqueException
508     {
509         return getResults(start, pageSize);
510     }
511 
512     /***
513      * Gets a block of rows starting at a specified row and containing a
514      * specified number of rows.
515      *
516      * @param start the starting row.
517      * @param size the number of rows.
518      * @return a <code>List</code> of query results containing a maximum of
519      * <code>pageSize</code> reslts.
520      * @throws IllegalArgumentException if <code>size &gt; memoryLimit</code> or
521      * <code>start</code> and <code>size</code> result in a situation that is
522      * not catered for.
523      * @throws TorqueException if invoking the <code>populateObjects()<code>
524      * method runs into problems or a sleep is unexpectedly interrupted.
525      */
526     private synchronized List getResults(int start, int size)
527             throws IllegalArgumentException, TorqueException
528     {
529         if (log.isDebugEnabled())
530         {
531             log.debug("getResults(start: " + start
532                     + ", size: " + size + ") invoked.");
533         }
534 
535         if (size > memoryLimit)
536         {
537             throw new IllegalArgumentException("size (" + size
538                     + ") exceeds memory limit (" + memoryLimit + ").");
539         }
540 
541         // Request was for a block of rows which should be in progess.
542         // If the rows have not yet been returned, wait for them to be
543         // retrieved.
544         if (start >= blockBegin && (start + size - 1) <= blockEnd)
545         {
546             if (log.isDebugEnabled())
547             {
548                 log.debug("getResults(): Sleeping until "
549                         + "start+size-1 (" + (start + size - 1)
550                         + ") > currentlyFilledTo (" + currentlyFilledTo
551                         + ") && !queryCompleted (!" + queryCompleted + ")");
552             }
553             while (((start + size - 1) > currentlyFilledTo) && !queryCompleted)
554             {
555                 try
556                 {
557                     Thread.sleep(500);
558                 }
559                 catch (InterruptedException e)
560                 {
561                     throw new TorqueException("Unexpected interruption", e);
562                 }
563             }
564         }
565 
566         // Going in reverse direction, trying to limit db hits so assume user
567         // might want at least 2 sets of data.
568         else if (start < blockBegin && start >= 0)
569         {
570             if (log.isDebugEnabled())
571             {
572                 log.debug("getResults(): Paging backwards as start (" + start
573                         + ") < blockBegin (" + blockBegin + ") && start >= 0");
574             }
575             stopQuery();
576             if (memoryLimit >= 2 * size)
577             {
578                 blockBegin = start - size;
579                 if (blockBegin < 0)
580                 {
581                     blockBegin = 0;
582                 }
583             }
584             else
585             {
586                 blockBegin = start;
587             }
588             blockEnd = blockBegin + memoryLimit - 1;
589             startQuery(size);
590             // Re-invoke getResults() to provide the wait processing.
591             return getResults(start, size);
592         }
593 
594         // Assume we are moving on, do not retrieve any records prior to start.
595         else if ((start + size - 1) > blockEnd)
596         {
597             if (log.isDebugEnabled())
598             {
599                 log.debug("getResults(): Paging past end of loaded data as "
600                         + "start+size-1 (" + (start + size - 1)
601                         + ") > blockEnd (" + blockEnd + ")");
602             }
603             stopQuery();
604             blockBegin = start;
605             blockEnd = blockBegin + memoryLimit - 1;
606             startQuery(size);
607             // Re-invoke getResults() to provide the wait processing.
608             return getResults(start, size);
609         }
610 
611         else
612         {
613             throw new IllegalArgumentException("Parameter configuration not "
614                     + "accounted for.");
615         }
616 
617         int fromIndex = start - blockBegin;
618         int toIndex = fromIndex + Math.min(size, results.size() - fromIndex);
619 
620         if (log.isDebugEnabled())
621         {
622             log.debug("getResults(): Retrieving records from results elements "
623                     + "start-blockBegin (" + fromIndex + ") through "
624                     + "fromIndex + Math.min(size, results.size() - fromIndex) ("
625                     + toIndex + ")");
626         }
627         List returnResults = results.subList(fromIndex, toIndex);
628 
629         if (null != returnBuilderClass)
630         {
631             // Invoke the populateObjects() method
632             Object[] theArgs = { returnResults };
633             try
634             {
635                 returnResults =
636                     (List) populateObjectsMethod.invoke(
637                         returnBuilderClass.newInstance(),
638                         theArgs);
639             }
640             catch (Exception e)
641             {
642                 throw new TorqueException("Unable to populate results", e);
643             }
644         }
645         position = start + size;
646         lastResults = returnResults;
647         return returnResults;
648     }
649 
650     /***
651      * A background thread that retrieves the rows.
652      */
653     public void run()
654     {
655         int size = pageSize;
656         /* The connection to the database. */
657         Connection conn = null;
658 
659         try
660         {
661             // Add 1 to memory limit to check if the query ends on a page break.
662             results = new ArrayList(memoryLimit + 1);
663 
664             // Use the criteria to limit the rows that are retrieved to the
665             // block of records that fit in the predefined memoryLimit.
666             criteria.setOffset(blockBegin);
667             // Add 1 to memory limit to check if the query ends on a page break.
668             criteria.setLimit(memoryLimit + 1);
669             query = BasePeer.createQueryString(criteria);
670 
671             // Get a connection to the db.
672             conn = Torque.getConnection(dbName);
673 
674             // Execute the query.
675             if (log.isDebugEnabled())
676             {
677                 log.debug("run(): query = " + query);
678                 log.debug("run(): memoryLimit = " + memoryLimit);
679                 log.debug("run(): blockBegin = " + blockBegin);
680                 log.debug("run(): blockEnd = " + blockEnd);
681             }
682             qds = new QueryDataSet(conn, query);
683 
684             // Continue getting rows one page at a time until the memory limit
685             // is reached, all results have been retrieved, or the rest
686             // of the results have been determined to be irrelevant.
687             while (!killThread
688                 && !qds.allRecordsRetrieved()
689                 && currentlyFilledTo + pageSize <= blockEnd)
690             {
691                 // This caters for when memoryLimit is not a multiple of
692                 //  pageSize which it never is because we always add 1 above.
693                 if ((currentlyFilledTo + pageSize) >= blockEnd)
694                 {
695                     // Add 1 to check if the query ends on a page break.
696                     size = blockEnd - currentlyFilledTo + 1;
697                 }
698 
699                 if (log.isDebugEnabled())
700                 {
701                     log.debug("run(): Invoking BasePeer.getSelectResults(qds, "
702                             + size + ", false)");
703                 }
704 
705                 List tempResults
706                         = BasePeer.getSelectResults(qds, size, false);
707 
708                 for (int i = 0, n = tempResults.size(); i < n; i++)
709                 {
710                     results.add(tempResults.get(i));
711                 }
712 
713                 currentlyFilledTo += tempResults.size();
714                 boolean perhapsLastPage = true;
715 
716                 // If the extra record was indeed found then we know we are not
717                 // on the last page but we must now get rid of it.
718                 if (results.size() == memoryLimit + 1)
719                 {
720                     results.remove(currentlyFilledTo--);
721                     perhapsLastPage = false;
722                 }
723 
724                 if (results.size() > 0
725                     && blockBegin + currentlyFilledTo >= totalRecords)
726                 {
727                     // Add 1 because index starts at 0
728                     totalRecords = blockBegin + currentlyFilledTo + 1;
729                 }
730 
731                 if (qds.allRecordsRetrieved())
732                 {
733                     queryCompleted = true;
734                     // The following ugly condition ensures that the totals are
735                     // not finalized when a user does something like requesting
736                     // a page greater than what exists in the database.
737                     if (perhapsLastPage
738                         && getCurrentPageNumber() <= getTotalPages())
739                     {
740                         totalsFinalized = true;
741                     }
742                 }
743                 qds.clearRecords();
744             }
745 
746             if (log.isDebugEnabled())
747             {
748                 log.debug("run(): While loop terminated because either:");
749                 log.debug("run(): 1. qds.allRecordsRetrieved(): "
750                         + qds.allRecordsRetrieved());
751                 log.debug("run(): 2. killThread: " + killThread);
752                 log.debug("run(): 3. !(currentlyFilledTo + size <= blockEnd): !"
753                         + (currentlyFilledTo + pageSize <= blockEnd));
754                 log.debug("run(): - currentlyFilledTo: " + currentlyFilledTo);
755                 log.debug("run(): - size: " + pageSize);
756                 log.debug("run(): - blockEnd: " + blockEnd);
757                 log.debug("run(): - results.size(): " + results.size());
758             }
759         }
760         catch (TorqueException e)
761         {
762             log.error(e);
763         }
764         catch (SQLException e)
765         {
766             log.error(e);
767         }
768         catch (DataSetException e)
769         {
770             log.error(e);
771         }
772         finally
773         {
774             try
775             {
776                 if (qds != null)
777                 {
778                     qds.close();
779                 }
780                 Torque.closeConnection(conn);
781             }
782             catch (SQLException e)
783             {
784                 log.error(e);
785             }
786             catch (DataSetException e)
787             {
788                 log.error(e);
789             }
790             threadRunning = false;
791         }
792     }
793 
794     /***
795      * Starts a new thread to retrieve the result set.
796      *
797      * @param initialSize the initial size for each block.
798      */
799     private synchronized void startQuery(int initialSize)
800     {
801         if (!threadRunning)
802         {
803             pageSize = initialSize;
804             currentlyFilledTo = -1;
805             queryCompleted = false;
806             thread = new Thread(this);
807             thread.start();
808             threadRunning = true;
809         }
810     }
811 
812     /***
813      * Used to stop filling the memory with the current block of results, if it
814      * has been determined that they are no longer relevant.
815      *
816      * @throws TorqueException if a sleep is interrupted.
817      */
818     private synchronized void stopQuery() throws TorqueException
819     {
820         if (threadRunning)
821         {
822             killThread = true;
823             while (thread.isAlive())
824             {
825                 try
826                 {
827                     Thread.sleep(100);
828                 }
829                 catch (InterruptedException e)
830                 {
831                     throw new TorqueException("Unexpected interruption", e);
832                 }
833             }
834             killThread = false;
835         }
836     }
837 
838     /***
839      * Retrieve the number of the current page.
840      *
841      * @return the current page number.
842      */
843     public int getCurrentPageNumber()
844     {
845         return currentPageNumber;
846     }
847 
848     /***
849      * Retrieve the total number of search result records that are known to
850      * exist (this will be the actual value when the query has completeted (see
851      * <code>getTotalsFinalized()</code>).  The convenience method
852      * <code>getRecordProgressText()</code> may be more useful for presenting to
853      * users.
854      *
855      * @return the number of result records known to exist (not accurate until
856      * <code>getTotalsFinalized()</code> returns <code>true</code>).
857      */
858     public int getTotalRecords()
859     {
860         return totalRecords;
861     }
862 
863     /***
864      * Provide an indication of whether or not paging of results will be
865      * required.
866      *
867      * @return <code>true</code> when multiple pages of results exist.
868      */
869     public boolean getPaginated()
870     {
871         // Handle a page memory limit of 1 page.
872         if (!getTotalsFinalized())
873         {
874             return true;
875         }
876         return blockBegin + currentlyFilledTo + 1 > pageSize;
877     }
878 
879     /***
880      * Retrieve the total number of pages of search results that are known to
881      * exist (this will be the actual value when the query has completeted (see
882      * <code>getQyeryCompleted()</code>).  The convenience method
883      * <code>getPageProgressText()</code> may be more useful for presenting to
884      * users.
885      *
886      * @return the number of pages of results known to exist (not accurate until
887      * <code>getTotalsFinalized()</code> returns <code>true</code>).
888      */
889     public int getTotalPages()
890     {
891         if (totalPages > -1)
892         {
893             return totalPages;
894         }
895 
896         int tempPageCount =
897             getTotalRecords() / pageSize
898                 + (getTotalRecords() % pageSize > 0 ? 1 : 0);
899 
900         if (getTotalsFinalized())
901         {
902             totalPages = tempPageCount;
903         }
904 
905         return tempPageCount;
906     }
907 
908     /***
909      * Retrieve the page size.
910      *
911      * @return the number of records returned on each invocation of
912      * <code>getNextResults()</code>/<code>getPreviousResults()</code>.
913      */
914     public int getPageSize()
915     {
916         return pageSize;
917     }
918 
919     /***
920      * Provide access to indicator that the total values for the number of
921      * records and pages are now accurate as opposed to known upper limits.
922      *
923      * @return <code>true</code> when the totals are known to have been fully
924      * computed.
925      */
926     public boolean getTotalsFinalized()
927     {
928         return totalsFinalized;
929     }
930 
931     /***
932      * Provide a way of changing the more pages/records indicator.
933      *
934      * @param moreIndicator the indicator to use in place of the default
935      * ("&gt;").
936      */
937     public static void setMoreIndicator(String moreIndicator)
938     {
939         LargeSelect.moreIndicator = moreIndicator;
940     }
941 
942     /***
943      * Retrieve the more pages/records indicator.
944      */
945     public static String getMoreIndicator()
946     {
947         return LargeSelect.moreIndicator;
948     }
949 
950     /***
951      * Sets the multiplier that will be used to compute the memory limit when a
952      * constructor with no memory page limit is used - the memory limit will be
953      * this number multiplied by the page size.
954      *
955      * @param memoryPageLimit the maximum number of pages to be in memory
956      * at one time.
957      */
958     public static void setMemoryPageLimit(int memoryPageLimit)
959     {
960         LargeSelect.memoryPageLimit = memoryPageLimit;
961     }
962 
963     /***
964      * Retrieves the multiplier that will be used to compute the memory limit
965      * when a constructor with no memory page limit is used - the memory limit
966      * will be this number multiplied by the page size.
967      */
968     public static int getMemoryPageLimit()
969     {
970         return LargeSelect.memoryPageLimit;
971     }
972 
973     /***
974      * A convenience method that provides text showing progress through the
975      * selected rows on a page basis.
976      *
977      * @return progress text in the form of "1 of &gt; 5" where "&gt;" can be
978      * configured using <code>setMoreIndicator()</code>.
979      */
980     public String getPageProgressText()
981     {
982         StringBuffer result = new StringBuffer();
983         result.append(getCurrentPageNumber());
984         result.append(" of ");
985         if (!totalsFinalized)
986         {
987             result.append(moreIndicator);
988             result.append(" ");
989         }
990         result.append(getTotalPages());
991         return result.toString();
992     }
993 
994     /***
995      * Provides a count of the number of rows to be displayed on the current
996      * page - for the last page this may be less than the configured page size.
997      *
998      * @return the number of records that are included on the current page of
999      * results.
1000      */
1001     public int getCurrentPageSize()
1002     {
1003         if (null == lastResults)
1004         {
1005             return 0;
1006         }
1007         return lastResults.size();
1008     }
1009 
1010     /***
1011      * Provide the record number of the first row included on the current page.
1012      *
1013      * @return The record number of the first row of the current page.
1014      */
1015     public int getFirstRecordNoForPage()
1016     {
1017         if (getCurrentPageNumber() < 1)
1018         {
1019             return 0;
1020         }
1021         return getCurrentPageNumber() * getPageSize() - getPageSize() + 1;
1022     }
1023 
1024     /***
1025      * Provide the record number of the last row included on the current page.
1026      *
1027      * @return the record number of the last row of the current page.
1028      */
1029     public int getLastRecordNoForPage()
1030     {
1031         if (0 == currentPageNumber)
1032         {
1033             return 0;
1034         }
1035         return (getCurrentPageNumber() - 1) * getPageSize()
1036                 + getCurrentPageSize();
1037     }
1038 
1039     /***
1040      * A convenience method that provides text showing progress through the
1041      * selected rows on a record basis.
1042      *
1043      * @return progress text in the form of "26 - 50 of &gt; 250" where "&gt;"
1044      * can be configured using <code>setMoreIndicator()</code>.
1045      */
1046     public String getRecordProgressText()
1047     {
1048         StringBuffer result = new StringBuffer();
1049         result.append(getFirstRecordNoForPage());
1050         result.append(" - ");
1051         result.append(getLastRecordNoForPage());
1052         result.append(" of ");
1053         if (!totalsFinalized)
1054         {
1055             result.append(moreIndicator);
1056             result.append(" ");
1057         }
1058         result.append(getTotalRecords());
1059         return result.toString();
1060     }
1061 
1062     /***
1063      * Indicates if further result pages are available.
1064      *
1065      * @return <code>true</code> when further results are available.
1066      */
1067     public boolean getNextResultsAvailable()
1068     {
1069         if (!totalsFinalized || getCurrentPageNumber() < getTotalPages())
1070         {
1071             return true;
1072         }
1073         return false;
1074     }
1075 
1076     /***
1077      * Indicates if previous results pages are available.
1078      *
1079      * @return <code>true</code> when previous results are available.
1080      */
1081     public boolean getPreviousResultsAvailable()
1082     {
1083         if (getCurrentPageNumber() <= 1)
1084         {
1085             return false;
1086         }
1087         return true;
1088     }
1089 
1090     /***
1091      * Indicates if any results are available.
1092      *
1093      * @return <code>true</code> of any results are available.
1094      */
1095     public boolean hasResultsAvailable()
1096     {
1097         return getTotalRecords() > 0;
1098     }
1099 
1100     /***
1101      * Clear the query result so that the query is reexecuted when the next page
1102      * is retrieved.  You may want to invoke this method if you are returning to
1103      * a page after performing an operation on an item in the result set.
1104      *
1105      * @throws TorqueException if a sleep is interrupted.
1106      */
1107     public synchronized void invalidateResult() throws TorqueException
1108     {
1109         stopQuery();
1110         blockBegin = 0;
1111         blockEnd = 0;
1112         currentlyFilledTo = -1;
1113         qds = null;
1114         results = null;
1115         position = 0;
1116         totalPages = -1;
1117         totalRecords = 0;
1118         // todo Perhaps store the oldPageNumber and immediately restart the
1119         // query. 
1120         // oldPageNumber = currentPageNumber;
1121         currentPageNumber = 0;
1122         queryCompleted = false;
1123         totalsFinalized = false;
1124         lastResults = null;
1125     }
1126 
1127     /***
1128      * Retrieve a search parameter.  This acts as a convenient place to store
1129      * parameters that relate to the LargeSelect to make it easy to get at them
1130      * in order to repopulate search parameters on a form when the next page of
1131      * results is retrieved - they in no way effect the operation of
1132      * LargeSelect.
1133      *
1134      * @param name the search parameter key to retrieve.
1135      * @return the value of the search parameter.
1136      */
1137     public String getSearchParam(String name)
1138     {
1139         return getSearchParam(name, null);
1140     }
1141 
1142     /***
1143      * Retrieve a search parameter.  This acts as a convenient place to store
1144      * parameters that relate to the LargeSelect to make it easy to get at them
1145      * in order to repopulate search parameters on a form when the next page of
1146      * results is retrieved - they in no way effect the operation of
1147      * LargeSelect.
1148      *
1149      * @param name the search parameter key to retrieve.
1150      * @param defaultValue the default value to return if the key is not found.
1151      * @return the value of the search parameter.
1152      */
1153     public String getSearchParam(String name, String defaultValue)
1154     {
1155         if (null == params)
1156         {
1157             return defaultValue;
1158         }
1159         String value = (String) params.get(name);
1160         return null == value ? defaultValue : value;
1161     }
1162 
1163     /***
1164      * Set a search parameter.  If the value is <code>null</code> then the 
1165      * key will be removed from the parameters.
1166      *
1167      * @param name the search parameter key to set.
1168      * @param value the value of the search parameter to store.
1169      */
1170     public void setSearchParam(String name, String value)
1171     {
1172         if (null == value)
1173         {
1174             removeSearchParam(name);
1175         }
1176         else
1177         {
1178             if (null != name)
1179             {
1180                 if (null == params)
1181                 {
1182                     params = new Hashtable();
1183                 }
1184                 params.put(name, value);
1185             }
1186         }
1187     }
1188 
1189     /***
1190      * Remove a value from the search parameters.
1191      *
1192      * @param name the search parameter key to remove.
1193      */
1194     public void removeSearchParam(String name)
1195     {
1196         if (null != params)
1197         {
1198             params.remove(name);
1199         }
1200     }
1201 
1202     /***
1203      * Provide something useful for debugging purposes.
1204      * 
1205      * @return some basic information about this instance of LargeSelect.
1206      */
1207     public String toString()
1208     {
1209         StringBuffer result = new StringBuffer();
1210         result.append("LargeSelect - TotalRecords: ");
1211         result.append(getTotalRecords());
1212         result.append(" TotalsFinalised: ");
1213         result.append(getTotalsFinalized());
1214         result.append("\nParameters:");
1215         if (null == params || params.size() == 0)
1216         {
1217             result.append(" No parameters have been set.");
1218         }
1219         else
1220         {
1221             Set keys = params.keySet();
1222             for (Iterator iter = keys.iterator(); iter.hasNext();)
1223             {
1224                 String key = (String) iter.next();
1225                 String val = (String) params.get(key);
1226                 result.append("\n ").append(key).append(": ").append(val);
1227             }
1228         }
1229         return result.toString();
1230     }
1231 
1232 }