001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.pool2.impl;
018
019import java.time.Duration;
020import java.util.ArrayList;
021import java.util.Deque;
022import java.util.HashMap;
023import java.util.Iterator;
024import java.util.List;
025import java.util.Map;
026import java.util.Map.Entry;
027import java.util.NoSuchElementException;
028import java.util.Objects;
029import java.util.TreeMap;
030import java.util.concurrent.ConcurrentHashMap;
031import java.util.concurrent.TimeUnit;
032import java.util.concurrent.atomic.AtomicInteger;
033import java.util.concurrent.atomic.AtomicLong;
034import java.util.concurrent.locks.Lock;
035import java.util.concurrent.locks.ReadWriteLock;
036import java.util.concurrent.locks.ReentrantReadWriteLock;
037import java.util.stream.Collectors;
038
039import org.apache.commons.pool2.DestroyMode;
040import org.apache.commons.pool2.KeyedObjectPool;
041import org.apache.commons.pool2.KeyedPooledObjectFactory;
042import org.apache.commons.pool2.PoolUtils;
043import org.apache.commons.pool2.PooledObject;
044import org.apache.commons.pool2.PooledObjectState;
045import org.apache.commons.pool2.SwallowedExceptionListener;
046import org.apache.commons.pool2.UsageTracking;
047
048/**
049 * A configurable {@code KeyedObjectPool} implementation.
050 * <p>
051 * When coupled with the appropriate {@link KeyedPooledObjectFactory},
052 * {@code GenericKeyedObjectPool} provides robust pooling functionality for
053 * keyed objects. A {@code GenericKeyedObjectPool} can be viewed as a map
054 * of sub-pools, keyed on the (unique) key values provided to the
055 * {@link #preparePool preparePool}, {@link #addObject addObject} or
056 * {@link #borrowObject borrowObject} methods. Each time a new key value is
057 * provided to one of these methods, a sub-new pool is created under the given
058 * key to be managed by the containing {@code GenericKeyedObjectPool.}
059 * </p>
060 * <p>
061 * Note that the current implementation uses a ConcurrentHashMap which uses
062 * equals() to compare keys.
063 * This means that distinct instance keys must be distinguishable using equals.
064 * </p>
065 * <p>
066 * Optionally, one may configure the pool to examine and possibly evict objects
067 * as they sit idle in the pool and to ensure that a minimum number of idle
068 * objects is maintained for each key. This is performed by an "idle object
069 * eviction" thread, which runs asynchronously. Caution should be used when
070 * configuring this optional feature. Eviction runs contend with client threads
071 * for access to objects in the pool, so if they run too frequently performance
072 * issues may result.
073 * </p>
074 * <p>
075 * Implementation note: To prevent possible deadlocks, care has been taken to
076 * ensure that no call to a factory method will occur within a synchronization
077 * block. See POOL-125 and DBCP-44 for more information.
078 * </p>
079 * <p>
080 * This class is intended to be thread-safe.
081 * </p>
082 *
083 * @see GenericObjectPool
084 *
085 * @param <K> The type of keys maintained by this pool.
086 * @param <T> Type of element pooled in this pool.
087 *
088 * @since 2.0
089 */
090public class GenericKeyedObjectPool<K, T> extends BaseGenericObjectPool<T>
091        implements KeyedObjectPool<K, T>, GenericKeyedObjectPoolMXBean<K>, UsageTracking<T> {
092
093    private static final Integer ZERO = Integer.valueOf(0);
094
095    /**
096     * Maintains information on the per key queue for a given key.
097     *
098     * @param <S> type of objects in the pool
099     */
100    private class ObjectDeque<S> {
101
102        private final LinkedBlockingDeque<PooledObject<S>> idleObjects;
103
104        /*
105         * Number of instances created - number destroyed.
106         * Invariant: createCount <= maxTotalPerKey
107         */
108        private final AtomicInteger createCount = new AtomicInteger(0);
109
110        private long makeObjectCount;
111        private final Object makeObjectCountLock = new Object();
112
113        /*
114         * The map is keyed on pooled instances, wrapped to ensure that
115         * they work properly as keys.
116         */
117        private final Map<IdentityWrapper<S>, PooledObject<S>> allObjects =
118                new ConcurrentHashMap<>();
119
120        /*
121         * Number of threads with registered interest in this key.
122         * register(K) increments this counter and deRegister(K) decrements it.
123         * Invariant: empty keyed pool will not be dropped unless numInterested
124         *            is 0.
125         */
126        private final AtomicLong numInterested = new AtomicLong();
127
128        /**
129         * Constructs a new ObjecDeque with the given fairness policy.
130         * @param fairness true means client threads waiting to borrow / return instances
131         * will be served as if waiting in a FIFO queue.
132         */
133        public ObjectDeque(final boolean fairness) {
134            idleObjects = new LinkedBlockingDeque<>(fairness);
135        }
136
137        /**
138         * Gets all the objects for the current key.
139         *
140         * @return All the objects
141         */
142        public Map<IdentityWrapper<S>, PooledObject<S>> getAllObjects() {
143            return allObjects;
144        }
145
146        /**
147         * Gets the count of the number of objects created for the current
148         * key.
149         *
150         * @return The number of objects created for this key
151         */
152        public AtomicInteger getCreateCount() {
153            return createCount;
154        }
155
156        /**
157         * Gets the idle objects for the current key.
158         *
159         * @return The idle objects
160         */
161        public LinkedBlockingDeque<PooledObject<S>> getIdleObjects() {
162            return idleObjects;
163        }
164
165        /**
166         * Gets the number of threads with an interest registered in this key.
167         *
168         * @return The number of threads with a registered interest in this key
169         */
170        public AtomicLong getNumInterested() {
171            return numInterested;
172        }
173
174        @Override
175        public String toString() {
176            final StringBuilder builder = new StringBuilder();
177            builder.append("ObjectDeque [idleObjects=");
178            builder.append(idleObjects);
179            builder.append(", createCount=");
180            builder.append(createCount);
181            builder.append(", allObjects=");
182            builder.append(allObjects);
183            builder.append(", numInterested=");
184            builder.append(numInterested);
185            builder.append("]");
186            return builder.toString();
187        }
188
189    }
190
191    // JMX specific attributes
192    private static final String ONAME_BASE =
193            "org.apache.commons.pool2:type=GenericKeyedObjectPool,name=";
194
195    //--- configuration attributes ---------------------------------------------
196    private volatile int maxIdlePerKey =
197            GenericKeyedObjectPoolConfig.DEFAULT_MAX_IDLE_PER_KEY;
198
199    private volatile int minIdlePerKey =
200            GenericKeyedObjectPoolConfig.DEFAULT_MIN_IDLE_PER_KEY;
201
202
203    private volatile int maxTotalPerKey =
204            GenericKeyedObjectPoolConfig.DEFAULT_MAX_TOTAL_PER_KEY;
205
206    private final KeyedPooledObjectFactory<K, T> factory;
207
208    private final boolean fairness;
209
210    /*
211     * My hash of sub-pools (ObjectQueue). The list of keys <b>must</b> be kept
212     * in step with {@link #poolKeyList} using {@link #keyLock} to ensure any
213     * changes to the list of current keys is made in a thread-safe manner.
214     */
215    private final Map<K, ObjectDeque<T>> poolMap =
216            new ConcurrentHashMap<>(); // @GuardedBy("keyLock") for write access (and some read access)
217
218    /*
219     * List of pool keys - used to control eviction order. The list of keys
220     * <b>must</b> be kept in step with {@link #poolMap} using {@link #keyLock}
221     * to ensure any changes to the list of current keys is made in a
222     * thread-safe manner.
223     */
224    private final List<K> poolKeyList = new ArrayList<>(); // @GuardedBy("keyLock")
225
226    private final ReadWriteLock keyLock = new ReentrantReadWriteLock(true);
227
228    /*
229     * The combined count of the currently active objects for all keys and those
230     * in the process of being created. Under load, it may exceed
231     * {@link #maxTotal} but there will never be more than {@link #maxTotal}
232     * created at any one time.
233     */
234    private final AtomicInteger numTotal = new AtomicInteger(0);
235
236    private Iterator<K> evictionKeyIterator; // @GuardedBy("evictionLock")
237
238
239    private K evictionKey; // @GuardedBy("evictionLock")
240
241    /**
242     * Constructs a new {@code GenericKeyedObjectPool} using defaults from
243     * {@link GenericKeyedObjectPoolConfig}.
244     * @param factory the factory to be used to create entries
245     */
246    public GenericKeyedObjectPool(final KeyedPooledObjectFactory<K, T> factory) {
247        this(factory, new GenericKeyedObjectPoolConfig<>());
248    }
249
250    /**
251     * Constructs a new {@code GenericKeyedObjectPool} using a specific
252     * configuration.
253     *
254     * @param factory the factory to be used to create entries
255     * @param config    The configuration to use for this pool instance. The
256     *                  configuration is used by value. Subsequent changes to
257     *                  the configuration object will not be reflected in the
258     *                  pool.
259     */
260    public GenericKeyedObjectPool(final KeyedPooledObjectFactory<K, T> factory,
261            final GenericKeyedObjectPoolConfig<T> config) {
262
263        super(config, ONAME_BASE, config.getJmxNamePrefix());
264
265        if (factory == null) {
266            jmxUnregister(); // tidy up
267            throw new IllegalArgumentException("Factory may not be null");
268        }
269        this.factory = factory;
270        this.fairness = config.getFairness();
271
272        setConfig(config);
273    }
274
275    /**
276     * Creates a new {@code GenericKeyedObjectPool} that tracks and destroys
277     * objects that are checked out, but never returned to the pool.
278     *
279     * @param factory   The object factory to be used to create object instances
280     *                  used by this pool
281     * @param config    The base pool configuration to use for this pool instance.
282     *                  The configuration is used by value. Subsequent changes to
283     *                  the configuration object will not be reflected in the
284     *                  pool.
285     * @param abandonedConfig  Configuration for abandoned object identification
286     *                         and removal.  The configuration is used by value.
287     * @since 2.10.0
288     */
289    public GenericKeyedObjectPool(final KeyedPooledObjectFactory<K, T> factory,
290            final GenericKeyedObjectPoolConfig<T> config, final AbandonedConfig abandonedConfig) {
291        this(factory, config);
292        setAbandonedConfig(abandonedConfig);
293    }
294
295    /**
296     * Add an object to the set of idle objects for a given key.
297     *
298     * @param key The key to associate with the idle object
299     * @param p The wrapped object to add.
300     *
301     * @throws Exception If the associated factory fails to passivate the object
302     */
303    private void addIdleObject(final K key, final PooledObject<T> p) throws Exception {
304
305        if (p != null) {
306            factory.passivateObject(key, p);
307            final LinkedBlockingDeque<PooledObject<T>> idleObjects =
308                    poolMap.get(key).getIdleObjects();
309            if (getLifo()) {
310                idleObjects.addFirst(p);
311            } else {
312                idleObjects.addLast(p);
313            }
314        }
315    }
316
317
318    /**
319     * Create an object using the {@link KeyedPooledObjectFactory#makeObject
320     * factory}, passivate it, and then place it in the idle object pool.
321     * {@code addObject} is useful for "pre-loading" a pool with idle
322     * objects.
323     *
324     * @param key the key a new instance should be added to
325     *
326     * @throws Exception when {@link KeyedPooledObjectFactory#makeObject}
327     *                   fails.
328     */
329    @Override
330    public void addObject(final K key) throws Exception {
331        assertOpen();
332        register(key);
333        try {
334            final PooledObject<T> p = create(key);
335            addIdleObject(key, p);
336        } finally {
337            deregister(key);
338        }
339    }
340
341
342    /**
343     * Equivalent to <code>{@link #borrowObject(Object, long) borrowObject}(key,
344     * {@link #getMaxWaitDuration()})</code>.
345     *
346     * {@inheritDoc}
347     */
348    @Override
349    public T borrowObject(final K key) throws Exception {
350        return borrowObject(key, getMaxWaitDuration().toMillis());
351    }
352
353
354    /**
355     * Borrows an object from the sub-pool associated with the given key using
356     * the specified waiting time which only applies if
357     * {@link #getBlockWhenExhausted()} is true.
358     * <p>
359     * If there is one or more idle instances available in the sub-pool
360     * associated with the given key, then an idle instance will be selected
361     * based on the value of {@link #getLifo()}, activated and returned.  If
362     * activation fails, or {@link #getTestOnBorrow() testOnBorrow} is set to
363     * {@code true} and validation fails, the instance is destroyed and the
364     * next available instance is examined.  This continues until either a valid
365     * instance is returned or there are no more idle instances available.
366     * </p>
367     * <p>
368     * If there are no idle instances available in the sub-pool associated with
369     * the given key, behavior depends on the {@link #getMaxTotalPerKey()
370     * maxTotalPerKey}, {@link #getMaxTotal() maxTotal}, and (if applicable)
371     * {@link #getBlockWhenExhausted()} and the value passed in to the
372     * {@code borrowMaxWaitMillis} parameter. If the number of instances checked
373     * out from the sub-pool under the given key is less than
374     * {@code maxTotalPerKey} and the total number of instances in
375     * circulation (under all keys) is less than {@code maxTotal}, a new
376     * instance is created, activated and (if applicable) validated and returned
377     * to the caller. If validation fails, a {@code NoSuchElementException}
378     * will be thrown.
379     * </p>
380     * <p>
381     * If the associated sub-pool is exhausted (no available idle instances and
382     * no capacity to create new ones), this method will either block
383     * ({@link #getBlockWhenExhausted()} is true) or throw a
384     * {@code NoSuchElementException}
385     * ({@link #getBlockWhenExhausted()} is false).
386     * The length of time that this method will block when
387     * {@link #getBlockWhenExhausted()} is true is determined by the value
388     * passed in to the {@code borrowMaxWait} parameter.
389     * </p>
390     * <p>
391     * When {@code maxTotal} is set to a positive value and this method is
392     * invoked when at the limit with no idle instances available under the requested
393     * key, an attempt is made to create room by clearing the oldest 15% of the
394     * elements from the keyed sub-pools.
395     * </p>
396     * <p>
397     * When the pool is exhausted, multiple calling threads may be
398     * simultaneously blocked waiting for instances to become available. A
399     * "fairness" algorithm has been implemented to ensure that threads receive
400     * available instances in request arrival order.
401     * </p>
402     *
403     * @param key pool key
404     * @param borrowMaxWaitMillis The time to wait in milliseconds for an object
405     *                            to become available
406     *
407     * @return object instance from the keyed pool
408     *
409     * @throws NoSuchElementException if a keyed object instance cannot be
410     *                                returned because the pool is exhausted.
411     *
412     * @throws Exception if a keyed object instance cannot be returned due to an
413     *                   error
414     */
415    public T borrowObject(final K key, final long borrowMaxWaitMillis) throws Exception {
416        assertOpen();
417
418        final AbandonedConfig ac = this.abandonedConfig;
419        if (ac != null && ac.getRemoveAbandonedOnBorrow() && (getNumIdle() < 2) &&
420                (getNumActive() > getMaxTotal() - 3)) {
421            removeAbandoned(ac);
422        }
423
424        PooledObject<T> p = null;
425
426        // Get local copy of current config so it is consistent for entire
427        // method execution
428        final boolean blockWhenExhausted = getBlockWhenExhausted();
429
430        boolean create;
431        final long waitTimeMillis = System.currentTimeMillis();
432        final ObjectDeque<T> objectDeque = register(key);
433
434        try {
435            while (p == null) {
436                create = false;
437                p = objectDeque.getIdleObjects().pollFirst();
438                if (p == null) {
439                    p = create(key);
440                    if (p != null) {
441                        create = true;
442                    }
443                }
444                if (blockWhenExhausted) {
445                    if (p == null) {
446                        if (borrowMaxWaitMillis < 0) {
447                            p = objectDeque.getIdleObjects().takeFirst();
448                        } else {
449                            p = objectDeque.getIdleObjects().pollFirst(borrowMaxWaitMillis, TimeUnit.MILLISECONDS);
450                        }
451                    }
452                    if (p == null) {
453                        throw new NoSuchElementException(appendStats(
454                                "Timeout waiting for idle object, borrowMaxWaitMillis=" + borrowMaxWaitMillis));
455                    }
456                } else if (p == null) {
457                    throw new NoSuchElementException(appendStats("Pool exhausted"));
458                }
459                if (!p.allocate()) {
460                    p = null;
461                }
462
463                if (p != null) {
464                    try {
465                        factory.activateObject(key, p);
466                    } catch (final Exception e) {
467                        try {
468                            destroy(key, p, true, DestroyMode.NORMAL);
469                        } catch (final Exception e1) {
470                            // Ignore - activation failure is more important
471                        }
472                        p = null;
473                        if (create) {
474                            final NoSuchElementException nsee = new NoSuchElementException(appendStats("Unable to activate object"));
475                            nsee.initCause(e);
476                            throw nsee;
477                        }
478                    }
479                    if (p != null && getTestOnBorrow()) {
480                        boolean validate = false;
481                        Throwable validationThrowable = null;
482                        try {
483                            validate = factory.validateObject(key, p);
484                        } catch (final Throwable t) {
485                            PoolUtils.checkRethrow(t);
486                            validationThrowable = t;
487                        }
488                        if (!validate) {
489                            try {
490                                destroy(key, p, true, DestroyMode.NORMAL);
491                                destroyedByBorrowValidationCount.incrementAndGet();
492                            } catch (final Exception e) {
493                                // Ignore - validation failure is more important
494                            }
495                            p = null;
496                            if (create) {
497                                final NoSuchElementException nsee = new NoSuchElementException(
498                                        appendStats("Unable to validate object"));
499                                nsee.initCause(validationThrowable);
500                                throw nsee;
501                            }
502                        }
503                    }
504                }
505            }
506        } finally {
507            deregister(key);
508        }
509
510        updateStatsBorrow(p, Duration.ofMillis(System.currentTimeMillis() - waitTimeMillis));
511
512        return p.getObject();
513    }
514
515    @Override
516    String getStatsString() {
517        // Simply listed in AB order.
518        return super.getStatsString() +
519                String.format(", fairness=%s, maxIdlePerKey%,d, maxTotalPerKey=%,d, minIdlePerKey=%,d, numTotal=%,d",
520                        fairness, maxIdlePerKey, maxTotalPerKey, minIdlePerKey, numTotal.get());
521    }
522
523    /**
524     * Calculate the number of objects that need to be created to attempt to
525     * maintain the minimum number of idle objects while not exceeded the limits
526     * on the maximum number of objects either per key or totally.
527     *
528     * @param objectDeque   The set of objects to check
529     *
530     * @return The number of new objects to create
531     */
532    private int calculateDeficit(final ObjectDeque<T> objectDeque) {
533
534        if (objectDeque == null) {
535            return getMinIdlePerKey();
536        }
537
538        // Used more than once so keep a local copy so the value is consistent
539        final int maxTotal = getMaxTotal();
540        final int maxTotalPerKeySave = getMaxTotalPerKey();
541
542        // Calculate no of objects needed to be created, in order to have
543        // the number of pooled objects < maxTotalPerKey();
544        int objectDefecit = getMinIdlePerKey() - objectDeque.getIdleObjects().size();
545        if (maxTotalPerKeySave > 0) {
546            final int growLimit = Math.max(0,
547                    maxTotalPerKeySave - objectDeque.getIdleObjects().size());
548            objectDefecit = Math.min(objectDefecit, growLimit);
549        }
550
551        // Take the maxTotal limit into account
552        if (maxTotal > 0) {
553            final int growLimit = Math.max(0, maxTotal - getNumActive() - getNumIdle());
554            objectDefecit = Math.min(objectDefecit, growLimit);
555        }
556
557        return objectDefecit;
558    }
559
560
561    /**
562     * Clears any objects sitting idle in the pool by removing them from the
563     * idle instance sub-pools and then invoking the configured
564     * PoolableObjectFactory's
565     * {@link KeyedPooledObjectFactory#destroyObject(Object, PooledObject)}
566     * method on each idle instance.
567     * <p>
568     * Implementation notes:
569     * </p>
570     * <ul>
571     * <li>This method does not destroy or effect in any way instances that are
572     * checked out when it is invoked.</li>
573     * <li>Invoking this method does not prevent objects being returned to the
574     * idle instance pool, even during its execution. Additional instances may
575     * be returned while removed items are being destroyed.</li>
576     * <li>Exceptions encountered destroying idle instances are swallowed
577     * but notified via a {@link SwallowedExceptionListener}.</li>
578     * </ul>
579     */
580    @Override
581    public void clear() {
582        poolMap.keySet().forEach(this::clear);
583    }
584
585
586    /**
587     * Clears the specified sub-pool, removing all pooled instances
588     * corresponding to the given {@code key}. Exceptions encountered
589     * destroying idle instances are swallowed but notified via a
590     * {@link SwallowedExceptionListener}.
591     *
592     * @param key the key to clear
593     */
594    @Override
595    public void clear(final K key) {
596
597        final ObjectDeque<T> objectDeque = register(key);
598
599        try {
600            final LinkedBlockingDeque<PooledObject<T>> idleObjects =
601                    objectDeque.getIdleObjects();
602
603            PooledObject<T> p = idleObjects.poll();
604
605            while (p != null) {
606                try {
607                    destroy(key, p, true, DestroyMode.NORMAL);
608                } catch (final Exception e) {
609                    swallowException(e);
610                }
611                p = idleObjects.poll();
612            }
613        } finally {
614            deregister(key);
615        }
616    }
617
618
619    /**
620     * Clears oldest 15% of objects in pool.  The method sorts the objects into
621     * a TreeMap and then iterates the first 15% for removal.
622     */
623    public void clearOldest() {
624
625        // build sorted map of idle objects
626        final Map<PooledObject<T>, K> map = new TreeMap<>();
627
628        poolMap.entrySet().forEach(entry -> {
629            final ObjectDeque<T> deque = entry.getValue();
630            // Protect against possible NPE if key has been removed in another
631            // thread. Not worth locking the keys while this loop completes.
632            if (deque != null) {
633                // Each item into the map using the PooledObject object as the
634                // key. It then gets sorted based on the idle time
635                deque.getIdleObjects().forEach(p -> map.put(p, entry.getKey()));
636            }
637        });
638
639        // Now iterate created map and kill the first 15% plus one to account
640        // for zero
641        int itemsToRemove = ((int) (map.size() * 0.15)) + 1;
642        final Iterator<Entry<PooledObject<T>, K>> iter = map.entrySet().iterator();
643
644        while (iter.hasNext() && itemsToRemove > 0) {
645            final Entry<PooledObject<T>, K> entry = iter.next();
646            // kind of backwards on naming.  In the map, each key is the
647            // PooledObject because it has the ordering with the timestamp
648            // value.  Each value that the key references is the key of the
649            // list it belongs to.
650            final K key = entry.getValue();
651            final PooledObject<T> p = entry.getKey();
652            // Assume the destruction succeeds
653            boolean destroyed = true;
654            try {
655                destroyed = destroy(key, p, false, DestroyMode.NORMAL);
656            } catch (final Exception e) {
657                swallowException(e);
658            }
659            if (destroyed) {
660                itemsToRemove--;
661            }
662        }
663    }
664
665
666    /**
667     * Closes the keyed object pool. Once the pool is closed,
668     * {@link #borrowObject(Object)} will fail with IllegalStateException, but
669     * {@link #returnObject(Object, Object)} and
670     * {@link #invalidateObject(Object, Object)} will continue to work, with
671     * returned objects destroyed on return.
672     * <p>
673     * Destroys idle instances in the pool by invoking {@link #clear()}.
674     * </p>
675     */
676    @Override
677    public void close() {
678        if (isClosed()) {
679            return;
680        }
681
682        synchronized (closeLock) {
683            if (isClosed()) {
684                return;
685            }
686
687            // Stop the evictor before the pool is closed since evict() calls
688            // assertOpen()
689            stopEvictor();
690
691            closed = true;
692            // This clear removes any idle objects
693            clear();
694
695            jmxUnregister();
696
697            // Release any threads that were waiting for an object
698            poolMap.values().forEach(e -> e.getIdleObjects().interuptTakeWaiters());
699            // This clear cleans up the keys now any waiting threads have been
700            // interrupted
701            clear();
702        }
703    }
704
705    /**
706     * Creates a new pooled object.
707     *
708     * @param key Key associated with new pooled object
709     *
710     * @return The new, wrapped pooled object
711     *
712     * @throws Exception If the objection creation fails
713     */
714    private PooledObject<T> create(final K key) throws Exception {
715        int maxTotalPerKeySave = getMaxTotalPerKey(); // Per key
716        if (maxTotalPerKeySave < 0) {
717            maxTotalPerKeySave = Integer.MAX_VALUE;
718        }
719        final int maxTotal = getMaxTotal();   // All keys
720
721        final ObjectDeque<T> objectDeque = poolMap.get(key);
722
723        // Check against the overall limit
724        boolean loop = true;
725
726        while (loop) {
727            final int newNumTotal = numTotal.incrementAndGet();
728            if (maxTotal > -1 && newNumTotal > maxTotal) {
729                numTotal.decrementAndGet();
730                if (getNumIdle() == 0) {
731                    return null;
732                }
733                clearOldest();
734            } else {
735                loop = false;
736            }
737        }
738
739        // Flag that indicates if create should:
740        // - TRUE:  call the factory to create an object
741        // - FALSE: return null
742        // - null:  loop and re-test the condition that determines whether to
743        //          call the factory
744        Boolean create = null;
745        while (create == null) {
746            synchronized (objectDeque.makeObjectCountLock) {
747                final long newCreateCount = objectDeque.getCreateCount().incrementAndGet();
748                // Check against the per key limit
749                if (newCreateCount > maxTotalPerKeySave) {
750                    // The key is currently at capacity or in the process of
751                    // making enough new objects to take it to capacity.
752                    objectDeque.getCreateCount().decrementAndGet();
753                    if (objectDeque.makeObjectCount == 0) {
754                        // There are no makeObject() calls in progress for this
755                        // key so the key is at capacity. Do not attempt to
756                        // create a new object. Return and wait for an object to
757                        // be returned.
758                        create = Boolean.FALSE;
759                    } else {
760                        // There are makeObject() calls in progress that might
761                        // bring the pool to capacity. Those calls might also
762                        // fail so wait until they complete and then re-test if
763                        // the pool is at capacity or not.
764                        objectDeque.makeObjectCountLock.wait();
765                    }
766                } else {
767                    // The pool is not at capacity. Create a new object.
768                    objectDeque.makeObjectCount++;
769                    create = Boolean.TRUE;
770                }
771            }
772        }
773
774        if (!create.booleanValue()) {
775            numTotal.decrementAndGet();
776            return null;
777        }
778
779        PooledObject<T> p = null;
780        try {
781            p = factory.makeObject(key);
782            if (getTestOnCreate() && !factory.validateObject(key, p)) {
783                numTotal.decrementAndGet();
784                objectDeque.getCreateCount().decrementAndGet();
785                return null;
786            }
787        } catch (final Exception e) {
788            numTotal.decrementAndGet();
789            objectDeque.getCreateCount().decrementAndGet();
790            throw e;
791        } finally {
792            synchronized (objectDeque.makeObjectCountLock) {
793                objectDeque.makeObjectCount--;
794                objectDeque.makeObjectCountLock.notifyAll();
795            }
796        }
797
798        final AbandonedConfig ac = this.abandonedConfig;
799        if (ac != null && ac.getLogAbandoned()) {
800            p.setLogAbandoned(true);
801            p.setRequireFullStackTrace(ac.getRequireFullStackTrace());
802        }
803
804        createdCount.incrementAndGet();
805        objectDeque.getAllObjects().put(new IdentityWrapper<>(p.getObject()), p);
806        return p;
807    }
808
809    /**
810     * De-register the use of a key by an object.
811     * <p>
812     * {@link #register(Object)} and {@link #deregister(Object)} must always be used as a pair.
813     * </p>
814     *
815     * @param k The key to de-register
816     */
817    private void deregister(final K k) {
818        Lock lock = keyLock.readLock();
819        try {
820            lock.lock();
821            final ObjectDeque<T> objectDeque = poolMap.get(k);
822            final long numInterested = objectDeque.getNumInterested().decrementAndGet();
823            if (numInterested == 0 && objectDeque.getCreateCount().get() == 0) {
824                // Potential to remove key
825                // Upgrade to write lock
826                lock.unlock();
827                lock = keyLock.writeLock();
828                lock.lock();
829                if (objectDeque.getCreateCount().get() == 0 && objectDeque.getNumInterested().get() == 0) {
830                    // NOTE: Keys must always be removed from both poolMap and
831                    //       poolKeyList at the same time while protected by
832                    //       keyLock.writeLock()
833                    poolMap.remove(k);
834                    poolKeyList.remove(k);
835                }
836            }
837        } finally {
838            lock.unlock();
839        }
840    }
841
842
843    /**
844     * Destroy the wrapped, pooled object.
845     *
846     * @param key The key associated with the object to destroy.
847     * @param toDestroy The wrapped object to be destroyed
848     * @param always Should the object be destroyed even if it is not currently
849     *               in the set of idle objects for the given key
850     * @param destroyMode DestroyMode context provided to the factory
851     *
852     * @return {@code true} if the object was destroyed, otherwise {@code false}
853     * @throws Exception If the object destruction failed
854     */
855    private boolean destroy(final K key, final PooledObject<T> toDestroy, final boolean always, final DestroyMode destroyMode)
856            throws Exception {
857
858        final ObjectDeque<T> objectDeque = register(key);
859
860        try {
861            boolean isIdle;
862            synchronized(toDestroy) {
863                // Check idle state directly
864                isIdle = toDestroy.getState().equals(PooledObjectState.IDLE);
865                // If idle, not under eviction test, or always is true, remove instance,
866                // updating isIdle if instance is found in idle objects
867                if (isIdle || always) {
868                    isIdle = objectDeque.getIdleObjects().remove(toDestroy);
869                }
870            }
871            if (isIdle || always) {
872                objectDeque.getAllObjects().remove(new IdentityWrapper<>(toDestroy.getObject()));
873                toDestroy.invalidate();
874
875                try {
876                    factory.destroyObject(key, toDestroy, destroyMode);
877                } finally {
878                    objectDeque.getCreateCount().decrementAndGet();
879                    destroyedCount.incrementAndGet();
880                    numTotal.decrementAndGet();
881                }
882                return true;
883            }
884            return false;
885        } finally {
886            deregister(key);
887        }
888    }
889
890    @Override
891    void ensureMinIdle() throws Exception {
892        final int minIdlePerKeySave = getMinIdlePerKey();
893        if (minIdlePerKeySave < 1) {
894            return;
895        }
896
897        for (final K k : poolMap.keySet()) {
898            ensureMinIdle(k);
899        }
900    }
901
902    /**
903     * Ensure that the configured number of minimum idle objects is available in
904     * the pool for the given key.
905     *
906     * @param key The key to check for idle objects
907     *
908     * @throws Exception If a new object is required and cannot be created
909     */
910    private void ensureMinIdle(final K key) throws Exception {
911        // Calculate current pool objects
912        ObjectDeque<T> objectDeque = poolMap.get(key);
913
914        // objectDeque == null is OK here. It is handled correctly by both
915        // methods called below.
916
917        // this method isn't synchronized so the
918        // calculateDeficit is done at the beginning
919        // as a loop limit and a second time inside the loop
920        // to stop when another thread already returned the
921        // needed objects
922        final int deficit = calculateDeficit(objectDeque);
923
924        for (int i = 0; i < deficit && calculateDeficit(objectDeque) > 0; i++) {
925            addObject(key);
926            // If objectDeque was null, it won't be any more. Obtain a reference
927            // to it so the deficit can be correctly calculated. It needs to
928            // take account of objects created in other threads.
929            if (objectDeque == null) {
930                objectDeque = poolMap.get(key);
931            }
932        }
933    }
934
935
936    /**
937     * {@inheritDoc}
938     * <p>
939     * Successive activations of this method examine objects in keyed sub-pools
940     * in sequence, cycling through the keys and examining objects in
941     * oldest-to-youngest order within the keyed sub-pools.
942     * </p>
943     */
944    @Override
945    public void evict() throws Exception {
946        assertOpen();
947
948        if (getNumIdle() > 0) {
949
950            PooledObject<T> underTest = null;
951            final EvictionPolicy<T> evictionPolicy = getEvictionPolicy();
952
953            synchronized (evictionLock) {
954                final EvictionConfig evictionConfig = new EvictionConfig(
955                        getMinEvictableIdleDuration(),
956                        getSoftMinEvictableIdleDuration(),
957                        getMinIdlePerKey());
958
959                final boolean testWhileIdle = getTestWhileIdle();
960
961                for (int i = 0, m = getNumTests(); i < m; i++) {
962                    if (evictionIterator == null || !evictionIterator.hasNext()) {
963                        if (evictionKeyIterator == null ||
964                                !evictionKeyIterator.hasNext()) {
965                            final List<K> keyCopy = new ArrayList<>();
966                            final Lock readLock = keyLock.readLock();
967                            readLock.lock();
968                            try {
969                                keyCopy.addAll(poolKeyList);
970                            } finally {
971                                readLock.unlock();
972                            }
973                            evictionKeyIterator = keyCopy.iterator();
974                        }
975                        while (evictionKeyIterator.hasNext()) {
976                            evictionKey = evictionKeyIterator.next();
977                            final ObjectDeque<T> objectDeque = poolMap.get(evictionKey);
978                            if (objectDeque == null) {
979                                continue;
980                            }
981
982                            final Deque<PooledObject<T>> idleObjects = objectDeque.getIdleObjects();
983                            evictionIterator = new EvictionIterator(idleObjects);
984                            if (evictionIterator.hasNext()) {
985                                break;
986                            }
987                            evictionIterator = null;
988                        }
989                    }
990                    if (evictionIterator == null) {
991                        // Pools exhausted
992                        return;
993                    }
994                    final Deque<PooledObject<T>> idleObjects;
995                    try {
996                        underTest = evictionIterator.next();
997                        idleObjects = evictionIterator.getIdleObjects();
998                    } catch (final NoSuchElementException nsee) {
999                        // Object was borrowed in another thread
1000                        // Don't count this as an eviction test so reduce i;
1001                        i--;
1002                        evictionIterator = null;
1003                        continue;
1004                    }
1005
1006                    if (!underTest.startEvictionTest()) {
1007                        // Object was borrowed in another thread
1008                        // Don't count this as an eviction test so reduce i;
1009                        i--;
1010                        continue;
1011                    }
1012
1013                    // User provided eviction policy could throw all sorts of
1014                    // crazy exceptions. Protect against such an exception
1015                    // killing the eviction thread.
1016                    boolean evict;
1017                    try {
1018                        evict = evictionPolicy.evict(evictionConfig, underTest,
1019                                poolMap.get(evictionKey).getIdleObjects().size());
1020                    } catch (final Throwable t) {
1021                        // Slightly convoluted as SwallowedExceptionListener
1022                        // uses Exception rather than Throwable
1023                        PoolUtils.checkRethrow(t);
1024                        swallowException(new Exception(t));
1025                        // Don't evict on error conditions
1026                        evict = false;
1027                    }
1028
1029                    if (evict) {
1030                        destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1031                        destroyedByEvictorCount.incrementAndGet();
1032                    } else {
1033                        if (testWhileIdle) {
1034                            boolean active = false;
1035                            try {
1036                                factory.activateObject(evictionKey, underTest);
1037                                active = true;
1038                            } catch (final Exception e) {
1039                                destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1040                                destroyedByEvictorCount.incrementAndGet();
1041                            }
1042                            if (active) {
1043                                boolean validate = false;
1044                                Throwable validationThrowable = null;
1045                                try {
1046                                    validate = factory.validateObject(evictionKey, underTest);
1047                                } catch (final Throwable t) {
1048                                    PoolUtils.checkRethrow(t);
1049                                    validationThrowable = t;
1050                                }
1051                                if (!validate) {
1052                                    destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1053                                    destroyedByEvictorCount.incrementAndGet();
1054                                    if (validationThrowable != null) {
1055                                        if (validationThrowable instanceof RuntimeException) {
1056                                            throw (RuntimeException) validationThrowable;
1057                                        }
1058                                        throw (Error) validationThrowable;
1059                                    }
1060                                } else {
1061                                    try {
1062                                        factory.passivateObject(evictionKey, underTest);
1063                                    } catch (final Exception e) {
1064                                        destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1065                                        destroyedByEvictorCount.incrementAndGet();
1066                                    }
1067                                }
1068                            }
1069                        }
1070                        if (!underTest.endEvictionTest(idleObjects)) {
1071                            // TODO - May need to add code here once additional
1072                            // states are used
1073                        }
1074                    }
1075                }
1076            }
1077        }
1078        final AbandonedConfig ac = this.abandonedConfig;
1079        if (ac != null && ac.getRemoveAbandonedOnMaintenance()) {
1080            removeAbandoned(ac);
1081        }
1082    }
1083
1084    /**
1085     * Gets a reference to the factory used to create, destroy and validate
1086     * the objects used by this pool.
1087     *
1088     * @return the factory
1089     */
1090    public KeyedPooledObjectFactory<K, T> getFactory() {
1091        return factory;
1092    }
1093
1094    /**
1095     * Gets the cap on the number of "idle" instances per key in the pool.
1096     * If maxIdlePerKey is set too low on heavily loaded systems it is possible
1097     * you will see objects being destroyed and almost immediately new objects
1098     * being created. This is a result of the active threads momentarily
1099     * returning objects faster than they are requesting them, causing the
1100     * number of idle objects to rise above maxIdlePerKey. The best value for
1101     * maxIdlePerKey for heavily loaded system will vary but the default is a
1102     * good starting point.
1103     *
1104     * @return the maximum number of "idle" instances that can be held in a
1105     *         given keyed sub-pool or a negative value if there is no limit
1106     *
1107     * @see #setMaxIdlePerKey
1108     */
1109    @Override
1110    public int getMaxIdlePerKey() {
1111        return maxIdlePerKey;
1112    }
1113
1114    /**
1115     * Gets the limit on the number of object instances allocated by the pool
1116     * (checked out or idle), per key. When the limit is reached, the sub-pool
1117     * is said to be exhausted. A negative value indicates no limit.
1118     *
1119     * @return the limit on the number of active instances per key
1120     *
1121     * @see #setMaxTotalPerKey
1122     */
1123    @Override
1124    public int getMaxTotalPerKey() {
1125        return maxTotalPerKey;
1126    }
1127
1128    /**
1129     * Gets the target for the minimum number of idle objects to maintain in
1130     * each of the keyed sub-pools. This setting only has an effect if it is
1131     * positive and {@link #getDurationBetweenEvictionRuns()} is greater than
1132     * zero. If this is the case, an attempt is made to ensure that each
1133     * sub-pool has the required minimum number of instances during idle object
1134     * eviction runs.
1135     * <p>
1136     * If the configured value of minIdlePerKey is greater than the configured
1137     * value for maxIdlePerKey then the value of maxIdlePerKey will be used
1138     * instead.
1139     * </p>
1140     *
1141     * @return minimum size of the each keyed pool
1142     *
1143     * @see #setTimeBetweenEvictionRunsMillis
1144     */
1145    @Override
1146    public int getMinIdlePerKey() {
1147        final int maxIdlePerKeySave = getMaxIdlePerKey();
1148        return this.minIdlePerKey > maxIdlePerKeySave ? maxIdlePerKeySave : minIdlePerKey;
1149    }
1150
1151    @Override
1152    public int getNumActive() {
1153        return numTotal.get() - getNumIdle();
1154    }
1155
1156    @Override
1157    public int getNumActive(final K key) {
1158        final ObjectDeque<T> objectDeque = poolMap.get(key);
1159        if (objectDeque != null) {
1160            return objectDeque.getAllObjects().size() -
1161                    objectDeque.getIdleObjects().size();
1162        }
1163        return 0;
1164    }
1165
1166    @Override
1167    public Map<String, Integer> getNumActivePerKey() {
1168        final HashMap<String, Integer> result = new HashMap<>();
1169
1170        poolMap.entrySet().forEach(entry -> {
1171            if (entry != null) {
1172                final K key = entry.getKey();
1173                final ObjectDeque<T> objectDequeue = entry.getValue();
1174                if (key != null && objectDequeue != null) {
1175                    result.put(key.toString(), Integer.valueOf(
1176                            objectDequeue.getAllObjects().size() -
1177                            objectDequeue.getIdleObjects().size()));
1178                }
1179            }
1180        });
1181        return result;
1182    }
1183
1184    @Override
1185    public int getNumIdle() {
1186        return poolMap.values().stream().mapToInt(e -> e.getIdleObjects().size()).sum();
1187    }
1188
1189
1190    //--- JMX support ----------------------------------------------------------
1191
1192    @Override
1193    public int getNumIdle(final K key) {
1194        final ObjectDeque<T> objectDeque = poolMap.get(key);
1195        return objectDeque != null ? objectDeque.getIdleObjects().size() : 0;
1196    }
1197
1198    /**
1199     * Calculate the number of objects to test in a run of the idle object
1200     * evictor.
1201     *
1202     * @return The number of objects to test for validity
1203     */
1204    private int getNumTests() {
1205        final int totalIdle = getNumIdle();
1206        final int numTests = getNumTestsPerEvictionRun();
1207        if (numTests >= 0) {
1208            return Math.min(numTests, totalIdle);
1209        }
1210        return (int) (Math.ceil(totalIdle / Math.abs((double) numTests)));
1211    }
1212
1213    /**
1214     * Return an estimate of the number of threads currently blocked waiting for
1215     * an object from the pool. This is intended for monitoring only, not for
1216     * synchronization control.
1217     *
1218     * @return The estimate of the number of threads currently blocked waiting
1219     *         for an object from the pool
1220     */
1221    @Override
1222    public int getNumWaiters() {
1223        if (getBlockWhenExhausted()) {
1224            // Assume no overflow
1225            return poolMap.values().stream().mapToInt(e -> e.getIdleObjects().getTakeQueueLength()).sum();
1226        }
1227        return 0;
1228    }
1229
1230    /**
1231     * Return an estimate of the number of threads currently blocked waiting for
1232     * an object from the pool for each key. This is intended for
1233     * monitoring only, not for synchronization control.
1234     *
1235     * @return The estimate of the number of threads currently blocked waiting
1236     *         for an object from the pool for each key
1237     */
1238    @Override
1239    public Map<String, Integer> getNumWaitersByKey() {
1240        final Map<String, Integer> result = new HashMap<>();
1241
1242        for (final Entry<K, ObjectDeque<T>> entry : poolMap.entrySet()) {
1243            final K k = entry.getKey();
1244            final ObjectDeque<T> deque = entry.getValue();
1245            if (deque != null) {
1246                result.put(k.toString(), getBlockWhenExhausted() ? Integer.valueOf(deque.getIdleObjects().getTakeQueueLength()) : ZERO);
1247            }
1248        }
1249        return result;
1250    }
1251
1252    /**
1253     * Checks to see if there are any threads currently waiting to borrow
1254     * objects but are blocked waiting for more objects to become available.
1255     *
1256     * @return {@code true} if there is at least one thread waiting otherwise
1257     *         {@code false}
1258     */
1259    private boolean hasBorrowWaiters() {
1260        return poolMap.values().stream().filter(deque -> deque != null && deque.getIdleObjects().hasTakeWaiters()).findFirst().isPresent();
1261    }
1262
1263    /**
1264     * {@inheritDoc}
1265     * <p>
1266     * Activation of this method decrements the active count associated with
1267     * the given keyed pool and attempts to destroy {@code obj.}
1268     * </p>
1269     *
1270     * @param key pool key
1271     * @param obj instance to invalidate
1272     *
1273     * @throws Exception             if an exception occurs destroying the
1274     *                               object
1275     * @throws IllegalStateException if obj does not belong to the pool
1276     *                               under the given key
1277     */
1278    @Override
1279    public void invalidateObject(final K key, final T obj) throws Exception {
1280        invalidateObject(key, obj, DestroyMode.NORMAL);
1281    }
1282
1283    /**
1284     * {@inheritDoc}
1285     * <p>
1286     * Activation of this method decrements the active count associated with
1287     * the given keyed pool and attempts to destroy {@code obj.}
1288     * </p>
1289     *
1290     * @param key pool key
1291     * @param obj instance to invalidate
1292     * @param destroyMode DestroyMode context provided to factory
1293     *
1294     * @throws Exception             if an exception occurs destroying the
1295     *                               object
1296     * @throws IllegalStateException if obj does not belong to the pool
1297     *                               under the given key
1298     * @since 2.9.0
1299     */
1300    @Override
1301    public void invalidateObject(final K key, final T obj, final DestroyMode destroyMode) throws Exception {
1302        final ObjectDeque<T> objectDeque = poolMap.get(key);
1303        final PooledObject<T> p = objectDeque.getAllObjects().get(new IdentityWrapper<>(obj));
1304        if (p == null) {
1305            throw new IllegalStateException(appendStats("Object not currently part of this pool"));
1306        }
1307        synchronized (p) {
1308            if (p.getState() != PooledObjectState.INVALID) {
1309                destroy(key, p, true, destroyMode);
1310            }
1311        }
1312        if (objectDeque.idleObjects.hasTakeWaiters()) {
1313            addObject(key);
1314        }
1315    }
1316
1317    /**
1318     * Provides information on all the objects in the pool, both idle (waiting
1319     * to be borrowed) and active (currently borrowed).
1320     * <p>
1321     * Note: This is named listAllObjects so it is presented as an operation via
1322     * JMX. That means it won't be invoked unless the explicitly requested
1323     * whereas all attributes will be automatically requested when viewing the
1324     * attributes for an object in a tool like JConsole.
1325     * </p>
1326     *
1327     * @return Information grouped by key on all the objects in the pool
1328     */
1329    @Override
1330    public Map<String, List<DefaultPooledObjectInfo>> listAllObjects() {
1331        final Map<String, List<DefaultPooledObjectInfo>> result = new HashMap<>();
1332
1333        poolMap.entrySet().forEach(entry -> {
1334            final K k = entry.getKey();
1335            final ObjectDeque<T> deque = entry.getValue();
1336            if (deque != null) {
1337                result.put(k.toString(), deque.getAllObjects().values().stream().map(DefaultPooledObjectInfo::new).collect(Collectors.toList()));
1338            }
1339        });
1340        return result;
1341    }
1342
1343    /**
1344     * Registers a key for pool control and ensures that
1345     * {@link #getMinIdlePerKey()} idle instances are created.
1346     *
1347     * @param key - The key to register for pool control.
1348     *
1349     * @throws Exception If the associated factory throws an exception
1350     */
1351    public void preparePool(final K key) throws Exception {
1352        final int minIdlePerKeySave = getMinIdlePerKey();
1353        if (minIdlePerKeySave < 1) {
1354            return;
1355        }
1356        ensureMinIdle(key);
1357    }
1358
1359    /**
1360     * Register the use of a key by an object.
1361     * <p>
1362     * {@link #register(Object)} and {@link #deregister(Object)} must always be used as a pair.
1363     * </p>
1364     *
1365     * @param k The key to register
1366     *
1367     * @return The objects currently associated with the given key. If this
1368     *         method returns without throwing an exception then it will never
1369     *         return null.
1370     */
1371    private ObjectDeque<T> register(final K k) {
1372        Lock lock = keyLock.readLock();
1373        ObjectDeque<T> objectDeque = null;
1374        try {
1375            lock.lock();
1376            objectDeque = poolMap.get(k);
1377            if (objectDeque == null) {
1378                // Upgrade to write lock
1379                lock.unlock();
1380                lock = keyLock.writeLock();
1381                lock.lock();
1382                objectDeque = poolMap.get(k);
1383                if (objectDeque == null) {
1384                    objectDeque = new ObjectDeque<>(fairness);
1385                    objectDeque.getNumInterested().incrementAndGet();
1386                    // NOTE: Keys must always be added to both poolMap and
1387                    //       poolKeyList at the same time while protected by
1388                    //       keyLock.writeLock()
1389                    poolMap.put(k, objectDeque);
1390                    poolKeyList.add(k);
1391                } else {
1392                    objectDeque.getNumInterested().incrementAndGet();
1393                }
1394            } else {
1395                objectDeque.getNumInterested().incrementAndGet();
1396            }
1397        } finally {
1398            lock.unlock();
1399        }
1400        return objectDeque;
1401    }
1402
1403    /**
1404     * Recovers abandoned objects which have been checked out but
1405     * not used since longer than the removeAbandonedTimeout.
1406     *
1407     * @param abandonedConfig The configuration to use to identify abandoned objects
1408     */
1409    @SuppressWarnings("resource") // The PrintWriter is managed elsewhere
1410    private void removeAbandoned(final AbandonedConfig abandonedConfig) {
1411        poolMap.entrySet().forEach(pool -> {
1412            // Generate a list of abandoned objects to remove
1413            final ArrayList<PooledObject<T>> remove = createRemoveList(abandonedConfig, pool.getValue().getAllObjects());
1414            // Now remove the abandoned objects
1415            remove.forEach(pooledObject -> {
1416                if (abandonedConfig.getLogAbandoned()) {
1417                    pooledObject.printStackTrace(abandonedConfig.getLogWriter());
1418                }
1419                try {
1420                    invalidateObject(pool.getKey(), pooledObject.getObject(), DestroyMode.ABANDONED);
1421                } catch (final Exception e) {
1422                    swallowException(e);
1423                }
1424            });
1425        });
1426    }
1427
1428    /**
1429     * Returns an object to a keyed sub-pool.
1430     * <p>
1431     * If {@link #getMaxIdlePerKey() maxIdle} is set to a positive value and the
1432     * number of idle instances under the given key has reached this value, the
1433     * returning instance is destroyed.
1434     * </p>
1435     * <p>
1436     * If {@link #getTestOnReturn() testOnReturn} == true, the returning
1437     * instance is validated before being returned to the idle instance sub-pool
1438     * under the given key. In this case, if validation fails, the instance is
1439     * destroyed.
1440     * </p>
1441     * <p>
1442     * Exceptions encountered destroying objects for any reason are swallowed
1443     * but notified via a {@link SwallowedExceptionListener}.
1444     * </p>
1445     *
1446     * @param key pool key
1447     * @param obj instance to return to the keyed pool
1448     *
1449     * @throws IllegalStateException if an object is returned to the pool that
1450     *                               was not borrowed from it or if an object is
1451     *                               returned to the pool multiple times
1452     */
1453    @Override
1454    public void returnObject(final K key, final T obj) {
1455
1456        final ObjectDeque<T> objectDeque = poolMap.get(key);
1457
1458        if (objectDeque == null) {
1459            throw new IllegalStateException("No keyed pool found under the given key.");
1460        }
1461
1462        final PooledObject<T> p = objectDeque.getAllObjects().get(new IdentityWrapper<>(obj));
1463
1464        if (p == null) {
1465            throw new IllegalStateException("Returned object not currently part of this pool");
1466        }
1467
1468        markReturningState(p);
1469
1470        final Duration activeTime = p.getActiveDuration();
1471
1472        try {
1473            if (getTestOnReturn() && !factory.validateObject(key, p)) {
1474                try {
1475                    destroy(key, p, true, DestroyMode.NORMAL);
1476                } catch (final Exception e) {
1477                    swallowException(e);
1478                }
1479                whenWaitersAddObject(key, objectDeque.idleObjects);
1480                return;
1481            }
1482
1483            try {
1484                factory.passivateObject(key, p);
1485            } catch (final Exception e1) {
1486                swallowException(e1);
1487                try {
1488                    destroy(key, p, true, DestroyMode.NORMAL);
1489                } catch (final Exception e) {
1490                    swallowException(e);
1491                }
1492                whenWaitersAddObject(key, objectDeque.idleObjects);
1493                return;
1494            }
1495
1496            if (!p.deallocate()) {
1497                throw new IllegalStateException("Object has already been returned to this pool");
1498            }
1499
1500            final int maxIdle = getMaxIdlePerKey();
1501            final LinkedBlockingDeque<PooledObject<T>> idleObjects =
1502                    objectDeque.getIdleObjects();
1503
1504            if (isClosed() || maxIdle > -1 && maxIdle <= idleObjects.size()) {
1505                try {
1506                    destroy(key, p, true, DestroyMode.NORMAL);
1507                } catch (final Exception e) {
1508                    swallowException(e);
1509                }
1510            } else {
1511                if (getLifo()) {
1512                    idleObjects.addFirst(p);
1513                } else {
1514                    idleObjects.addLast(p);
1515                }
1516                if (isClosed()) {
1517                    // Pool closed while object was being added to idle objects.
1518                    // Make sure the returned object is destroyed rather than left
1519                    // in the idle object pool (which would effectively be a leak)
1520                    clear(key);
1521                }
1522            }
1523        } finally {
1524            if (hasBorrowWaiters()) {
1525                reuseCapacity();
1526            }
1527            updateStatsReturn(activeTime);
1528        }
1529    }
1530
1531    /**
1532     * Attempt to create one new instance to serve from the most heavily
1533     * loaded pool that can add a new instance.
1534     *
1535     * This method exists to ensure liveness in the pool when threads are
1536     * parked waiting and capacity to create instances under the requested keys
1537     * subsequently becomes available.
1538     *
1539     * This method is not guaranteed to create an instance and its selection
1540     * of the most loaded pool that can create an instance may not always be
1541     * correct, since it does not lock the pool and instances may be created,
1542     * borrowed, returned or destroyed by other threads while it is executing.
1543     */
1544    private void reuseCapacity() {
1545        final int maxTotalPerKeySave = getMaxTotalPerKey();
1546
1547        // Find the most loaded pool that could take a new instance
1548        int maxQueueLength = 0;
1549        LinkedBlockingDeque<PooledObject<T>> mostLoaded = null;
1550        K loadedKey = null;
1551        for (final Entry<K, GenericKeyedObjectPool<K, T>.ObjectDeque<T>> entry : poolMap.entrySet()) {
1552            final K k = entry.getKey();
1553            final ObjectDeque<T> deque = entry.getValue();
1554            if (deque != null) {
1555                final LinkedBlockingDeque<PooledObject<T>> pool = deque.getIdleObjects();
1556                final int queueLength = pool.getTakeQueueLength();
1557                if (getNumActive(k) < maxTotalPerKeySave && queueLength > maxQueueLength) {
1558                    maxQueueLength = queueLength;
1559                    mostLoaded = pool;
1560                    loadedKey = k;
1561                }
1562            }
1563        }
1564
1565        // Attempt to add an instance to the most loaded pool
1566        if (mostLoaded != null) {
1567            register(loadedKey);
1568            try {
1569                final PooledObject<T> p = create(loadedKey);
1570                if (p != null) {
1571                    addIdleObject(loadedKey, p);
1572                }
1573            } catch (final Exception e) {
1574                swallowException(e);
1575            } finally {
1576                deregister(loadedKey);
1577            }
1578        }
1579    }
1580
1581    /**
1582     * Sets the configuration.
1583     *
1584     * @param conf the new configuration to use. This is used by value.
1585     *
1586     * @see GenericKeyedObjectPoolConfig
1587     */
1588    public void setConfig(final GenericKeyedObjectPoolConfig<T> conf) {
1589        super.setConfig(conf);
1590        setMaxIdlePerKey(conf.getMaxIdlePerKey());
1591        setMaxTotalPerKey(conf.getMaxTotalPerKey());
1592        setMaxTotal(conf.getMaxTotal());
1593        setMinIdlePerKey(conf.getMinIdlePerKey());
1594    }
1595
1596    /**
1597     * Sets the cap on the number of "idle" instances per key in the pool.
1598     * If maxIdlePerKey is set too low on heavily loaded systems it is possible
1599     * you will see objects being destroyed and almost immediately new objects
1600     * being created. This is a result of the active threads momentarily
1601     * returning objects faster than they are requesting them, causing the
1602     * number of idle objects to rise above maxIdlePerKey. The best value for
1603     * maxIdlePerKey for heavily loaded system will vary but the default is a
1604     * good starting point.
1605     *
1606     * @param maxIdlePerKey the maximum number of "idle" instances that can be
1607     *                      held in a given keyed sub-pool. Use a negative value
1608     *                      for no limit
1609     *
1610     * @see #getMaxIdlePerKey
1611     */
1612    public void setMaxIdlePerKey(final int maxIdlePerKey) {
1613        this.maxIdlePerKey = maxIdlePerKey;
1614    }
1615
1616    /**
1617     * Sets the limit on the number of object instances allocated by the pool
1618     * (checked out or idle), per key. When the limit is reached, the sub-pool
1619     * is said to be exhausted. A negative value indicates no limit.
1620     *
1621     * @param maxTotalPerKey the limit on the number of active instances per key
1622     *
1623     * @see #getMaxTotalPerKey
1624     */
1625    public void setMaxTotalPerKey(final int maxTotalPerKey) {
1626        this.maxTotalPerKey = maxTotalPerKey;
1627    }
1628
1629    /**
1630     * Sets the target for the minimum number of idle objects to maintain in
1631     * each of the keyed sub-pools. This setting only has an effect if it is
1632     * positive and {@link #getDurationBetweenEvictionRuns()} is greater than
1633     * zero. If this is the case, an attempt is made to ensure that each
1634     * sub-pool has the required minimum number of instances during idle object
1635     * eviction runs.
1636     * <p>
1637     * If the configured value of minIdlePerKey is greater than the configured
1638     * value for maxIdlePerKey then the value of maxIdlePerKey will be used
1639     * instead.
1640     * </p>
1641     *
1642     * @param minIdlePerKey The minimum size of the each keyed pool
1643     *
1644     * @see #getMinIdlePerKey()
1645     * @see #getMaxIdlePerKey()
1646     * @see #setTimeBetweenEvictionRuns(Duration)
1647     */
1648    public void setMinIdlePerKey(final int minIdlePerKey) {
1649        this.minIdlePerKey = minIdlePerKey;
1650    }
1651
1652    @Override
1653    protected void toStringAppendFields(final StringBuilder builder) {
1654        super.toStringAppendFields(builder);
1655        builder.append(", maxIdlePerKey=");
1656        builder.append(maxIdlePerKey);
1657        builder.append(", minIdlePerKey=");
1658        builder.append(minIdlePerKey);
1659        builder.append(", maxTotalPerKey=");
1660        builder.append(maxTotalPerKey);
1661        builder.append(", factory=");
1662        builder.append(factory);
1663        builder.append(", fairness=");
1664        builder.append(fairness);
1665        builder.append(", poolMap=");
1666        builder.append(poolMap);
1667        builder.append(", poolKeyList=");
1668        builder.append(poolKeyList);
1669        builder.append(", keyLock=");
1670        builder.append(keyLock);
1671        builder.append(", numTotal=");
1672        builder.append(numTotal);
1673        builder.append(", evictionKeyIterator=");
1674        builder.append(evictionKeyIterator);
1675        builder.append(", evictionKey=");
1676        builder.append(evictionKey);
1677        builder.append(", abandonedConfig=");
1678        builder.append(abandonedConfig);
1679    }
1680
1681    /**
1682     * Whether there is at least one thread waiting on this deque, add an pool object.
1683     * @param key pool key.
1684     * @param idleObjects list of idle pool objects.
1685     */
1686    private void whenWaitersAddObject(final K key, final LinkedBlockingDeque<PooledObject<T>> idleObjects) {
1687        if (idleObjects.hasTakeWaiters()) {
1688            try {
1689                addObject(key);
1690            } catch (final Exception e) {
1691                swallowException(e);
1692            }
1693        }
1694    }
1695
1696    /**
1697     * @since 2.10.0
1698     */
1699    @Override
1700    public void use(final T pooledObject) {
1701        final AbandonedConfig abandonedCfg = this.abandonedConfig;
1702        if (abandonedCfg != null && abandonedCfg.getUseUsageTracking()) {
1703            poolMap.values().stream()
1704                .map(pool -> pool.getAllObjects().get(new IdentityWrapper<>(pooledObject)))
1705                .filter(Objects::nonNull)
1706                .findFirst()
1707                .ifPresent(PooledObject::use);
1708        }
1709    }
1710
1711}