1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase;
22  
23  import com.sun.management.UnixOperatingSystemMXBean;
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
27  
28  import java.lang.management.ManagementFactory;
29  import java.lang.management.OperatingSystemMXBean;
30  import java.util.*;
31  
32  
33  /**
34   * Check the resources used:
35   * - threads
36   * - file descriptor
37   */
38  public class ResourceChecker {
39    private static final Log LOG = LogFactory.getLog(ResourceChecker.class);
40  
41    enum Phase {
42      INITIAL, INTERMEDIATE, END
43    }
44    private static Set<String> initialThreadNames = new HashSet<String>();
45  
46    /**
47     * On unix, we know how to get the number of open file descriptor
48     */
49    private static class ResourceAnalyzer {
50      private static final OperatingSystemMXBean osStats;
51      private static final UnixOperatingSystemMXBean unixOsStats;
52  
53      public long getThreadsCount(Phase phase) {
54        Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
55        if (phase == Phase.INITIAL) {
56          for (Thread t : stackTraces.keySet()) {
57            initialThreadNames.add(t.getName());
58          }
59        }
60        return stackTraces.size();
61      }
62  
63      public long getOpenFileDescriptorCount() {
64        if (unixOsStats == null) {
65          return 0;
66        } else {
67          return unixOsStats.getOpenFileDescriptorCount();
68        }
69      }
70  
71      public long getMaxFileDescriptorCount() {
72        if (unixOsStats == null) {
73          return 0;
74        } else {
75          return unixOsStats.getMaxFileDescriptorCount();
76        }
77      }
78  
79      public long getConnectionCount(){
80        return HConnectionTestingUtility.getConnectionCount();
81      }
82  
83      static {
84        osStats =
85          ManagementFactory.getOperatingSystemMXBean();
86        if (osStats instanceof UnixOperatingSystemMXBean) {
87          unixOsStats = (UnixOperatingSystemMXBean) osStats;
88        } else {
89          unixOsStats = null;
90        }
91      }
92    }
93  
94    private static final ResourceAnalyzer rc = new ResourceAnalyzer();
95  
96    /**
97     * Maximum we set for the thread. Will get a warning in logs
98     * if we go other this limit
99     */
100   private static final long MAX_THREADS_COUNT = 500;
101 
102   /**
103    * Maximum we set for the thread. Will get a warning in logs
104    * if we go other this limit
105    */
106   private static final long MAX_FILE_HANDLES_COUNT = 1024;
107 
108 
109   private long initialThreadsCount;
110   private long initialFileHandlesCount;
111   private long initialConnectionCount;
112 
113 
114   public boolean checkThreads(String tagLine) {
115     boolean isOk = true;
116     long threadCount = rc.getThreadsCount(Phase.INTERMEDIATE);
117 
118     if (threadCount > MAX_THREADS_COUNT) {
119       LOG.error(
120         tagLine + ": too many threads used. We use " +
121           threadCount + " our max is " + MAX_THREADS_COUNT);
122       isOk = false;
123     }
124     return isOk;
125   }
126 
127   public boolean check(String tagLine) {
128 
129     boolean isOk = checkThreads(tagLine);
130     if (!checkFileHandles(tagLine)) isOk = false;
131 
132     return isOk;
133   }
134 
135   public ResourceChecker(String tagLine) {
136     init(tagLine);
137   }
138 
139   public final void init(String tagLine) {
140     if (rc.getMaxFileDescriptorCount() < MAX_FILE_HANDLES_COUNT) {
141       LOG.error(
142         "Bad configuration: the operating systems file handles maximum is " +
143           rc.getMaxFileDescriptorCount() + " our is " + MAX_FILE_HANDLES_COUNT);
144     }
145 
146     logInfo(Phase.INITIAL, tagLine);
147 
148     initialThreadsCount = rc.getThreadsCount(Phase.INITIAL);
149     initialFileHandlesCount = rc.getOpenFileDescriptorCount();
150     initialConnectionCount= rc.getConnectionCount();
151 
152     check(tagLine);
153   }
154 
155   public void logInfo(Phase phase, String tagLine) {
156     long threadCount = rc.getThreadsCount(phase);
157     LOG.info(
158         tagLine + ": " +
159         threadCount + " threads" +
160         (initialThreadsCount > 0 ?
161           " (was " + initialThreadsCount + "), " : ", ") +
162         rc.getOpenFileDescriptorCount() + " file descriptors" +
163         (initialFileHandlesCount > 0 ?
164           " (was " + initialFileHandlesCount + "). " : " ") +
165         rc.getConnectionCount() + " connections" +
166         (initialConnectionCount > 0 ?
167           " (was " + initialConnectionCount + "), " : ", ") +
168         (initialThreadsCount > 0 && threadCount > initialThreadsCount ?
169           " -thread leak?- " : "") +
170         (initialFileHandlesCount > 0 &&
171           rc.getOpenFileDescriptorCount() > initialFileHandlesCount ?
172           " -file handle leak?- " : "") +
173         (initialConnectionCount > 0 &&
174           rc.getConnectionCount() > initialConnectionCount ?
175           " -connection leak?- " : "" )
176     );
177     if (phase == Phase.END) {
178       Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
179       if (stackTraces.size() > initialThreadNames.size()) {
180         for (Thread t : stackTraces.keySet()) {
181           if (!initialThreadNames.contains(t.getName())) {
182             LOG.info(tagLine + ": potentially hanging thread - " + t.getName());
183             StackTraceElement[] stackElements = stackTraces.get(t);
184             for (StackTraceElement ele : stackElements) {
185               LOG.info("\t" + ele);
186             }
187           }
188         }
189       }
190     }
191   }
192 
193 
194   public boolean checkFileHandles(String tagLine) {
195     boolean isOk = true;
196 
197     if (rc.getOpenFileDescriptorCount() > MAX_FILE_HANDLES_COUNT) {
198       LOG.error(
199         tagLine + ": too many file handles used. We use " +
200           rc.getOpenFileDescriptorCount() + " our max is " +
201           MAX_FILE_HANDLES_COUNT);
202       isOk = false;
203     }
204 
205     return isOk;
206   }
207 
208   /**
209    * Helper function: print the threads
210    */
211   public static void printThreads(){
212     Set<Thread> threads = Thread.getAllStackTraces().keySet();
213     System.out.println("name; state; isDameon; isAlive; isInterrupted");
214     for (Thread t: threads){
215       System.out.println(
216         t.getName()+";"+t.getState()+";"+t.isDaemon()+";"+t.isAlive()+
217           ";"+t.isInterrupted()
218       );
219     }
220   }
221 }