1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.monitoring;
20
21 import java.io.PrintWriter;
22 import java.lang.ref.WeakReference;
23 import java.lang.reflect.InvocationHandler;
24 import java.lang.reflect.Method;
25 import java.lang.reflect.Proxy;
26 import java.util.ArrayList;
27 import java.util.Iterator;
28 import java.util.List;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.classification.InterfaceAudience;
33
34 import com.google.common.annotations.VisibleForTesting;
35 import com.google.common.collect.Lists;
36
37
38
39
40
41
42 @InterfaceAudience.Private
43 public class TaskMonitor {
44 private static final Log LOG = LogFactory.getLog(TaskMonitor.class);
45
46
47
48 private static final long EXPIRATION_TIME = 60*1000;
49
50 @VisibleForTesting
51 static final int MAX_TASKS = 1000;
52
53 private static TaskMonitor instance;
54 private List<TaskAndWeakRefPair> tasks =
55 Lists.newArrayList();
56
57
58
59
60
61 public static synchronized TaskMonitor get() {
62 if (instance == null) {
63 instance = new TaskMonitor();
64 }
65 return instance;
66 }
67
68 public synchronized MonitoredTask createStatus(String description) {
69 MonitoredTask stat = new MonitoredTaskImpl();
70 stat.setDescription(description);
71 MonitoredTask proxy = (MonitoredTask) Proxy.newProxyInstance(
72 stat.getClass().getClassLoader(),
73 new Class<?>[] { MonitoredTask.class },
74 new PassthroughInvocationHandler<MonitoredTask>(stat));
75 TaskAndWeakRefPair pair = new TaskAndWeakRefPair(stat, proxy);
76 synchronized (this) {
77 tasks.add(pair);
78 }
79 return proxy;
80 }
81
82 public synchronized MonitoredRPCHandler createRPCStatus(String description) {
83 MonitoredRPCHandler stat = new MonitoredRPCHandlerImpl();
84 stat.setDescription(description);
85 MonitoredRPCHandler proxy = (MonitoredRPCHandler) Proxy.newProxyInstance(
86 stat.getClass().getClassLoader(),
87 new Class<?>[] { MonitoredRPCHandler.class },
88 new PassthroughInvocationHandler<MonitoredRPCHandler>(stat));
89 TaskAndWeakRefPair pair = new TaskAndWeakRefPair(stat, proxy);
90 synchronized (this) {
91 tasks.add(pair);
92 }
93 return proxy;
94 }
95
96 private synchronized void purgeExpiredTasks() {
97 int size = 0;
98
99 for (Iterator<TaskAndWeakRefPair> it = tasks.iterator();
100 it.hasNext();) {
101 TaskAndWeakRefPair pair = it.next();
102 MonitoredTask stat = pair.get();
103
104 if (pair.isDead()) {
105
106
107 if (stat.getState() == MonitoredTaskImpl.State.RUNNING) {
108 LOG.warn("Status " + stat + " appears to have been leaked");
109 stat.cleanup();
110 }
111 }
112
113 if (canPurge(stat)) {
114 it.remove();
115 } else {
116 size++;
117 }
118 }
119
120 if (size > MAX_TASKS) {
121 LOG.warn("Too many actions in action monitor! Purging some.");
122 tasks = tasks.subList(size - MAX_TASKS, size);
123 }
124 }
125
126
127
128
129
130
131 public synchronized List<MonitoredTask> getTasks() {
132 purgeExpiredTasks();
133 ArrayList<MonitoredTask> ret = Lists.newArrayListWithCapacity(tasks.size());
134 for (TaskAndWeakRefPair pair : tasks) {
135 MonitoredTask t = pair.get();
136 ret.add(t.clone());
137 }
138 return ret;
139 }
140
141 private boolean canPurge(MonitoredTask stat) {
142 long cts = stat.getCompletionTimestamp();
143 return (cts > 0 && System.currentTimeMillis() - cts > EXPIRATION_TIME);
144 }
145
146
147 public void dumpAsText(PrintWriter out) {
148 long now = System.currentTimeMillis();
149
150 List<MonitoredTask> tasks = getTasks();
151 for (MonitoredTask task : tasks) {
152 out.println("Task: " + task.getDescription());
153 out.println("Status: " + task.getState() + ":" + task.getStatus());
154 long running = (now - task.getStartTime())/1000;
155 if (task.getCompletionTimestamp() != -1) {
156 long completed = (now - task.getCompletionTimestamp()) / 1000;
157 out.println("Completed " + completed + "s ago");
158 out.println("Ran for " +
159 (task.getCompletionTimestamp() - task.getStartTime())/1000
160 + "s");
161 } else {
162 out.println("Running for " + running + "s");
163 }
164 out.println();
165 }
166 }
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186 private static class TaskAndWeakRefPair {
187 private MonitoredTask impl;
188 private WeakReference<MonitoredTask> weakProxy;
189
190 public TaskAndWeakRefPair(MonitoredTask stat,
191 MonitoredTask proxy) {
192 this.impl = stat;
193 this.weakProxy = new WeakReference<MonitoredTask>(proxy);
194 }
195
196 public MonitoredTask get() {
197 return impl;
198 }
199
200 public boolean isDead() {
201 return weakProxy.get() == null;
202 }
203 }
204
205
206
207
208
209 private static class PassthroughInvocationHandler<T> implements InvocationHandler {
210 private T delegatee;
211
212 public PassthroughInvocationHandler(T delegatee) {
213 this.delegatee = delegatee;
214 }
215
216 @Override
217 public Object invoke(Object proxy, Method method, Object[] args)
218 throws Throwable {
219 return method.invoke(delegatee, args);
220 }
221 }
222 }