1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.util;
18
19 import java.io.IOException;
20 import java.util.Collection;
21 import java.util.concurrent.atomic.AtomicInteger;
22 import java.util.concurrent.atomic.AtomicLong;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.util.StringUtils;
28
29
30
31
32
33 public abstract class MultiThreadedAction {
34 private static final Log LOG = LogFactory.getLog(MultiThreadedAction.class);
35
36 protected final byte[] tableName;
37 protected final byte[] columnFamily;
38 protected final Configuration conf;
39
40 protected int numThreads = 1;
41
42
43 protected long startKey = 0;
44
45
46 protected long endKey = 1;
47
48 protected AtomicInteger numThreadsWorking = new AtomicInteger();
49 protected AtomicLong numKeys = new AtomicLong();
50 protected AtomicLong numCols = new AtomicLong();
51 protected AtomicLong totalOpTimeMs = new AtomicLong();
52 protected boolean verbose = false;
53
54 protected int minDataSize = 256;
55 protected int maxDataSize = 1024;
56
57
58 private String actionLetter;
59
60
61 private boolean streamingCounters;
62
63 public static final int REPORTING_INTERVAL_MS = 5000;
64
65 public MultiThreadedAction(Configuration conf, byte[] tableName,
66 byte[] columnFamily, String actionLetter) {
67 this.conf = conf;
68 this.tableName = tableName;
69 this.columnFamily = columnFamily;
70 this.actionLetter = actionLetter;
71 }
72
73 public void start(long startKey, long endKey, int numThreads)
74 throws IOException {
75 this.startKey = startKey;
76 this.endKey = endKey;
77 this.numThreads = numThreads;
78 (new Thread(new ProgressReporter(actionLetter))).start();
79 }
80
81 private static String formatTime(long elapsedTime) {
82 String format = String.format("%%0%dd", 2);
83 elapsedTime = elapsedTime / 1000;
84 String seconds = String.format(format, elapsedTime % 60);
85 String minutes = String.format(format, (elapsedTime % 3600) / 60);
86 String hours = String.format(format, elapsedTime / 3600);
87 String time = hours + ":" + minutes + ":" + seconds;
88 return time;
89 }
90
91
92 private class ProgressReporter implements Runnable {
93
94 private String reporterId = "";
95
96 public ProgressReporter(String id) {
97 this.reporterId = id;
98 }
99
100 @Override
101 public void run() {
102 long startTime = System.currentTimeMillis();
103 long priorNumKeys = 0;
104 long priorCumulativeOpTime = 0;
105 int priorAverageKeysPerSecond = 0;
106
107
108 Threads.sleep(REPORTING_INTERVAL_MS);
109
110 while (numThreadsWorking.get() != 0) {
111 String threadsLeft =
112 "[" + reporterId + ":" + numThreadsWorking.get() + "] ";
113 if (numKeys.get() == 0) {
114 LOG.info(threadsLeft + "Number of keys = 0");
115 } else {
116 long numKeys = MultiThreadedAction.this.numKeys.get();
117 long time = System.currentTimeMillis() - startTime;
118 long totalOpTime = totalOpTimeMs.get();
119
120 long numKeysDelta = numKeys - priorNumKeys;
121 long totalOpTimeDelta = totalOpTime - priorCumulativeOpTime;
122
123 double averageKeysPerSecond =
124 (time > 0) ? (numKeys * 1000 / time) : 0;
125
126 LOG.info(threadsLeft
127 + "Keys="
128 + numKeys
129 + ", cols="
130 + StringUtils.humanReadableInt(numCols.get())
131 + ", time="
132 + formatTime(time)
133 + ((numKeys > 0 && time > 0) ? (" Overall: [" + "keys/s= "
134 + numKeys * 1000 / time + ", latency=" + totalOpTime
135 / numKeys + " ms]") : "")
136 + ((numKeysDelta > 0) ? (" Current: [" + "keys/s="
137 + numKeysDelta * 1000 / REPORTING_INTERVAL_MS + ", latency="
138 + totalOpTimeDelta / numKeysDelta + " ms]") : "")
139 + progressInfo());
140
141 if (streamingCounters) {
142 printStreamingCounters(numKeysDelta,
143 averageKeysPerSecond - priorAverageKeysPerSecond);
144 }
145
146 priorNumKeys = numKeys;
147 priorCumulativeOpTime = totalOpTime;
148 priorAverageKeysPerSecond = (int) averageKeysPerSecond;
149 }
150
151 Threads.sleep(REPORTING_INTERVAL_MS);
152 }
153 }
154
155 private void printStreamingCounters(long numKeysDelta,
156 double avgKeysPerSecondDelta) {
157
158
159 System.err.println("reporter:counter:numKeys," + reporterId + ","
160 + numKeysDelta);
161 System.err.println("reporter:counter:numCols," + reporterId + ","
162 + numCols.get());
163 System.err.println("reporter:counter:avgKeysPerSecond," + reporterId
164 + "," + (long) (avgKeysPerSecondDelta));
165 }
166 }
167
168 public void setDataSize(int minDataSize, int maxDataSize) {
169 this.minDataSize = minDataSize;
170 this.maxDataSize = maxDataSize;
171 }
172
173 public void waitForFinish() {
174 while (numThreadsWorking.get() != 0) {
175 Threads.sleepWithoutInterrupt(1000);
176 }
177 }
178
179 protected void startThreads(Collection<? extends Thread> threads) {
180 numThreadsWorking.addAndGet(threads.size());
181 for (Thread thread : threads) {
182 thread.start();
183 }
184 }
185
186
187 public long getEndKey() {
188 return endKey;
189 }
190
191
192 protected abstract String progressInfo();
193
194 protected static void appendToStatus(StringBuilder sb, String desc,
195 long v) {
196 if (v == 0) {
197 return;
198 }
199 sb.append(", ");
200 sb.append(desc);
201 sb.append("=");
202 sb.append(v);
203 }
204
205 }