1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.IOException;
22 import java.io.PrintWriter;
23 import java.io.StringWriter;
24 import java.util.PriorityQueue;
25 import java.util.Queue;
26 import java.util.Set;
27 import java.util.concurrent.ArrayBlockingQueue;
28 import java.util.concurrent.BlockingQueue;
29 import java.util.concurrent.ConcurrentSkipListSet;
30 import java.util.concurrent.TimeUnit;
31 import java.util.concurrent.atomic.AtomicLong;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.hbase.HRegionLocation;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.client.HTable;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
41 import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
42 import org.apache.hadoop.util.StringUtils;
43
44
45 public abstract class MultiThreadedWriterBase extends MultiThreadedAction {
46 private static final Log LOG = LogFactory.getLog(MultiThreadedWriterBase.class);
47
48
49
50
51
52
53
54 protected BlockingQueue<Long> wroteKeys = new ArrayBlockingQueue<Long>(10000);
55
56
57
58
59
60 protected AtomicLong nextKeyToWrite = new AtomicLong();
61
62
63
64
65 protected AtomicLong wroteUpToKey = new AtomicLong();
66
67
68 protected Set<Long> failedKeySet = new ConcurrentSkipListSet<Long>();
69
70
71
72
73
74
75 protected AtomicLong wroteKeyQueueSize = new AtomicLong();
76
77
78 protected boolean trackWroteKeys;
79
80 public MultiThreadedWriterBase(LoadTestDataGenerator dataGen, Configuration conf,
81 TableName tableName, String actionLetter) {
82 super(dataGen, conf, tableName, actionLetter);
83 }
84
85 @Override
86 public void start(long startKey, long endKey, int numThreads)
87 throws IOException {
88 super.start(startKey, endKey, numThreads);
89
90 nextKeyToWrite.set(startKey);
91 wroteUpToKey.set(startKey - 1);
92
93 if (trackWroteKeys) {
94 new Thread(new WroteKeysTracker()).start();
95 numThreadsWorking.incrementAndGet();
96 }
97 }
98
99 protected String getRegionDebugInfoSafe(HTable table, byte[] rowKey) {
100 HRegionLocation cached = null, real = null;
101 try {
102 cached = table.getRegionLocation(rowKey, false);
103 real = table.getRegionLocation(rowKey, true);
104 } catch (Throwable t) {
105
106 }
107 String result = "no information can be obtained";
108 if (cached != null) {
109 result = "cached: " + cached.toString();
110 }
111 if (real != null) {
112 if (real.equals(cached)) {
113 result += "; cache is up to date";
114 } else {
115 result = (cached != null) ? (result + "; ") : "";
116 result += "real: " + real.toString();
117 }
118 }
119 return result;
120 }
121
122
123
124
125
126 private class WroteKeysTracker implements Runnable {
127
128 @Override
129 public void run() {
130 Thread.currentThread().setName(getClass().getSimpleName());
131 try {
132 long expectedKey = startKey;
133 Queue<Long> sortedKeys = new PriorityQueue<Long>();
134 while (expectedKey < endKey) {
135
136 Long k;
137 try {
138 k = wroteKeys.poll(1, TimeUnit.SECONDS);
139 } catch (InterruptedException e) {
140 LOG.info("Inserted key tracker thread interrupted", e);
141 break;
142 }
143 if (k == null) {
144 continue;
145 }
146 if (k == expectedKey) {
147
148 wroteUpToKey.set(k);
149 ++expectedKey;
150 } else {
151 sortedKeys.add(k);
152 }
153
154
155 while (!sortedKeys.isEmpty()
156 && ((k = sortedKeys.peek()) == expectedKey)) {
157 sortedKeys.poll();
158 wroteUpToKey.set(k);
159 ++expectedKey;
160 }
161
162 wroteKeyQueueSize.set(wroteKeys.size() + sortedKeys.size());
163 }
164 } catch (Exception ex) {
165 LOG.error("Error in inserted/updaed key tracker", ex);
166 } finally {
167 numThreadsWorking.decrementAndGet();
168 }
169 }
170 }
171
172 public int getNumWriteFailures() {
173 return failedKeySet.size();
174 }
175
176 public void insert(HTable table, Put put, long keyBase) {
177 long start = System.currentTimeMillis();
178 try {
179 table.put(put);
180 totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
181 } catch (IOException e) {
182 failedKeySet.add(keyBase);
183 String exceptionInfo;
184 if (e instanceof RetriesExhaustedWithDetailsException) {
185 RetriesExhaustedWithDetailsException aggEx = (RetriesExhaustedWithDetailsException)e;
186 exceptionInfo = aggEx.getExhaustiveDescription();
187 } else {
188 StringWriter stackWriter = new StringWriter();
189 PrintWriter pw = new PrintWriter(stackWriter);
190 e.printStackTrace(pw);
191 pw.flush();
192 exceptionInfo = StringUtils.stringifyException(e);
193 }
194 LOG.error("Failed to insert: " + keyBase + " after " + (System.currentTimeMillis() - start) +
195 "ms; region information: " + getRegionDebugInfoSafe(table, put.getRow()) + "; errors: "
196 + exceptionInfo);
197 }
198 }
199
200
201
202
203
204 public long wroteUpToKey() {
205 return wroteUpToKey.get();
206 }
207
208 public boolean failedToWriteKey(long k) {
209 return failedKeySet.contains(k);
210 }
211
212 @Override
213 protected String progressInfo() {
214 StringBuilder sb = new StringBuilder();
215 appendToStatus(sb, "wroteUpTo", wroteUpToKey.get());
216 appendToStatus(sb, "wroteQSize", wroteKeyQueueSize.get());
217 return sb.toString();
218 }
219
220
221
222
223
224
225 public void setTrackWroteKeys(boolean enable) {
226 trackWroteKeys = enable;
227 }
228 }