View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.IOException;
22  import java.io.PrintWriter;
23  import java.io.StringWriter;
24  import java.util.PriorityQueue;
25  import java.util.Queue;
26  import java.util.Set;
27  import java.util.concurrent.ArrayBlockingQueue;
28  import java.util.concurrent.BlockingQueue;
29  import java.util.concurrent.ConcurrentSkipListSet;
30  import java.util.concurrent.TimeUnit;
31  import java.util.concurrent.atomic.AtomicLong;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.HRegionLocation;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
41  import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
42  import org.apache.hadoop.util.StringUtils;
43  
44  /** Creates multiple threads that write key/values into the */
45  public abstract class MultiThreadedWriterBase extends MultiThreadedAction {
46    private static final Log LOG = LogFactory.getLog(MultiThreadedWriterBase.class);
47  
48    /**
49     * A temporary place to keep track of inserted/updated keys. This is written to by
50     * all writers and is drained on a separate thread that populates
51     * {@link #wroteUpToKey}, the maximum key in the contiguous range of keys
52     * being inserted/updated. This queue is supposed to stay small.
53     */
54    protected BlockingQueue<Long> wroteKeys = new ArrayBlockingQueue<Long>(10000);
55  
56    /**
57     * This is the current key to be inserted/updated by any thread. Each thread does an
58     * atomic get and increment operation and inserts the current value.
59     */
60    protected AtomicLong nextKeyToWrite = new AtomicLong();
61  
62    /**
63     * The highest key in the contiguous range of keys .
64     */
65    protected AtomicLong wroteUpToKey = new AtomicLong();
66  
67    /** The sorted set of keys NOT inserted/updated by the writers */
68    protected Set<Long> failedKeySet = new ConcurrentSkipListSet<Long>();
69  
70    /**
71     * The total size of the temporary inserted/updated key set that have not yet lined
72     * up in a our contiguous sequence starting from startKey. Supposed to stay
73     * small.
74     */
75    protected AtomicLong wroteKeyQueueSize = new AtomicLong();
76  
77    /** Enable this if used in conjunction with a concurrent reader. */
78    protected boolean trackWroteKeys;
79  
80    public MultiThreadedWriterBase(LoadTestDataGenerator dataGen, Configuration conf,
81        TableName tableName, String actionLetter) {
82      super(dataGen, conf, tableName, actionLetter);
83    }
84  
85    @Override
86    public void start(long startKey, long endKey, int numThreads)
87        throws IOException {
88      super.start(startKey, endKey, numThreads);
89  
90      nextKeyToWrite.set(startKey);
91      wroteUpToKey.set(startKey - 1);
92  
93      if (trackWroteKeys) {
94        new Thread(new WroteKeysTracker()).start();
95        numThreadsWorking.incrementAndGet();
96      }
97    }
98  
99    protected String getRegionDebugInfoSafe(HTable table, byte[] rowKey) {
100     HRegionLocation cached = null, real = null;
101     try {
102       cached = table.getRegionLocation(rowKey, false);
103       real = table.getRegionLocation(rowKey, true);
104     } catch (Throwable t) {
105       // Cannot obtain region information for another catch block - too bad!
106     }
107     String result = "no information can be obtained";
108     if (cached != null) {
109       result = "cached: " + cached.toString();
110     }
111     if (real != null) {
112       if (real.equals(cached)) {
113         result += "; cache is up to date";
114       } else {
115         result = (cached != null) ? (result + "; ") : "";
116         result += "real: " + real.toString();
117       }
118     }
119     return result;
120   }
121 
122   /**
123    * A thread that keeps track of the highest key in the contiguous range of
124    * inserted/updated keys.
125    */
126   private class WroteKeysTracker implements Runnable {
127 
128     @Override
129     public void run() {
130       Thread.currentThread().setName(getClass().getSimpleName());
131       try {
132         long expectedKey = startKey;
133         Queue<Long> sortedKeys = new PriorityQueue<Long>();
134         while (expectedKey < endKey) {
135           // Block until a new element is available.
136           Long k;
137           try {
138             k = wroteKeys.poll(1, TimeUnit.SECONDS);
139           } catch (InterruptedException e) {
140             LOG.info("Inserted key tracker thread interrupted", e);
141             break;
142           }
143           if (k == null) {
144             continue;
145           }
146           if (k == expectedKey) {
147             // Skip the "sorted key" queue and consume this key.
148             wroteUpToKey.set(k);
149             ++expectedKey;
150           } else {
151             sortedKeys.add(k);
152           }
153 
154           // See if we have a sequence of contiguous keys lined up.
155           while (!sortedKeys.isEmpty()
156               && ((k = sortedKeys.peek()) == expectedKey)) {
157             sortedKeys.poll();
158             wroteUpToKey.set(k);
159             ++expectedKey;
160           }
161 
162           wroteKeyQueueSize.set(wroteKeys.size() + sortedKeys.size());
163         }
164       } catch (Exception ex) {
165         LOG.error("Error in inserted/updaed key tracker", ex);
166       } finally {
167         numThreadsWorking.decrementAndGet();
168       }
169     }
170   }
171 
172   public int getNumWriteFailures() {
173     return failedKeySet.size();
174   }
175 
176   public void insert(HTable table, Put put, long keyBase) {
177     long start = System.currentTimeMillis();
178     try {
179       table.put(put);
180       totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
181     } catch (IOException e) {
182       failedKeySet.add(keyBase);
183       String exceptionInfo;
184       if (e instanceof RetriesExhaustedWithDetailsException) {
185         RetriesExhaustedWithDetailsException aggEx = (RetriesExhaustedWithDetailsException)e;
186         exceptionInfo = aggEx.getExhaustiveDescription();
187       } else {
188         StringWriter stackWriter = new StringWriter();
189         PrintWriter pw = new PrintWriter(stackWriter);
190         e.printStackTrace(pw);
191         pw.flush();
192         exceptionInfo = StringUtils.stringifyException(e);
193       }
194       LOG.error("Failed to insert: " + keyBase + " after " + (System.currentTimeMillis() - start) +
195         "ms; region information: " + getRegionDebugInfoSafe(table, put.getRow()) + "; errors: "
196           + exceptionInfo);
197     }
198   }
199 
200   /**
201    * The max key until which all keys have been inserted/updated (successfully or not).
202    * @return the last key that we have inserted/updated all keys up to (inclusive)
203    */
204   public long wroteUpToKey() {
205     return wroteUpToKey.get();
206   }
207 
208   public boolean failedToWriteKey(long k) {
209     return failedKeySet.contains(k);
210   }
211 
212   @Override
213   protected String progressInfo() {
214     StringBuilder sb = new StringBuilder();
215     appendToStatus(sb, "wroteUpTo", wroteUpToKey.get());
216     appendToStatus(sb, "wroteQSize", wroteKeyQueueSize.get());
217     return sb.toString();
218   }
219 
220   /**
221    * Used for a joint write/read workload. Enables tracking the last inserted/updated
222    * key, which requires a blocking queue and a consumer thread.
223    * @param enable whether to enable tracking the last inserted/updated key
224    */
225   public void setTrackWroteKeys(boolean enable) {
226     trackWroteKeys = enable;
227   }
228 }