View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.thrift;
20  
21  import java.io.IOException;
22  import java.util.Arrays;
23  import java.util.List;
24  import java.util.Set;
25  import java.util.concurrent.Callable;
26  import java.util.concurrent.ConcurrentHashMap;
27  import java.util.concurrent.ConcurrentMap;
28  import java.util.concurrent.LinkedBlockingQueue;
29  import java.util.concurrent.ThreadFactory;
30  import java.util.concurrent.ThreadPoolExecutor;
31  import java.util.concurrent.TimeUnit;
32  import java.util.concurrent.atomic.AtomicInteger;
33  import java.util.concurrent.atomic.AtomicLong;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.hbase.KeyValue;
38  import org.apache.hadoop.hbase.client.HTableInterface;
39  import org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler;
40  import org.apache.hadoop.hbase.thrift.generated.TIncrement;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.Threads;
43  import org.apache.hadoop.metrics.util.MBeanUtil;
44  import org.apache.thrift.TException;
45  
46  /**
47   * This class will coalesce increments from a thift server if
48   * hbase.regionserver.thrift.coalesceIncrement is set to true. Turning this
49   * config to true will cause the thrift server to queue increments into an
50   * instance of this class. The thread pool associated with this class will drain
51   * the coalesced increments as the thread is able. This can cause data loss if the
52   * thrift server dies or is shut down before everything in the queue is drained.
53   *
54   */
55  public class IncrementCoalescer implements IncrementCoalescerMBean {
56  
57    /**
58     * Used to identify a cell that will be incremented.
59     *
60     */
61    static class FullyQualifiedRow {
62      private byte[] table;
63      private byte[] rowKey;
64      private byte[] family;
65      private byte[] qualifier;
66  
67      public FullyQualifiedRow(byte[] table, byte[] rowKey, byte[] fam, byte[] qual) {
68        super();
69        this.table = table;
70        this.rowKey = rowKey;
71        this.family = fam;
72        this.qualifier = qual;
73      }
74  
75      public byte[] getTable() {
76        return table;
77      }
78  
79      public void setTable(byte[] table) {
80        this.table = table;
81      }
82  
83      public byte[] getRowKey() {
84        return rowKey;
85      }
86  
87      public void setRowKey(byte[] rowKey) {
88        this.rowKey = rowKey;
89      }
90  
91      public byte[] getFamily() {
92        return family;
93      }
94  
95      public void setFamily(byte[] fam) {
96        this.family = fam;
97      }
98  
99      public byte[] getQualifier() {
100       return qualifier;
101     }
102 
103     public void setQualifier(byte[] qual) {
104       this.qualifier = qual;
105     }
106 
107     @Override
108     public int hashCode() {
109       final int prime = 31;
110       int result = 1;
111       result = prime * result + Arrays.hashCode(family);
112       result = prime * result + Arrays.hashCode(qualifier);
113       result = prime * result + Arrays.hashCode(rowKey);
114       result = prime * result + Arrays.hashCode(table);
115       return result;
116     }
117 
118     @Override
119     public boolean equals(Object obj) {
120       if (this == obj) return true;
121       if (obj == null) return false;
122       if (getClass() != obj.getClass()) return false;
123       FullyQualifiedRow other = (FullyQualifiedRow) obj;
124       if (!Arrays.equals(family, other.family)) return false;
125       if (!Arrays.equals(qualifier, other.qualifier)) return false;
126       if (!Arrays.equals(rowKey, other.rowKey)) return false;
127       if (!Arrays.equals(table, other.table)) return false;
128       return true;
129     }
130 
131   }
132 
133   static class DaemonThreadFactory implements ThreadFactory {
134     static final AtomicInteger poolNumber = new AtomicInteger(1);
135     final ThreadGroup group;
136     final AtomicInteger threadNumber = new AtomicInteger(1);
137     final String namePrefix;
138 
139     DaemonThreadFactory() {
140       SecurityManager s = System.getSecurityManager();
141       group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
142       namePrefix = "ICV-" + poolNumber.getAndIncrement() + "-thread-";
143     }
144 
145     public Thread newThread(Runnable r) {
146       Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0);
147       if (!t.isDaemon()) t.setDaemon(true);
148       if (t.getPriority() != Thread.NORM_PRIORITY) t.setPriority(Thread.NORM_PRIORITY);
149       return t;
150     }
151   }
152 
153   private final AtomicLong failedIncrements = new AtomicLong();
154   private final AtomicLong successfulCoalescings = new AtomicLong();
155   private final AtomicLong totalIncrements = new AtomicLong();
156   private final ConcurrentMap<FullyQualifiedRow, Long> countersMap =
157       new ConcurrentHashMap<FullyQualifiedRow, Long>(100000, 0.75f, 1500);
158   private final ThreadPoolExecutor pool;
159   private final HBaseHandler handler;
160 
161   private int maxQueueSize = 500000;
162   private static final int CORE_POOL_SIZE = 1;
163 
164   protected final Log LOG = LogFactory.getLog(this.getClass().getName());
165 
166   public IncrementCoalescer(HBaseHandler hand) {
167     this.handler = hand;
168     LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>();
169     pool =
170         new ThreadPoolExecutor(CORE_POOL_SIZE, CORE_POOL_SIZE, 50, TimeUnit.MILLISECONDS, queue,
171             Threads.newDaemonThreadFactory("IncrementCoalescer"));
172 
173     MBeanUtil.registerMBean("thrift", "Thrift", this);
174   }
175 
176   public boolean queueIncrement(TIncrement inc) throws TException {
177     if (!canQueue()) {
178       failedIncrements.incrementAndGet();
179       return false;
180     }
181     return internalQueueTincrement(inc);
182   }
183 
184   public boolean queueIncrements(List<TIncrement> incs) throws TException {
185     if (!canQueue()) {
186       failedIncrements.incrementAndGet();
187       return false;
188     }
189 
190     for (TIncrement tinc : incs) {
191       internalQueueTincrement(tinc);
192     }
193     return true;
194 
195   }
196 
197   private boolean internalQueueTincrement(TIncrement inc) throws TException {
198     byte[][] famAndQf = KeyValue.parseColumn(inc.getColumn());
199     if (famAndQf.length != 2) return false;
200 
201     return internalQueueIncrement(inc.getTable(), inc.getRow(), famAndQf[0], famAndQf[1],
202       inc.getAmmount());
203   }
204 
205   private boolean internalQueueIncrement(byte[] tableName, byte[] rowKey, byte[] fam,
206       byte[] qual, long ammount) throws TException {
207     int countersMapSize = countersMap.size();
208 
209 
210     //Make sure that the number of threads is scaled.
211     dynamicallySetCoreSize(countersMapSize);
212 
213     totalIncrements.incrementAndGet();
214 
215     FullyQualifiedRow key = new FullyQualifiedRow(tableName, rowKey, fam, qual);
216 
217     long currentAmount = ammount;
218     // Spin until able to insert the value back without collisions
219     while (true) {
220       Long value = countersMap.remove(key);
221       if (value == null) {
222         // There was nothing there, create a new value
223         value = Long.valueOf(currentAmount);
224       } else {
225         value += currentAmount;
226         successfulCoalescings.incrementAndGet();
227       }
228       // Try to put the value, only if there was none
229       Long oldValue = countersMap.putIfAbsent(key, value);
230       if (oldValue == null) {
231         // We were able to put it in, we're done
232         break;
233       }
234       // Someone else was able to put a value in, so let's remember our
235       // current value (plus what we picked up) and retry to add it in
236       currentAmount = value;
237     }
238 
239     // We limit the size of the queue simply because all we need is a
240     // notification that something needs to be incremented. No need
241     // for millions of callables that mean the same thing.
242     if (pool.getQueue().size() <= 1000) {
243       // queue it up
244       Callable<Integer> callable = createIncCallable();
245       pool.submit(callable);
246     }
247 
248     return true;
249   }
250 
251   public boolean canQueue() {
252     return countersMap.size() < maxQueueSize;
253   }
254 
255   private Callable<Integer> createIncCallable() {
256     return new Callable<Integer>() {
257       @Override
258       public Integer call() throws Exception {
259         int failures = 0;
260         Set<FullyQualifiedRow> keys = countersMap.keySet();
261         for (FullyQualifiedRow row : keys) {
262           Long counter = countersMap.remove(row);
263           if (counter == null) {
264             continue;
265           }
266           HTableInterface table = null;
267           try {
268             table = handler.getTable(row.getTable());
269             if (failures > 2) {
270               throw new IOException("Auto-Fail rest of ICVs");
271             }
272             table.incrementColumnValue(row.getRowKey(), row.getFamily(), row.getQualifier(),
273               counter);
274           } catch (IOException e) {
275             // log failure of increment
276             failures++;
277             LOG.error("FAILED_ICV: " + Bytes.toString(row.getTable()) + ", "
278                 + Bytes.toStringBinary(row.getRowKey()) + ", "
279                 + Bytes.toStringBinary(row.getFamily()) + ", "
280                 + Bytes.toStringBinary(row.getQualifier()) + ", " + counter, e);
281           } finally{
282             if(table != null){
283               table.close();
284             }
285           }
286         }
287         return failures;
288       }
289     };
290   }
291 
292   /**
293    * This method samples the incoming requests and, if selected, will check if
294    * the corePoolSize should be changed.
295    * @param countersMapSize
296    */
297   private void dynamicallySetCoreSize(int countersMapSize) {
298     // Here we are using countersMapSize as a random number, meaning this
299     // could be a Random object
300     if (countersMapSize % 10 != 0) {
301       return;
302     }
303     double currentRatio = (double) countersMapSize / (double) maxQueueSize;
304     int newValue = 1;
305     if (currentRatio < 0.1) {
306       // it's 1
307     } else if (currentRatio < 0.3) {
308       newValue = 2;
309     } else if (currentRatio < 0.5) {
310       newValue = 4;
311     } else if (currentRatio < 0.7) {
312       newValue = 8;
313     } else if (currentRatio < 0.9) {
314       newValue = 14;
315     } else {
316       newValue = 22;
317     }
318     if (pool.getCorePoolSize() != newValue) {
319       pool.setCorePoolSize(newValue);
320     }
321   }
322 
323   // MBean get/set methods
324   public int getQueueSize() {
325     return pool.getQueue().size();
326   }
327   public int getMaxQueueSize() {
328     return this.maxQueueSize;
329   }
330   public void setMaxQueueSize(int newSize) {
331     this.maxQueueSize = newSize;
332   }
333 
334   public long getPoolCompletedTaskCount() {
335     return pool.getCompletedTaskCount();
336   }
337   public long getPoolTaskCount() {
338     return pool.getTaskCount();
339   }
340   public int getPoolLargestPoolSize() {
341     return pool.getLargestPoolSize();
342   }
343   public int getCorePoolSize() {
344     return pool.getCorePoolSize();
345   }
346   public void setCorePoolSize(int newCoreSize) {
347     pool.setCorePoolSize(newCoreSize);
348   }
349   public int getMaxPoolSize() {
350     return pool.getMaximumPoolSize();
351   }
352   public void setMaxPoolSize(int newMaxSize) {
353     pool.setMaximumPoolSize(newMaxSize);
354   }
355   public long getFailedIncrements() {
356     return failedIncrements.get();
357   }
358 
359   public long getSuccessfulCoalescings() {
360     return successfulCoalescings.get();
361   }
362 
363   public long getTotalIncrements() {
364     return totalIncrements.get();
365   }
366 
367   public long getCountersMapSize() {
368     return countersMap.size();
369   }
370 
371 }