View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.replication.regionserver;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.Map.Entry;
26  import java.util.TreeMap;
27  import java.util.concurrent.ExecutorService;
28  import java.util.concurrent.SynchronousQueue;
29  import java.util.concurrent.ThreadPoolExecutor;
30  import java.util.concurrent.TimeUnit;
31  import java.util.concurrent.atomic.AtomicLong;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.classification.InterfaceAudience;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.HBaseConfiguration;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.KeyValue;
40  import org.apache.hadoop.hbase.Stoppable;
41  import org.apache.hadoop.hbase.client.Delete;
42  import org.apache.hadoop.hbase.client.HConnection;
43  import org.apache.hadoop.hbase.client.HConnectionManager;
44  import org.apache.hadoop.hbase.client.HTable;
45  import org.apache.hadoop.hbase.client.HTableInterface;
46  import org.apache.hadoop.hbase.client.Put;
47  import org.apache.hadoop.hbase.client.Row;
48  import org.apache.hadoop.hbase.regionserver.wal.HLog;
49  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.Threads;
52  
53  /**
54   * This class is responsible for replicating the edits coming
55   * from another cluster.
56   * <p/>
57   * This replication process is currently waiting for the edits to be applied
58   * before the method can return. This means that the replication of edits
59   * is synchronized (after reading from HLogs in ReplicationSource) and that a
60   * single region server cannot receive edits from two sources at the same time
61   * <p/>
62   * This class uses the native HBase client in order to replicate entries.
63   * <p/>
64   *
65   * TODO make this class more like ReplicationSource wrt log handling
66   */
67  @InterfaceAudience.Private
68  public class ReplicationSink {
69  
70    private static final Log LOG = LogFactory.getLog(ReplicationSink.class);
71    // Name of the HDFS directory that contains the temporary rep logs
72    public static final String REPLICATION_LOG_DIR = ".replogs";
73    private final Configuration conf;
74    private final ExecutorService sharedThreadPool;
75    private final HConnection sharedHtableCon;
76    private final MetricsSink metrics;
77    private final AtomicLong totalReplicatedEdits = new AtomicLong();
78  
79    /**
80     * Create a sink for replication
81     *
82     * @param conf                conf object
83     * @param stopper             boolean to tell this thread to stop
84     * @throws IOException thrown when HDFS goes bad or bad file name
85     */
86    public ReplicationSink(Configuration conf, Stoppable stopper)
87        throws IOException {
88      this.conf = HBaseConfiguration.create(conf);
89      decorateConf();
90      this.metrics = new MetricsSink();
91      this.sharedHtableCon = HConnectionManager.createConnection(this.conf);
92      this.sharedThreadPool = new ThreadPoolExecutor(1,
93          conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE),
94          conf.getLong("hbase.htable.threads.keepalivetime", 60), TimeUnit.SECONDS,
95          new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("hbase-repl"));
96      ((ThreadPoolExecutor) this.sharedThreadPool).allowCoreThreadTimeOut(true);
97    }
98  
99    /**
100    * decorate the Configuration object to make replication more receptive to delays:
101    * lessen the timeout and numTries.
102    */
103   private void decorateConf() {
104     this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
105         this.conf.getInt("replication.sink.client.retries.number", 4));
106     this.conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
107         this.conf.getInt("replication.sink.client.ops.timeout", 10000));
108    }
109 
110   /**
111    * Replicate this array of entries directly into the local cluster
112    * using the native client.
113    *
114    * @param entries
115    * @throws IOException
116    */
117   public void replicateEntries(HLog.Entry[] entries)
118       throws IOException {
119     if (entries.length == 0) {
120       return;
121     }
122     // Very simple optimization where we batch sequences of rows going
123     // to the same table.
124     try {
125       long totalReplicated = 0;
126       // Map of table => list of Rows, we only want to flushCommits once per
127       // invocation of this method per table.
128       Map<byte[], List<Row>> rows = new TreeMap<byte[], List<Row>>(Bytes.BYTES_COMPARATOR);
129       for (HLog.Entry entry : entries) {
130         WALEdit edit = entry.getEdit();
131         byte[] table = entry.getKey().getTablename();
132         Put put = null;
133         Delete del = null;
134         KeyValue lastKV = null;
135         List<KeyValue> kvs = edit.getKeyValues();
136         for (KeyValue kv : kvs) {
137           if (lastKV == null || lastKV.getType() != kv.getType() || !lastKV.matchingRow(kv)) {
138             if (kv.isDelete()) {
139               del = new Delete(kv.getRow());
140               del.setClusterId(entry.getKey().getClusterId());
141               addToMultiMap(rows, table, del);
142             } else {
143               put = new Put(kv.getRow());
144               put.setClusterId(entry.getKey().getClusterId());
145               addToMultiMap(rows, table, put);
146             }
147           }
148           if (kv.isDelete()) {
149             del.addDeleteMarker(kv);
150           } else {
151             put.add(kv);
152           }
153           lastKV = kv;
154         }
155         totalReplicated++;
156       }
157       for (Entry<byte[], List<Row>> entry : rows.entrySet()) {
158         batch(entry.getKey(), entry.getValue());
159       }
160       this.metrics.setAgeOfLastAppliedOp(
161           entries[entries.length-1].getKey().getWriteTime());
162       this.metrics.applyBatch(entries.length);
163       this.totalReplicatedEdits.addAndGet(totalReplicated);
164     } catch (IOException ex) {
165       LOG.error("Unable to accept edit because:", ex);
166       throw ex;
167     }
168   }
169 
170   /**
171    * Simple helper to a map from key to (a list of) values
172    * TODO: Make a general utility method
173    * @param map
174    * @param key
175    * @param value
176    * @return
177    */
178   private <K, V> List<V> addToMultiMap(Map<K, List<V>> map, K key, V value) {
179     List<V> values = map.get(key);
180     if (values == null) {
181       values = new ArrayList<V>();
182       map.put(key, values);
183     }
184     values.add(value);
185     return values;
186   }
187 
188   /**
189    * stop the thread pool executor. It is called when the regionserver is stopped.
190    */
191   public void stopReplicationSinkServices() {
192     try {
193       this.sharedThreadPool.shutdown();
194       if (!this.sharedThreadPool.awaitTermination(60000, TimeUnit.MILLISECONDS)) {
195         this.sharedThreadPool.shutdownNow();
196       }
197     }  catch (InterruptedException e) {
198       LOG.warn("Interrupted while closing the table pool", e); // ignoring it as we are closing.
199       Thread.currentThread().interrupt();
200     }
201     try {
202       this.sharedHtableCon.close();
203     } catch (IOException e) {
204       LOG.warn("IOException while closing the connection", e); // ignoring as we are closing.
205     }
206   }
207 
208 
209   /**
210    * Do the changes and handle the pool
211    * @param tableName table to insert into
212    * @param rows list of actions
213    * @throws IOException
214    */
215   private void batch(byte[] tableName, List<Row> rows) throws IOException {
216     if (rows.isEmpty()) {
217       return;
218     }
219     HTableInterface table = null;
220     try {
221       table = new HTable(tableName, this.sharedHtableCon, this.sharedThreadPool);
222       table.batch(rows);
223     } catch (InterruptedException ix) {
224       throw new IOException(ix);
225     } finally {
226       if (table != null) {
227         table.close();
228       }
229     }
230   }
231 
232   /**
233    * Get a string representation of this sink's metrics
234    * @return string with the total replicated edits count and the date
235    * of the last edit that was applied
236    */
237   public String getStats() {
238     return this.totalReplicatedEdits.get() == 0 ? "" : "Sink: " +
239       "age in ms of last applied edit: " + this.metrics.refreshAgeOfLastAppliedOp() +
240       ", total replicated edits: " + this.totalReplicatedEdits;
241   }
242 }