View Javadoc

1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.replication.regionserver;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.HBaseConfiguration;
26  import org.apache.hadoop.hbase.HConstants;
27  import org.apache.hadoop.hbase.KeyValue;
28  import org.apache.hadoop.hbase.client.Delete;
29  import org.apache.hadoop.hbase.client.HConnection;
30  import org.apache.hadoop.hbase.client.HConnectionManager;
31  import org.apache.hadoop.hbase.client.HTable;
32  import org.apache.hadoop.hbase.client.HTableInterface;
33  import org.apache.hadoop.hbase.client.Put;
34  import org.apache.hadoop.hbase.client.Row;
35  import org.apache.hadoop.hbase.regionserver.wal.HLog;
36  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.apache.hadoop.hbase.util.Threads;
39  import org.apache.hadoop.hbase.Stoppable;
40  
41  import java.io.IOException;
42  import java.util.ArrayList;
43  import java.util.List;
44  import java.util.Map;
45  import java.util.TreeMap;
46  import java.util.concurrent.ExecutorService;
47  import java.util.concurrent.SynchronousQueue;
48  import java.util.concurrent.ThreadPoolExecutor;
49  import java.util.concurrent.TimeUnit;
50  
51  /**
52   * This class is responsible for replicating the edits coming
53   * from another cluster.
54   * <p/>
55   * This replication process is currently waiting for the edits to be applied
56   * before the method can return. This means that the replication of edits
57   * is synchronized (after reading from HLogs in ReplicationSource) and that a
58   * single region server cannot receive edits from two sources at the same time
59   * <p/>
60   * This class uses the native HBase client in order to replicate entries.
61   * <p/>
62   *
63   * TODO make this class more like ReplicationSource wrt log handling
64   */
65  public class ReplicationSink {
66  
67    private static final Log LOG = LogFactory.getLog(ReplicationSink.class);
68    // Name of the HDFS directory that contains the temporary rep logs
69    public static final String REPLICATION_LOG_DIR = ".replogs";
70    private final Configuration conf;
71    private final ExecutorService sharedThreadPool;
72    private final HConnection sharedHtableCon;
73    private final ReplicationSinkMetrics metrics;
74  
75    /**
76     * Create a sink for replication
77     *
78     * @param conf                conf object
79     * @param stopper             boolean to tell this thread to stop
80     * @throws IOException thrown when HDFS goes bad or bad file name
81     */
82    public ReplicationSink(Configuration conf, Stoppable stopper)
83        throws IOException {
84      this.conf = HBaseConfiguration.create(conf);
85      decorateConf();
86      this.sharedHtableCon = HConnectionManager.createConnection(this.conf);
87      this.sharedThreadPool = new ThreadPoolExecutor(1, 
88          conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE), 
89          conf.getLong("hbase.htable.threads.keepalivetime", 60), TimeUnit.SECONDS,
90          new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("hbase-repl"));
91      ((ThreadPoolExecutor)this.sharedThreadPool).allowCoreThreadTimeOut(true);
92      this.metrics = new ReplicationSinkMetrics();
93    }
94  
95    /**
96     * decorate the Configuration object to make replication more receptive to
97     * delays: lessen the timeout and numTries.
98     */
99    private void decorateConf() {
100     this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
101         this.conf.getInt("replication.sink.client.retries.number", 4));
102     this.conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
103         this.conf.getInt("replication.sink.client.ops.timeout", 10000));
104   }
105 
106   /**
107    * Replicate this array of entries directly into the local cluster
108    * using the native client.
109    *
110    * @param entries
111    * @throws IOException
112    */
113   public void replicateEntries(HLog.Entry[] entries)
114       throws IOException {
115     if (entries.length == 0) {
116       return;
117     }
118     // Very simple optimization where we batch sequences of rows going
119     // to the same table.
120     try {
121       long totalReplicated = 0;
122       // Map of table => list of Rows, we only want to flushCommits once per
123       // invocation of this method per table.
124       Map<byte[], List<Row>> rows = new TreeMap<byte[], List<Row>>(Bytes.BYTES_COMPARATOR);
125       for (HLog.Entry entry : entries) {
126         WALEdit edit = entry.getEdit();
127         byte[] table = entry.getKey().getTablename();
128         Put put = null;
129         Delete del = null;
130         KeyValue lastKV = null;
131         List<KeyValue> kvs = edit.getKeyValues();
132         for (KeyValue kv : kvs) {
133           if (lastKV == null || lastKV.getType() != kv.getType() || !lastKV.matchingRow(kv)) {
134             if (kv.isDelete()) {
135               del = new Delete(kv.getRow());
136               del.setClusterId(entry.getKey().getClusterId());
137               addToMultiMap(rows, table, del);
138             } else {
139               put = new Put(kv.getRow());
140               put.setClusterId(entry.getKey().getClusterId());
141               addToMultiMap(rows, table, put);
142             }
143           }
144           if (kv.isDelete()) {
145             del.addDeleteMarker(kv);
146           } else {
147             put.add(kv);
148           }
149           lastKV = kv;
150         }
151         totalReplicated++;
152       }
153       for(byte [] table : rows.keySet()) {
154         batch(table, rows.get(table));
155       }
156       this.metrics.setAgeOfLastAppliedOp(
157           entries[entries.length-1].getKey().getWriteTime());
158       this.metrics.appliedBatchesRate.inc(1);
159       LOG.info("Total replicated: " + totalReplicated);
160     } catch (IOException ex) {
161       LOG.error("Unable to accept edit because:", ex);
162       throw ex;
163     }
164   }
165 
166   /**
167    * Simple helper to a map from key to (a list of) values
168    * TODO: Make a general utility method
169    * @param map
170    * @param key
171    * @param value
172    * @return
173    */
174   private <K, V> List<V> addToMultiMap(Map<K, List<V>> map, K key, V value) {
175     List<V> values = map.get(key);
176     if (values == null) {
177       values = new ArrayList<V>();
178       map.put(key, values);
179     }
180     values.add(value);
181     return values;
182   }
183 
184   /**
185    * stop the thread pool executor. It is called when the regionserver is stopped.
186    */
187   public void stopReplicationSinkServices() {
188     try {
189       this.sharedThreadPool.shutdown();
190       if (!this.sharedThreadPool.awaitTermination(60000, TimeUnit.MILLISECONDS)) {
191         this.sharedThreadPool.shutdownNow();
192       }
193     } catch (InterruptedException e) {
194       LOG.warn("Interrupted while closing the table pool", e); // ignoring it as we are closing.
195       Thread.currentThread().interrupt();
196     }
197     try {
198       this.sharedHtableCon.close();
199     } catch (IOException e) {
200       LOG.warn("IOException while closing the connection", e); // ignoring as we are closing.
201     }
202   }  
203 
204   /**
205    * Do the changes and handle the pool
206    * @param tableName table to insert into
207    * @param rows list of actions
208    * @throws IOException
209    */
210   private void batch(byte[] tableName, List<Row> rows) throws IOException {
211     if (rows.isEmpty()) {
212       return;
213     }
214     HTableInterface table = null;
215     try {
216       table = new HTable(tableName, this.sharedHtableCon, this.sharedThreadPool);
217       table.batch(rows);
218       this.metrics.appliedOpsRate.inc(rows.size());
219     } catch (InterruptedException ix) {
220       throw new IOException(ix);
221     } finally {
222       if (table != null) {
223         table.close();
224       }
225     }
226   }
227 }