View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.protobuf;
21  
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Iterator;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.NavigableMap;
29  import java.util.TreeMap;
30  import java.util.UUID;
31  
32  import org.apache.hadoop.classification.InterfaceAudience;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.CellScanner;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.io.SizedCellScanner;
37  import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
38  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
39  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
40  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
41  import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
42  import org.apache.hadoop.hbase.regionserver.wal.HLog;
43  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
44  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.Pair;
47  
48  import com.google.protobuf.ByteString;
49  import com.google.protobuf.ServiceException;
50  
51  @InterfaceAudience.Private
52  public class ReplicationProtbufUtil {
53    /**
54     * Get the HLog entries from a list of protocol buffer WALEntry
55     *
56     * @param protoList the list of protocol buffer WALEntry
57     * @return an array of HLog entries
58     */
59    public static HLog.Entry[]
60        toHLogEntries(final List<AdminProtos.WALEntry> protoList) throws IOException {
61      List<HLog.Entry> entries = new ArrayList<HLog.Entry>();
62      for (AdminProtos.WALEntry entry: protoList) {
63        WALProtos.WALKey walKey = entry.getKey();
64        HLogKey key = new HLogKey(walKey);
65        WALEdit edit = new WALEdit();
66        for (ByteString keyValue: entry.getKeyValueBytesList()) {
67          edit.add(new KeyValue(keyValue.toByteArray()));
68        }
69        if (walKey.getScopesCount() > 0) {
70          TreeMap<byte[], Integer> scopes =
71            new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
72          for (WALProtos.FamilyScope scope: walKey.getScopesList()) {
73            scopes.put(scope.getFamily().toByteArray(),
74              Integer.valueOf(scope.getScopeType().ordinal()));
75          }
76          key.setScopes(scopes);
77        }
78        entries.add(new HLog.Entry(key, edit));
79      }
80      return entries.toArray(new HLog.Entry[entries.size()]);
81    }
82  
83    /**
84     * A helper to replicate a list of HLog entries using admin protocol.
85     *
86     * @param admin
87     * @param entries
88     * @throws java.io.IOException
89     */
90    public static void replicateWALEntry(final AdminService.BlockingInterface admin,
91        final HLog.Entry[] entries) throws IOException {
92      Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner> p =
93        buildReplicateWALEntryRequest(entries);
94      try {
95        PayloadCarryingRpcController controller = new PayloadCarryingRpcController(p.getSecond());
96        admin.replicateWALEntry(controller, p.getFirst());
97      } catch (ServiceException se) {
98        throw ProtobufUtil.getRemoteException(se);
99      }
100   }
101 
102   /**
103    * Create a new ReplicateWALEntryRequest from a list of HLog entries
104    *
105    * @param entries the HLog entries to be replicated
106    * @return a pair of ReplicateWALEntryRequest and a CellScanner over all the WALEdit values
107    * found.
108    */
109   public static Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>
110       buildReplicateWALEntryRequest(final HLog.Entry[] entries) {
111     // Accumulate all the KVs seen in here.
112     List<List<? extends Cell>> allkvs = new ArrayList<List<? extends Cell>>(entries.length);
113     int size = 0;
114     WALProtos.FamilyScope.Builder scopeBuilder = WALProtos.FamilyScope.newBuilder();
115     AdminProtos.WALEntry.Builder entryBuilder = AdminProtos.WALEntry.newBuilder();
116     AdminProtos.ReplicateWALEntryRequest.Builder builder =
117       AdminProtos.ReplicateWALEntryRequest.newBuilder();
118     for (HLog.Entry entry: entries) {
119       entryBuilder.clear();
120       WALProtos.WALKey.Builder keyBuilder = entryBuilder.getKeyBuilder();
121       HLogKey key = entry.getKey();
122       keyBuilder.setEncodedRegionName(
123         ByteString.copyFrom(key.getEncodedRegionName()));
124       keyBuilder.setTableName(ByteString.copyFrom(key.getTablename().getName()));
125       keyBuilder.setLogSequenceNumber(key.getLogSeqNum());
126       keyBuilder.setWriteTime(key.getWriteTime());
127       UUID clusterId = key.getClusterId();
128       if (clusterId != null) {
129         HBaseProtos.UUID.Builder uuidBuilder = keyBuilder.getClusterIdBuilder();
130         uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits());
131         uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits());
132       }
133       WALEdit edit = entry.getEdit();
134       NavigableMap<byte[], Integer> scopes = key.getScopes();
135       if (scopes != null && !scopes.isEmpty()) {
136         for (Map.Entry<byte[], Integer> scope: scopes.entrySet()) {
137           scopeBuilder.setFamily(ByteString.copyFrom(scope.getKey()));
138           WALProtos.ScopeType scopeType =
139               WALProtos.ScopeType.valueOf(scope.getValue().intValue());
140           scopeBuilder.setScopeType(scopeType);
141           keyBuilder.addScopes(scopeBuilder.build());
142         }
143       }
144       List<KeyValue> kvs = edit.getKeyValues();
145       // Add up the size.  It is used later serializing out the kvs.
146       for (KeyValue kv: kvs) {
147         size += kv.getLength();
148       }
149       // Collect up the kvs
150       allkvs.add(kvs);
151       // Write out how many kvs associated with this entry.
152       entryBuilder.setAssociatedCellCount(kvs.size());
153       builder.addEntry(entryBuilder.build());
154     }
155     return new Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>(builder.build(),
156       getCellScanner(allkvs, size));
157   }
158 
159   /**
160    * @param cells
161    * @return <code>cells</code> packaged as a CellScanner
162    */
163   static CellScanner getCellScanner(final List<List<? extends Cell>> cells, final int size) {
164     return new SizedCellScanner() {
165       private final Iterator<List<? extends Cell>> entries = cells.iterator();
166       private Iterator<? extends Cell> currentIterator = null;
167       private Cell currentCell;
168 
169       @Override
170       public Cell current() {
171         return this.currentCell;
172       }
173 
174       @Override
175       public boolean advance() {
176         if (this.currentIterator == null) {
177           if (!this.entries.hasNext()) return false;
178           this.currentIterator = this.entries.next().iterator();
179         }
180         if (this.currentIterator.hasNext()) {
181           this.currentCell = this.currentIterator.next();
182           return true;
183         }
184         this.currentCell = null;
185         this.currentIterator = null;
186         return advance();
187       }
188 
189       @Override
190       public long heapSize() {
191         return size;
192       }
193     };
194   }
195 }