1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.protobuf;
21
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Iterator;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.NavigableMap;
29 import java.util.TreeMap;
30 import java.util.UUID;
31
32 import org.apache.hadoop.classification.InterfaceAudience;
33 import org.apache.hadoop.hbase.Cell;
34 import org.apache.hadoop.hbase.CellScanner;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.io.SizedCellScanner;
37 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
38 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
39 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
40 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
41 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
42 import org.apache.hadoop.hbase.regionserver.wal.HLog;
43 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
44 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.Pair;
47
48 import com.google.protobuf.ByteString;
49 import com.google.protobuf.ServiceException;
50
51 @InterfaceAudience.Private
52 public class ReplicationProtbufUtil {
53
54
55
56
57
58
59 public static HLog.Entry[]
60 toHLogEntries(final List<AdminProtos.WALEntry> protoList) throws IOException {
61 List<HLog.Entry> entries = new ArrayList<HLog.Entry>();
62 for (AdminProtos.WALEntry entry: protoList) {
63 WALProtos.WALKey walKey = entry.getKey();
64 HLogKey key = new HLogKey(walKey);
65 WALEdit edit = new WALEdit();
66 for (ByteString keyValue: entry.getKeyValueBytesList()) {
67 edit.add(new KeyValue(keyValue.toByteArray()));
68 }
69 if (walKey.getScopesCount() > 0) {
70 TreeMap<byte[], Integer> scopes =
71 new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
72 for (WALProtos.FamilyScope scope: walKey.getScopesList()) {
73 scopes.put(scope.getFamily().toByteArray(),
74 Integer.valueOf(scope.getScopeType().ordinal()));
75 }
76 key.setScopes(scopes);
77 }
78 entries.add(new HLog.Entry(key, edit));
79 }
80 return entries.toArray(new HLog.Entry[entries.size()]);
81 }
82
83
84
85
86
87
88
89
90 public static void replicateWALEntry(final AdminService.BlockingInterface admin,
91 final HLog.Entry[] entries) throws IOException {
92 Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner> p =
93 buildReplicateWALEntryRequest(entries);
94 try {
95 PayloadCarryingRpcController controller = new PayloadCarryingRpcController(p.getSecond());
96 admin.replicateWALEntry(controller, p.getFirst());
97 } catch (ServiceException se) {
98 throw ProtobufUtil.getRemoteException(se);
99 }
100 }
101
102
103
104
105
106
107
108
109 public static Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>
110 buildReplicateWALEntryRequest(final HLog.Entry[] entries) {
111
112 List<List<? extends Cell>> allkvs = new ArrayList<List<? extends Cell>>(entries.length);
113 int size = 0;
114 WALProtos.FamilyScope.Builder scopeBuilder = WALProtos.FamilyScope.newBuilder();
115 AdminProtos.WALEntry.Builder entryBuilder = AdminProtos.WALEntry.newBuilder();
116 AdminProtos.ReplicateWALEntryRequest.Builder builder =
117 AdminProtos.ReplicateWALEntryRequest.newBuilder();
118 for (HLog.Entry entry: entries) {
119 entryBuilder.clear();
120 WALProtos.WALKey.Builder keyBuilder = entryBuilder.getKeyBuilder();
121 HLogKey key = entry.getKey();
122 keyBuilder.setEncodedRegionName(
123 ByteString.copyFrom(key.getEncodedRegionName()));
124 keyBuilder.setTableName(ByteString.copyFrom(key.getTablename().getName()));
125 keyBuilder.setLogSequenceNumber(key.getLogSeqNum());
126 keyBuilder.setWriteTime(key.getWriteTime());
127 UUID clusterId = key.getClusterId();
128 if (clusterId != null) {
129 HBaseProtos.UUID.Builder uuidBuilder = keyBuilder.getClusterIdBuilder();
130 uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits());
131 uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits());
132 }
133 WALEdit edit = entry.getEdit();
134 NavigableMap<byte[], Integer> scopes = key.getScopes();
135 if (scopes != null && !scopes.isEmpty()) {
136 for (Map.Entry<byte[], Integer> scope: scopes.entrySet()) {
137 scopeBuilder.setFamily(ByteString.copyFrom(scope.getKey()));
138 WALProtos.ScopeType scopeType =
139 WALProtos.ScopeType.valueOf(scope.getValue().intValue());
140 scopeBuilder.setScopeType(scopeType);
141 keyBuilder.addScopes(scopeBuilder.build());
142 }
143 }
144 List<KeyValue> kvs = edit.getKeyValues();
145
146 for (KeyValue kv: kvs) {
147 size += kv.getLength();
148 }
149
150 allkvs.add(kvs);
151
152 entryBuilder.setAssociatedCellCount(kvs.size());
153 builder.addEntry(entryBuilder.build());
154 }
155 return new Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>(builder.build(),
156 getCellScanner(allkvs, size));
157 }
158
159
160
161
162
163 static CellScanner getCellScanner(final List<List<? extends Cell>> cells, final int size) {
164 return new SizedCellScanner() {
165 private final Iterator<List<? extends Cell>> entries = cells.iterator();
166 private Iterator<? extends Cell> currentIterator = null;
167 private Cell currentCell;
168
169 @Override
170 public Cell current() {
171 return this.currentCell;
172 }
173
174 @Override
175 public boolean advance() {
176 if (this.currentIterator == null) {
177 if (!this.entries.hasNext()) return false;
178 this.currentIterator = this.entries.next().iterator();
179 }
180 if (this.currentIterator.hasNext()) {
181 this.currentCell = this.currentIterator.next();
182 return true;
183 }
184 this.currentCell = null;
185 this.currentIterator = null;
186 return advance();
187 }
188
189 @Override
190 public long heapSize() {
191 return size;
192 }
193 };
194 }
195 }