View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import java.io.DataInput;
22  import java.io.DataOutput;
23  import java.io.EOFException;
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.HashMap;
27  import java.util.Iterator;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.NavigableMap;
31  import java.util.TreeMap;
32  import java.util.UUID;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.classification.InterfaceAudience;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
41  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FamilyScope;
42  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.ScopeType;
43  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALKey;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.io.WritableComparable;
46  import org.apache.hadoop.io.WritableUtils;
47  
48  import com.google.protobuf.ByteString;
49  import com.google.protobuf.ZeroCopyLiteralByteString;
50  
51  /**
52   * A Key for an entry in the change log.
53   *
54   * The log intermingles edits to many tables and rows, so each log entry
55   * identifies the appropriate table and row.  Within a table and row, they're
56   * also sorted.
57   *
58   * <p>Some Transactional edits (START, COMMIT, ABORT) will not have an
59   * associated row.
60   */
61  // TODO: Key and WALEdit are never used separately, or in one-to-many relation, for practical
62  //       purposes. They need to be merged into HLogEntry.
63  @InterfaceAudience.Private
64  public class HLogKey implements WritableComparable<HLogKey> {
65    public static final Log LOG = LogFactory.getLog(HLogKey.class);
66  
67    // should be < 0 (@see #readFields(DataInput))
68    // version 2 supports HLog compression
69    enum Version {
70      UNVERSIONED(0),
71      // Initial number we put on HLogKey when we introduced versioning.
72      INITIAL(-1),
73      // Version -2 introduced a dictionary compression facility.  Only this
74      // dictionary-based compression is available in version -2.
75      COMPRESSED(-2);
76  
77      final int code;
78      static final Version[] byCode;
79      static {
80        byCode = Version.values();
81        for (int i = 0; i < byCode.length; i++) {
82          if (byCode[i].code != -1 * i) {
83            throw new AssertionError("Values in this enum should be descending by one");
84          }
85        }
86      }
87  
88      Version(int code) {
89        this.code = code;
90      }
91  
92      boolean atLeast(Version other) {
93        return code <= other.code;
94      }
95  
96      static Version fromCode(int code) {
97        return byCode[code * -1];
98      }
99    }
100 
101   /*
102    * This is used for reading the log entries created by the previous releases
103    * (0.94.11) which write the clusters information to the scopes of WALEdit.
104    */
105   private static final String PREFIX_CLUSTER_KEY = ".";
106 
107 
108   private static final Version VERSION = Version.COMPRESSED;
109 
110   //  The encoded region name.
111   private byte [] encodedRegionName;
112   private TableName tablename;
113   private long logSeqNum;
114   // Time at which this edit was written.
115   private long writeTime;
116 
117   // The first element in the list is the cluster id on which the change has originated
118   private List<UUID> clusterIds;
119 
120   private NavigableMap<byte[], Integer> scopes;
121 
122   private CompressionContext compressionContext;
123 
124   public HLogKey() {
125     init(null, null, 0L, HConstants.LATEST_TIMESTAMP,
126         new ArrayList<UUID>());
127   }
128 
129   public HLogKey(final byte[] encodedRegionName, final TableName tablename, long logSeqNum,
130       final long now, UUID clusterId) {
131     List<UUID> clusterIds = new ArrayList<UUID>();
132     clusterIds.add(clusterId);
133     init(encodedRegionName, tablename, logSeqNum, now, clusterIds);
134   }
135 
136   /**
137    * Create the log key for writing to somewhere.
138    * We maintain the tablename mainly for debugging purposes.
139    * A regionName is always a sub-table object.
140    *
141    * @param encodedRegionName Encoded name of the region as returned by
142    * <code>HRegionInfo#getEncodedNameAsBytes()</code>.
143    * @param tablename   - name of table
144    * @param logSeqNum   - log sequence number
145    * @param now Time at which this edit was written.
146    * @param clusterIds the clusters that have consumed the change(used in Replication)
147    */
148   public HLogKey(final byte [] encodedRegionName, final TableName tablename,
149       long logSeqNum, final long now, List<UUID> clusterIds){
150     init(encodedRegionName, tablename, logSeqNum, now, clusterIds);
151   }
152 
153   protected void init(final byte [] encodedRegionName, final TableName tablename,
154       long logSeqNum, final long now, List<UUID> clusterIds) {
155     this.logSeqNum = logSeqNum;
156     this.writeTime = now;
157     this.clusterIds = clusterIds;
158     this.encodedRegionName = encodedRegionName;
159     this.tablename = tablename;
160   }
161 
162   /**
163    * @param compressionContext Compression context to use
164    */
165   public void setCompressionContext(CompressionContext compressionContext) {
166     this.compressionContext = compressionContext;
167   }
168 
169   /** @return encoded region name */
170   public byte [] getEncodedRegionName() {
171     return encodedRegionName;
172   }
173 
174   /** @return table name */
175   public TableName getTablename() {
176     return tablename;
177   }
178 
179   /** @return log sequence number */
180   public long getLogSeqNum() {
181     return this.logSeqNum;
182   }
183 
184   /**
185    * @return the write time
186    */
187   public long getWriteTime() {
188     return this.writeTime;
189   }
190 
191   public NavigableMap<byte[], Integer> getScopes() {
192     return scopes;
193   }
194 
195   public void setScopes(NavigableMap<byte[], Integer> scopes) {
196     this.scopes = scopes;
197   }
198 
199   public void readOlderScopes(NavigableMap<byte[], Integer> scopes) {
200     if (scopes != null) {
201       Iterator<Map.Entry<byte[], Integer>> iterator = scopes.entrySet()
202           .iterator();
203       while (iterator.hasNext()) {
204         Map.Entry<byte[], Integer> scope = iterator.next();
205         String key = Bytes.toString(scope.getKey());
206         if (key.startsWith(PREFIX_CLUSTER_KEY)) {
207           addClusterId(UUID.fromString(key.substring(PREFIX_CLUSTER_KEY
208               .length())));
209           iterator.remove();
210         }
211       }
212       if (scopes.size() > 0) {
213         this.scopes = scopes;
214       }
215     }
216   }
217 
218   /**
219    * Marks that the cluster with the given clusterId has consumed the change
220    */
221   public void addClusterId(UUID clusterId) {
222     if (!clusterIds.contains(clusterId)) {
223       clusterIds.add(clusterId);
224     }
225   }
226 
227   /**
228    * @return the set of cluster Ids that have consumed the change
229    */
230   public List<UUID> getClusterIds() {
231     return clusterIds;
232   }
233 
234   /**
235    * @return the cluster id on which the change has originated. It there is no such cluster, it
236    *         returns DEFAULT_CLUSTER_ID (cases where replication is not enabled)
237    */
238   public UUID getOriginatingClusterId(){
239     return clusterIds.isEmpty() ? HConstants.DEFAULT_CLUSTER_ID : clusterIds.get(0);
240   }
241 
242   @Override
243   public String toString() {
244     return tablename + "/" + Bytes.toString(encodedRegionName) + "/" +
245       logSeqNum;
246   }
247 
248   /**
249    * Produces a string map for this key. Useful for programmatic use and
250    * manipulation of the data stored in an HLogKey, for example, printing
251    * as JSON.
252    *
253    * @return a Map containing data from this key
254    */
255   public Map<String, Object> toStringMap() {
256     Map<String, Object> stringMap = new HashMap<String, Object>();
257     stringMap.put("table", tablename);
258     stringMap.put("region", Bytes.toStringBinary(encodedRegionName));
259     stringMap.put("sequence", logSeqNum);
260     return stringMap;
261   }
262 
263   @Override
264   public boolean equals(Object obj) {
265     if (this == obj) {
266       return true;
267     }
268     if (obj == null || getClass() != obj.getClass()) {
269       return false;
270     }
271     return compareTo((HLogKey)obj) == 0;
272   }
273 
274   @Override
275   public int hashCode() {
276     int result = Bytes.hashCode(this.encodedRegionName);
277     result ^= this.logSeqNum;
278     result ^= this.writeTime;
279     return result;
280   }
281 
282   public int compareTo(HLogKey o) {
283     int result = Bytes.compareTo(this.encodedRegionName, o.encodedRegionName);
284     if (result == 0) {
285       if (this.logSeqNum < o.logSeqNum) {
286         result = -1;
287       } else if (this.logSeqNum  > o.logSeqNum ) {
288         result = 1;
289       }
290       if (result == 0) {
291         if (this.writeTime < o.writeTime) {
292           result = -1;
293         } else if (this.writeTime > o.writeTime) {
294           return 1;
295         }
296       }
297     }
298     // why isn't cluster id accounted for?
299     return result;
300   }
301 
302   /**
303    * Drop this instance's tablename byte array and instead
304    * hold a reference to the provided tablename. This is not
305    * meant to be a general purpose setter - it's only used
306    * to collapse references to conserve memory.
307    */
308   void internTableName(TableName tablename) {
309     // We should not use this as a setter - only to swap
310     // in a new reference to the same table name.
311     assert tablename.equals(this.tablename);
312     this.tablename = tablename;
313   }
314 
315   /**
316    * Drop this instance's region name byte array and instead
317    * hold a reference to the provided region name. This is not
318    * meant to be a general purpose setter - it's only used
319    * to collapse references to conserve memory.
320    */
321   void internEncodedRegionName(byte []encodedRegionName) {
322     // We should not use this as a setter - only to swap
323     // in a new reference to the same table name.
324     assert Bytes.equals(this.encodedRegionName, encodedRegionName);
325     this.encodedRegionName = encodedRegionName;
326   }
327 
328   @Override
329   @Deprecated
330   public void write(DataOutput out) throws IOException {
331     LOG.warn("HLogKey is being serialized to writable - only expected in test code");
332     WritableUtils.writeVInt(out, VERSION.code);
333     if (compressionContext == null) {
334       Bytes.writeByteArray(out, this.encodedRegionName);
335       Bytes.writeByteArray(out, this.tablename.getName());
336     } else {
337       Compressor.writeCompressed(this.encodedRegionName, 0,
338           this.encodedRegionName.length, out,
339           compressionContext.regionDict);
340       Compressor.writeCompressed(this.tablename.getName(), 0, this.tablename.getName().length, out,
341           compressionContext.tableDict);
342     }
343     out.writeLong(this.logSeqNum);
344     out.writeLong(this.writeTime);
345     // Don't need to write the clusters information as we are using protobufs from 0.95
346     // Writing only the first clusterId for testing the legacy read
347     Iterator<UUID> iterator = clusterIds.iterator();
348     if(iterator.hasNext()){
349       out.writeBoolean(true);
350       UUID clusterId = iterator.next();
351       out.writeLong(clusterId.getMostSignificantBits());
352       out.writeLong(clusterId.getLeastSignificantBits());
353     } else {
354       out.writeBoolean(false);
355     }
356   }
357 
358   @Override
359   public void readFields(DataInput in) throws IOException {
360     Version version = Version.UNVERSIONED;
361     // HLogKey was not versioned in the beginning.
362     // In order to introduce it now, we make use of the fact
363     // that encodedRegionName was written with Bytes.writeByteArray,
364     // which encodes the array length as a vint which is >= 0.
365     // Hence if the vint is >= 0 we have an old version and the vint
366     // encodes the length of encodedRegionName.
367     // If < 0 we just read the version and the next vint is the length.
368     // @see Bytes#readByteArray(DataInput)
369     this.scopes = null; // writable HLogKey does not contain scopes
370     int len = WritableUtils.readVInt(in);
371     byte[] tablenameBytes = null;
372     if (len < 0) {
373       // what we just read was the version
374       version = Version.fromCode(len);
375       // We only compress V2 of HLogkey.
376       // If compression is on, the length is handled by the dictionary
377       if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) {
378         len = WritableUtils.readVInt(in);
379       }
380     }
381     if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) {
382       this.encodedRegionName = new byte[len];
383       in.readFully(this.encodedRegionName);
384       tablenameBytes = Bytes.readByteArray(in);
385     } else {
386       this.encodedRegionName = Compressor.readCompressed(in, compressionContext.regionDict);
387       tablenameBytes = Compressor.readCompressed(in, compressionContext.tableDict);
388     }
389 
390     this.logSeqNum = in.readLong();
391     this.writeTime = in.readLong();
392 
393     this.clusterIds.clear();
394     if (version.atLeast(Version.INITIAL)) {
395       if (in.readBoolean()) {
396         // read the older log
397         // Definitely is the originating cluster
398         clusterIds.add(new UUID(in.readLong(), in.readLong()));
399       }
400     } else {
401       try {
402         // dummy read (former byte cluster id)
403         in.readByte();
404       } catch(EOFException e) {
405         // Means it's a very old key, just continue
406       }
407     }
408     try {
409       this.tablename = TableName.valueOf(tablenameBytes);
410     } catch (IllegalArgumentException iae) {
411       if (Bytes.toString(tablenameBytes).equals(TableName.OLD_META_STR)) {
412         // It is a pre-namespace meta table edit, continue with new format.
413         LOG.info("Got an old .META. edit, continuing with new format ");
414         this.tablename = TableName.META_TABLE_NAME;
415         this.encodedRegionName = HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes();
416       } else if (Bytes.toString(tablenameBytes).equals(TableName.OLD_ROOT_STR)) {
417         this.tablename = TableName.OLD_ROOT_TABLE_NAME;
418          throw iae;
419       } else throw iae;
420     }
421     // Do not need to read the clusters information as we are using protobufs from 0.95
422   }
423 
424   public WALKey.Builder getBuilder(
425       WALCellCodec.ByteStringCompressor compressor) throws IOException {
426     WALKey.Builder builder = WALKey.newBuilder();
427     if (compressionContext == null) {
428       builder.setEncodedRegionName(ZeroCopyLiteralByteString.wrap(this.encodedRegionName));
429       builder.setTableName(ZeroCopyLiteralByteString.wrap(this.tablename.getName()));
430     } else {
431       builder.setEncodedRegionName(
432           compressor.compress(this.encodedRegionName, compressionContext.regionDict));
433       builder.setTableName(compressor.compress(this.tablename.getName(),
434           compressionContext.tableDict));
435     }
436     builder.setLogSequenceNumber(this.logSeqNum);
437     builder.setWriteTime(writeTime);
438     HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder();
439     for (UUID clusterId : clusterIds) {
440       uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits());
441       uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits());
442       builder.addClusterIds(uuidBuilder.build());
443     }
444     if (scopes != null) {
445       for (Map.Entry<byte[], Integer> e : scopes.entrySet()) {
446         ByteString family = (compressionContext == null) ? ZeroCopyLiteralByteString.wrap(e.getKey())
447             : compressor.compress(e.getKey(), compressionContext.familyDict);
448         builder.addScopes(FamilyScope.newBuilder()
449             .setFamily(family).setScopeType(ScopeType.valueOf(e.getValue())));
450       }
451     }
452     return builder;
453   }
454 
455   public void readFieldsFromPb(
456       WALKey walKey, WALCellCodec.ByteStringUncompressor uncompressor) throws IOException {
457     if (this.compressionContext != null) {
458       this.encodedRegionName = uncompressor.uncompress(
459           walKey.getEncodedRegionName(), compressionContext.regionDict);
460       byte[] tablenameBytes = uncompressor.uncompress(
461           walKey.getTableName(), compressionContext.tableDict);
462       this.tablename = TableName.valueOf(tablenameBytes);
463     } else {
464       this.encodedRegionName = walKey.getEncodedRegionName().toByteArray();
465       this.tablename = TableName.valueOf(walKey.getTableName().toByteArray());
466     }
467     clusterIds.clear();
468     if (walKey.hasClusterId()) {
469       //When we are reading the older log (0.95.1 release)
470       //This is definitely the originating cluster
471       clusterIds.add(new UUID(walKey.getClusterId().getMostSigBits(), walKey.getClusterId()
472           .getLeastSigBits()));
473     }
474     for (HBaseProtos.UUID clusterId : walKey.getClusterIdsList()) {
475       clusterIds.add(new UUID(clusterId.getMostSigBits(), clusterId.getLeastSigBits()));
476     }
477     this.scopes = null;
478     if (walKey.getScopesCount() > 0) {
479       this.scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
480       for (FamilyScope scope : walKey.getScopesList()) {
481         byte[] family = (compressionContext == null) ? scope.getFamily().toByteArray() :
482           uncompressor.uncompress(scope.getFamily(), compressionContext.familyDict);
483         this.scopes.put(family, scope.getScopeType().getNumber());
484       }
485     }
486     this.logSeqNum = walKey.getLogSequenceNumber();
487     this.writeTime = walKey.getWriteTime();
488   }
489 }