View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import java.io.DataInput;
22  import java.io.DataOutput;
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.NavigableMap;
27  import java.util.TreeMap;
28  
29  import org.apache.hadoop.classification.InterfaceAudience;
30  import org.apache.hadoop.hbase.io.HeapSize;
31  import org.apache.hadoop.hbase.KeyValue;
32  import org.apache.hadoop.hbase.util.Bytes;
33  import org.apache.hadoop.hbase.util.ClassSize;
34  import org.apache.hadoop.io.Writable;
35  
36  /**
37   * WALEdit: Used in HBase's transaction log (WAL) to represent
38   * the collection of edits (KeyValue objects) corresponding to a
39   * single transaction. The class implements "Writable" interface
40   * for serializing/deserializing a set of KeyValue items.
41   *
42   * Previously, if a transaction contains 3 edits to c1, c2, c3 for a row R,
43   * the HLog would have three log entries as follows:
44   *
45   *    <logseq1-for-edit1>:<KeyValue-for-edit-c1>
46   *    <logseq2-for-edit2>:<KeyValue-for-edit-c2>
47   *    <logseq3-for-edit3>:<KeyValue-for-edit-c3>
48   *
49   * This presents problems because row level atomicity of transactions
50   * was not guaranteed. If we crash after few of the above appends make
51   * it, then recovery will restore a partial transaction.
52   *
53   * In the new world, all the edits for a given transaction are written
54   * out as a single record, for example:
55   *
56   *   <logseq#-for-entire-txn>:<WALEdit-for-entire-txn>
57   *
58   * where, the WALEdit is serialized as:
59   *   <-1, # of edits, <KeyValue>, <KeyValue>, ... >
60   * For example:
61   *   <-1, 3, <Keyvalue-for-edit-c1>, <KeyValue-for-edit-c2>, <KeyValue-for-edit-c3>>
62   *
63   * The -1 marker is just a special way of being backward compatible with
64   * an old HLog which would have contained a single <KeyValue>.
65   *
66   * The deserializer for WALEdit backward compatibly detects if the record
67   * is an old style KeyValue or the new style WALEdit.
68   *
69   */
70  @InterfaceAudience.Private
71  public class WALEdit implements Writable, HeapSize {
72  
73    private final int VERSION_2 = -1;
74  
75    private final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
76    private NavigableMap<byte[], Integer> scopes;
77  
78    private CompressionContext compressionContext;
79  
80    public WALEdit() {
81    }
82  
83    public void setCompressionContext(final CompressionContext compressionContext) {
84      this.compressionContext = compressionContext;
85    }
86  
87    public void add(KeyValue kv) {
88      this.kvs.add(kv);
89    }
90  
91    public boolean isEmpty() {
92      return kvs.isEmpty();
93    }
94  
95    public int size() {
96      return kvs.size();
97    }
98  
99    public List<KeyValue> getKeyValues() {
100     return kvs;
101   }
102 
103   public NavigableMap<byte[], Integer> getScopes() {
104     return scopes;
105   }
106 
107 
108   public void setScopes (NavigableMap<byte[], Integer> scopes) {
109     // We currently process the map outside of WALEdit,
110     // TODO revisit when replication is part of core
111     this.scopes = scopes;
112   }
113 
114   public void readFields(DataInput in) throws IOException {
115     kvs.clear();
116     if (scopes != null) {
117       scopes.clear();
118     }
119     int versionOrLength = in.readInt();
120     // TODO: Change version when we protobuf.  Also, change way we serialize KV!  Pb it too.
121     if (versionOrLength == VERSION_2) {
122       // this is new style HLog entry containing multiple KeyValues.
123       int numEdits = in.readInt();
124       for (int idx = 0; idx < numEdits; idx++) {
125         if (compressionContext != null) {
126           this.add(KeyValueCompression.readKV(in, compressionContext));
127         } else {
128           this.add(KeyValue.create(in));
129     	  }
130       }
131       int numFamilies = in.readInt();
132       if (numFamilies > 0) {
133         if (scopes == null) {
134           scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
135         }
136         for (int i = 0; i < numFamilies; i++) {
137           byte[] fam = Bytes.readByteArray(in);
138           int scope = in.readInt();
139           scopes.put(fam, scope);
140         }
141       }
142     } else {
143       // this is an old style HLog entry. The int that we just
144       // read is actually the length of a single KeyValue
145       this.add(KeyValue.create(versionOrLength, in));
146     }
147 
148   }
149 
150   public void write(DataOutput out) throws IOException {
151     out.writeInt(VERSION_2);
152     out.writeInt(kvs.size());
153     // We interleave the two lists for code simplicity
154     for (KeyValue kv : kvs) {
155       if (compressionContext != null) {
156         KeyValueCompression.writeKV(out, kv, compressionContext);
157       } else{
158         KeyValue.write(kv, out);
159       }
160     }
161     if (scopes == null) {
162       out.writeInt(0);
163     } else {
164       out.writeInt(scopes.size());
165       for (byte[] key : scopes.keySet()) {
166         Bytes.writeByteArray(out, key);
167         out.writeInt(scopes.get(key));
168       }
169     }
170   }
171 
172   public long heapSize() {
173     long ret = 0;
174     for (KeyValue kv : kvs) {
175       ret += kv.heapSize();
176     }
177     if (scopes != null) {
178       ret += ClassSize.TREEMAP;
179       ret += ClassSize.align(scopes.size() * ClassSize.MAP_ENTRY);
180       // TODO this isn't quite right, need help here
181     }
182     return ret;
183   }
184 
185   public String toString() {
186     StringBuilder sb = new StringBuilder();
187 
188     sb.append("[#edits: " + kvs.size() + " = <");
189     for (KeyValue kv : kvs) {
190       sb.append(kv.toString());
191       sb.append("; ");
192     }
193     if (scopes != null) {
194       sb.append(" scopes: " + scopes.toString());
195     }
196     sb.append(">]");
197     return sb.toString();
198   }
199 
200 }