View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import java.util.HashMap;
22  
23  import org.apache.hadoop.classification.InterfaceAudience;
24  import org.apache.hadoop.hbase.util.Bytes;
25  
26  import com.google.common.base.Preconditions;
27  
28  /**
29   * WALDictionary using an LRU eviction algorithm. Uses a linked list running
30   * through a hashtable.  Currently has max of 2^15 entries.  Will start
31   * evicting if exceeds this number  The maximum memory we expect this dictionary
32   * to take in the worst case is about:
33   * <code>(2 ^ 15) * 5 (Regionname, Row key, CF, Column qual, table) * 100 bytes (these are some big names) = ~16MB</code>.
34   * If you want to get silly, even at 1kb entries, it maxes out at 160 megabytes.
35   */
36  @InterfaceAudience.Private
37  public class LRUDictionary implements Dictionary {
38    private final BidirectionalLRUMap backingStore = new BidirectionalLRUMap();
39  
40    @Override
41    public byte[] getEntry(short idx) {
42      return backingStore.get(idx);
43    }
44  
45    @Override
46    public short findEntry(byte[] data, int offset, int length) {
47      short ret = backingStore.findIdx(data, offset, length);
48      if (ret == NOT_IN_DICTIONARY) {
49        addEntry(data, offset, length);
50      }
51      return ret;
52    }
53  
54    @Override
55    public short addEntry(byte[] data, int offset, int length) {
56      if (length <= 0) return NOT_IN_DICTIONARY;
57      return backingStore.put(data, offset, length);
58    }
59  
60    @Override
61    public void clear() {
62      backingStore.clear();
63    }
64  
65    /*
66     * Internal class used to implement LRU eviction and dual lookup (by key and
67     * value).
68     * 
69     * This is not thread safe. Don't use in multi-threaded applications.
70     */
71    static class BidirectionalLRUMap {
72      static final int MAX_SIZE = Short.MAX_VALUE;
73      private int currSize = 0;
74  
75      // Head and tail of the LRU list.
76      private Node head;
77      private Node tail;
78  
79      private HashMap<Node, Short> nodeToIndex = new HashMap<Node, Short>();
80      private Node[] indexToNode = new Node[MAX_SIZE];
81  
82      public BidirectionalLRUMap() {
83        for (int i = 0; i < MAX_SIZE; i++) {
84          indexToNode[i] = new Node();
85        }
86      }
87  
88      private short put(byte[] array, int offset, int length) {
89        // We copy the bytes we want, otherwise we might be holding references to
90        // massive arrays in our dictionary (or those arrays might change)
91        byte[] stored = new byte[length];
92        Bytes.putBytes(stored, 0, array, offset, length);
93  
94        if (currSize < MAX_SIZE) {
95          // There is space to add without evicting.
96          indexToNode[currSize].setContents(stored, 0, stored.length);
97          setHead(indexToNode[currSize]);
98          short ret = (short) currSize++;
99          nodeToIndex.put(indexToNode[ret], ret);
100         return ret;
101       } else {
102         short s = nodeToIndex.remove(tail);
103         tail.setContents(stored, 0, stored.length);
104         // we need to rehash this.
105         nodeToIndex.put(tail, s);
106         moveToHead(tail);
107         return s;
108       }
109     }
110 
111     private short findIdx(byte[] array, int offset, int length) {
112       Short s;
113       final Node comparisonNode = new Node();
114       comparisonNode.setContents(array, offset, length);
115       if ((s = nodeToIndex.get(comparisonNode)) != null) {
116         moveToHead(indexToNode[s]);
117         return s;
118       } else {
119         return -1;
120       }
121     }
122 
123     private byte[] get(short idx) {
124       Preconditions.checkElementIndex(idx, currSize);
125       moveToHead(indexToNode[idx]);
126       return indexToNode[idx].container;
127     }
128 
129     private void moveToHead(Node n) {
130       if (head == n) {
131         // no-op -- it's already the head.
132         return;
133       }
134       // At this point we definitely have prev, since it's not the head.
135       assert n.prev != null;
136       // Unlink prev.
137       n.prev.next = n.next;
138 
139       // Unlink next
140       if (n.next != null) {
141         n.next.prev = n.prev;
142       } else {
143         assert n == tail;
144         tail = n.prev;
145       }
146       // Node is now removed from the list. Re-add it at the head.
147       setHead(n);
148     }
149     
150     private void setHead(Node n) {
151       // assume it's already unlinked from the list at this point.
152       n.prev = null;
153       n.next = head;
154       if (head != null) {
155         assert head.prev == null;
156         head.prev = n;
157       }
158 
159       head = n;
160 
161       // First entry
162       if (tail == null) {
163         tail = n;
164       }
165     }
166 
167     private void clear() {
168       currSize = 0;
169       nodeToIndex.clear();
170       tail = null;
171       head = null;
172 
173       for (Node n : indexToNode) {
174         n.container = null;
175       }
176 
177       for (int i = 0; i < MAX_SIZE; i++) {
178         indexToNode[i].next = null;
179         indexToNode[i].prev = null;
180       }
181     }
182 
183     private static class Node {
184       byte[] container;
185       int offset;
186       int length;
187       Node next; // link towards the tail
188       Node prev; // link towards the head
189 
190       public Node() {
191       }
192 
193       private void setContents(byte[] container, int offset, int length) {
194         this.container = container;
195         this.offset = offset;
196         this.length = length;
197       }
198 
199       @Override
200       public int hashCode() {
201         return Bytes.hashCode(container, offset, length);
202       }
203 
204       @Override
205       public boolean equals(Object other) {
206         if (!(other instanceof Node)) {
207           return false;
208         }
209 
210         Node casted = (Node) other;
211         return Bytes.equals(container, offset, length, casted.container,
212             casted.offset, casted.length);
213       }
214     }
215   }
216 }