1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.io.hfile.bucket;
22
23 import java.util.Arrays;
24 import java.util.Map;
25 import java.util.concurrent.atomic.AtomicLong;
26
27 import org.apache.commons.collections.map.LinkedMap;
28 import com.google.common.base.Objects;
29 import com.google.common.base.Preconditions;
30 import com.google.common.primitives.Ints;
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
35 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
36 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.BucketEntry;
37 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
38
39
40
41
42
43
44
45
46
47 @InterfaceAudience.Private
48 @JsonIgnoreProperties({"indexStatistics", "freeSize", "usedSize"})
49 public final class BucketAllocator {
50 static final Log LOG = LogFactory.getLog(BucketAllocator.class);
51
52 @JsonIgnoreProperties({"completelyFree", "uninstantiated"})
53 public final static class Bucket {
54 private long baseOffset;
55 private int itemAllocationSize, sizeIndex;
56 private int itemCount;
57 private int freeList[];
58 private int freeCount, usedCount;
59
60 public Bucket(long offset) {
61 baseOffset = offset;
62 sizeIndex = -1;
63 }
64
65 void reconfigure(int sizeIndex, int[] bucketSizes, long bucketCapacity) {
66 Preconditions.checkElementIndex(sizeIndex, bucketSizes.length);
67 this.sizeIndex = sizeIndex;
68 itemAllocationSize = bucketSizes[sizeIndex];
69 itemCount = (int) (bucketCapacity / (long) itemAllocationSize);
70 freeCount = itemCount;
71 usedCount = 0;
72 freeList = new int[itemCount];
73 for (int i = 0; i < freeCount; ++i)
74 freeList[i] = i;
75 }
76
77 public boolean isUninstantiated() {
78 return sizeIndex == -1;
79 }
80
81 public int sizeIndex() {
82 return sizeIndex;
83 }
84
85 public int getItemAllocationSize() {
86 return itemAllocationSize;
87 }
88
89 public boolean hasFreeSpace() {
90 return freeCount > 0;
91 }
92
93 public boolean isCompletelyFree() {
94 return usedCount == 0;
95 }
96
97 public int freeCount() {
98 return freeCount;
99 }
100
101 public int usedCount() {
102 return usedCount;
103 }
104
105 public int getFreeBytes() {
106 return freeCount * itemAllocationSize;
107 }
108
109 public int getUsedBytes() {
110 return usedCount * itemAllocationSize;
111 }
112
113 public long getBaseOffset() {
114 return baseOffset;
115 }
116
117
118
119
120
121
122 public long allocate() {
123 assert freeCount > 0;
124 assert sizeIndex != -1;
125 ++usedCount;
126 long offset = baseOffset + (freeList[--freeCount] * itemAllocationSize);
127 assert offset >= 0;
128 return offset;
129 }
130
131 public void addAllocation(long offset) throws BucketAllocatorException {
132 offset -= baseOffset;
133 if (offset < 0 || offset % itemAllocationSize != 0)
134 throw new BucketAllocatorException(
135 "Attempt to add allocation for bad offset: " + offset + " base="
136 + baseOffset + ", bucket size=" + itemAllocationSize);
137 int idx = (int) (offset / itemAllocationSize);
138 boolean matchFound = false;
139 for (int i = 0; i < freeCount; ++i) {
140 if (matchFound) freeList[i - 1] = freeList[i];
141 else if (freeList[i] == idx) matchFound = true;
142 }
143 if (!matchFound)
144 throw new BucketAllocatorException("Couldn't find match for index "
145 + idx + " in free list");
146 ++usedCount;
147 --freeCount;
148 }
149
150 private void free(long offset) {
151 offset -= baseOffset;
152 assert offset >= 0;
153 assert offset < itemCount * itemAllocationSize;
154 assert offset % itemAllocationSize == 0;
155 assert usedCount > 0;
156 assert freeCount < itemCount;
157 int item = (int) (offset / (long) itemAllocationSize);
158 assert !freeListContains(item);
159 --usedCount;
160 freeList[freeCount++] = item;
161 }
162
163 private boolean freeListContains(int blockNo) {
164 for (int i = 0; i < freeCount; ++i) {
165 if (freeList[i] == blockNo) return true;
166 }
167 return false;
168 }
169 }
170
171 final class BucketSizeInfo {
172
173
174 private LinkedMap bucketList, freeBuckets, completelyFreeBuckets;
175 private int sizeIndex;
176
177 BucketSizeInfo(int sizeIndex) {
178 bucketList = new LinkedMap();
179 freeBuckets = new LinkedMap();
180 completelyFreeBuckets = new LinkedMap();
181 this.sizeIndex = sizeIndex;
182 }
183
184 public synchronized void instantiateBucket(Bucket b) {
185 assert b.isUninstantiated() || b.isCompletelyFree();
186 b.reconfigure(sizeIndex, bucketSizes, bucketCapacity);
187 bucketList.put(b, b);
188 freeBuckets.put(b, b);
189 completelyFreeBuckets.put(b, b);
190 }
191
192 public int sizeIndex() {
193 return sizeIndex;
194 }
195
196
197
198
199
200 public long allocateBlock() {
201 Bucket b = null;
202 if (freeBuckets.size() > 0) {
203
204 b = (Bucket) freeBuckets.lastKey();
205 }
206 if (b == null) {
207 b = grabGlobalCompletelyFreeBucket();
208 if (b != null) instantiateBucket(b);
209 }
210 if (b == null) return -1;
211 long result = b.allocate();
212 blockAllocated(b);
213 return result;
214 }
215
216 void blockAllocated(Bucket b) {
217 if (!b.isCompletelyFree()) completelyFreeBuckets.remove(b);
218 if (!b.hasFreeSpace()) freeBuckets.remove(b);
219 }
220
221 public Bucket findAndRemoveCompletelyFreeBucket() {
222 Bucket b = null;
223 assert bucketList.size() > 0;
224 if (bucketList.size() == 1) {
225
226 return null;
227 }
228
229 if (completelyFreeBuckets.size() > 0) {
230 b = (Bucket) completelyFreeBuckets.firstKey();
231 removeBucket(b);
232 }
233 return b;
234 }
235
236 private synchronized void removeBucket(Bucket b) {
237 assert b.isCompletelyFree();
238 bucketList.remove(b);
239 freeBuckets.remove(b);
240 completelyFreeBuckets.remove(b);
241 }
242
243 public void freeBlock(Bucket b, long offset) {
244 assert bucketList.containsKey(b);
245
246 assert (!completelyFreeBuckets.containsKey(b));
247 b.free(offset);
248 if (!freeBuckets.containsKey(b)) freeBuckets.put(b, b);
249 if (b.isCompletelyFree()) completelyFreeBuckets.put(b, b);
250 }
251
252 public synchronized IndexStatistics statistics() {
253 long free = 0, used = 0;
254 for (Object obj : bucketList.keySet()) {
255 Bucket b = (Bucket) obj;
256 free += b.freeCount();
257 used += b.usedCount();
258 }
259 return new IndexStatistics(free, used, bucketSizes[sizeIndex]);
260 }
261
262 @Override
263 public String toString() {
264 return Objects.toStringHelper(this.getClass())
265 .add("sizeIndex", sizeIndex)
266 .add("bucketSize", bucketSizes[sizeIndex])
267 .toString();
268 }
269 }
270
271
272
273
274 private static final int DEFAULT_BUCKET_SIZES[] = { 4 * 1024 + 1024, 8 * 1024 + 1024,
275 16 * 1024 + 1024, 32 * 1024 + 1024, 40 * 1024 + 1024, 48 * 1024 + 1024,
276 56 * 1024 + 1024, 64 * 1024 + 1024, 96 * 1024 + 1024, 128 * 1024 + 1024,
277 192 * 1024 + 1024, 256 * 1024 + 1024, 384 * 1024 + 1024,
278 512 * 1024 + 1024 };
279
280
281
282
283
284 public BucketSizeInfo roundUpToBucketSizeInfo(int blockSize) {
285 for (int i = 0; i < bucketSizes.length; ++i)
286 if (blockSize <= bucketSizes[i])
287 return bucketSizeInfos[i];
288 return null;
289 }
290
291 static public final int FEWEST_ITEMS_IN_BUCKET = 4;
292
293 private final int[] bucketSizes;
294 private final int bigItemSize;
295
296 private final long bucketCapacity;
297 private Bucket[] buckets;
298 private BucketSizeInfo[] bucketSizeInfos;
299 private final long totalSize;
300 private long usedSize = 0;
301
302 BucketAllocator(long availableSpace, int[] bucketSizes)
303 throws BucketAllocatorException {
304 this.bucketSizes = bucketSizes == null ? DEFAULT_BUCKET_SIZES : bucketSizes;
305 Arrays.sort(this.bucketSizes);
306 this.bigItemSize = Ints.max(this.bucketSizes);
307 this.bucketCapacity = FEWEST_ITEMS_IN_BUCKET * bigItemSize;
308 buckets = new Bucket[(int) (availableSpace / bucketCapacity)];
309 if (buckets.length < this.bucketSizes.length)
310 throw new BucketAllocatorException(
311 "Bucket allocator size too small - must have room for at least "
312 + this.bucketSizes.length + " buckets");
313 bucketSizeInfos = new BucketSizeInfo[this.bucketSizes.length];
314 for (int i = 0; i < this.bucketSizes.length; ++i) {
315 bucketSizeInfos[i] = new BucketSizeInfo(i);
316 }
317 for (int i = 0; i < buckets.length; ++i) {
318 buckets[i] = new Bucket(bucketCapacity * i);
319 bucketSizeInfos[i < this.bucketSizes.length ? i : this.bucketSizes.length - 1]
320 .instantiateBucket(buckets[i]);
321 }
322 this.totalSize = ((long) buckets.length) * bucketCapacity;
323 }
324
325
326
327
328
329
330
331
332
333 BucketAllocator(long availableSpace, int[] bucketSizes, Map<BlockCacheKey, BucketEntry> map,
334 AtomicLong realCacheSize) throws BucketAllocatorException {
335 this(availableSpace, bucketSizes);
336
337
338
339
340
341 boolean[] reconfigured = new boolean[buckets.length];
342 for (Map.Entry<BlockCacheKey, BucketEntry> entry : map.entrySet()) {
343 long foundOffset = entry.getValue().offset();
344 int foundLen = entry.getValue().getLength();
345 int bucketSizeIndex = -1;
346 for (int i = 0; i < bucketSizes.length; ++i) {
347 if (foundLen <= bucketSizes[i]) {
348 bucketSizeIndex = i;
349 break;
350 }
351 }
352 if (bucketSizeIndex == -1) {
353 throw new BucketAllocatorException(
354 "Can't match bucket size for the block with size " + foundLen);
355 }
356 int bucketNo = (int) (foundOffset / bucketCapacity);
357 if (bucketNo < 0 || bucketNo >= buckets.length)
358 throw new BucketAllocatorException("Can't find bucket " + bucketNo
359 + ", total buckets=" + buckets.length
360 + "; did you shrink the cache?");
361 Bucket b = buckets[bucketNo];
362 if (reconfigured[bucketNo]) {
363 if (b.sizeIndex() != bucketSizeIndex)
364 throw new BucketAllocatorException(
365 "Inconsistent allocation in bucket map;");
366 } else {
367 if (!b.isCompletelyFree())
368 throw new BucketAllocatorException("Reconfiguring bucket "
369 + bucketNo + " but it's already allocated; corrupt data");
370
371
372 BucketSizeInfo bsi = bucketSizeInfos[bucketSizeIndex];
373 BucketSizeInfo oldbsi = bucketSizeInfos[b.sizeIndex()];
374 oldbsi.removeBucket(b);
375 bsi.instantiateBucket(b);
376 reconfigured[bucketNo] = true;
377 }
378 realCacheSize.addAndGet(foundLen);
379 buckets[bucketNo].addAllocation(foundOffset);
380 usedSize += buckets[bucketNo].getItemAllocationSize();
381 bucketSizeInfos[bucketSizeIndex].blockAllocated(b);
382 }
383 }
384
385 public String toString() {
386 StringBuilder sb = new StringBuilder(1024);
387 for (int i = 0; i < buckets.length; ++i) {
388 Bucket b = buckets[i];
389 if (i > 0) sb.append(", ");
390 sb.append("bucket.").append(i).append(": size=").append(b.getItemAllocationSize());
391 sb.append(", freeCount=").append(b.freeCount()).append(", used=").append(b.usedCount());
392 }
393 return sb.toString();
394 }
395
396 public long getUsedSize() {
397 return this.usedSize;
398 }
399
400 public long getFreeSize() {
401 return this.totalSize - getUsedSize();
402 }
403
404 public long getTotalSize() {
405 return this.totalSize;
406 }
407
408
409
410
411
412
413
414 public synchronized long allocateBlock(int blockSize) throws CacheFullException,
415 BucketAllocatorException {
416 assert blockSize > 0;
417 BucketSizeInfo bsi = roundUpToBucketSizeInfo(blockSize);
418 if (bsi == null) {
419 throw new BucketAllocatorException("Allocation too big size=" + blockSize +
420 "; adjust BucketCache sizes " + CacheConfig.BUCKET_CACHE_BUCKETS_KEY +
421 " to accomodate if size seems reasonable and you want it cached.");
422 }
423 long offset = bsi.allocateBlock();
424
425
426 if (offset < 0)
427 throw new CacheFullException(blockSize, bsi.sizeIndex());
428 usedSize += bucketSizes[bsi.sizeIndex()];
429 return offset;
430 }
431
432 private Bucket grabGlobalCompletelyFreeBucket() {
433 for (BucketSizeInfo bsi : bucketSizeInfos) {
434 Bucket b = bsi.findAndRemoveCompletelyFreeBucket();
435 if (b != null) return b;
436 }
437 return null;
438 }
439
440
441
442
443
444
445 public synchronized int freeBlock(long offset) {
446 int bucketNo = (int) (offset / bucketCapacity);
447 assert bucketNo >= 0 && bucketNo < buckets.length;
448 Bucket targetBucket = buckets[bucketNo];
449 bucketSizeInfos[targetBucket.sizeIndex()].freeBlock(targetBucket, offset);
450 usedSize -= targetBucket.getItemAllocationSize();
451 return targetBucket.getItemAllocationSize();
452 }
453
454 public int sizeIndexOfAllocation(long offset) {
455 int bucketNo = (int) (offset / bucketCapacity);
456 assert bucketNo >= 0 && bucketNo < buckets.length;
457 Bucket targetBucket = buckets[bucketNo];
458 return targetBucket.sizeIndex();
459 }
460
461 public int sizeOfAllocation(long offset) {
462 int bucketNo = (int) (offset / bucketCapacity);
463 assert bucketNo >= 0 && bucketNo < buckets.length;
464 Bucket targetBucket = buckets[bucketNo];
465 return targetBucket.getItemAllocationSize();
466 }
467
468 static class IndexStatistics {
469 private long freeCount, usedCount, itemSize, totalCount;
470
471 public long freeCount() {
472 return freeCount;
473 }
474
475 public long usedCount() {
476 return usedCount;
477 }
478
479 public long totalCount() {
480 return totalCount;
481 }
482
483 public long freeBytes() {
484 return freeCount * itemSize;
485 }
486
487 public long usedBytes() {
488 return usedCount * itemSize;
489 }
490
491 public long totalBytes() {
492 return totalCount * itemSize;
493 }
494
495 public long itemSize() {
496 return itemSize;
497 }
498
499 public IndexStatistics(long free, long used, long itemSize) {
500 setTo(free, used, itemSize);
501 }
502
503 public IndexStatistics() {
504 setTo(-1, -1, 0);
505 }
506
507 public void setTo(long free, long used, long itemSize) {
508 this.itemSize = itemSize;
509 this.freeCount = free;
510 this.usedCount = used;
511 this.totalCount = free + used;
512 }
513 }
514
515 public Bucket [] getBuckets() {
516 return this.buckets;
517 }
518
519 public void dumpToLog() {
520 logStatistics();
521 StringBuilder sb = new StringBuilder();
522 for (Bucket b : buckets) {
523 sb.append("Bucket:").append(b.baseOffset).append('\n');
524 sb.append(" Size index: " + b.sizeIndex() + "; Free:" + b.freeCount
525 + "; used:" + b.usedCount + "; freelist\n");
526 for (int i = 0; i < b.freeCount(); ++i)
527 sb.append(b.freeList[i]).append(',');
528 sb.append('\n');
529 }
530 LOG.info(sb);
531 }
532
533 public void logStatistics() {
534 IndexStatistics total = new IndexStatistics();
535 IndexStatistics[] stats = getIndexStatistics(total);
536 LOG.info("Bucket allocator statistics follow:\n");
537 LOG.info(" Free bytes=" + total.freeBytes() + "+; used bytes="
538 + total.usedBytes() + "; total bytes=" + total.totalBytes());
539 for (IndexStatistics s : stats) {
540 LOG.info(" Object size " + s.itemSize() + " used=" + s.usedCount()
541 + "; free=" + s.freeCount() + "; total=" + s.totalCount());
542 }
543 }
544
545 public IndexStatistics[] getIndexStatistics(IndexStatistics grandTotal) {
546 IndexStatistics[] stats = getIndexStatistics();
547 long totalfree = 0, totalused = 0;
548 for (IndexStatistics stat : stats) {
549 totalfree += stat.freeBytes();
550 totalused += stat.usedBytes();
551 }
552 grandTotal.setTo(totalfree, totalused, 1);
553 return stats;
554 }
555
556 public IndexStatistics[] getIndexStatistics() {
557 IndexStatistics[] stats = new IndexStatistics[bucketSizes.length];
558 for (int i = 0; i < stats.length; ++i)
559 stats[i] = bucketSizeInfos[i].statistics();
560 return stats;
561 }
562
563 public long freeBlock(long freeList[]) {
564 long sz = 0;
565 for (int i = 0; i < freeList.length; ++i)
566 sz += freeBlock(freeList[i]);
567 return sz;
568 }
569
570 }