View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one or more
5    * contributor license agreements. See the NOTICE file distributed with this
6    * work for additional information regarding copyright ownership. The ASF
7    * licenses this file to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance with the License.
9    * You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16   * License for the specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile.bucket;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.FileNotFoundException;
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.concurrent.BlockingQueue;
29  import java.util.concurrent.atomic.AtomicLong;
30  
31  import org.apache.hadoop.hbase.SmallTests;
32  import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
33  import org.apache.hadoop.hbase.io.hfile.Cacheable;
34  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.BucketEntry;
35  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.RAMQueueEntry;
36  import org.apache.hadoop.hbase.util.Threads;
37  import org.junit.After;
38  import org.junit.Before;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  import org.mockito.Mockito;
42  
43  @Category(SmallTests.class)
44  public class TestBucketWriterThread {
45    private BucketCache bc;
46    private BucketCache.WriterThread wt;
47    private BlockingQueue<RAMQueueEntry> q;
48    private Cacheable plainCacheable;
49    private BlockCacheKey plainKey;
50  
51    /**
52     * Set up variables and get BucketCache and WriterThread into state where tests can  manually
53     * control the running of WriterThread and BucketCache is empty.
54     * @throws Exception
55     */
56    @Before
57    public void setUp() throws Exception {
58      // Arbitrary capacity.
59      final int capacity = 16;
60      // Run with one writer thread only. Means there will be one writer queue only too.  We depend
61      // on this in below.
62      final int writerThreadsCount = 1;
63      this.bc = new BucketCache("heap", capacity, 1, new int [] {1}, writerThreadsCount,
64        capacity, null, 100/*Tolerate ioerrors for 100ms*/);
65      assertEquals(writerThreadsCount, bc.writerThreads.length);
66      assertEquals(writerThreadsCount, bc.writerQueues.size());
67      // Get reference to our single WriterThread instance.
68      this.wt = bc.writerThreads[0];
69      this.q = bc.writerQueues.get(0);
70      // On construction bucketcache WriterThread is blocked on the writer queue so it will not
71      // notice the disabling of the writer until after it has processed an entry.  Lets pass one
72      // through after setting disable flag on the writer. We want to disable the WriterThread so
73      // we can run the doDrain manually so we can watch it working and assert it doing right thing.
74      wt.disableWriter();
75      this.plainKey = new BlockCacheKey("f", 0);
76      this.plainCacheable = Mockito.mock(Cacheable.class);
77      bc.cacheBlock(this.plainKey, plainCacheable);
78      while(!bc.ramCache.isEmpty()) Threads.sleep(1);
79      assertTrue(q.isEmpty());
80      // Now writer thread should be disabled.
81    }
82  
83    @After
84    public void tearDown() throws Exception {
85      if (this.bc != null) this.bc.shutdown();
86    }
87  
88    /**
89     * Test non-error case just works.
90     * @throws FileNotFoundException
91     * @throws IOException
92     * @throws InterruptedException
93     */
94    @Test (timeout=30000)
95    public void testNonErrorCase()
96    throws FileNotFoundException, IOException, InterruptedException {
97      bc.cacheBlock(this.plainKey, this.plainCacheable);
98      doDrainOfOneEntry(this.bc, this.wt, this.q);
99    }
100 
101   /**
102    * Pass through a too big entry and ensure it is cleared from queues and ramCache.
103    * Manually run the WriterThread.
104    * @throws InterruptedException 
105    */
106   @Test
107   public void testTooBigEntry() throws InterruptedException {
108     Cacheable tooBigCacheable = Mockito.mock(Cacheable.class);
109     Mockito.when(tooBigCacheable.getSerializedLength()).thenReturn(Integer.MAX_VALUE);
110     this.bc.cacheBlock(this.plainKey, tooBigCacheable);
111     doDrainOfOneEntry(this.bc, this.wt, this.q);
112   }
113 
114   /**
115    * Do IOE. Take the RAMQueueEntry that was on the queue, doctor it to throw exception, then
116    * put it back and process it.
117    * @throws IOException 
118    * @throws BucketAllocatorException 
119    * @throws CacheFullException 
120    * @throws InterruptedException 
121    */
122   @SuppressWarnings("unchecked")
123   @Test (timeout=30000)
124   public void testIOE()
125   throws CacheFullException, BucketAllocatorException, IOException, InterruptedException {
126     this.bc.cacheBlock(this.plainKey, plainCacheable);
127     RAMQueueEntry rqe = q.remove();
128     RAMQueueEntry spiedRqe = Mockito.spy(rqe);
129     Mockito.doThrow(new IOException("Mocked!")).when(spiedRqe).
130       writeToCache((IOEngine)Mockito.any(), (BucketAllocator)Mockito.any(),
131         (UniqueIndexMap<Integer>)Mockito.any(), (AtomicLong)Mockito.any());
132     this.q.add(spiedRqe);
133     doDrainOfOneEntry(bc, wt, q);
134     // Cache disabled when ioes w/o ever healing.
135     assertTrue(!bc.isCacheEnabled());
136   }
137 
138   /**
139    * Do Cache full exception
140    * @throws IOException 
141    * @throws BucketAllocatorException 
142    * @throws CacheFullException 
143    * @throws InterruptedException 
144    */
145   @Test (timeout=30000)
146   public void testCacheFullException()
147   throws CacheFullException, BucketAllocatorException, IOException, InterruptedException {
148     this.bc.cacheBlock(this.plainKey, plainCacheable);
149     RAMQueueEntry rqe = q.remove();
150     RAMQueueEntry spiedRqe = Mockito.spy(rqe);
151     final CacheFullException cfe = new CacheFullException(0, 0);
152     BucketEntry mockedBucketEntry = Mockito.mock(BucketEntry.class);
153     Mockito.doThrow(cfe).
154       doReturn(mockedBucketEntry).
155       when(spiedRqe).writeToCache((IOEngine)Mockito.any(), (BucketAllocator)Mockito.any(),
156         (UniqueIndexMap<Integer>)Mockito.any(), (AtomicLong)Mockito.any());
157     this.q.add(spiedRqe);
158     doDrainOfOneEntry(bc, wt, q);
159   }
160 
161   private static void doDrainOfOneEntry(final BucketCache bc, final BucketCache.WriterThread wt,
162       final BlockingQueue<RAMQueueEntry> q)
163   throws InterruptedException {
164     List<RAMQueueEntry> rqes = BucketCache.getRAMQueueEntries(q, new ArrayList<RAMQueueEntry>(1));
165     wt.doDrain(rqes);
166     assertTrue(q.isEmpty());
167     assertTrue(bc.ramCache.isEmpty());
168     assertEquals(0, bc.heapSize());
169   }
170 }