View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import static org.junit.Assert.assertEquals;
22  
23  import java.io.IOException;
24  
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HRegionInfo;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.testclassification.MediumTests;
33  import org.apache.hadoop.hbase.TableName;
34  import org.apache.hadoop.hbase.client.Durability;
35  import org.apache.hadoop.hbase.client.Increment;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.regionserver.HRegion;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hbase.util.FSUtils;
41  import org.apache.hadoop.hbase.wal.DefaultWALProvider;
42  import org.apache.hadoop.hbase.wal.WAL;
43  import org.apache.hadoop.hbase.wal.WALFactory;
44  import org.apache.hadoop.hdfs.MiniDFSCluster;
45  import org.junit.AfterClass;
46  import org.junit.BeforeClass;
47  import org.junit.Test;
48  import org.junit.experimental.categories.Category;
49  
50  /**
51   * Tests for WAL write durability
52   */
53  @Category(MediumTests.class)
54  public class TestDurability {
55    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56    private static FileSystem FS;
57    private static MiniDFSCluster CLUSTER;
58    private static Configuration CONF;
59    private static Path DIR;
60  
61    private static byte[] FAMILY = Bytes.toBytes("family");
62    private static byte[] ROW = Bytes.toBytes("row");
63    private static byte[] COL = Bytes.toBytes("col");
64  
65  
66    @BeforeClass
67    public static void setUpBeforeClass() throws Exception {
68      CONF = TEST_UTIL.getConfiguration();
69      TEST_UTIL.startMiniDFSCluster(1);
70  
71      CLUSTER = TEST_UTIL.getDFSCluster();
72      FS = CLUSTER.getFileSystem();
73      DIR = TEST_UTIL.getDataTestDirOnTestFS("TestDurability");
74      FSUtils.setRootDir(CONF, DIR);
75    }
76  
77    @AfterClass
78    public static void tearDownAfterClass() throws Exception {
79      TEST_UTIL.shutdownMiniCluster();
80    }
81  
82    @Test
83    public void testDurability() throws Exception {
84      final WALFactory wals = new WALFactory(CONF, null, "TestDurability");
85      byte[] tableName = Bytes.toBytes("TestDurability");
86      final WAL wal = wals.getWAL(tableName);
87      HRegion region = createHRegion(tableName, "region", wal, Durability.USE_DEFAULT);
88      HRegion deferredRegion = createHRegion(tableName, "deferredRegion", wal, Durability.ASYNC_WAL);
89  
90      region.put(newPut(null));
91      verifyWALCount(wals, wal, 1);
92  
93      // a put through the deferred table does not write to the wal immediately,
94      // but maybe has been successfully sync-ed by the underlying AsyncWriter +
95      // AsyncFlusher thread
96      deferredRegion.put(newPut(null));
97      // but will after we sync the wal
98      wal.sync();
99      verifyWALCount(wals, wal, 2);
100 
101     // a put through a deferred table will be sync with the put sync'ed put
102     deferredRegion.put(newPut(null));
103     wal.sync();
104     verifyWALCount(wals, wal, 3);
105     region.put(newPut(null));
106     verifyWALCount(wals, wal, 4);
107 
108     // a put through a deferred table will be sync with the put sync'ed put
109     deferredRegion.put(newPut(Durability.USE_DEFAULT));
110     wal.sync();
111     verifyWALCount(wals, wal, 5);
112     region.put(newPut(Durability.USE_DEFAULT));
113     verifyWALCount(wals, wal, 6);
114 
115     // SKIP_WAL never writes to the wal
116     region.put(newPut(Durability.SKIP_WAL));
117     deferredRegion.put(newPut(Durability.SKIP_WAL));
118     verifyWALCount(wals, wal, 6);
119     wal.sync();
120     verifyWALCount(wals, wal, 6);
121 
122     // Async overrides sync table default
123     region.put(newPut(Durability.ASYNC_WAL));
124     deferredRegion.put(newPut(Durability.ASYNC_WAL));
125     wal.sync();
126     verifyWALCount(wals, wal, 8);
127 
128     // sync overrides async table default
129     region.put(newPut(Durability.SYNC_WAL));
130     deferredRegion.put(newPut(Durability.SYNC_WAL));
131     verifyWALCount(wals, wal, 10);
132 
133     // fsync behaves like sync
134     region.put(newPut(Durability.FSYNC_WAL));
135     deferredRegion.put(newPut(Durability.FSYNC_WAL));
136     verifyWALCount(wals, wal, 12);
137   }
138 
139   @Test
140   public void testIncrement() throws Exception {
141     byte[] row1 = Bytes.toBytes("row1");
142     byte[] col1 = Bytes.toBytes("col1");
143     byte[] col2 = Bytes.toBytes("col2");
144     byte[] col3 = Bytes.toBytes("col3");
145 
146     // Setting up region
147     final WALFactory wals = new WALFactory(CONF, null, "TestIncrement");
148     byte[] tableName = Bytes.toBytes("TestIncrement");
149     final WAL wal = wals.getWAL(tableName);
150     HRegion region = createHRegion(tableName, "increment", wal, Durability.USE_DEFAULT);
151 
152     // col1: amount = 1, 1 write back to WAL
153     Increment inc1 = new Increment(row1);
154     inc1.addColumn(FAMILY, col1, 1);
155     Result res = region.increment(inc1);
156     assertEquals(1, res.size());
157     assertEquals(1, Bytes.toLong(res.getValue(FAMILY, col1)));
158     verifyWALCount(wals, wal, 1);
159 
160     // col1: amount = 0, 0 write back to WAL
161     inc1 = new Increment(row1);
162     inc1.addColumn(FAMILY, col1, 0);
163     res = region.increment(inc1);
164     assertEquals(1, res.size());
165     assertEquals(1, Bytes.toLong(res.getValue(FAMILY, col1)));
166     verifyWALCount(wals, wal, 1);
167 
168     // col1: amount = 0, col2: amount = 0, col3: amount = 0
169     // 0 write back to WAL
170     inc1 = new Increment(row1);
171     inc1.addColumn(FAMILY, col1, 0);
172     inc1.addColumn(FAMILY, col2, 0);
173     inc1.addColumn(FAMILY, col3, 0);
174     res = region.increment(inc1);
175     assertEquals(3, res.size());
176     assertEquals(1, Bytes.toLong(res.getValue(FAMILY, col1)));
177     assertEquals(0, Bytes.toLong(res.getValue(FAMILY, col2)));
178     assertEquals(0, Bytes.toLong(res.getValue(FAMILY, col3)));
179     verifyWALCount(wals, wal, 1);
180 
181     // col1: amount = 5, col2: amount = 4, col3: amount = 3
182     // 1 write back to WAL
183     inc1 = new Increment(row1);
184     inc1.addColumn(FAMILY, col1, 5);
185     inc1.addColumn(FAMILY, col2, 4);
186     inc1.addColumn(FAMILY, col3, 3);
187     res = region.increment(inc1);
188     assertEquals(3, res.size());
189     assertEquals(6, Bytes.toLong(res.getValue(FAMILY, col1)));
190     assertEquals(4, Bytes.toLong(res.getValue(FAMILY, col2)));
191     assertEquals(3, Bytes.toLong(res.getValue(FAMILY, col3)));
192     verifyWALCount(wals, wal, 2);
193   }
194 
195   private Put newPut(Durability durability) {
196     Put p = new Put(ROW);
197     p.add(FAMILY, COL, COL);
198     if (durability != null) {
199       p.setDurability(durability);
200     }
201     return p;
202   }
203 
204   private void verifyWALCount(WALFactory wals, WAL log, int expected) throws Exception {
205     Path walPath = DefaultWALProvider.getCurrentFileName(log);
206     WAL.Reader reader = wals.createReader(FS, walPath);
207     int count = 0;
208     WAL.Entry entry = new WAL.Entry();
209     while (reader.next(entry) != null) count++;
210     reader.close();
211     assertEquals(expected, count);
212   }
213 
214   // lifted from TestAtomicOperation
215   private HRegion createHRegion (byte [] tableName, String callingMethod,
216       WAL log, Durability durability)
217     throws IOException {
218       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
219       htd.setDurability(durability);
220       HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
221       htd.addFamily(hcd);
222       HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
223       Path path = new Path(DIR + callingMethod);
224       if (FS.exists(path)) {
225         if (!FS.delete(path, true)) {
226           throw new IOException("Failed delete of " + path);
227         }
228       }
229       return HRegion.createHRegion(info, path, CONF, htd, log);
230     }
231 
232 }