1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import static org.junit.Assert.assertEquals;
22
23 import java.io.IOException;
24
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HRegionInfo;
31 import org.apache.hadoop.hbase.HTableDescriptor;
32 import org.apache.hadoop.hbase.testclassification.MediumTests;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.client.Durability;
35 import org.apache.hadoop.hbase.client.Increment;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Result;
38 import org.apache.hadoop.hbase.regionserver.HRegion;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.apache.hadoop.hbase.util.FSUtils;
41 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
42 import org.apache.hadoop.hbase.wal.WAL;
43 import org.apache.hadoop.hbase.wal.WALFactory;
44 import org.apache.hadoop.hdfs.MiniDFSCluster;
45 import org.junit.AfterClass;
46 import org.junit.BeforeClass;
47 import org.junit.Test;
48 import org.junit.experimental.categories.Category;
49
50
51
52
53 @Category(MediumTests.class)
54 public class TestDurability {
55 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56 private static FileSystem FS;
57 private static MiniDFSCluster CLUSTER;
58 private static Configuration CONF;
59 private static Path DIR;
60
61 private static byte[] FAMILY = Bytes.toBytes("family");
62 private static byte[] ROW = Bytes.toBytes("row");
63 private static byte[] COL = Bytes.toBytes("col");
64
65
66 @BeforeClass
67 public static void setUpBeforeClass() throws Exception {
68 CONF = TEST_UTIL.getConfiguration();
69 TEST_UTIL.startMiniDFSCluster(1);
70
71 CLUSTER = TEST_UTIL.getDFSCluster();
72 FS = CLUSTER.getFileSystem();
73 DIR = TEST_UTIL.getDataTestDirOnTestFS("TestDurability");
74 FSUtils.setRootDir(CONF, DIR);
75 }
76
77 @AfterClass
78 public static void tearDownAfterClass() throws Exception {
79 TEST_UTIL.shutdownMiniCluster();
80 }
81
82 @Test
83 public void testDurability() throws Exception {
84 final WALFactory wals = new WALFactory(CONF, null, "TestDurability");
85 byte[] tableName = Bytes.toBytes("TestDurability");
86 final WAL wal = wals.getWAL(tableName);
87 HRegion region = createHRegion(tableName, "region", wal, Durability.USE_DEFAULT);
88 HRegion deferredRegion = createHRegion(tableName, "deferredRegion", wal, Durability.ASYNC_WAL);
89
90 region.put(newPut(null));
91 verifyWALCount(wals, wal, 1);
92
93
94
95
96 deferredRegion.put(newPut(null));
97
98 wal.sync();
99 verifyWALCount(wals, wal, 2);
100
101
102 deferredRegion.put(newPut(null));
103 wal.sync();
104 verifyWALCount(wals, wal, 3);
105 region.put(newPut(null));
106 verifyWALCount(wals, wal, 4);
107
108
109 deferredRegion.put(newPut(Durability.USE_DEFAULT));
110 wal.sync();
111 verifyWALCount(wals, wal, 5);
112 region.put(newPut(Durability.USE_DEFAULT));
113 verifyWALCount(wals, wal, 6);
114
115
116 region.put(newPut(Durability.SKIP_WAL));
117 deferredRegion.put(newPut(Durability.SKIP_WAL));
118 verifyWALCount(wals, wal, 6);
119 wal.sync();
120 verifyWALCount(wals, wal, 6);
121
122
123 region.put(newPut(Durability.ASYNC_WAL));
124 deferredRegion.put(newPut(Durability.ASYNC_WAL));
125 wal.sync();
126 verifyWALCount(wals, wal, 8);
127
128
129 region.put(newPut(Durability.SYNC_WAL));
130 deferredRegion.put(newPut(Durability.SYNC_WAL));
131 verifyWALCount(wals, wal, 10);
132
133
134 region.put(newPut(Durability.FSYNC_WAL));
135 deferredRegion.put(newPut(Durability.FSYNC_WAL));
136 verifyWALCount(wals, wal, 12);
137 }
138
139 @Test
140 public void testIncrement() throws Exception {
141 byte[] row1 = Bytes.toBytes("row1");
142 byte[] col1 = Bytes.toBytes("col1");
143 byte[] col2 = Bytes.toBytes("col2");
144 byte[] col3 = Bytes.toBytes("col3");
145
146
147 final WALFactory wals = new WALFactory(CONF, null, "TestIncrement");
148 byte[] tableName = Bytes.toBytes("TestIncrement");
149 final WAL wal = wals.getWAL(tableName);
150 HRegion region = createHRegion(tableName, "increment", wal, Durability.USE_DEFAULT);
151
152
153 Increment inc1 = new Increment(row1);
154 inc1.addColumn(FAMILY, col1, 1);
155 Result res = region.increment(inc1);
156 assertEquals(1, res.size());
157 assertEquals(1, Bytes.toLong(res.getValue(FAMILY, col1)));
158 verifyWALCount(wals, wal, 1);
159
160
161 inc1 = new Increment(row1);
162 inc1.addColumn(FAMILY, col1, 0);
163 res = region.increment(inc1);
164 assertEquals(1, res.size());
165 assertEquals(1, Bytes.toLong(res.getValue(FAMILY, col1)));
166 verifyWALCount(wals, wal, 1);
167
168
169
170 inc1 = new Increment(row1);
171 inc1.addColumn(FAMILY, col1, 0);
172 inc1.addColumn(FAMILY, col2, 0);
173 inc1.addColumn(FAMILY, col3, 0);
174 res = region.increment(inc1);
175 assertEquals(3, res.size());
176 assertEquals(1, Bytes.toLong(res.getValue(FAMILY, col1)));
177 assertEquals(0, Bytes.toLong(res.getValue(FAMILY, col2)));
178 assertEquals(0, Bytes.toLong(res.getValue(FAMILY, col3)));
179 verifyWALCount(wals, wal, 1);
180
181
182
183 inc1 = new Increment(row1);
184 inc1.addColumn(FAMILY, col1, 5);
185 inc1.addColumn(FAMILY, col2, 4);
186 inc1.addColumn(FAMILY, col3, 3);
187 res = region.increment(inc1);
188 assertEquals(3, res.size());
189 assertEquals(6, Bytes.toLong(res.getValue(FAMILY, col1)));
190 assertEquals(4, Bytes.toLong(res.getValue(FAMILY, col2)));
191 assertEquals(3, Bytes.toLong(res.getValue(FAMILY, col3)));
192 verifyWALCount(wals, wal, 2);
193 }
194
195 private Put newPut(Durability durability) {
196 Put p = new Put(ROW);
197 p.add(FAMILY, COL, COL);
198 if (durability != null) {
199 p.setDurability(durability);
200 }
201 return p;
202 }
203
204 private void verifyWALCount(WALFactory wals, WAL log, int expected) throws Exception {
205 Path walPath = DefaultWALProvider.getCurrentFileName(log);
206 WAL.Reader reader = wals.createReader(FS, walPath);
207 int count = 0;
208 WAL.Entry entry = new WAL.Entry();
209 while (reader.next(entry) != null) count++;
210 reader.close();
211 assertEquals(expected, count);
212 }
213
214
215 private HRegion createHRegion (byte [] tableName, String callingMethod,
216 WAL log, Durability durability)
217 throws IOException {
218 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
219 htd.setDurability(durability);
220 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
221 htd.addFamily(hcd);
222 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
223 Path path = new Path(DIR + callingMethod);
224 if (FS.exists(path)) {
225 if (!FS.delete(path, true)) {
226 throw new IOException("Failed delete of " + path);
227 }
228 }
229 return HRegion.createHRegion(info, path, CONF, htd, log);
230 }
231
232 }