1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   * http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.coprocessor;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.hbase.*;
29  import org.apache.hadoop.hbase.regionserver.HRegion;
30  import org.apache.hadoop.hbase.client.Put;
31  import org.apache.hadoop.hbase.regionserver.wal.HLog;
32  import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
33  import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost;
34  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
35  import org.apache.hadoop.hbase.security.User;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.hbase.util.EnvironmentEdge;
38  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
39  import org.apache.hadoop.hdfs.MiniDFSCluster;
40  import org.junit.After;
41  import org.junit.AfterClass;
42  import org.junit.Before;
43  import org.junit.BeforeClass;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  
47  import java.io.IOException;
48  import java.security.PrivilegedExceptionAction;
49  import java.util.Arrays;
50  import java.util.List;
51  import java.util.Map;
52  
53  import static org.junit.Assert.*;
54  
55  /**
56   * Tests invocation of the {@link org.apache.hadoop.hbase.coprocessor.MasterObserver}
57   * interface hooks at all appropriate times during normal HMaster operations.
58   */
59  @Category(MediumTests.class)
60  public class TestWALObserver {
61    private static final Log LOG = LogFactory.getLog(TestWALObserver.class);
62    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63  
64    private static byte[] TEST_TABLE = Bytes.toBytes("observedTable");
65    private static byte[][] TEST_FAMILY = { Bytes.toBytes("fam1"),
66      Bytes.toBytes("fam2"),
67      Bytes.toBytes("fam3"),
68    };
69    private static byte[][] TEST_QUALIFIER = { Bytes.toBytes("q1"),
70      Bytes.toBytes("q2"),
71      Bytes.toBytes("q3"),
72    };
73    private static byte[][] TEST_VALUE = { Bytes.toBytes("v1"),
74      Bytes.toBytes("v2"),
75      Bytes.toBytes("v3"),
76    };
77    private static byte[] TEST_ROW = Bytes.toBytes("testRow");
78  
79    private Configuration conf;
80    private FileSystem fs;
81    private Path dir;
82    private MiniDFSCluster cluster;
83    private Path hbaseRootDir;
84    private Path oldLogDir;
85    private Path logDir;
86  
87    @BeforeClass
88    public static void setupBeforeClass() throws Exception {
89      Configuration conf = TEST_UTIL.getConfiguration();
90      conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
91          SampleRegionWALObserver.class.getName());
92      conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
93          SampleRegionWALObserver.class.getName());
94      conf.setBoolean("dfs.support.append", true);
95      conf.setInt("dfs.client.block.recovery.retries", 2);
96  
97      TEST_UTIL.startMiniCluster(1);
98      Path hbaseRootDir =
99        TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
100     LOG.info("hbase.rootdir=" + hbaseRootDir);
101     conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString());
102   }
103 
104   @AfterClass
105   public static void teardownAfterClass() throws Exception {
106     TEST_UTIL.shutdownMiniCluster();
107   }
108 
109   @Before
110   public void setUp() throws Exception {
111     this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
112     //this.cluster = TEST_UTIL.getDFSCluster();
113     this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
114     this.hbaseRootDir = new Path(conf.get(HConstants.HBASE_DIR));
115     this.dir = new Path(this.hbaseRootDir, TestWALObserver.class.getName());
116     this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
117     this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME);
118 
119     if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
120       TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
121     }
122   }
123 
124   @After
125   public void tearDown() throws Exception {
126     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
127   }
128 
129   /**
130    * Test WAL write behavior with WALObserver. The coprocessor monitors
131    * a WALEdit written to WAL, and ignore, modify, and add KeyValue's for the
132    * WALEdit.
133    */
134   @Test
135   public void testWALObserverWriteToWAL() throws Exception {
136 
137     HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE));
138     final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
139     HRegion region2 = HRegion.createHRegion(hri,
140             hbaseRootDir, this.conf, htd);
141 
142     Path basedir = new Path(this.hbaseRootDir, Bytes.toString(TEST_TABLE));
143     deleteDir(basedir);
144     fs.mkdirs(new Path(basedir, hri.getEncodedName()));
145 
146     HLog log = new HLog(this.fs, this.dir, this.oldLogDir, this.conf);
147     SampleRegionWALObserver cp = getCoprocessor(log);
148 
149     // TEST_FAMILY[0] shall be removed from WALEdit.
150     // TEST_FAMILY[1] value shall be changed.
151     // TEST_FAMILY[2] shall be added to WALEdit, although it's not in the put.
152     cp.setTestValues(TEST_TABLE, TEST_ROW, TEST_FAMILY[0], TEST_QUALIFIER[0],
153         TEST_FAMILY[1], TEST_QUALIFIER[1],
154         TEST_FAMILY[2], TEST_QUALIFIER[2]);
155 
156     assertFalse(cp.isPreWALWriteCalled());
157     assertFalse(cp.isPostWALWriteCalled());
158 
159     // TEST_FAMILY[2] is not in the put, however it shall be added by the tested
160     // coprocessor.
161     // Use a Put to create familyMap.
162     Put p = creatPutWith2Families(TEST_ROW);
163 
164     Map<byte [], List<KeyValue>> familyMap = p.getFamilyMap();
165     WALEdit edit = new WALEdit();
166     addFamilyMapToWALEdit(familyMap, edit);
167 
168     boolean foundFamily0 = false;
169     boolean foundFamily2 = false;
170     boolean modifiedFamily1 = false;
171 
172     List<KeyValue> kvs = edit.getKeyValues();
173 
174     for (KeyValue kv : kvs) {
175       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[0])) {
176         foundFamily0 = true;
177       }
178       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[2])) {
179         foundFamily2 = true;
180       }
181       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[1])) {
182         if (!Arrays.equals(kv.getValue(), TEST_VALUE[1])) {
183           modifiedFamily1 = true;
184         }
185       }
186     }
187     assertTrue(foundFamily0);
188     assertFalse(foundFamily2);
189     assertFalse(modifiedFamily1);
190 
191     // it's where WAL write cp should occur.
192     long now = EnvironmentEdgeManager.currentTimeMillis();
193     log.append(hri, hri.getTableName(), edit, now, htd);
194 
195     // the edit shall have been change now by the coprocessor.
196     foundFamily0 = false;
197     foundFamily2 = false;
198     modifiedFamily1 = false;
199     for (KeyValue kv : kvs) {
200       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[0])) {
201         foundFamily0 = true;
202       }
203       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[2])) {
204         foundFamily2 = true;
205       }
206       if (Arrays.equals(kv.getFamily(), TEST_FAMILY[1])) {
207         if (!Arrays.equals(kv.getValue(), TEST_VALUE[1])) {
208           modifiedFamily1 = true;
209         }
210       }
211     }
212     assertFalse(foundFamily0);
213     assertTrue(foundFamily2);
214     assertTrue(modifiedFamily1);
215 
216     assertTrue(cp.isPreWALWriteCalled());
217     assertTrue(cp.isPostWALWriteCalled());
218   }
219 
220   /**
221    * Test WAL replay behavior with WALObserver.
222    */
223   @Test
224   public void testWALCoprocessorReplay() throws Exception {
225     // WAL replay is handled at HRegion::replayRecoveredEdits(), which is
226     // ultimately called by HRegion::initialize()
227     byte[] tableName = Bytes.toBytes("testWALCoprocessorReplay");
228     final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(Bytes.toString(tableName));
229     //final HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
230     //final HRegionInfo hri1 = createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
231     final HRegionInfo hri = new HRegionInfo(tableName, null, null);
232 
233     final Path basedir = new Path(this.hbaseRootDir, Bytes.toString(tableName));
234     deleteDir(basedir);
235     fs.mkdirs(new Path(basedir, hri.getEncodedName()));
236 
237     final Configuration newConf = HBaseConfiguration.create(this.conf);
238 
239     HRegion region2 = HRegion.createHRegion(hri,
240         hbaseRootDir, newConf,htd);
241 
242 
243     //HLog wal = new HLog(this.fs, this.dir, this.oldLogDir, this.conf);
244     HLog wal = createWAL(this.conf);
245     //Put p = creatPutWith2Families(TEST_ROW);
246     WALEdit edit = new WALEdit();
247     long now = EnvironmentEdgeManager.currentTimeMillis();
248     //addFamilyMapToWALEdit(p.getFamilyMap(), edit);
249     final int countPerFamily = 1000;
250     //for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
251     for (HColumnDescriptor hcd: htd.getFamilies()) {
252           //addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
253           //EnvironmentEdgeManager.getDelegate(), wal);
254       addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
255       EnvironmentEdgeManager.getDelegate(), wal, htd);
256     }
257     wal.append(hri, tableName, edit, now, htd);
258     // sync to fs.
259     wal.sync();
260 
261     User user = HBaseTestingUtility.getDifferentUser(newConf,
262         ".replay.wal.secondtime");
263     user.runAs(new PrivilegedExceptionAction() {
264       public Object run() throws Exception {
265         Path p = runWALSplit(newConf);
266         LOG.info("WALSplit path == " + p);
267         FileSystem newFS = FileSystem.get(newConf);
268         // Make a new wal for new region open.
269         HLog wal2 = createWAL(newConf);
270         Path tableDir =
271           HTableDescriptor.getTableDir(hbaseRootDir, hri.getTableName());
272         HRegion region = new HRegion(tableDir, wal2, FileSystem.get(newConf),
273           newConf, hri, htd, TEST_UTIL.getHBaseCluster().getRegionServer(0));
274 
275         long seqid2 = region.initialize();
276         SampleRegionWALObserver cp2 =
277           (SampleRegionWALObserver)region.getCoprocessorHost().findCoprocessor(
278               SampleRegionWALObserver.class.getName());
279         // TODO: asserting here is problematic.
280         assertNotNull(cp2);
281         assertTrue(cp2.isPreWALRestoreCalled());
282         assertTrue(cp2.isPostWALRestoreCalled());
283         region.close();
284         wal2.closeAndDelete();
285         return null;
286       }
287     });
288   }
289 
290   /**
291    * Test to see CP loaded successfully or not. There is a duplication
292    * at TestHLog, but the purpose of that one is to see whether the loaded
293    * CP will impact existing HLog tests or not.
294    */
295   @Test
296   public void testWALObserverLoaded() throws Exception {
297     HLog log = new HLog(fs, dir, oldLogDir, conf);
298     assertNotNull(getCoprocessor(log));
299   }
300 
301   private SampleRegionWALObserver getCoprocessor(HLog wal) throws Exception {
302     WALCoprocessorHost host = wal.getCoprocessorHost();
303     Coprocessor c = host.findCoprocessor(SampleRegionWALObserver.class.getName());
304     return (SampleRegionWALObserver)c;
305   }
306 
307   /*
308    * Creates an HRI around an HTD that has <code>tableName</code> and three
309    * column families named.
310    * @param tableName Name of table to use when we create HTableDescriptor.
311    */
312   private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) {
313     HTableDescriptor htd = new HTableDescriptor(tableName);
314 
315     for (int i = 0; i < TEST_FAMILY.length; i++ ) {
316       HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]);
317       htd.addFamily(a);
318     }
319     return new HRegionInfo(htd.getName(), null, null, false);
320   }
321 
322   /*
323    * @param p Directory to cleanup
324    */
325   private void deleteDir(final Path p) throws IOException {
326     if (this.fs.exists(p)) {
327       if (!this.fs.delete(p, true)) {
328         throw new IOException("Failed remove of " + p);
329       }
330     }
331   }
332 
333   private Put creatPutWith2Families(byte[] row) throws IOException {
334     Put p = new Put(row);
335     for (int i = 0; i < TEST_FAMILY.length-1; i++ ) {
336       p.add(TEST_FAMILY[i], TEST_QUALIFIER[i],
337           TEST_VALUE[i]);
338     }
339     return p;
340   }
341 
342   /**
343    * Copied from HRegion.
344    *
345    * @param familyMap map of family->edits
346    * @param walEdit the destination entry to append into
347    */
348   private void addFamilyMapToWALEdit(Map<byte[], List<KeyValue>> familyMap,
349       WALEdit walEdit) {
350     for (List<KeyValue> edits : familyMap.values()) {
351       for (KeyValue kv : edits) {
352         walEdit.add(kv);
353       }
354     }
355   }
356   private Path runWALSplit(final Configuration c) throws IOException {
357     FileSystem fs = FileSystem.get(c);
358     HLogSplitter logSplitter = HLogSplitter.createLogSplitter(c,
359         this.hbaseRootDir, this.logDir, this.oldLogDir, fs);
360     List<Path> splits = logSplitter.splitLog();
361     // Split should generate only 1 file since there's only 1 region
362     assertEquals(1, splits.size());
363     // Make sure the file exists
364     assertTrue(fs.exists(splits.get(0)));
365     LOG.info("Split file=" + splits.get(0));
366     return splits.get(0);
367   }
368   private HLog createWAL(final Configuration c) throws IOException {
369     HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c);
370     return wal;
371   }
372   private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
373       final byte [] rowName, final byte [] family,
374       final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd)
375   throws IOException {
376     String familyStr = Bytes.toString(family);
377     for (int j = 0; j < count; j++) {
378       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
379       byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
380       WALEdit edit = new WALEdit();
381       edit.add(new KeyValue(rowName, family, qualifierBytes,
382         ee.currentTimeMillis(), columnBytes));
383       wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd);
384     }
385   }
386   private HTableDescriptor getBasic3FamilyHTableDescriptor(
387       final String tableName) {
388     HTableDescriptor htd = new HTableDescriptor(tableName);
389 
390     for (int i = 0; i < TEST_FAMILY.length; i++ ) {
391       HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]);
392       htd.addFamily(a);
393     }
394     return htd;
395   }
396 
397   private HTableDescriptor createBasic3FamilyHTD(final String tableName) {
398     HTableDescriptor htd = new HTableDescriptor(tableName);
399     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
400     htd.addFamily(a);
401     HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
402     htd.addFamily(b);
403     HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
404     htd.addFamily(c);
405     return htd;
406   }
407 
408 
409   @org.junit.Rule
410   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
411     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
412 }
413 
414