View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.replication.regionserver;
20  
21  import org.apache.hadoop.conf.Configuration;
22  import org.apache.hadoop.fs.FileSystem;
23  import org.apache.hadoop.fs.Path;
24  import org.apache.hadoop.hbase.HBaseTestingUtility;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.HRegionInfo;
27  import org.apache.hadoop.hbase.HTableDescriptor;
28  import org.apache.hadoop.hbase.KeyValue;
29  import org.apache.hadoop.hbase.LargeTests;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.regionserver.wal.HLog;
32  import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
33  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
34  import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
35  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.hdfs.MiniDFSCluster;
38  import org.junit.After;
39  import org.junit.AfterClass;
40  import org.junit.Before;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  import org.junit.runner.RunWith;
45  import org.junit.runners.Parameterized;
46  import org.junit.runners.Parameterized.Parameters;
47  
48  import static org.junit.Assert.*;
49  
50  import java.io.EOFException;
51  import java.io.IOException;
52  import java.util.ArrayList;
53  import java.util.Collection;
54  import java.util.List;
55  
56  @Category(LargeTests.class)
57  @RunWith(Parameterized.class)
58  public class TestReplicationHLogReaderManager {
59  
60    private static HBaseTestingUtility TEST_UTIL;
61    private static Configuration conf;
62    private static Path hbaseDir;
63    private static FileSystem fs;
64    private static MiniDFSCluster cluster;
65    private static final TableName tableName = TableName.valueOf("tablename");
66    private static final byte [] family = Bytes.toBytes("column");
67    private static final byte [] qualifier = Bytes.toBytes("qualifier");
68    private static final HRegionInfo info = new HRegionInfo(tableName,
69        HConstants.EMPTY_START_ROW, HConstants.LAST_ROW, false);
70    private static final HTableDescriptor htd = new HTableDescriptor(tableName);
71  
72    private HLog log;
73    private ReplicationHLogReaderManager logManager;
74    private PathWatcher pathWatcher;
75    private int nbRows;
76    private int walEditKVs;
77  
78    @Parameters
79    public static Collection<Object[]> parameters() {
80      // Try out different combinations of row count and KeyValue count
81      int[] NB_ROWS = { 1500, 60000 };
82      int[] NB_KVS = { 1, 100 };
83      // whether compression is used
84      Boolean[] BOOL_VALS = { false, true };
85      List<Object[]> parameters = new ArrayList<Object[]>();
86      for (int nbRows : NB_ROWS) {
87        for (int walEditKVs : NB_KVS) {
88          for (boolean b : BOOL_VALS) {
89            Object[] arr = new Object[3];
90            arr[0] = nbRows;
91            arr[1] = walEditKVs;
92            arr[2] = b;
93            parameters.add(arr);
94          }
95        }
96      }
97      return parameters;
98    }
99  
100   public TestReplicationHLogReaderManager(int nbRows, int walEditKVs, boolean enableCompression) {
101     this.nbRows = nbRows;
102     this.walEditKVs = walEditKVs;
103     TEST_UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION,
104       enableCompression);
105   }
106   
107   @BeforeClass
108   public static void setUpBeforeClass() throws Exception {
109     TEST_UTIL = new HBaseTestingUtility();
110     conf = TEST_UTIL.getConfiguration();
111     TEST_UTIL.startMiniDFSCluster(3);
112 
113     hbaseDir = TEST_UTIL.createRootDir();
114     cluster = TEST_UTIL.getDFSCluster();
115     fs = cluster.getFileSystem();
116   }
117 
118   @AfterClass
119   public static void tearDownAfterClass() throws Exception {
120     TEST_UTIL.shutdownMiniCluster();
121   }
122 
123   @Before
124   public void setUp() throws Exception {
125     logManager = new ReplicationHLogReaderManager(fs, conf);
126     List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
127     pathWatcher = new PathWatcher();
128     listeners.add(pathWatcher);
129     log = HLogFactory.createHLog(fs, hbaseDir, "test", conf, listeners, "some server");
130   }
131 
132   @After
133   public void tearDown() throws Exception {
134     log.closeAndDelete();
135   }
136 
137   @Test
138   public void test() throws Exception {
139     // Grab the path that was generated when the log rolled as part of its creation
140     Path path = pathWatcher.currentPath;
141 
142     assertEquals(0, logManager.getPosition());
143 
144     appendToLog();
145 
146     // There's one edit in the log, read it. Reading past it needs to return nulls
147     assertNotNull(logManager.openReader(path));
148     logManager.seek();
149     HLog.Entry entry = logManager.readNextAndSetPosition();
150     assertNotNull(entry);
151     entry = logManager.readNextAndSetPosition();
152     assertNull(entry);
153     logManager.closeReader();
154     long oldPos = logManager.getPosition();
155 
156     appendToLog();
157 
158     // Read the newly added entry, make sure we made progress
159     assertNotNull(logManager.openReader(path));
160     logManager.seek();
161     entry = logManager.readNextAndSetPosition();
162     assertNotEquals(oldPos, logManager.getPosition());
163     assertNotNull(entry);
164     logManager.closeReader();
165     oldPos = logManager.getPosition();
166 
167     log.rollWriter();
168 
169     // We rolled but we still should see the end of the first log and not get data
170     assertNotNull(logManager.openReader(path));
171     logManager.seek();
172     entry = logManager.readNextAndSetPosition();
173     assertEquals(oldPos, logManager.getPosition());
174     assertNull(entry);
175     logManager.finishCurrentFile();
176 
177     path = pathWatcher.currentPath;
178 
179     for (int i = 0; i < nbRows; i++) { appendToLogPlus(walEditKVs); }
180     log.rollWriter();
181     logManager.openReader(path);
182     logManager.seek();
183     for (int i = 0; i < nbRows; i++) {
184       HLog.Entry e = logManager.readNextAndSetPosition();
185       if (e == null) {
186         fail("Should have enough entries");
187       }
188     }
189   }
190 
191   private void appendToLog() throws IOException {
192     appendToLogPlus(1);
193   }
194 
195   private void appendToLogPlus(int count) throws IOException {
196     log.append(info, tableName, getWALEdits(count), System.currentTimeMillis(), htd);
197   }
198 
199   private WALEdit getWALEdits(int count) {
200     WALEdit edit = new WALEdit();
201     for (int i = 0; i < count; i++) {
202       edit.add(new KeyValue(Bytes.toBytes(System.currentTimeMillis()), family, qualifier,
203         System.currentTimeMillis(), qualifier));
204     }
205     return edit;
206   }
207 
208   class PathWatcher implements WALActionsListener {
209 
210     Path currentPath;
211 
212     @Override
213     public void preLogRoll(Path oldPath, Path newPath) throws IOException {
214       currentPath = newPath;
215     }
216 
217     @Override
218     public void postLogRoll(Path oldPath, Path newPath) throws IOException {}
219 
220     @Override
221     public void preLogArchive(Path oldPath, Path newPath) throws IOException {}
222 
223     @Override
224     public void postLogArchive(Path oldPath, Path newPath) throws IOException {}
225 
226     @Override
227     public void logRollRequested() {}
228 
229     @Override
230     public void logCloseRequested() {}
231 
232     @Override
233     public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) {}
234 
235     @Override
236     public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) {}
237   }
238 }