View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.replication.regionserver;
20  
21  import org.apache.hadoop.conf.Configuration;
22  import org.apache.hadoop.fs.FileSystem;
23  import org.apache.hadoop.fs.Path;
24  import org.apache.hadoop.hbase.HBaseTestingUtility;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.HRegionInfo;
27  import org.apache.hadoop.hbase.HTableDescriptor;
28  import org.apache.hadoop.hbase.KeyValue;
29  import org.apache.hadoop.hbase.LargeTests;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.regionserver.wal.HLog;
32  import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
33  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
34  import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
35  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.hdfs.MiniDFSCluster;
38  import org.junit.After;
39  import org.junit.AfterClass;
40  import org.junit.Before;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  import org.junit.runner.RunWith;
45  import org.junit.runners.Parameterized;
46  import org.junit.runners.Parameterized.Parameters;
47  
48  import static org.junit.Assert.*;
49  
50  import java.io.EOFException;
51  import java.io.IOException;
52  import java.util.ArrayList;
53  import java.util.Collection;
54  import java.util.List;
55  
56  @Category(LargeTests.class)
57  @RunWith(Parameterized.class)
58  public class TestReplicationHLogReaderManager {
59  
60    private static HBaseTestingUtility TEST_UTIL;
61    private static Configuration conf;
62    private static Path hbaseDir;
63    private static FileSystem fs;
64    private static MiniDFSCluster cluster;
65    private static final TableName tableName = TableName.valueOf("tablename");
66    private static final byte [] family = Bytes.toBytes("column");
67    private static final byte [] qualifier = Bytes.toBytes("qualifier");
68    private static final HRegionInfo info = new HRegionInfo(tableName,
69        HConstants.EMPTY_START_ROW, HConstants.LAST_ROW, false);
70    private static final HTableDescriptor htd = new HTableDescriptor(tableName);
71  
72    private HLog log;
73    private ReplicationHLogReaderManager logManager;
74    private PathWatcher pathWatcher;
75    private int nbRows;
76    private int walEditKVs;
77  
78    @Parameters
79    public static Collection<Object[]> parameters() {
80      // Try out different combinations of row count and KeyValue count
81      int[] NB_ROWS = { 1500, 60000 };
82      int[] NB_KVS = { 1, 100 };
83      // whether compression is used
84      Boolean[] BOOL_VALS = { false, true };
85      List<Object[]> parameters = new ArrayList<Object[]>();
86      for (int nbRows : NB_ROWS) {
87        for (int walEditKVs : NB_KVS) {
88          for (boolean b : BOOL_VALS) {
89            Object[] arr = new Object[3];
90            arr[0] = nbRows;
91            arr[1] = walEditKVs;
92            arr[2] = b;
93            parameters.add(arr);
94          }
95        }
96      }
97      return parameters;
98    }
99  
100   public TestReplicationHLogReaderManager(int nbRows, int walEditKVs, boolean enableCompression) {
101     this.nbRows = nbRows;
102     this.walEditKVs = walEditKVs;
103     TEST_UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION,
104       enableCompression);
105   }
106   
107   @BeforeClass
108   public static void setUpBeforeClass() throws Exception {
109     TEST_UTIL = new HBaseTestingUtility();
110     conf = TEST_UTIL.getConfiguration();
111     TEST_UTIL.startMiniDFSCluster(3);
112 
113     hbaseDir = TEST_UTIL.createRootDir();
114     cluster = TEST_UTIL.getDFSCluster();
115     fs = cluster.getFileSystem();
116   }
117 
118   @AfterClass
119   public static void tearDownAfterClass() throws Exception {
120     TEST_UTIL.shutdownMiniCluster();
121   }
122 
123   @Before
124   public void setUp() throws Exception {
125     logManager = new ReplicationHLogReaderManager(fs, conf);
126     List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
127     pathWatcher = new PathWatcher();
128     listeners.add(pathWatcher);
129     log = HLogFactory.createHLog(fs, hbaseDir, "test", conf, listeners, "some server");
130   }
131 
132   @After
133   public void tearDown() throws Exception {
134     log.closeAndDelete();
135   }
136 
137   @Test
138   public void test() throws Exception {
139     // Grab the path that was generated when the log rolled as part of its creation
140     Path path = pathWatcher.currentPath;
141 
142     // open it, it's empty so it fails
143     try {
144       logManager.openReader(path);
145       fail("Shouldn't be able to open an empty file");
146     } catch (EOFException ex) {}
147 
148     assertEquals(0, logManager.getPosition());
149 
150     appendToLog();
151 
152     // There's one edit in the log, read it. Reading past it needs to return nulls
153     assertNotNull(logManager.openReader(path));
154     logManager.seek();
155     HLog.Entry[] entriesArray = new HLog.Entry[1];
156     HLog.Entry entry = logManager.readNextAndSetPosition(entriesArray, 0);
157     assertNotNull(entry);
158     entry = logManager.readNextAndSetPosition(entriesArray, 0);
159     assertNull(entry);
160     logManager.closeReader();
161     long oldPos = logManager.getPosition();
162 
163     appendToLog();
164 
165     // Read the newly added entry, make sure we made progress
166     assertNotNull(logManager.openReader(path));
167     logManager.seek();
168     entry = logManager.readNextAndSetPosition(entriesArray, 0);
169     assertNotEquals(oldPos, logManager.getPosition());
170     assertNotNull(entry);
171     logManager.closeReader();
172     oldPos = logManager.getPosition();
173 
174     log.rollWriter();
175 
176     // We rolled but we still should see the end of the first log and not get data
177     assertNotNull(logManager.openReader(path));
178     logManager.seek();
179     entry = logManager.readNextAndSetPosition(entriesArray, 0);
180     assertEquals(oldPos, logManager.getPosition());
181     assertNull(entry);
182     logManager.finishCurrentFile();
183 
184     path = pathWatcher.currentPath;
185 
186     // Finally we have a new empty log, which should still give us EOFs
187     try {
188       logManager.openReader(path);
189       fail();
190     } catch (EOFException ex) {}
191 
192     for (int i = 0; i < nbRows; i++) { appendToLogPlus(walEditKVs); }
193     log.rollWriter();
194     logManager.openReader(path);
195     logManager.seek();
196     for (int i = 0; i < nbRows; i++) {
197       HLog.Entry e = logManager.readNextAndSetPosition(entriesArray, 0);
198       if (e == null) {
199         fail("Should have enough entries");
200       }
201     }
202   }
203 
204   private void appendToLog() throws IOException {
205     appendToLogPlus(1);
206   }
207 
208   private void appendToLogPlus(int count) throws IOException {
209     log.append(info, tableName, getWALEdits(count), System.currentTimeMillis(), htd);
210   }
211 
212   private WALEdit getWALEdits(int count) {
213     WALEdit edit = new WALEdit();
214     for (int i = 0; i < count; i++) {
215       edit.add(new KeyValue(Bytes.toBytes(System.currentTimeMillis()), family, qualifier,
216         System.currentTimeMillis(), qualifier));
217     }
218     return edit;
219   }
220 
221   class PathWatcher implements WALActionsListener {
222 
223     Path currentPath;
224 
225     @Override
226     public void preLogRoll(Path oldPath, Path newPath) throws IOException {
227       currentPath = newPath;
228     }
229 
230     @Override
231     public void postLogRoll(Path oldPath, Path newPath) throws IOException {}
232 
233     @Override
234     public void preLogArchive(Path oldPath, Path newPath) throws IOException {}
235 
236     @Override
237     public void postLogArchive(Path oldPath, Path newPath) throws IOException {}
238 
239     @Override
240     public void logRollRequested() {}
241 
242     @Override
243     public void logCloseRequested() {}
244 
245     @Override
246     public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) {}
247 
248     @Override
249     public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) {}
250   }
251 }