1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertArrayEquals;
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.Map;
27 import java.util.TreeMap;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileStatus;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.KeyValue;
38 import org.apache.hadoop.hbase.SmallTests;
39 import org.apache.hadoop.hbase.regionserver.wal.HLog;
40 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
41 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.FSUtils;
44 import org.junit.*;
45 import org.junit.experimental.categories.Category;
46
47
48
49
50 @Category(SmallTests.class)
51 public class TestSnapshotLogSplitter {
52 final Log LOG = LogFactory.getLog(getClass());
53
54 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55
56 private byte[] TEST_QUALIFIER = Bytes.toBytes("q");
57 private byte[] TEST_FAMILY = Bytes.toBytes("f");
58
59 private Configuration conf;
60 private FileSystem fs;
61 private Path logFile;
62
63 @Before
64 public void setup() throws Exception {
65 conf = TEST_UTIL.getConfiguration();
66 fs = FileSystem.get(conf);
67 logFile = new Path(TEST_UTIL.getDataTestDir(), "test.log");
68 writeTestLog(logFile);
69 }
70
71 @After
72 public void tearDown() throws Exception {
73 fs.delete(logFile, false);
74 }
75
76 @Test
77 public void testSplitLogs() throws IOException {
78 Map<byte[], byte[]> regionsMap = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
79 splitTestLogs(getTableName(5), regionsMap);
80 }
81
82 @Test
83 public void testSplitLogsOnDifferentTable() throws IOException {
84 byte[] tableName = getTableName(1);
85 Map<byte[], byte[]> regionsMap = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
86 for (int j = 0; j < 10; ++j) {
87 byte[] regionName = getRegionName(tableName, j);
88 byte[] newRegionName = getNewRegionName(tableName, j);
89 regionsMap.put(regionName, newRegionName);
90 }
91 splitTestLogs(tableName, regionsMap);
92 }
93
94
95
96
97 private void splitTestLogs(final byte[] tableName, final Map<byte[], byte[]> regionsMap)
98 throws IOException {
99 Path tableDir = new Path(TEST_UTIL.getDataTestDir(), Bytes.toString(tableName));
100 SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
101 tableName, regionsMap);
102 try {
103 logSplitter.splitLog(logFile);
104 } finally {
105 logSplitter.close();
106 }
107 verifyRecoverEdits(tableDir, tableName, regionsMap);
108 }
109
110
111
112
113 private void verifyRecoverEdits(final Path tableDir, final byte[] tableName,
114 final Map<byte[], byte[]> regionsMap) throws IOException {
115 for (FileStatus regionStatus: FSUtils.listStatus(fs, tableDir)) {
116 assertTrue(regionStatus.getPath().getName().startsWith(Bytes.toString(tableName)));
117 Path regionEdits = HLog.getRegionDirRecoveredEditsDir(regionStatus.getPath());
118 byte[] regionName = Bytes.toBytes(regionStatus.getPath().getName());
119 assertFalse(regionsMap.containsKey(regionName));
120 for (FileStatus logStatus: FSUtils.listStatus(fs, regionEdits)) {
121 HLog.Reader reader = HLog.getReader(fs, logStatus.getPath(), conf);
122 try {
123 HLog.Entry entry;
124 while ((entry = reader.next()) != null) {
125 HLogKey key = entry.getKey();
126 assertArrayEquals(tableName, key.getTablename());
127 assertArrayEquals(regionName, key.getEncodedRegionName());
128 }
129 } finally {
130 reader.close();
131 }
132 }
133 }
134 }
135
136
137
138
139
140
141
142 private void writeTestLog(final Path logFile) throws IOException {
143 fs.mkdirs(logFile.getParent());
144 HLog.Writer writer = HLog.createWriter(fs, logFile, conf);
145 try {
146 for (int i = 0; i < 7; ++i) {
147 byte[] tableName = getTableName(i);
148 for (int j = 0; j < 10; ++j) {
149 byte[] regionName = getRegionName(tableName, j);
150 for (int k = 0; k < 50; ++k) {
151 byte[] rowkey = Bytes.toBytes("row-" + k);
152 HLogKey key = new HLogKey(regionName, tableName, (long)k,
153 System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
154 WALEdit edit = new WALEdit();
155 edit.add(new KeyValue(rowkey, TEST_FAMILY, TEST_QUALIFIER, rowkey));
156 writer.append(new HLog.Entry(key, edit));
157 }
158 }
159 }
160 } finally {
161 writer.close();
162 }
163 }
164
165 private byte[] getTableName(int tableId) {
166 return Bytes.toBytes("testtb-" + tableId);
167 }
168
169 private byte[] getRegionName(final byte[] tableName, int regionId) {
170 return Bytes.toBytes(Bytes.toString(tableName) + "-region-" + regionId);
171 }
172
173 private byte[] getNewRegionName(final byte[] tableName, int regionId) {
174 return Bytes.toBytes(Bytes.toString(tableName) + "-new-region-" + regionId);
175 }
176 }