1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.File;
28  import java.io.IOException;
29  import java.util.UUID;
30  import java.util.Set;
31  import java.util.HashSet;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.fs.FSDataInputStream;
37  import org.apache.hadoop.fs.FSDataOutputStream;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.fs.permission.FsPermission;
42  import org.apache.hadoop.hbase.HBaseConfiguration;
43  import org.apache.hadoop.hbase.HBaseTestingUtility;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
46  import org.apache.hadoop.hbase.MediumTests;
47  import org.apache.hadoop.hbase.regionserver.wal.HLog;
48  import org.apache.hadoop.hbase.util.MD5Hash;
49  import org.apache.hadoop.hbase.util.FSUtils;
50  import org.junit.*;
51  import org.junit.experimental.categories.Category;
52  
53  /**
54   * Test {@link FSUtils}.
55   */
56  @Category(MediumTests.class)
57  public class TestFSVisitor {
58    final Log LOG = LogFactory.getLog(getClass());
59  
60    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61  
62    private final String TABLE_NAME = "testtb";
63  
64    private Set<String> tableFamilies;
65    private Set<String> tableRegions;
66    private Set<String> recoveredEdits;
67    private Set<String> tableHFiles;
68    private Set<String> regionServers;
69    private Set<String> serverLogs;
70  
71    private FileSystem fs;
72    private Path tableDir;
73    private Path logsDir;
74    private Path rootDir;
75  
76    @Before
77    public void setUp() throws Exception {
78      fs = FileSystem.get(TEST_UTIL.getConfiguration());
79      rootDir = TEST_UTIL.getDataTestDir("hbase");
80      logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
81  
82      tableFamilies = new HashSet<String>();
83      tableRegions = new HashSet<String>();
84      recoveredEdits = new HashSet<String>();
85      tableHFiles = new HashSet<String>();
86      regionServers = new HashSet<String>();
87      serverLogs = new HashSet<String>();
88      tableDir = createTableFiles(rootDir, TABLE_NAME, tableRegions, tableFamilies, tableHFiles);
89      createRecoverEdits(tableDir, tableRegions, recoveredEdits);
90      createLogs(logsDir, regionServers, serverLogs);
91      FSUtils.logFileSystemState(fs, rootDir, LOG);
92    }
93  
94    @After
95    public void tearDown() throws Exception {
96      fs.delete(rootDir);
97    }
98  
99    @Test
100   public void testVisitStoreFiles() throws IOException {
101     final Set<String> regions = new HashSet<String>();
102     final Set<String> families = new HashSet<String>();
103     final Set<String> hfiles = new HashSet<String>();
104     FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
105       public void storeFile(final String region, final String family, final String hfileName)
106           throws IOException {
107         regions.add(region);
108         families.add(family);
109         hfiles.add(hfileName);
110       }
111     });
112     assertEquals(tableRegions, regions);
113     assertEquals(tableFamilies, families);
114     assertEquals(tableHFiles, hfiles);
115   }
116 
117   @Test
118   public void testVisitRecoveredEdits() throws IOException {
119     final Set<String> regions = new HashSet<String>();
120     final Set<String> edits = new HashSet<String>();
121     FSVisitor.visitTableRecoveredEdits(fs, tableDir, new FSVisitor.RecoveredEditsVisitor() {
122       public void recoveredEdits (final String region, final String logfile)
123           throws IOException {
124         regions.add(region);
125         edits.add(logfile);
126       }
127     });
128     assertEquals(tableRegions, regions);
129     assertEquals(recoveredEdits, edits);
130   }
131 
132   @Test
133   public void testVisitLogFiles() throws IOException {
134     final Set<String> servers = new HashSet<String>();
135     final Set<String> logs = new HashSet<String>();
136     FSVisitor.visitLogFiles(fs, rootDir, new FSVisitor.LogFileVisitor() {
137       public void logFile (final String server, final String logfile) throws IOException {
138         servers.add(server);
139         logs.add(logfile);
140       }
141     });
142     assertEquals(regionServers, servers);
143     assertEquals(serverLogs, logs);
144   }
145 
146 
147   /*
148    * |-testtb/
149    * |----f1d3ff8443297732862df21dc4e57262/
150    * |-------f1/
151    * |----------d0be84935ba84b66b1e866752ec5d663
152    * |----------9fc9d481718f4878b29aad0a597ecb94
153    * |-------f2/
154    * |----------4b0fe6068c564737946bcf4fd4ab8ae1
155    */
156   private Path createTableFiles(final Path rootDir, final String tableName,
157       final Set<String> tableRegions, final Set<String> tableFamilies,
158       final Set<String> tableHFiles) throws IOException {
159     Path tableDir = new Path(rootDir, tableName);
160     for (int r = 0; r < 10; ++r) {
161       String regionName = MD5Hash.getMD5AsHex(Bytes.toBytes(r));
162       tableRegions.add(regionName);
163       Path regionDir = new Path(tableDir, regionName);
164       for (int f = 0; f < 3; ++f) {
165         String familyName = "f" + f;
166         tableFamilies.add(familyName);
167         Path familyDir = new Path(regionDir, familyName);
168         fs.mkdirs(familyDir);
169         for (int h = 0; h < 5; ++h) {
170          String hfileName = UUID.randomUUID().toString().replaceAll("-", "");
171          tableHFiles.add(hfileName);
172          fs.createNewFile(new Path(familyDir, hfileName));
173         }
174       }
175     }
176     return tableDir;
177   }
178 
179   /*
180    * |-testtb/
181    * |----f1d3ff8443297732862df21dc4e57262/
182    * |-------recovered.edits/
183    * |----------0000001351969633479
184    * |----------0000001351969633481
185    */
186   private void createRecoverEdits(final Path tableDir, final Set<String> tableRegions,
187       final Set<String> recoverEdits) throws IOException {
188     for (String region: tableRegions) {
189       Path regionEditsDir = HLog.getRegionDirRecoveredEditsDir(new Path(tableDir, region));
190       long seqId = System.currentTimeMillis();
191       for (int i = 0; i < 3; ++i) {
192         String editName = String.format("%019d", seqId + i);
193         recoverEdits.add(editName);
194         FSDataOutputStream stream = fs.create(new Path(regionEditsDir, editName));
195         stream.write(Bytes.toBytes("test"));
196         stream.close();
197       }
198     }
199   }
200 
201   /*
202    * |-.logs/
203    * |----server5,5,1351969633508/
204    * |-------server5,5,1351969633508.0
205    * |----server6,6,1351969633512/
206    * |-------server6,6,1351969633512.0
207    * |-------server6,6,1351969633512.3
208    */
209   private void createLogs(final Path logDir, final Set<String> servers,
210       final Set<String> logs) throws IOException {
211     for (int s = 0; s < 7; ++s) {
212       String server = String.format("server%d,%d,%d", s, s, System.currentTimeMillis());
213       servers.add(server);
214       Path serverLogDir = new Path(logDir, server);
215       fs.mkdirs(serverLogDir);
216       for (int i = 0; i < 5; ++i) {
217         String logfile = server + '.' + i;
218         logs.add(logfile);
219         FSDataOutputStream stream = fs.create(new Path(serverLogDir, logfile));
220         stream.write(Bytes.toBytes("test"));
221         stream.close();
222       }
223     }
224   }
225 }