1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  import static org.junit.Assert.*;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.net.URI;
25  import java.util.UUID;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FSDataInputStream;
31  import org.apache.hadoop.fs.FSDataOutputStream;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.permission.FsPermission;
36  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
37  import org.apache.hadoop.hbase.util.FSUtils;
38  import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
39  import org.apache.hadoop.util.Progressable;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  @Category(MediumTests.class)
45  public class TestHBaseFileSystem {
46    public static final Log LOG = LogFactory.getLog(TestHBaseFileSystem.class);
47  
48    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
49    private static Configuration conf;
50  
51    @BeforeClass
52    public static void setUpBeforeClass() throws Exception {
53      conf = TEST_UTIL.getConfiguration();
54      conf.setBoolean("dfs.support.append", true);
55      // The below config supported by 0.20-append and CDH3b2
56      conf.setInt("dfs.client.block.recovery.retries", 2);
57      TEST_UTIL.startMiniDFSCluster(3);
58      Path hbaseRootDir =
59        TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
60      LOG.info("hbase.rootdir=" + hbaseRootDir);
61      conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString());
62      conf.setInt("hdfs.client.retries.number", 10);
63      HBaseFileSystem.setRetryCounts(conf);
64    }
65  
66    
67    @Test
68    public void testNonIdempotentOpsWithRetries() throws IOException {
69      LOG.info("testNonIdempotentOpsWithRetries");
70  
71      Path rootDir = new Path(TestHBaseFileSystem.conf.get(HConstants.HBASE_DIR));
72      FileSystem fs = TEST_UTIL.getTestFileSystem();
73      // Create a Region
74      assertTrue(HBaseFileSystem.createPathOnFileSystem(fs, rootDir, 
75        true) != null);
76  
77      boolean result = HBaseFileSystem.makeDirOnFileSystem(new MockFileSystemForCreate(), 
78        new Path("/a"));
79      assertTrue("Couldn't create the directory", result);
80  
81      try {
82        HBaseFileSystem.createPathOnFileSystem(new MockFileSystemForCreate(), 
83          new Path("/A"), false);
84       assertTrue(false);// control should not come here.
85      } catch (Exception e) {
86        LOG.info(e);
87      }
88  
89      result = HBaseFileSystem.renameDirForFileSystem(new MockFileSystem(), new Path("/a"),
90        new Path("/b"));
91      assertTrue("Couldn't rename the directory", result);
92  
93      result = HBaseFileSystem.deleteDirFromFileSystem(new MockFileSystem(), 
94        new Path("/a"));
95  
96      assertTrue("Couldn't delete the directory", result);
97      fs.delete(rootDir, true);
98    }
99    
100   @Test
101   public void testRenameAndSetModifyTime() throws Exception {
102     assertTrue(FSUtils.isHDFS(conf));
103 
104     FileSystem fs = FileSystem.get(conf);
105     Path testDir = TEST_UTIL.getDataTestDir("testArchiveFile");
106 
107     String file = UUID.randomUUID().toString();
108     Path p = new Path(testDir, file);
109 
110     FSDataOutputStream out = fs.create(p);
111     out.close();
112     assertTrue("The created file should be present", FSUtils.isExists(fs, p));
113 
114     long expect = System.currentTimeMillis() + 1000;
115     assertFalse(expect == fs.getFileStatus(p).getModificationTime());
116 
117     ManualEnvironmentEdge mockEnv = new ManualEnvironmentEdge();
118     mockEnv.setValue(expect);
119     EnvironmentEdgeManager.injectEdge(mockEnv);
120 
121     String dstFile = UUID.randomUUID().toString();
122     Path dst = new Path(testDir , dstFile);
123 
124     assertTrue(HBaseFileSystem.renameAndSetModifyTime(fs, p, dst));
125     assertFalse("The moved file should not be present", FSUtils.isExists(fs,
126                                                                          p));
127     assertTrue("The dst file should be present", FSUtils.isExists(fs, dst));
128 
129     assertEquals(expect, fs.getFileStatus(dst).getModificationTime());
130   }
131 
132   
133   static class MockFileSystemForCreate extends MockFileSystem {
134     @Override
135     public boolean exists(Path path) {
136       if ("/A".equals(path.toString())) return true;
137       return false;
138     }
139   }
140 
141   /**
142    * a mock fs which throws exception for first 3 times, and then process the call (returns the
143    * excepted result).
144    */
145   static class MockFileSystem extends FileSystem {
146     int retryCount;
147     final static int successRetryCount = 3;
148 
149     public MockFileSystem() {
150       retryCount = 0;
151     }
152 
153     @Override
154     public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException {
155       throw new IOException("");
156     }
157 
158     @Override
159     public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3,
160         short arg4, long arg5, Progressable arg6) throws IOException {
161       LOG.debug("Create, " + retryCount);
162       if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
163       return null;
164     }
165 
166     @Override
167     public boolean delete(Path arg0) throws IOException {
168       if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
169       return true;
170     }
171 
172     @Override
173     public boolean delete(Path arg0, boolean arg1) throws IOException {
174       if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
175       return true;
176     }
177 
178     @Override
179     public FileStatus getFileStatus(Path arg0) throws IOException {
180       FileStatus fs = new FileStatus();
181       return fs;
182     }
183 
184     @Override
185     public boolean exists(Path path) {
186       return true;
187     }
188 
189     @Override
190     public URI getUri() {
191       throw new RuntimeException("Something bad happen");
192     }
193 
194     @Override
195     public Path getWorkingDirectory() {
196       throw new RuntimeException("Something bad happen");
197     }
198 
199     @Override
200     public FileStatus[] listStatus(Path arg0) throws IOException {
201       throw new IOException("Something bad happen");
202     }
203 
204     @Override
205     public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException {
206       LOG.debug("mkdirs, " + retryCount);
207       if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
208       return true;
209     }
210 
211     @Override
212     public FSDataInputStream open(Path arg0, int arg1) throws IOException {
213       throw new IOException("Something bad happen");
214     }
215 
216     @Override
217     public boolean rename(Path arg0, Path arg1) throws IOException {
218       LOG.debug("rename, " + retryCount);
219       if (retryCount++ < successRetryCount) throw new IOException("Something bad happen");
220       return true;
221     }
222 
223     @Override
224     public void setWorkingDirectory(Path arg0) {
225       throw new RuntimeException("Something bad happen");
226     }
227   }
228 
229 }