View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.util.Map;
22  import java.util.Map.Entry;
23  import java.util.Set;
24  import java.util.concurrent.ConcurrentHashMap;
25  import java.util.concurrent.ConcurrentMap;
26  import java.util.concurrent.ConcurrentSkipListMap;
27  import java.util.concurrent.ConcurrentSkipListSet;
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.hbase.classification.InterfaceAudience;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.HRegionLocation;
34  import org.apache.hadoop.hbase.RegionLocations;
35  import org.apache.hadoop.hbase.ServerName;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.util.Bytes;
38  
39  /**
40   * A cache implementation for region locations from meta.
41   */
42  @InterfaceAudience.Private
43  public class MetaCache {
44  
45    private static final Log LOG = LogFactory.getLog(MetaCache.class);
46  
47    /**
48     * Map of table to table {@link HRegionLocation}s.
49     */
50    private final ConcurrentMap<TableName, ConcurrentSkipListMap<byte[], RegionLocations>>
51    cachedRegionLocations =
52    new ConcurrentHashMap<TableName, ConcurrentSkipListMap<byte[], RegionLocations>>();
53  
54    // The presence of a server in the map implies it's likely that there is an
55    // entry in cachedRegionLocations that map to this server; but the absence
56    // of a server in this map guarentees that there is no entry in cache that
57    // maps to the absent server.
58    // The access to this attribute must be protected by a lock on cachedRegionLocations
59    private final Set<ServerName> cachedServers = new ConcurrentSkipListSet<ServerName>();
60  
61    /**
62     * Search the cache for a location that fits our table and row key.
63     * Return null if no suitable region is located.
64     *
65     *
66     * @param tableName
67     * @param row
68     * @return Null or region location found in cache.
69     */
70    public RegionLocations getCachedLocation(final TableName tableName, final byte [] row) {
71      ConcurrentSkipListMap<byte[], RegionLocations> tableLocations =
72        getTableLocations(tableName);
73  
74      Entry<byte[], RegionLocations> e = tableLocations.floorEntry(row);
75      if (e == null) {
76        return null;
77      }
78      RegionLocations possibleRegion = e.getValue();
79  
80      // make sure that the end key is greater than the row we're looking
81      // for, otherwise the row actually belongs in the next region, not
82      // this one. the exception case is when the endkey is
83      // HConstants.EMPTY_END_ROW, signifying that the region we're
84      // checking is actually the last region in the table.
85      byte[] endKey = possibleRegion.getRegionLocation().getRegionInfo().getEndKey();
86      if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
87          tableName.getRowComparator().compareRows(
88              endKey, 0, endKey.length, row, 0, row.length) > 0) {
89        return possibleRegion;
90      }
91  
92      // Passed all the way through, so we got nothing - complete cache miss
93      return null;
94    }
95  
96    /**
97     * Put a newly discovered HRegionLocation into the cache.
98     * @param tableName The table name.
99     * @param source the source of the new location
100    * @param location the new location
101    */
102   public void cacheLocation(final TableName tableName, final ServerName source,
103       final HRegionLocation location) {
104     assert source != null;
105     byte [] startKey = location.getRegionInfo().getStartKey();
106     ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName);
107     RegionLocations locations = new RegionLocations(new HRegionLocation[] {location}) ;
108     RegionLocations oldLocations = tableLocations.putIfAbsent(startKey, locations);
109     boolean isNewCacheEntry = (oldLocations == null);
110     if (isNewCacheEntry) {
111       if (LOG.isTraceEnabled()) {
112         LOG.trace("Cached location: " + location);
113       }
114       addToCachedServers(locations);
115       return;
116     }
117 
118     // If the server in cache sends us a redirect, assume it's always valid.
119     HRegionLocation oldLocation = oldLocations.getRegionLocation(
120       location.getRegionInfo().getReplicaId());
121     boolean force = oldLocation != null && oldLocation.getServerName() != null
122         && oldLocation.getServerName().equals(source);
123 
124     // For redirect if the number is equal to previous
125     // record, the most common case is that first the region was closed with seqNum, and then
126     // opened with the same seqNum; hence we will ignore the redirect.
127     // There are so many corner cases with various combinations of opens and closes that
128     // an additional counter on top of seqNum would be necessary to handle them all.
129     RegionLocations updatedLocations = oldLocations.updateLocation(location, false, force);
130     if (oldLocations != updatedLocations) {
131       boolean replaced = tableLocations.replace(startKey, oldLocations, updatedLocations);
132       if (replaced && LOG.isTraceEnabled()) {
133         LOG.trace("Changed cached location to: " + location);
134       }
135       addToCachedServers(updatedLocations);
136     }
137   }
138 
139   /**
140    * Put a newly discovered HRegionLocation into the cache.
141    * @param tableName The table name.
142    * @param locations the new locations
143    */
144   public void cacheLocation(final TableName tableName, final RegionLocations locations) {
145     byte [] startKey = locations.getRegionLocation().getRegionInfo().getStartKey();
146     ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName);
147     RegionLocations oldLocation = tableLocations.putIfAbsent(startKey, locations);
148     boolean isNewCacheEntry = (oldLocation == null);
149     if (isNewCacheEntry) {
150       if (LOG.isTraceEnabled()) {
151         LOG.trace("Cached location: " + locations);
152       }
153       addToCachedServers(locations);
154       return;
155     }
156 
157     // merge old and new locations and add it to the cache
158     // Meta record might be stale - some (probably the same) server has closed the region
159     // with later seqNum and told us about the new location.
160     RegionLocations mergedLocation = oldLocation.mergeLocations(locations);
161     boolean replaced = tableLocations.replace(startKey, oldLocation, mergedLocation);
162     if (replaced && LOG.isTraceEnabled()) {
163       LOG.trace("Merged cached locations: " + mergedLocation);
164     }
165     addToCachedServers(locations);
166   }
167 
168   private void addToCachedServers(RegionLocations locations) {
169     for (HRegionLocation loc : locations.getRegionLocations()) {
170       if (loc != null) {
171         cachedServers.add(loc.getServerName());
172       }
173     }
174   }
175 
176   /**
177    * @param tableName
178    * @return Map of cached locations for passed <code>tableName</code>
179    */
180   private ConcurrentSkipListMap<byte[], RegionLocations>
181     getTableLocations(final TableName tableName) {
182     // find the map of cached locations for this table
183     ConcurrentSkipListMap<byte[], RegionLocations> result;
184     result = this.cachedRegionLocations.get(tableName);
185     // if tableLocations for this table isn't built yet, make one
186     if (result == null) {
187       result = new ConcurrentSkipListMap<byte[], RegionLocations>(Bytes.BYTES_COMPARATOR);
188       ConcurrentSkipListMap<byte[], RegionLocations> old =
189           this.cachedRegionLocations.putIfAbsent(tableName, result);
190       if (old != null) {
191         return old;
192       }
193     }
194     return result;
195   }
196 
197   /**
198    * Check the region cache to see whether a region is cached yet or not.
199    * @param tableName tableName
200    * @param row row
201    * @return Region cached or not.
202    */
203   public boolean isRegionCached(TableName tableName, final byte[] row) {
204     RegionLocations location = getCachedLocation(tableName, row);
205     return location != null;
206   }
207 
208   /**
209    * Return the number of cached region for a table. It will only be called
210    * from a unit test.
211    */
212   public int getNumberOfCachedRegionLocations(final TableName tableName) {
213     Map<byte[], RegionLocations> tableLocs = this.cachedRegionLocations.get(tableName);
214     if (tableLocs == null) {
215       return 0;
216     }
217     int numRegions = 0;
218     for (RegionLocations tableLoc : tableLocs.values()) {
219       numRegions += tableLoc.numNonNullElements();
220     }
221     return numRegions;
222   }
223 
224   /**
225    * Delete all cached entries.
226    */
227   public void clearCache() {
228     this.cachedRegionLocations.clear();
229     this.cachedServers.clear();
230   }
231 
232   /**
233    * Delete all cached entries of a server.
234    */
235   public void clearCache(final ServerName serverName) {
236     if (!this.cachedServers.contains(serverName)) {
237       return;
238     }
239 
240     boolean deletedSomething = false;
241     synchronized (this.cachedServers) {
242       // We block here, because if there is an error on a server, it's likely that multiple
243       //  threads will get the error  simultaneously. If there are hundreds of thousand of
244       //  region location to check, it's better to do this only once. A better pattern would
245       //  be to check if the server is dead when we get the region location.
246       if (!this.cachedServers.contains(serverName)) {
247         return;
248       }
249       for (ConcurrentMap<byte[], RegionLocations> tableLocations : cachedRegionLocations.values()){
250         for (Entry<byte[], RegionLocations> e : tableLocations.entrySet()) {
251           RegionLocations regionLocations = e.getValue();
252           if (regionLocations != null) {
253             RegionLocations updatedLocations = regionLocations.removeByServer(serverName);
254             if (updatedLocations != regionLocations) {
255               if (updatedLocations.isEmpty()) {
256                 deletedSomething |= tableLocations.remove(e.getKey(), regionLocations);
257               } else {
258                 deletedSomething |= tableLocations.replace(e.getKey(), regionLocations, updatedLocations);
259               }
260             }
261           }
262         }
263       }
264       this.cachedServers.remove(serverName);
265     }
266     if (deletedSomething && LOG.isTraceEnabled()) {
267       LOG.trace("Removed all cached region locations that map to " + serverName);
268     }
269   }
270 
271   /**
272    * Delete all cached entries of a table.
273    */
274   public void clearCache(final TableName tableName) {
275     if (LOG.isTraceEnabled()) {
276       LOG.trace("Removed all cached region locations for table " + tableName);
277     }
278     this.cachedRegionLocations.remove(tableName);
279   }
280 
281   /**
282    * Delete a cached location, no matter what it is. Called when we were told to not use cache.
283    * @param tableName tableName
284    * @param row
285    */
286   public void clearCache(final TableName tableName, final byte [] row, int replicaId) {
287     ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName);
288 
289     boolean removed = false;
290     RegionLocations regionLocations = getCachedLocation(tableName, row);
291     if (regionLocations != null) {
292       HRegionLocation toBeRemoved = regionLocations.getRegionLocation(replicaId);
293       RegionLocations updatedLocations = regionLocations.remove(replicaId);
294       if (updatedLocations != regionLocations) {
295         byte[] startKey = regionLocations.getRegionLocation().getRegionInfo().getStartKey();
296         if (updatedLocations.isEmpty()) {
297           removed = tableLocations.remove(startKey, regionLocations);
298         } else {
299           removed = tableLocations.replace(startKey, regionLocations, updatedLocations);
300         }
301       }
302 
303       if (removed && LOG.isTraceEnabled() && toBeRemoved != null) {
304         LOG.trace("Removed " + toBeRemoved + " from cache");
305       }
306     }
307   }
308 
309   /**
310    * Delete a cached location, no matter what it is. Called when we were told to not use cache.
311    * @param tableName tableName
312    * @param row
313    */
314   public void clearCache(final TableName tableName, final byte [] row) {
315     ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName);
316 
317     RegionLocations regionLocations = getCachedLocation(tableName, row);
318     if (regionLocations != null) {
319       byte[] startKey = regionLocations.getRegionLocation().getRegionInfo().getStartKey();
320       boolean removed = tableLocations.remove(startKey, regionLocations);
321       if (removed && LOG.isTraceEnabled()) {
322         LOG.trace("Removed " + regionLocations + " from cache");
323       }
324     }
325   }
326 
327   /**
328    * Delete a cached location for a table, row and server
329    */
330   public void clearCache(final TableName tableName, final byte [] row, ServerName serverName) {
331     ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName);
332 
333     RegionLocations regionLocations = getCachedLocation(tableName, row);
334     if (regionLocations != null) {
335       RegionLocations updatedLocations = regionLocations.removeByServer(serverName);
336       if (updatedLocations != regionLocations) {
337         byte[] startKey = regionLocations.getRegionLocation().getRegionInfo().getStartKey();
338         boolean removed = false;
339         if (updatedLocations.isEmpty()) {
340           removed = tableLocations.remove(startKey, regionLocations);
341         } else {
342           removed = tableLocations.replace(startKey, regionLocations, updatedLocations);
343         }
344         if (removed && LOG.isTraceEnabled()) {
345           LOG.trace("Removed locations of table: " + tableName + " ,row: " + Bytes.toString(row)
346             + " mapping to server: " + serverName + " from cache");
347         }
348       }
349     }
350   }
351 
352   /**
353    * Deletes the cached location of the region if necessary, based on some error from source.
354    * @param hri The region in question.
355    */
356   public void clearCache(HRegionInfo hri) {
357     ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(hri.getTable());
358     RegionLocations regionLocations = tableLocations.get(hri.getStartKey());
359     if (regionLocations != null) {
360       HRegionLocation oldLocation = regionLocations.getRegionLocation(hri.getReplicaId());
361       if (oldLocation == null) return;
362       RegionLocations updatedLocations = regionLocations.remove(oldLocation);
363       boolean removed = false;
364       if (updatedLocations != regionLocations) {
365         if (updatedLocations.isEmpty()) {
366           removed = tableLocations.remove(hri.getStartKey(), regionLocations);
367         } else {
368           removed = tableLocations.replace(hri.getStartKey(), regionLocations, updatedLocations);
369         }
370         if (removed && LOG.isTraceEnabled()) {
371           LOG.trace("Removed " + oldLocation + " from cache");
372         }
373       }
374     }
375   }
376 
377   public void clearCache(final HRegionLocation location) {
378     if (location == null) {
379       return;
380     }
381     TableName tableName = location.getRegionInfo().getTable();
382     ConcurrentMap<byte[], RegionLocations> tableLocations = getTableLocations(tableName);
383     RegionLocations regionLocations = tableLocations.get(location.getRegionInfo().getStartKey());
384     if (regionLocations != null) {
385       RegionLocations updatedLocations = regionLocations.remove(location);
386       boolean removed = false;
387       if (updatedLocations != regionLocations) {
388         if (updatedLocations.isEmpty()) {
389           removed = tableLocations.remove(location.getRegionInfo().getStartKey(), regionLocations);
390         } else {
391           removed = tableLocations.replace(location.getRegionInfo().getStartKey(), regionLocations, updatedLocations);
392         }
393         if (removed && LOG.isTraceEnabled()) {
394           LOG.trace("Removed " + location + " from cache");
395         }
396       }
397     }
398   }
399 }