View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.Set;
26  import java.util.UUID;
27  import java.util.concurrent.locks.Lock;
28  import java.util.concurrent.locks.ReentrantLock;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.fs.FileStatus;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.HColumnDescriptor;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.InvalidFamilyOperationException;
41  import org.apache.hadoop.hbase.RemoteExceptionHandler;
42  import org.apache.hadoop.hbase.Server;
43  import org.apache.hadoop.hbase.ServerName;
44  import org.apache.hadoop.hbase.backup.HFileArchiver;
45  import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
46  import org.apache.hadoop.hbase.regionserver.HRegion;
47  import org.apache.hadoop.hbase.regionserver.wal.HLog;
48  import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
49  import org.apache.hadoop.hbase.regionserver.wal.OrphanHLogAfterSplitException;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
52  import org.apache.hadoop.hbase.util.FSTableDescriptors;
53  import org.apache.hadoop.hbase.util.FSUtils;
54  
55  /**
56   * This class abstracts a bunch of operations the HMaster needs to interact with
57   * the underlying file system, including splitting log files, checking file
58   * system status, etc.
59   */
60  public class MasterFileSystem {
61    private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
62    // HBase configuration
63    Configuration conf;
64    // master status
65    Server master;
66    // metrics for master
67    MasterMetrics metrics;
68    // Persisted unique cluster ID
69    private String clusterId;
70    // Keep around for convenience.
71    private final FileSystem fs;
72    // Is the fileystem ok?
73    private volatile boolean fsOk = true;
74    // The Path to the old logs dir
75    private final Path oldLogDir;
76    // root hbase directory on the FS
77    private final Path rootdir;
78    // hbase temp directory used for table construction and deletion
79    private final Path tempdir;
80    // create the split log lock
81    final Lock splitLogLock = new ReentrantLock();
82    final boolean distributedLogSplitting;
83    final SplitLogManager splitLogManager;
84    private final MasterServices services;
85  
86    public MasterFileSystem(Server master, MasterServices services,
87        MasterMetrics metrics, boolean masterRecovery)
88    throws IOException {
89      this.conf = master.getConfiguration();
90      this.master = master;
91      this.services = services;
92      this.metrics = metrics;
93      // Set filesystem to be that of this.rootdir else we get complaints about
94      // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
95      // default localfs.  Presumption is that rootdir is fully-qualified before
96      // we get to here with appropriate fs scheme.
97      this.rootdir = FSUtils.getRootDir(conf);
98      this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
99      // Cover both bases, the old way of setting default fs and the new.
100     // We're supposed to run on 0.20 and 0.21 anyways.
101     this.fs = this.rootdir.getFileSystem(conf);
102     String fsUri = this.fs.getUri().toString();
103     conf.set("fs.default.name", fsUri);
104     conf.set("fs.defaultFS", fsUri);
105     // make sure the fs has the same conf
106     fs.setConf(conf);
107     this.distributedLogSplitting =
108       conf.getBoolean("hbase.master.distributed.log.splitting", true);
109     if (this.distributedLogSplitting) {
110       this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
111           master.getConfiguration(), master, master.getServerName().toString());
112       this.splitLogManager.finishInitialization(masterRecovery);
113     } else {
114       this.splitLogManager = null;
115     }
116     // setup the filesystem variable
117     // set up the archived logs path
118     this.oldLogDir = createInitialFileSystemLayout();
119   }
120 
121   /**
122    * Create initial layout in filesystem.
123    * <ol>
124    * <li>Check if the root region exists and is readable, if not create it.
125    * Create hbase.version and the -ROOT- directory if not one.
126    * </li>
127    * <li>Create a log archive directory for RS to put archived logs</li>
128    * </ol>
129    * Idempotent.
130    */
131   private Path createInitialFileSystemLayout() throws IOException {
132     // check if the root directory exists
133     checkRootDir(this.rootdir, conf, this.fs);
134 
135     // check if temp directory exists and clean it
136     checkTempDir(this.tempdir, conf, this.fs);
137 
138     Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
139 
140     // Make sure the region servers can archive their old logs
141     if(!this.fs.exists(oldLogDir)) {
142       this.fs.mkdirs(oldLogDir);
143     }
144 
145     return oldLogDir;
146   }
147 
148   public FileSystem getFileSystem() {
149     return this.fs;
150   }
151 
152   /**
153    * Get the directory where old logs go
154    * @return the dir
155    */
156   public Path getOldLogDir() {
157     return this.oldLogDir;
158   }
159 
160   /**
161    * Checks to see if the file system is still accessible.
162    * If not, sets closed
163    * @return false if file system is not available
164    */
165   public boolean checkFileSystem() {
166     if (this.fsOk) {
167       try {
168         FSUtils.checkFileSystemAvailable(this.fs);
169         FSUtils.checkDfsSafeMode(this.conf);
170       } catch (IOException e) {
171         master.abort("Shutting down HBase cluster: file system not available", e);
172         this.fsOk = false;
173       }
174     }
175     return this.fsOk;
176   }
177 
178   /**
179    * @return HBase root dir.
180    */
181   public Path getRootDir() {
182     return this.rootdir;
183   }
184 
185   /**
186    * @return HBase temp dir.
187    */
188   public Path getTempDir() {
189     return this.tempdir;
190   }
191 
192   /**
193    * @return The unique identifier generated for this cluster
194    */
195   public String getClusterId() {
196     return clusterId;
197   }
198 
199   /**
200    * Inspect the log directory to recover any log file without
201    * an active region server.
202    */
203   void splitLogAfterStartup() {
204     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
205         HLog.SPLIT_SKIP_ERRORS_DEFAULT);
206     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
207     do {
208       if (master.isStopped()) {
209         LOG.warn("Master stopped while splitting logs");
210         break;
211       }
212       List<ServerName> serverNames = new ArrayList<ServerName>();
213       try {
214         if (!this.fs.exists(logsDirPath)) return;
215         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
216         // Get online servers after getting log folders to avoid log folder deletion of newly
217         // checked in region servers . see HBASE-5916
218         Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
219             .keySet();
220 
221         if (logFolders == null || logFolders.length == 0) {
222           LOG.debug("No log files to split, proceeding...");
223           return;
224         }
225         for (FileStatus status : logFolders) {
226           String sn = status.getPath().getName();
227           // truncate splitting suffix if present (for ServerName parsing)
228           if (sn.endsWith(HLog.SPLITTING_EXT)) {
229             sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
230           }
231           ServerName serverName = ServerName.parseServerName(sn);
232           if (!onlineServers.contains(serverName)) {
233             LOG.info("Log folder " + status.getPath() + " doesn't belong "
234                 + "to a known region server, splitting");
235             serverNames.add(serverName);
236           } else {
237             LOG.info("Log folder " + status.getPath()
238                 + " belongs to an existing region server");
239           }
240         }
241         splitLog(serverNames);
242         retrySplitting = false;
243       } catch (IOException ioe) {
244         LOG.warn("Failed splitting of " + serverNames, ioe);
245         if (!checkFileSystem()) {
246           LOG.warn("Bad Filesystem, exiting");
247           Runtime.getRuntime().halt(1);
248         }
249         try {
250           if (retrySplitting) {
251             Thread.sleep(conf.getInt(
252               "hbase.hlog.split.failure.retry.interval", 30 * 1000));
253           }
254         } catch (InterruptedException e) {
255           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
256           Thread.currentThread().interrupt();
257           retrySplitting = false;
258           Runtime.getRuntime().halt(1);
259         }
260       }
261     } while (retrySplitting);
262   }
263 
264   public void splitLog(final ServerName serverName) throws IOException {
265     List<ServerName> serverNames = new ArrayList<ServerName>();
266     serverNames.add(serverName);
267     splitLog(serverNames);
268   }
269 
270   public void splitLog(final List<ServerName> serverNames) throws IOException {
271     long splitTime = 0, splitLogSize = 0;
272     List<Path> logDirs = new ArrayList<Path>();
273     for(ServerName serverName: serverNames){
274       Path logDir = new Path(this.rootdir,
275         HLog.getHLogDirectoryName(serverName.toString()));
276       Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
277       // rename the directory so a rogue RS doesn't create more HLogs
278       if (fs.exists(logDir)) {
279         if (!this.fs.rename(logDir, splitDir)) {
280           throw new IOException("Failed fs.rename for log split: " + logDir);
281         }
282         logDir = splitDir;
283         LOG.debug("Renamed region directory: " + splitDir);
284       } else if (!fs.exists(splitDir)) {
285         LOG.info("Log dir for server " + serverName + " does not exist");
286         continue;
287       }
288       logDirs.add(splitDir);
289     }
290 
291     if (logDirs.isEmpty()) {
292       LOG.info("No logs to split");
293       return;
294     }
295 
296     if (distributedLogSplitting) {
297       splitLogManager.handleDeadWorkers(serverNames);
298       splitTime = EnvironmentEdgeManager.currentTimeMillis();
299       splitLogSize = splitLogManager.splitLogDistributed(logDirs);
300       splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
301     } else {
302       for(Path logDir: logDirs){
303         // splitLogLock ensures that dead region servers' logs are processed
304         // one at a time
305         this.splitLogLock.lock();
306         try {
307           HLogSplitter splitter = HLogSplitter.createLogSplitter(
308             conf, rootdir, logDir, oldLogDir, this.fs);
309           try {
310             // If FS is in safe mode, just wait till out of it.
311             FSUtils.waitOnSafeMode(conf, conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 1000));
312             splitter.splitLog();
313           } catch (OrphanHLogAfterSplitException e) {
314             LOG.warn("Retrying splitting because of:", e);
315             //An HLogSplitter instance can only be used once.  Get new instance.
316             splitter = HLogSplitter.createLogSplitter(conf, rootdir, logDir,
317               oldLogDir, this.fs);
318             splitter.splitLog();
319           }
320           splitTime = splitter.getTime();
321           splitLogSize = splitter.getSize();
322         } finally {
323           this.splitLogLock.unlock();
324         }
325       }
326     }
327 
328     if (this.metrics != null) {
329       this.metrics.addSplit(splitTime, splitLogSize);
330     }
331   }
332 
333   /**
334    * Get the rootdir.  Make sure its wholesome and exists before returning.
335    * @param rd
336    * @param conf
337    * @param fs
338    * @return hbase.rootdir (after checks for existence and bootstrapping if
339    * needed populating the directory with necessary bootup files).
340    * @throws IOException
341    */
342   private Path checkRootDir(final Path rd, final Configuration c,
343     final FileSystem fs)
344   throws IOException {
345     // If FS is in safe mode wait till out of it.
346     FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
347         10 * 1000));
348     // Filesystem is good. Go ahead and check for hbase.rootdir.
349     try {
350       if (!fs.exists(rd)) {
351         fs.mkdirs(rd);
352         // DFS leaves safe mode with 0 DNs when there are 0 blocks.
353         // We used to handle this by checking the current DN count and waiting until
354         // it is nonzero. With security, the check for datanode count doesn't work --
355         // it is a privileged op. So instead we adopt the strategy of the jobtracker
356         // and simply retry file creation during bootstrap indefinitely. As soon as
357         // there is one datanode it will succeed. Permission problems should have
358         // already been caught by mkdirs above.
359         FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
360           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
361         		  HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
362       } else {
363         if (!fs.isDirectory(rd)) {
364           throw new IllegalArgumentException(rd.toString() + " is not a directory");
365         }
366         // as above
367         FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
368           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
369         		  HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
370       }
371     } catch (IllegalArgumentException iae) {
372       LOG.fatal("Please fix invalid configuration for "
373         + HConstants.HBASE_DIR + " " + rd.toString(), iae);
374       throw iae;
375     }
376     // Make sure cluster ID exists
377     if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
378         HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
379       FSUtils.setClusterId(fs, rd, UUID.randomUUID().toString(), c.getInt(
380           HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
381     }
382     clusterId = FSUtils.getClusterId(fs, rd);
383 
384     // Make sure the root region directory exists!
385     if (!FSUtils.rootRegionExists(fs, rd)) {
386       bootstrap(rd, c);
387     }
388     createRootTableInfo(rd);
389     return rd;
390   }
391 
392   private void createRootTableInfo(Path rd) throws IOException {
393     // Create ROOT tableInfo if required.
394     if (!FSTableDescriptors.isTableInfoExists(fs, rd,
395         Bytes.toString(HRegionInfo.ROOT_REGIONINFO.getTableName()))) {
396       FSTableDescriptors.createTableDescriptor(HTableDescriptor.ROOT_TABLEDESC, this.conf);
397     }
398   }
399 
400   /**
401    * Make sure the hbase temp directory exists and is empty.
402    * NOTE that this method is only executed once just after the master becomes the active one.
403    */
404   private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
405       throws IOException {
406     // If the temp directory exists, clear the content (left over, from the previous run)
407     if (fs.exists(tmpdir)) {
408       // Archive table in temp, maybe left over from failed deletion,
409       // if not the cleaner will take care of them.
410       for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
411         for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
412           HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
413         }
414       }
415       if (!fs.delete(tmpdir, true)) {
416         throw new IOException("Unable to clean the temp directory: " + tmpdir);
417       }
418     }
419 
420     // Create the temp directory
421     if (!fs.mkdirs(tmpdir)) {
422       throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
423     }
424   }
425 
426   private static void bootstrap(final Path rd, final Configuration c)
427   throws IOException {
428     LOG.info("BOOTSTRAP: creating ROOT and first META regions");
429     try {
430       // Bootstrapping, make sure blockcache is off.  Else, one will be
431       // created here in bootstap and it'll need to be cleaned up.  Better to
432       // not make it in first place.  Turn off block caching for bootstrap.
433       // Enable after.
434       HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
435       setInfoFamilyCachingForRoot(false);
436       HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
437       setInfoFamilyCachingForMeta(false);
438       HRegion root = HRegion.createHRegion(rootHRI, rd, c,
439           HTableDescriptor.ROOT_TABLEDESC);
440       HRegion meta = HRegion.createHRegion(metaHRI, rd, c,
441           HTableDescriptor.META_TABLEDESC);
442       setInfoFamilyCachingForRoot(true);
443       setInfoFamilyCachingForMeta(true);
444       // Add first region from the META table to the ROOT region.
445       HRegion.addRegionToMETA(root, meta);
446       root.close();
447       root.getLog().closeAndDelete();
448       meta.close();
449       meta.getLog().closeAndDelete();
450     } catch (IOException e) {
451       e = RemoteExceptionHandler.checkIOException(e);
452       LOG.error("bootstrap", e);
453       throw e;
454     }
455   }
456 
457   /**
458    * Enable in-memory caching for -ROOT-
459    */
460   public static void setInfoFamilyCachingForRoot(final boolean b) {
461     for (HColumnDescriptor hcd:
462         HTableDescriptor.ROOT_TABLEDESC.getColumnFamilies()) {
463        if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
464          hcd.setBlockCacheEnabled(b);
465          hcd.setInMemory(b);
466      }
467     }
468   }
469 
470   /**
471    * Enable in memory caching for .META.
472    */
473   public static void setInfoFamilyCachingForMeta(final boolean b) {
474     for (HColumnDescriptor hcd:
475         HTableDescriptor.META_TABLEDESC.getColumnFamilies()) {
476       if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
477         hcd.setBlockCacheEnabled(b);
478         hcd.setInMemory(b);
479       }
480     }
481   }
482 
483 
484   public void deleteRegion(HRegionInfo region) throws IOException {
485     HFileArchiver.archiveRegion(conf, fs, region);
486   }
487 
488   public void deleteTable(byte[] tableName) throws IOException {
489     fs.delete(new Path(rootdir, Bytes.toString(tableName)), true);
490   }
491 
492   /**
493    * Move the specified file/directory to the hbase temp directory.
494    * @param path The path of the file/directory to move
495    * @return The temp location of the file/directory moved
496    * @throws IOException in case of file-system failure
497    */
498   public Path moveToTemp(final Path path) throws IOException {
499     Path tempPath = new Path(this.tempdir, path.getName());
500 
501     // Ensure temp exists
502     if (!fs.exists(tempdir) && !fs.mkdirs(tempdir)) {
503       throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
504     }
505 
506     if (!fs.rename(path, tempPath)) {
507       throw new IOException("Unable to move '" + path + "' to temp '" + tempPath + "'");
508     }
509 
510     return tempPath;
511   }
512 
513   /**
514    * Move the specified table to the hbase temp directory
515    * @param tableName Table name to move
516    * @return The temp location of the table moved
517    * @throws IOException in case of file-system failure
518    */
519   public Path moveTableToTemp(byte[] tableName) throws IOException {
520     return moveToTemp(HTableDescriptor.getTableDir(this.rootdir, tableName));
521   }
522 
523   public void updateRegionInfo(HRegionInfo region) {
524     // TODO implement this.  i think this is currently broken in trunk i don't
525     //      see this getting updated.
526     //      @see HRegion.checkRegioninfoOnFilesystem()
527   }
528 
529   public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
530       throws IOException {
531     // archive family store files
532     Path tableDir = new Path(rootdir, region.getTableNameAsString());
533     HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
534 
535     // delete the family folder
536     Path familyDir = new Path(tableDir,
537       new Path(region.getEncodedName(), Bytes.toString(familyName)));
538     if (fs.delete(familyDir, true) == false) {
539       throw new IOException("Could not delete family "
540           + Bytes.toString(familyName) + " from FileSystem for region "
541           + region.getRegionNameAsString() + "(" + region.getEncodedName()
542           + ")");
543     }
544   }
545 
546   public void stop() {
547     if (splitLogManager != null) {
548       this.splitLogManager.stop();
549     }
550   }
551 
552   /**
553    * Create new HTableDescriptor in HDFS.
554    *
555    * @param htableDescriptor
556    */
557   public void createTableDescriptor(HTableDescriptor htableDescriptor)
558       throws IOException {
559     FSTableDescriptors.createTableDescriptor(htableDescriptor, conf);
560   }
561 
562   /**
563    * Delete column of a table
564    * @param tableName
565    * @param familyName
566    * @return Modified HTableDescriptor with requested column deleted.
567    * @throws IOException
568    */
569   public HTableDescriptor deleteColumn(byte[] tableName, byte[] familyName)
570       throws IOException {
571     LOG.info("DeleteColumn. Table = " + Bytes.toString(tableName)
572         + " family = " + Bytes.toString(familyName));
573     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
574     htd.removeFamily(familyName);
575     this.services.getTableDescriptors().add(htd);
576     return htd;
577   }
578 
579   /**
580    * Modify Column of a table
581    * @param tableName
582    * @param hcd HColumnDesciptor
583    * @return Modified HTableDescriptor with the column modified.
584    * @throws IOException
585    */
586   public HTableDescriptor modifyColumn(byte[] tableName, HColumnDescriptor hcd)
587       throws IOException {
588     LOG.info("AddModifyColumn. Table = " + Bytes.toString(tableName)
589         + " HCD = " + hcd.toString());
590 
591     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
592     byte [] familyName = hcd.getName();
593     if(!htd.hasFamily(familyName)) {
594       throw new InvalidFamilyOperationException("Family '" +
595         Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
596     }
597     htd.addFamily(hcd);
598     this.services.getTableDescriptors().add(htd);
599     return htd;
600   }
601 
602   /**
603    * Add column to a table
604    * @param tableName
605    * @param hcd
606    * @return Modified HTableDescriptor with new column added.
607    * @throws IOException
608    */
609   public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
610       throws IOException {
611     LOG.info("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
612       hcd.toString());
613     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
614     if (htd == null) {
615       throw new InvalidFamilyOperationException("Family '" +
616         hcd.getNameAsString() + "' cannot be modified as HTD is null");
617     }
618     htd.addFamily(hcd);
619     this.services.getTableDescriptors().add(htd);
620     return htd;
621   }
622 }