org.apache.hadoop.hive.metastore
Class MetaStoreUtils

java.lang.Object
  extended by org.apache.hadoop.hive.metastore.MetaStoreUtils

public class MetaStoreUtils
extends Object


Field Summary
static String DATABASE_WAREHOUSE_SUFFIX
           
static String DEFAULT_DATABASE_COMMENT
           
static String DEFAULT_DATABASE_NAME
           
protected static org.apache.commons.logging.Log LOG
           
 
Constructor Summary
MetaStoreUtils()
           
 
Method Summary
static Table createColumnsetSchema(String name, List<String> columns, List<String> partCols, org.apache.hadoop.conf.Configuration conf)
           
static void deleteWHDirectory(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf, boolean use_trash)
           
static int findFreePort()
          Finds a free port on the machine.
static Class<?> getClass(String rawStoreClassName)
           
static String getColumnNamesFromFieldSchema(List<FieldSchema> fieldSchemas)
          Convert FieldSchemas to columnNames.
static String getColumnTypesFromFieldSchema(List<FieldSchema> fieldSchemas)
          Convert FieldSchemas to columnTypes.
static String getDDLFromFieldSchema(String structName, List<FieldSchema> fieldSchemas)
          Convert FieldSchemas to Thrift DDL.
static Deserializer getDeserializer(org.apache.hadoop.conf.Configuration conf, Partition part, Table table)
          getDeserializer Get the Deserializer for a partition.
static Deserializer getDeserializer(org.apache.hadoop.conf.Configuration conf, Properties schema)
          getDeserializer Get the Deserializer for a table given its name and properties.
static Deserializer getDeserializer(org.apache.hadoop.conf.Configuration conf, Table table)
          getDeserializer Get the Deserializer for a table.
static FieldSchema getFieldSchemaFromTypeInfo(String fieldName, TypeInfo typeInfo)
          Convert TypeInfo to FieldSchema.
static List<FieldSchema> getFieldsFromDeserializer(String tableName, Deserializer deserializer)
           
static String getFullDDLFromFieldSchema(String structName, List<FieldSchema> fieldSchemas)
          Convert FieldSchemas to Thrift DDL + column names and column types
static String getIndexTableName(String dbName, String baseTblName, String indexName)
           
static String getListType(String t)
           
static String getMapType(String k, String v)
           
static org.apache.hadoop.fs.Path getOriginalLocation(Partition part)
           
static Properties getPartSchemaFromTableSchema(StorageDescriptor sd, StorageDescriptor tblsd, Map<String,String> parameters, String databaseName, String tableName, List<FieldSchema> partitionKeys, Properties tblSchema)
          Get partition level schema from table level schema.
static Properties getSchema(Partition part, Table table)
           
static Properties getSchema(StorageDescriptor sd, StorageDescriptor tblsd, Map<String,String> parameters, String databaseName, String tableName, List<FieldSchema> partitionKeys)
           
static Properties getSchema(Table table)
           
static boolean isArchived(Partition part)
           
static boolean isExternalTable(Table table)
          Determines whether a table is an external table.
static boolean isIndexTable(Table table)
           
static boolean isNonNativeTable(Table table)
           
static void makeDir(org.apache.hadoop.fs.Path path, HiveConf hiveConf)
           
static String makeFilterStringFromMap(Map<String,String> m)
          Given a map of partition column names to values, this creates a filter string that can be used to call the *byFilter methods
static
<T> T
newInstance(Class<T> theClass, Class<?>[] parameterTypes, Object[] initargs)
          Create an object of the given class.
static void printStackTrace(Exception e)
          printStackTrace Helper function to print an exception stack trace to the log and not stderr
static boolean pvalMatches(List<String> partial, List<String> full)
          Returns true if partial has the same values as full for all values that aren't empty in partial.
static void recursiveDelete(File f)
          recursiveDelete just recursively deletes a dir - you'd think Java would have something to do this??
static void setSerdeParam(SerDeInfo sdi, Properties schema, String param)
           
static void startMetaStore(int port, HadoopThriftAuthBridge bridge)
           
static String typeToThriftType(String type)
          Convert type to ThriftType.
static boolean validateColNames(List<FieldSchema> cols)
           
static boolean validateName(String name)
          validateName Checks the name conforms to our standars which are: "[a-zA-z_0-9]+".
static boolean validateSkewedColNames(List<String> cols)
           
static boolean validateSkewedColNamesSubsetCol(List<String> skewedColNames, List<FieldSchema> cols)
           
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Field Detail

LOG

protected static final org.apache.commons.logging.Log LOG

DEFAULT_DATABASE_NAME

public static final String DEFAULT_DATABASE_NAME
See Also:
Constant Field Values

DEFAULT_DATABASE_COMMENT

public static final String DEFAULT_DATABASE_COMMENT
See Also:
Constant Field Values

DATABASE_WAREHOUSE_SUFFIX

public static final String DATABASE_WAREHOUSE_SUFFIX
See Also:
Constant Field Values
Constructor Detail

MetaStoreUtils

public MetaStoreUtils()
Method Detail

printStackTrace

public static void printStackTrace(Exception e)
printStackTrace Helper function to print an exception stack trace to the log and not stderr

Parameters:
e - the exception

createColumnsetSchema

public static Table createColumnsetSchema(String name,
                                          List<String> columns,
                                          List<String> partCols,
                                          org.apache.hadoop.conf.Configuration conf)
                                   throws MetaException
Throws:
MetaException

recursiveDelete

public static void recursiveDelete(File f)
                            throws IOException
recursiveDelete just recursively deletes a dir - you'd think Java would have something to do this??

Parameters:
f - - the file/dir to delete
Throws:
IOException - propogate f.delete() exceptions

getDeserializer

public static Deserializer getDeserializer(org.apache.hadoop.conf.Configuration conf,
                                           Properties schema)
                                    throws MetaException
getDeserializer Get the Deserializer for a table given its name and properties.

Parameters:
conf - hadoop config
schema - the properties to use to instantiate the deserializer
Returns:
the Deserializer
Throws:
MetaException - if any problems instantiating the Deserializer todo - this should move somewhere into serde.jar

getDeserializer

public static Deserializer getDeserializer(org.apache.hadoop.conf.Configuration conf,
                                           Table table)
                                    throws MetaException
getDeserializer Get the Deserializer for a table.

Parameters:
conf - - hadoop config
table - the table
Returns:
the Deserializer
Throws:
MetaException - if any problems instantiating the Deserializer todo - this should move somewhere into serde.jar

getDeserializer

public static Deserializer getDeserializer(org.apache.hadoop.conf.Configuration conf,
                                           Partition part,
                                           Table table)
                                    throws MetaException
getDeserializer Get the Deserializer for a partition.

Parameters:
conf - - hadoop config
part - the partition
table - the table
Returns:
the Deserializer
Throws:
MetaException - if any problems instantiating the Deserializer

deleteWHDirectory

public static void deleteWHDirectory(org.apache.hadoop.fs.Path path,
                                     org.apache.hadoop.conf.Configuration conf,
                                     boolean use_trash)
                              throws MetaException
Throws:
MetaException

validateName

public static boolean validateName(String name)
validateName Checks the name conforms to our standars which are: "[a-zA-z_0-9]+". checks this is just characters and numbers and _

Parameters:
name - the name to validate
Returns:
true or false depending on conformance
Throws:
MetaException - if it doesn't match the pattern.

validateColNames

public static boolean validateColNames(List<FieldSchema> cols)

validateSkewedColNames

public static boolean validateSkewedColNames(List<String> cols)

validateSkewedColNamesSubsetCol

public static boolean validateSkewedColNamesSubsetCol(List<String> skewedColNames,
                                                      List<FieldSchema> cols)

getListType

public static String getListType(String t)

getMapType

public static String getMapType(String k,
                                String v)

setSerdeParam

public static void setSerdeParam(SerDeInfo sdi,
                                 Properties schema,
                                 String param)

typeToThriftType

public static String typeToThriftType(String type)
Convert type to ThriftType. We do that by tokenizing the type and convert each token.


getFullDDLFromFieldSchema

public static String getFullDDLFromFieldSchema(String structName,
                                               List<FieldSchema> fieldSchemas)
Convert FieldSchemas to Thrift DDL + column names and column types

Parameters:
structName - The name of the table
fieldSchemas - List of fields along with their schemas
Returns:
String containing "Thrift DDL#comma-separated-column-names#colon-separated-columntypes Example: "struct result { a string, map b}#a,b#string:map"

getDDLFromFieldSchema

public static String getDDLFromFieldSchema(String structName,
                                           List<FieldSchema> fieldSchemas)
Convert FieldSchemas to Thrift DDL.


getSchema

public static Properties getSchema(Table table)

getSchema

public static Properties getSchema(Partition part,
                                   Table table)

getPartSchemaFromTableSchema

public static Properties getPartSchemaFromTableSchema(StorageDescriptor sd,
                                                      StorageDescriptor tblsd,
                                                      Map<String,String> parameters,
                                                      String databaseName,
                                                      String tableName,
                                                      List<FieldSchema> partitionKeys,
                                                      Properties tblSchema)
Get partition level schema from table level schema. This function will use the same column names, column types and partition keys for each partition Properties. Their values are copied from the table Properties. This is mainly to save CPU and memory. CPU is saved because the first time the StorageDescriptor column names are accessed, JDO needs to execute a SQL query to retrieve the data. If we know the data will be the same as the table level schema and they are immutable, we should just reuse the table level schema objects.

Parameters:
sd - The Partition level Storage Descriptor.
tblsd - The Table level Storage Descriptor.
parameters - partition level parameters
databaseName - DB name
tableName - table name
partitionKeys - partition columns
tblSchema - The table level schema from which this partition should be copied.
Returns:
the properties

getSchema

public static Properties getSchema(StorageDescriptor sd,
                                   StorageDescriptor tblsd,
                                   Map<String,String> parameters,
                                   String databaseName,
                                   String tableName,
                                   List<FieldSchema> partitionKeys)

getColumnNamesFromFieldSchema

public static String getColumnNamesFromFieldSchema(List<FieldSchema> fieldSchemas)
Convert FieldSchemas to columnNames.


getColumnTypesFromFieldSchema

public static String getColumnTypesFromFieldSchema(List<FieldSchema> fieldSchemas)
Convert FieldSchemas to columnTypes.


makeDir

public static void makeDir(org.apache.hadoop.fs.Path path,
                           HiveConf hiveConf)
                    throws MetaException
Throws:
MetaException

startMetaStore

public static void startMetaStore(int port,
                                  HadoopThriftAuthBridge bridge)
                           throws Exception
Throws:
Exception

findFreePort

public static int findFreePort()
                        throws IOException
Finds a free port on the machine.

Returns:
Throws:
IOException

getFieldsFromDeserializer

public static List<FieldSchema> getFieldsFromDeserializer(String tableName,
                                                          Deserializer deserializer)
                                                   throws SerDeException,
                                                          MetaException
Parameters:
tableName -
deserializer -
Returns:
the list of fields
Throws:
SerDeException
MetaException

getFieldSchemaFromTypeInfo

public static FieldSchema getFieldSchemaFromTypeInfo(String fieldName,
                                                     TypeInfo typeInfo)
Convert TypeInfo to FieldSchema.


isExternalTable

public static boolean isExternalTable(Table table)
Determines whether a table is an external table.

Parameters:
table - table of interest
Returns:
true if external

isArchived

public static boolean isArchived(Partition part)

getOriginalLocation

public static org.apache.hadoop.fs.Path getOriginalLocation(Partition part)

isNonNativeTable

public static boolean isNonNativeTable(Table table)

pvalMatches

public static boolean pvalMatches(List<String> partial,
                                  List<String> full)
Returns true if partial has the same values as full for all values that aren't empty in partial.


getIndexTableName

public static String getIndexTableName(String dbName,
                                       String baseTblName,
                                       String indexName)

isIndexTable

public static boolean isIndexTable(Table table)

makeFilterStringFromMap

public static String makeFilterStringFromMap(Map<String,String> m)
Given a map of partition column names to values, this creates a filter string that can be used to call the *byFilter methods

Parameters:
m -
Returns:
the filter string

getClass

public static Class<?> getClass(String rawStoreClassName)
                         throws MetaException
Throws:
MetaException

newInstance

public static <T> T newInstance(Class<T> theClass,
                                Class<?>[] parameterTypes,
                                Object[] initargs)
Create an object of the given class.

Parameters:
theClass -
parameterTypes - an array of parameterTypes for the constructor
initargs - the list of arguments for the constructor


Copyright © 2011 The Apache Software Foundation