public final class SparkEvaluator extends Evaluator implements Serializable
Modifier and Type | Field and Description |
---|---|
(package private) static org.apache.spark.TaskContext |
context |
(package private) static String |
core_jar |
(package private) static String |
data_source_dir_name |
(package private) static String |
data_source_directory |
(package private) static org.apache.spark.broadcast.Broadcast<String> |
data_source_directory_b |
(package private) static Bag |
empty_bag |
(package private) static String |
gen_jar |
(package private) static String |
scanner_jar |
static org.apache.spark.api.java.JavaSparkContext |
spark_context |
(package private) static String |
spark_jar |
counter_container, counter_key, evaluator, value_container
coerce_method, false_name, false_value, global_env, identity_mapper, is_dataset, null_name, null_value, query_plan, query_type, tab_count, true_name, true_value
algebraic_operators, data_constructors, functions, global_datatype_env, global_functions, global_macros, global_type_env, global_vars, identity, impure_functions, monoids, plan_names, plans_with_distributed_lambdas, repeat_variables, st, type_env, type_names
Constructor and Description |
---|
SparkEvaluator() |
Modifier and Type | Method and Description |
---|---|
MRData |
aggregate(Tree acc_fnc,
Tree zero,
Tree plan,
Environment env)
The Aggregate physical operator
|
static Bag |
bag(org.apache.spark.api.java.JavaRDD<MRData> rdd)
Convert a Spark RDD into a lazy bag
|
Class<? extends MRQLFileInputFormat> |
binaryInputFormat()
return the FileInputFormat for binary files
|
static void |
dump_source_dir()
used by the master to send parsing details (eg, record types) to workers
|
void |
dump_text(String file,
Tree type,
MRData data)
dump MRQL data into text a CVS file
|
void |
dump(String file,
Tree type,
MRData data)
dump MRQL data into a sequence file
|
org.apache.spark.api.java.JavaRDD<MRData> |
eval(Tree e,
Environment env)
Evaluate an MRQL physical plan using Spark and print tracing info
|
DataSet |
eval(Tree e,
Environment env,
String counter)
Evaluate an MRQL physical plan using Spark
|
org.apache.spark.api.java.JavaRDD<MRData> |
evalD(Tree e,
Environment env)
Evaluate MRQL physical operators using Spark
|
Class<? extends MRQLFileInputFormat> |
generatorInputFormat()
return the FileInputFormat for data generator files
|
void |
init(org.apache.hadoop.conf.Configuration conf)
initialize the Spark evaluator
|
void |
initialize_query() |
static void |
load_source_dir()
executed by a worker when reading parsed input (see SparkParsedInputFormat)
|
Tuple |
loop(Tree e,
Environment env)
Evaluate a loop a fixed # of times
|
org.apache.hadoop.conf.Configuration |
new_configuration() |
Class<? extends MRQLFileInputFormat> |
parsedInputFormat()
return the FileInputFormat for parsed files (CSV, XML, JSON, etc)
|
void |
shutdown(org.apache.hadoop.conf.Configuration conf)
shutdown the Spark evaluator
|
, bsp, distribute, synchronize
closure, evalE, evalE, evalF, evalM, evalMM, evalS, lookup_distributed_binding, lookup_global_binding, new_distributed_binding, new_global_binding, remove_global_binding, set_global_bindings, tabs, translate_expression, variable_lookup
compare_types, equal_types, expand, make_persistent_type, normalize_type, subtype_unify, subtype, subtype, subtype, type_error, type_inference, type_inference2, unify, unify
alpha_equivalent, alpha_equivalent, collection_type, error, free_variables, global_reset, identity, is_collection, is_persistent_collection, is_pure, new_var, occurences, pattern_variables, persistent_collection, print_aggregates, reify, reify, reset, subst_list, subst_var, subst, template, top_level, transient_collection, translate_select, union
print_plan, print_query, print_type, print, reify
public static org.apache.spark.api.java.JavaSparkContext spark_context
static final String spark_jar
static final String core_jar
static final String gen_jar
static final String scanner_jar
static String data_source_directory
static final Bag empty_bag
static org.apache.spark.broadcast.Broadcast<String> data_source_directory_b
static final String data_source_dir_name
static final org.apache.spark.TaskContext context
public final void init(org.apache.hadoop.conf.Configuration conf)
public final void shutdown(org.apache.hadoop.conf.Configuration conf)
public final void initialize_query()
initialize_query
in class Evaluator
public final org.apache.hadoop.conf.Configuration new_configuration()
new_configuration
in class Evaluator
public final Class<? extends MRQLFileInputFormat> parsedInputFormat()
parsedInputFormat
in class Evaluator
public final Class<? extends MRQLFileInputFormat> binaryInputFormat()
binaryInputFormat
in class Evaluator
public final Class<? extends MRQLFileInputFormat> generatorInputFormat()
generatorInputFormat
in class Evaluator
public static void dump_source_dir() throws IOException
IOException
public static void load_source_dir() throws IOException
IOException
public final void dump(String file, Tree type, MRData data) throws Exception
public final void dump_text(String file, Tree type, MRData data) throws Exception
public final MRData aggregate(Tree acc_fnc, Tree zero, Tree plan, Environment env) throws Exception
aggregate
in class Evaluator
acc_fnc
- the accumulator function from (T,T) to Tzero
- the zero element of type Tplan
- the plan that constructs the dataset that contains the bag of values {T}env
- contains bindings fro variables to values (MRData)Exception
public final Tuple loop(Tree e, Environment env) throws Exception
public static Bag bag(org.apache.spark.api.java.JavaRDD<MRData> rdd)
rdd
- the Spark RDDpublic final DataSet eval(Tree e, Environment env, String counter)
public final org.apache.spark.api.java.JavaRDD<MRData> eval(Tree e, Environment env)
e
- the physical planenv
- contains bindings from variables to values (MRData)public final org.apache.spark.api.java.JavaRDD<MRData> evalD(Tree e, Environment env)
e
- the physical planenv
- contains bindings fro variables to values (MRData)Copyright © 2013–2014 The Apache Software Foundation. All rights reserved.