// CSC 369: Distributed Computing
// Alex Dekhtyar

//  Two Hadoop Jobs Chained together
//  Find which words appear more often than the average

// Section 1: Imports


                  // Data containers for Map() and Reduce() functions

                  // You would import the data types needed for your keys and values
import org.apache.hadoop.io.IntWritable; // Hadoop's serialized int wrapper class
import org.apache.hadoop.io.LongWritable; // Hadoop's serialized int wrapper class
import org.apache.hadoop.io.Text;        // Hadoop's serialized String wrapper class


                 // For Map and Reduce jobs

import org.apache.hadoop.mapreduce.Mapper; // Mapper class to be extended by our Map function
import org.apache.hadoop.mapreduce.Reducer; // Reducer class to be extended by our Reduce function

                 // To start the MapReduce process

import org.apache.hadoop.mapreduce.Job; // the MapReduce job class that is used a the driver


                // For File "I/O"

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; // class for "pointing" at input file(s)
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; // class for "pointing" at output file
import org.apache.hadoop.fs.Path;                // Hadoop's implementation of directory path/filename


// Exception handling

import java.io.IOException;


public class filter {


// Mapper  Class Template


public static class counterMapper     // Need to replace the four type labels there with actual Java class names
     extends Mapper< LongWritable, Text, Text, LongWritable > {

// start counting word occurrences


public void map(LongWritable key, Text value, Context context)
      throws IOException, InterruptedException {


   context.write(value, new LongWritable(1));

 } // map

} //counterMapper

//  Reducer Class Template

public static class counterReducer   // needs to replace the four type labels with actual Java class names
      extends  Reducer< Text, LongWritable, Text, LongWritable> {

 // note: InValueType is a type of a single value Reducer will work with
 //       the parameter to reduce() method will be Iterable<InValueType> - i.e. a list of these values

@Override  // we are overriding the Reducer's reduce() method


public void reduce( Text key, Iterable<LongWritable> values, Context context)
     throws IOException, InterruptedException {

// output the word with the number of its occurrences

  long sum = 0; 

 for(LongWritable one : values) {
   sum = sum+ one.get();
 }
 
 context.write(key, new LongWritable(sum));

 } // reduce


} // reducer




public static class totalCountMapper     // Need to replace the four type labels there with actual Java class names
     extends Mapper< LongWritable, Text, LongWritable, LongWritable > {

// start counting word occurrences


public void map(LongWritable key, Text value, Context context)
      throws IOException, InterruptedException {


   context.write(new LongWritable(1), new LongWritable(1));


 } // map



} // totalCountMapper

//  Reducer Class Template

public static class totalCountReducer   // needs to replace the four type labels with actual Java class names
      extends  Reducer< LongWritable, LongWritable, LongWritable, LongWritable> {

 // note: InValueType is a type of a single value Reducer will work with
 //       the parameter to reduce() method will be Iterable<InValueType> - i.e. a list of these values

@Override  // we are overriding the Reducer's reduce() method


public void reduce( LongWritable key, Iterable<LongWritable> values, Context context)
     throws IOException, InterruptedException {

// output the word with the number of its occurrences

  long sum = 0; 

 for(LongWritable one : values) {
   sum = sum+ one.get();
 }
 
 context.write(key, new LongWritable(sum));

 } // reduce


} // reducer



//  MapReduce Driver

public static void main(String[] args) throws Exception {

     // step 1: get a new MapReduce Job object
     Job  job = Job.getInstance();  //  job = new Job() is now deprecated
     
    // step 2: register the MapReduce class
      job.setJarByClass(filter.class);  

   //  step 3:  Set Input and Output files
       FileInputFormat.addInputPath(job, new Path("./test/", "words")); // put what you need as input file
       FileOutputFormat.setOutputPath(job, new Path("./test/","counts")); // put what you need as output file

   // step 4:  Register mapper and reducer
      job.setMapperClass(counterMapper.class);
      job.setReducerClass(counterReducer.class);
  
   //  step 5: Set up output information
       job.setOutputKeyClass(Text.class); // specify the output class (what reduce() emits) for key
       job.setOutputValueClass(LongWritable.class); // specify the output class (what reduce() emits) for value

   // step 6: Set up other job parameters at will
      job.setJobName("Chains");

   // step 7:  ?

   // step 8: profit
      job.waitForCompletion(true);

   Job countAllJob = Job.getInstance();
   countAllJob.setJarByClass(filter.class);
   FileInputFormat.addInputPath(countAllJob, new Path("./test/counts", "part-r-00000")); // put what you need as input file
   FileOutputFormat.setOutputPath(countAllJob, new Path("./test/","totals")); // put what you need as output file
   countAllJob.setMapperClass(totalCountMapper.class);
   countAllJob.setReducerClass(totalCountReducer.class);
   countAllJob.setOutputKeyClass(LongWritable.class);
   countAllJob.setOutputValueClass(LongWritable.class);
   countAllJob.setJobName("Count em All!");
    
   System.exit(countAllJob.waitForCompletion(true) ? 0: 1);
   

  } // main()


} // MyMapReduceDriver