2021-11-05

WordCount的实现

map端

public static class WordCountMapper extends Mapper{

        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

            String[] words = value.toString().split(" ");

            for (String word:words) {

                context.write(new Text(word),new IntWritable(1));

            }

        }

    }

reduce端

public static class WordCountReducer extends Reducer{

        protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {

            int sum = 0;

            for (IntWritable value:values) {

                sum+=value;

            }

            context.write(key,new IntWritable(sum));

        }

    }

job端

public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

        Configuration conf = new Configuration();

Job job = Job.getInstance(conf);

        job.setJarByClass(WordCountDemo.class);

        job.setMapperClass(WordCountMapper.class);

        job.setReducerClass(WordCountReducer.class);

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(IntWritable.class);

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(IntWritable.class);

        FileSystem fs = FileSystem.get(conf);

        Path path = new Path("D:\hadoop\mapreduce\wordcount\output");

        if (fs.exists(path)){

          fs.delete(path,true)；}FileInputFormat.setInputPaths(job,new  Path("D:\hadoop\mapreduce\wordcount\input\test.txt"));

        FileOutputFormat.setOutputPath(job,path);

        boolean isDone = job.waitForCompletion(true);

        System.exit(isDone ? 0 : 1);

    }