这里要实现的案例是学生基本信息的表与成绩表的join操作。与之前案例不同的是,前面的案例都是单个文件作为map端的数据输入,而在此案例中我们要将学生基本信息表和成绩表两个数据文件作为map端的数据输入。
- 准备数据
上传至HDFS:
[root@hadoop01 test_data]# hdfs dfs -mkdir /test_reduce_join_input [root@hadoop01 test_data]# hdfs dfs -put reduce* /test_reduce_join_input
新建project:
- 引入pom依赖
4.0.0 wyh.test mapreduce_reduce_join1.0-SNAPSHOT 8 8 jar org.apache.hadoop hadoop-common2.7.5 org.apache.hadoop hadoop-client2.7.5 org.apache.hadoop hadoop-hdfs2.7.5 org.apache.hadoop hadoop-mapreduce-client-core2.7.5 junit junitRELEASE org.apache.maven.plugins maven-compiler-plugin3.1 1.8 1.8 UTF-8 org.apache.maven.plugins maven-shade-plugin2.4.3 package shade true
- 自定义Mapper
package wyh.test.reduce.join; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import java.io.IOException; public class ReduceJoinMapper extends Mapper{ @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { //获取数据文件的文件对象 FileSplit fileSplit = (FileSplit) context.getInputSplit(); //获取输入文件的文件名 String inputFileName = fileSplit.getPath().getName(); //根据不同的文件名分别做不同的K2,V2处理 if(inputFileName.equals("reduce_join_student_info.txt")){ //数据来自学生基本信息表 String[] splitInfo = value.toString().split(","); String studentId = splitInfo[0]; context.write(new Text(studentId), value); }else{ //数据来自成绩表 String[] splitGrade = value.toString().split(","); String studentId = splitGrade[1]; context.write(new Text(studentId), value); } } }
- 自定义Reducer
package wyh.test.reduce.join; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Reducer; import java.io.IOException; public class ReduceJoinReducer extends Reducer{ @Override protected void reduce(Text key, Iterable values, Reducer .Context context) throws IOException, InterruptedException { String studentInfo = ""; String studentGrade = ""; for (Text student : values) { if(student.toString().startsWith("s")){ //该数据为学生基本信息 studentInfo = student.toString(); }else{ //该数据为成绩 studentGrade = student.toString(); } } String result = studentInfo+" # "+studentGrade; context.write(key, new Text(result)); } }
- 自定义主类
package wyh.test.reduce.join;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class ReduceJoinJobMain extends Configured implements Tool {
@Override
public int run(String[] strings) throws Exception {
Job job = Job.getInstance(super.getConf(), "test_reduce_join");
job.setJarByClass(ReduceJoinJobMain.class);
job.setInputFormatClass(TextInputFormat.class);
TextInputFormat.addInputPath(job, new Path("hdfs://192.168.126.132:8020/test_reduce_join_input"));
job.setMapperClass(ReduceJoinMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setReducerClass(ReduceJoinReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setOutputFormatClass(TextOutputFormat.class);
TextOutputFormat.setOutputPath(job, new Path("hdfs://192.168.126.132:8020/test_reduce_join_output"));
boolean b = job.waitForCompletion(true);
return b?0:1;
}
public static void main(String[] args) throws Exception {
Configuration configuration = new Configuration();
int runStatus = ToolRunner.run(configuration, new ReduceJoinJobMain(), args);
System.exit(runStatus);
}
}
- 打包
- 将jar包上传至服务器并运行
[root@hadoop01 test_jar]# hadoop jar mapreduce_reduce_join-1.0-SNAPSHOT.jar wyh.test.reduce.join.ReduceJoinJobMain
- 查看输出结果
[root@hadoop01 test_jar]# hdfs dfs -cat /test_reduce_join_output/part-r-00000 s002 s002,Tom,boy,11 # c002,s002,96 s003 s003,Alice,girl,12 # c001,s003,84
这样就简单地实现了MapReduce中reduce端join多文件的功能。



