Hadoop集成Spring的使用详细教程(快速入门大数据)

https://spring.io/projects/spring-hadoop

添加依赖

 
  org.springframework.data
  spring-data-hadoop
  2.5.0.RELEASE
 

新建资源文件beans.xml

 
  fs.defaultFS=hdfs://hadoop01:9000
  hadoop.tmp.dir=/tmp/hadoop
  electric=sea
 

 

测试文件

package com.bennyrhys.hadoop.spring;

import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import java.io.IOException;


public class SpringHadoopHDFSApp {
 private ApplicationContext ctx;
 // apache hadoop
 private FileSystem fileSystem;

 
 @Test
 public void testMkdirs() throws Exception {
  fileSystem.mkdirs(new Path("/springhdfs"));
 }

 
 @Test
 public void testText() throws Exception {
  FSDataInputStream in = fileSystem.open(new Path("/springhdfs/hello.txt"));
  IOUtils.copyBytes(in, System.out, 1024);
  in.close();
 }

 @Before
 public void setUp() {
  ctx = new ClassPathXmlApplicationContext("beans.xml");
  fileSystem = (FileSystem) ctx.getBean("fileSystem");
 }

 @After
 public void tearDown() throws Exception {
  ctx = null;
  fileSystem.close();
 }
}

提取变量

使用xml中的头文件替换bean，使其允许使用上下文
${}导入变量

新建配置文件application.properties

spring.hadoop.fsUri=hdfs://hadoop01:9000

获取context上下文引入变量
beans.xml

 
  fs.defaultFS=${spring.hadoop.fsUri}
  hadoop.tmp.dir=/tmp/hadoop
  electric=sea
 
 

 

pom.xml

 
  org.springframework.data
  spring-data-hadoop-boot
  2.5.0.RELEASE
 

SpringBootHDFSApp

package com.bennyrhys.hadoop.spring;

import org.apache.hadoop.fs.FileStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.data.hadoop.fs.FsShell;


@SpringBootApplication
public class SpringBootHDFSApp implements CommandLineRunner {

 @Autowired
 FsShell fsShell; //引入spring的

 @Override
 public void run(String... strings) throws Exception {
  for (FileStatus fileStatus : fsShell.lsr("/springhdfs")) {
   System.out.println("> " + fileStatus.getPath());
  }
 }

 
 public static void main(String[] args) {
  SpringApplication.run(SpringBootHDFSApp.class, args);
 }
}

到此这篇关于Hadoop集成Spring的使用详细教程(快速入门大数据)的文章就介绍到这了,更多相关Hadoop集成Spring的使用内容请搜索考高分网以前的文章或继续浏览下面的相关文章希望大家以后多多支持考高分网！