近期因进行项目全链路压测，需要对服务部分接口进行业务日志提取，用来做参数化测试数据。据了解，服务日志存储于elk日志平台中。于是为了避免每次手工提取费时费事，且每次不能批量提取大量日志，决定使用脚本实现自动提取，并保存至特定目录文件中。
前提：
- elk日志平台地址
- 查询索引名
- 查询条件
以下为具体java代码，供参考：
ImSearchExtractData类
package com.Fulllink.imsearch;
import com.Fulllink.utils.HttpGetTest;
import com.Fulllink.utils.IsOverdueByAt;
import com.Fulllink.utils.WriteFile;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.UnsupportedEncodingException;
public class ImSearchExtractData {
Logger logger = LoggerFactory.getLogger(ImSearchExtractData.class);
HttpGetTest httpGetTest = new HttpGetTest();
IsOverdueByAt isOverdueByAt = new IsOverdueByAt();
WriteFile writeFile = new WriteFile();
public Boolean metaDataSource(String filePath,String index,String queryCondition,int expctLogNum,String methodName){
//int actualLogNum = 0; //实际的提取的条数
final int indexStart = 1; //起始查询的索引(常量)
final int size = 100; //每次请求接口查询100条数据(常量)
int fetchCount = expctLogNum/size;//访问ELK接口次数,用于计算实际查询的条数
File fileName = new File(filePath);
String indexStr = index.trim();
int nums = 0;
String queryConditionStr = queryCondition;
String thrift_params = null;
String thrift_at = null;
Long atTime = null;
String accessTokenEndTime = null;
JSonObject jsonObjectAt = null;
if(indexStr!=null && indexStr!="" && queryConditionStr !=null && queryConditionStr!="" && expctLogNum >0 ){
for(int m = 0; m < fetchCount; m++){
//组装URL请求串
String url = "http://xx-xx-elk.zpidc.com/" + index;
String passportUrl ="http://p-xx-xx.zpidc.com/passport-service/userService/getUserTokenByAt?at=";
String param = queryConditionStr + "&from=" + indexStart*m*size + "&size=" + size + "&sort=@timestamp:desc";
try {
JSonObject jsonObject = httpGetTest.doGet(url,param);//调用提取数据url-每次取100条
if(jsonObject.getJSonObject("hits")!=null && jsonObject.getJSonObject("hits").getJSonArray("hits")!=null) {
JSonArray jsonArray = jsonObject.getJSonObject("hits").getJSonArray("hits");
StringBuffer sb = new StringBuffer();
for (int i = 0; i < jsonArray.size(); i++) {
if (jsonArray.getJSonObject(i).getJSonObject("_source").size() >= 1) {
if(methodName.equals("getTags")){
thrift_params = jsonArray.getJSonObject(i).getJSonObject("_source").getString("thrift.Params");
sb.append(""" + thrift_params + """+ "rn");//post请求 换行
nums++;
}else {
thrift_at = jsonArray.getJSonObject(i).getJSonObject("_source").getString("thrift.Headers_.x-zp-at");
thrift_params = jsonArray.getJSonObject(i).getJSonObject("_source").getString("thrift.Body");
jsonObjectAt = httpGetTest.doGet(passportUrl, thrift_at);
if (jsonObjectAt.getJSonObject("data") != null) {
try {
accessTokenEndTime = jsonObjectAt.getJSonObject("data").getString("accessTokenEndTime");
if (accessTokenEndTime != null) {
atTime = Long.parseLong(accessTokenEndTime);
if (atTime != null && isOverdueByAt.validateByAt(atTime)) {//token失效日期大于2天
thrift_params = StringUtils.replace(thrift_params, """, """");
sb.append(""" + thrift_params + """ + "," + thrift_at + "rn");//post请求 换行
nums++;
}
}
} catch (NumberFormatException e) {
e.printStackTrace();
}
}
}
}
}
writeFile.writeFile(sb.toString(),fileName);//每100条写入一次文件
sb.delete(0,sb.length());
System.out.println(methodName+"接口写入记录:"+ nums+"条！");
}
//logger.info("*******************写入记录{%d}条！",fetchCount * size);
} catch (Exception e) {
logger.error("写入文件异常!");
e.printStackTrace();
return false;
}
}
}
return true;
}
public static void main(String [] args){
ImSearchExtractData extractData = new ImSearchExtractData();
//查询条件-接口名称
String [] methodNames ={"getSessionById","getTags","listUsableSession","listStaffSessionV2","countStaffSessionV2"};
String index = null;
String queryConditionHead = null;
String query = null;
for (int t = 0; t
HttpGetTest类
package com.Fulllink.utils;
import com.alibaba.fastjson.JSONObject;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
public class HttpGetTest {
Logger logger = LoggerFactory.getLogger(HttpGetTest.class);
public JSonObject doGet(String url, String param) throws Exception{
CloseableHttpClient httpClient = null;
CloseableHttpResponse httpResponse =null;
String strResult = "";
try {
httpClient = HttpClients.createDefault();//创建一个httpClient实例
org.apache.http.client.methods.HttpGet httpGet =new org.apache.http.client.methods.HttpGet(url+param);//创建httpGet远程连接实例
// 设置请求头信息
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(60000).//连接主机服务器时间
setConnectionRequestTimeout(60000).//请求超时时间
setSocketTimeout(60000).//数据读取超时时间
build();
httpGet.setConfig(requestConfig);//为httpGet实例设置配置信息
httpResponse = httpClient.execute(httpGet);//通过get请求得到返回对象
//发送请求成功并得到响应
if(httpResponse.getStatusLine().getStatusCode()==200){
strResult = EntityUtils.toString(httpResponse.getEntity());
JSonObject resultJsonObject = JSONObject.parseObject(strResult);//获取请求返回结果
return resultJsonObject;
}else{
logger.error("请求{%s}失败,状态码为{%d}",url,httpResponse.getStatusLine().getStatusCode());
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException i){
i.printStackTrace();
logger.error("IOException异常：{%s}",i.getMessage());
} finally {
if(httpResponse!=null){
try {
httpResponse.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(httpClient!=null){
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return null;
}
}
WriteFile 类
package com.Fulllink.utils;
import java.io.*;
public class WriteFile {
public void writeFile(String str, File fileName){
if((str!=null || !str.trim().equals("")) && fileName!=null ) {
try {
FileOutputStream fos = new FileOutputStream(fileName,true);
Writer w = new OutputStreamWriter(fos,"UTF-8");
try {
str = str.replaceAll("%(?![0-9a-fA-F]{2})", "%25");
str = str.replaceAll("\+", "%2B");
w.write(java.net.URLDecoder.decode(str, "UTF-8"));
w.flush();
w.close();
fos.close();
} catch (IOException e) {
e.printStackTrace();
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
}
}
IsOverdueByAt 类
package com.Fulllink.utils;
public class IsOverdueByAt {
public Boolean validateByAt(Long at){
if(at!=null){
long days = 0;
Long todayTs = System.currentTimeMillis(); // 当前时间戳
days = (at-todayTs)/(24*60*60*1000);
if(days>=2){
return true;
}else return false;
}
return null;
}
}
最后生成参数化文件
