es的使用:
安装:docker pull elasticsearch:7.12.1
创建目录:
mkdir /docker/es
mkdir /docker/es/conf
mkdir /docker/es/data
mkdir /docker/es/plugins
创建文件:
touch /docker/es/conf/elasticsearch.yml
配置文件：
cluster.name: my-application #集群名称
node.name: node-1 #节点名称
#数据和日志的存储目录
path.data: /usr/share/elasticsearch/data
path.logs: /usr/share/elasticsearch/logs
##设置绑定的ip，设置为0.0.0.0以后就可以让任何计算机节点访问到了
network.host: 0.0.0.0
http.port: 9200 #端口
##设置在集群中的所有节点名称，这个节点名称就是之前所修改的，当然你也可以采用默认的也行，目前 是单机，放入一个节点即可
cluster.initial_master_nodes: ["node-1"]
启动镜像:
docker run -p 9200:9200 -d --name es -e ES_JAVA_OPTS="-Xms512m -Xmx512m" -v /docker/es/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /docker/es/data:/usr/share/elasticsearch/data -v /docker/es/plugins:/usr/share/elasticsearch/plugins --privileged=true elasticsearch:7.12.1
报错处理:
出现异常：max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
解决：
- 修改配置sysctl.conf vi /etc/sysctl.conf
- 在尾行添加以下内容 vm.max_map_count=655300
- 执行命令 sysctl -p
kibana的安装:(管理es的工具)
docker pull kibana:7.12.1
创建文件:
mkidr /docker/kibana
mkdir /docker/kibana/conf
touch /docker/kibana/conf/kibana.yml
配置文件:
server.name: kibana
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://你的es地址:9200"] xpack.monitoring.ui.container.elasticsearch.enabled: true
启动:
docker run -p 5601:5601 -d --name -v /docker/kibana/conf/kibana.yml:/usr/share/kibana/config/kibana.yml --privileged=true kibana:7.12.1
配置中文分词：
解压安装包：放到定义的plugins目录中的ik文件夹中
mysql的binlog日志必须是row模式的
go-mysql-es：（mysql与es同步的工具）
安装:
docker pull gozer/go-mysql-elasticsearch；
创建go-mysql-river.toml 配置文件
my_addr = ""
my_user = ""
my_pass = ""
my_charset = "utf8"
enable-relay = true
es_addr = "172.17.0.2:9200"
es_user = ""
es_pass = ""
data_dir = "/docker/data"
stat_addr = "127.0.0.1:12800"
stat_path = "/metrics"
server_id = 1001
flavor = "mysql"
mysqldump = "mysqldump"
#skip_master_data = false
bulk_size = 128
flush_bulk_time = "200ms"
skip_no_pk_table = false
[[source]]
schema = "yjh_ydzykj_com"
tables = ["fa_posts"]
[[rule]]
schema = "yjh_ydzykj_com"
table = "fa_posts"
index = "posts"
type = "_doc"
filter = ["id", "user_id","circle_id","cat_id","content","images","label","is_top","status","zan_num","shoucan_num","pinglun_num","zhaunfa_num","createtime"]
[rule.filed]
mysql = "id"
elastic = "poid"
启动:
docker run -p 12345:12345 -d --name go-mysql-es -v /root/docker-go-mysql-es/go-mysql-river.toml:/config/river.toml:ro --privileged=true gozer/go-mysql-elasticsearch
logstash增量同步mysql到es配置详解
docker 镜像安装
docker pull logstash:7.12.1
创建文件在root目录里 logstash
toch logstash.conf
下载与mysql对应的mysql-connector-java-5.1.49.jar
docker run -p 9900:9900 -d --name logstash -v /root/logstash:/etc/logstash/pipeline --privileged=true logstash:7.12.1
进入容器安装插件
docker exec -it logstash bash
cd bin目录
logstash-plugin install logstash-input-jdbc
logstash-plugin install logstash-output-elasticsearch
修改 logstash.yml
cd /usr/share/logstash/config
配置 es的ip地址
修改 pipelines.yml中的config /etc/logstash/pipeline/logstash.conf
- 其中分词有两种设置方法，ik_max_word和ik_smart，他们的区别如下，可以根据自己项目的情况进行选择：ik_max_word: 会将文本做最细粒度的拆分，比如会将“中华人民共和国国歌”拆分为“中华人民共和国,中华人民,中华,华人,人民共和国,人民,人,民,共和国,共和,和,国国,国歌”，会穷尽各种可能的组合；ik_smart: 会做最粗粒度的拆分，比如会将“中华人民共和国国歌”拆分为“中华人民共和国,国歌”。
- tp5 先引入扩展composer require elasticsearch/elasticsearch
- 链接类代码
-
setHost(); $this->_client = ClientBuilder::create()->setHosts($this->_host)->build(); } private function setHost() { //实例化一个客户端 $cfg = Config::get('eshosts'); $this->_host = [ $cfg['scheme'] . '://' . $cfg['user'] . ':' . $cfg['pass'] . '@' . $cfg['host'] . ':' . $cfg['port'] ]; } public function getClient() { return $this->_client; } public function delDoc($index, $id) { $chk = $this->checkParams($index, $id, ['def_key' => 'only_for_delete_doc']); //参数3无实际意义,仅用过能通过验证而已 if ($chk === false) { return false; } try { $data = [ 'index' => $index, 'id' => $id, 'type'=>'_doc' ]; $this->_client->delete($data); } catch (Exception $e) { $this->error = $e->getMessage(); return false; } return true; } private function checkParams($index, $id, $body) { $result = true; foreach ($body as $key => $value) { if (!isset($key) || empty($key)) { $result = false; break; } } if ($result === false) { $this->error = '索引文档的主体内容输入不正确'; return false; } return true; } public function getError() { return $this->error; } } -
搜索类代码
-
'posts',//索引 'type' => '_doc',//类型 'body' => [ "query" => [ "bool" => [ 'filter' => [], 'must' => [] ] ] ] ]; public function __construct($index) { $this->params["index"] = $index; return $this; } public function source($fields) { $fields = is_array($fields)?$fields:[$fields]; $this->params["body"]["_source"] = $fields; return $this; } public function paginate($size,$page) { $this->params['body']['from'] = ($page - 1) * $size; $this->params['body']['size'] = $size; return $this; } public function IsStatus() { $this->params['body']['query']['bool']['filter'][] = ['term' => ['status' => true]]; return $this; } public function province($province) { $this->params['body']['query']['bool']['filter'][] = ['term' => ['province' => $province]]; return $this; } public function Isnotnull() { $this->params['body']['query']['bool']['must_not'][] = ['exists' => ['field' => 'deletetime']]; return $this; } public function Isshield($ids) { $this->params['body']['query']['bool']['must_not'][] = ['terms' => ['user_id' => $ids]]; return $this; } public function shield($ids) { $this->params['body']['query']['bool']['must'][] = ['terms' => ['user_id' => $ids]]; return $this; } public function getlists($ids) { $this->params['body']['query']['bool']['must'][] = ['terms' => ['id' => $ids]]; return $this; } public function follow($ids) { $this->params['body']['query']['bool']['filter'][] = ['terms' => ['user_id' => $ids]]; return $this; } public function circle($circle_id) { $this->params['body']['query']['bool']['filter'][] = ['term' => ['circle_id' => $circle_id]]; return $this; } public function category($category) { $this->params['body']['query']['bool']['filter'][] = ['term' => ['cat_id' => $category]]; return $this; } public function myposts($user_id) { $this->params['body']['query']['bool']['filter'][] = ['term' => ['user_id' => $user_id]]; return $this; } public function keywords($keywords) { //如果不是数组需要转为数组 $keywords = is_array($keywords) ? $keywords : [$keywords]; foreach ($keywords as $keyword){ $this->params['body']['query']['bool']['must'][] = [ 'multi_match' => [ 'query' => $keyword, 'fields' => [ 'content^3', ] ] ]; } return $this; } public function orderBy($filed,$lon,$lat) { if (!isset($this->params['body']['sort'])){ $this->params['body']['sort'] = []; } $this->params['body']['sort'][] = ['is_top' => 'desc','pinglun_num'=>'desc']; if($filed=='location'){ $this->params['body']['sort'][] = ['_geo_distance' =>[$filed=>['lat'=>$lat,'lon'=>$lon],'order'=>'asc','unit'=>'km']]; }elseif($filed=='createtime'){ $this->params['body']['sort'][] = [$filed => 'desc']; } return $this; } public function getParams() { return $this->params; } //查询 public function search() { $searchParams['index'] = 'posts'; $searchParams['type'] = '_doc'; $searchParams['from'] = 0; $searchParams['size'] = 100; $searchParams['body']['query']['match']['content'] = 'ces'; $retDoc = $this->_client->search($searchParams); return $retDoc; } public function getError() { return $this->error; } } -
控制器中使用
public function posts_list() { $ElasticClient=new ElasticClient(); $page = $this->request->get('page',1); $perPage = 10; $builder = (new ElasticSearch("posts"))->IsStatus()->Isnotnull()->Isshield($this->getshield($this->userinfo->id))->circle($this->request->get('circle'))->paginate($perPage,$page); //分类搜索 if ($category=$this->request->get('category_id')){ $builder->category($category); } //城市选择搜搜 if ($province=$this->request->get('province')){ $builder->province($province); } //关键词搜索 if ($search = $this->request->get('search','')){ $keywords = array_filter(explode(' ',$search)); $builder->keywords($keywords); } //关注数据搜索 if ($follow= $this->request->get('follow')){ $builder->follow($this->getfollow($this->userinfo->id)); } //最新、离我最近的数据搜索 if ($order = $this->request->get('order','pinglun_num')){ if (in_array($order,['createtime','location','is_top','pinglun_num'])){ $builder->orderBy($order,$this->request->get('lon'),$this->request->get('lat')); } } $data['list']=$ElasticClient->getClient()->search($builder->getParams())['hits']['hits']; $data['followids']=$this->getfollow($this->userinfo->id); $data['zanids']=appadminmodelpostsZan::where(['user_id'=>$this->userinfo->id])->column('posts_id'); $data['collectionids']=appadminmodelpostsCollection::where(['user_id'=>$this->userinfo->id])->column('posts_id'); $this->success($data); } /**
