Elastic Stack也被称为ELK Stack，干货其能够安全可靠地获取任何来源、基于践任何格式的志分数据，然后实时地对数据进行搜索、析系分析和可视化。统实上一篇文章为大家简单的干货介绍了一下Elastic Stack以及ElasticSearch 和 Lucene 的关系等内容，接下来为大家介绍一下ElasticSearch 单点部署和ElasticStack 分布式集群部署等内容，基于践希望对你们有用。志分
推荐阅读：干货 | 基于Elastic Stack的析系日志分析系统实践
ElasticSearch单点部署
下载指定的ES版本
# 详细步骤见视频。 # 参考链接:https://www.elastic.co/cn/downloads/elasticsearch部署JDK环境-可选步骤
单点部署elasticsearch
# 安装服务 yum -y localinstal elasticsearch-7.17.3-x86_64.rpm # 修改配置文件 egrep -v "^#|^$" /etc/elasticsearch/elasticsearch.yml cluster.name: oldboyedu-elk node.name: oldboyedu-elk103 path.data: /var/lib/elasticsearch path.logs: /var/log/elasticsearch network.host: 10.0.0.103 discovery.seed_hosts: ["10.0.0.103"] # 启动服务 systemctl start elasticsearch.serviceOpenJDK切换Oracle JDK并修改堆内存大小
# 修改es的统实环境变量配置文件 vim /etc/sysconfig/elasticsearch ... ES_JAVA_HOME=/oldboyedu/softwares/jdk # 修改堆内存大小 vim /etc/elasticsearch/jvm.options ... -Xms256m -Xmx256m # 验证堆内存大小 jmap -heap `ps -ef | grep java | grep -v grep | awk {print $2}` # 同步配置文件到其他节点 data_rsync.sh /etc/sysconfig/elasticsearch data_rsync.sh /etc/elasticsearch/jvm.optionsElasticStack分布式集群部署
elk101修改配置文件
egrep -v "^$|^#" /etc/elasticsearch/elasticsearch.yml ... cluster.name: oldboyedu-elk node.name: elk101 path.data: /var/lib/elasticsearch path.logs: /var/log/elasticsearch network.host: 0.0.0.0 discovery.seed_hosts: ["elk101","elk102","elk103"] cluster.initial_master_nodes: ["elk101","elk102","elk103"] # 温馨提示:"node.name"各个节点配置要区分清楚，建议写对应的干货主机名称。b2b信息网同步配置文件到集群的基于践其他节点
# elk101同步配置文件到集群的其他节点 data_rsync.sh /etc/elasticsearch/elasticsearch.yml # elk102节点配置 vim /etc/elasticsearch/elasticsearch.yml ... node.name: elk102 # elk103节点配置 vim /etc/elasticsearch/elasticsearch.yml ... node.name: elk103所有节点删除之前的临时数据
pkill java rm -rf /var/{lib,log}/elasticsearch/* /tmp/* ll /var/{lib,log}/elasticsearch/ /tmp/所有节点启动服务
# 所有节点启动服务 systemctl start elasticsearch # 启动过程中建议查看日志 tail -100f /var/log/elasticsearch/oldboyedu-elk.log验证集群是否正常
curl elk103:9200/_cat/nodes?v
部署kibana服务
本地安装kibana
yum -y localinstall kibana-7.17.3-x86_64.rpm修改kibana的配置文件
vim /etc/kibana/kibana.yml ... server.host: "10.0.0.101" server.name: "oldboyedu-kibana-server" elasticsearch.hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] i18n.locale: "zh-CN"启动kibana服务
systemctl enable --now kibana systemctl status kibanafilebeat部署及基础使用
部署filebeat环境
yum -y localinstall filebeat-7.17.3-x86_64.rpm # 温馨提示:elk102节点操作。修改filebeat的志分配置文件
# 编写测试的配置文件 mkdir /etc/filebeat/config cat > /etc/filebeat/config/01-stdin-to-console.yml <<EOF # 指定输入的类型 filebeat.inputs: # 指定输入的类型为"stdin",表示标准输入 - type: stdin # 指定输出的类型 output.console: # 打印漂亮的格式 pretty: true EOF # 运行filebeat实例 filebeat -e -c /etc/filebeat/config/01-stdin-to-console.yml # 测试#见视频。input的析系log类型
filebeat.inputs: - type: log paths: - /tmp/test.log output.console: pretty: trueinput的通配符案例
filebeat.inputs: - type: log paths: - /tmp/test.log - /tmp/*.txt output.console: pretty: trueinput的通用字段案例
filebeat.inputs: - type: log # 是站群服务器否启动当前的输入类型，默认值为true enabled: true # 指定数据路径 paths: - /tmp/test.log - /tmp/*.txt # 给当前的统实输入类型搭上标签 tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"] # 自定义字段 fields: school: "北京昌平区沙河镇" class: "linux80" - type: log enabled: true paths: - /tmp/test/*/*.log tags: ["oldboyedu-python","云原生开发"] fields: name: "oldboy" hobby: "linux,抖音" # 将自定义字段的key-value放到顶级字段. # 默认值为false，会将数据放在一个叫"fields"字段的下面. fields_under_root: true output.console: pretty: true将数据写入es案例
filebeat.inputs: - type: log enabled: true paths: - /tmp/test.log - /tmp/*.txt tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"] fields: school: "北京昌平区沙河镇" class: "linux80" - type: log enabled: true paths: - /tmp/test/*/*.log tags: ["oldboyedu-python","云原生开发"] fields: name: "oldboy" hobby: "linux,抖音" fields_under_root: true output.elasticsearch: hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"]自定义es索引名称
filebeat.inputs: - type: log enabled: true paths: - /tmp/test.log - /tmp/*.txt tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"] fields: school: "北京昌平区沙河镇" class: "linux80" - type: log enabled: true paths: - /tmp/test/*/*.log tags: ["oldboyedu-python","云原生开发"] fields: name: "oldboy" hobby: "linux,抖音" fields_under_root: true output.elasticsearch: enabled: true hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] index: "oldboyedu-linux-elk-%{+yyyy.MM.dd}" # 禁用索引生命周期管理 setup.ilm.enabled: false # 设置索引模板的名称 setup.template.name: "oldboyedu-linux" # 设置索引模板的匹配模式 setup.template.pattern: "oldboyedu-linux*"多个索引写入案例
filebeat.inputs: - type: log enabled: true paths: - /tmp/test.log - /tmp/*.txt tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"] fields: school: "北京昌平区沙河镇" class: "linux80" - type: log enabled: true paths: - /tmp/test/*/*.log tags: ["oldboyedu-python","云原生开发"] fields: name: "oldboy" hobby: "linux,抖音" fields_under_root: true output.elasticsearch: enabled: true hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] # index: "oldboyedu-linux-elk-%{+yyyy.MM.dd}" indices: - index: "oldboyedu-linux-elk-%{+yyyy.MM.dd}" # 匹配指定字段包含的内容 when.contains: tags: "oldboyedu-linux80" - index: "oldboyedu-linux-python-%{+yyyy.MM.dd}" when.contains: tags: "oldboyedu-python" # 禁用索引生命周期管理 setup.ilm.enabled: false # 设置索引模板的名称 setup.template.name: "oldboyedu-linux" # 设置索引模板的匹配模式 setup.template.pattern: "oldboyedu-linux*"自定义分片和副本案例
filebeat.inputs: - type: log enabled: true paths: - /tmp/test.log - /tmp/*.txt tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"] fields: school: "北京昌平区沙河镇" class: "linux80" - type: log enabled: true paths: - /tmp/test/*/*.log tags: ["oldboyedu-python","云原生开发"] fields: name: "oldboy" hobby: "linux,抖音" fields_under_root: true output.elasticsearch: enabled: true hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] # index: "oldboyedu-linux-elk-%{+yyyy.MM.dd}" indices: - index: "oldboyedu-linux-elk-%{+yyyy.MM.dd}" # 匹配指定字段包含的内容 when.contains: tags: "oldboyedu-linux80" - index: "oldboyedu-linux-python-%{+yyyy.MM.dd}" when.contains: tags: "oldboyedu-python" # 禁用索引生命周期管理 setup.ilm.enabled: false # 设置索引模板的名称 setup.template.name: "oldboyedu-linux" # 设置索引模板的匹配模式 setup.template.pattern: "oldboyedu-linux*" # 覆盖已有的索引模板 setup.template.overwrite: false # 配置索引模板 setup.template.settings: # 设置分片数量 index.number_of_shards: 3 # 设置副本数量，要求小于集群的免费信息发布网数量 index.number_of_replicas: 2
亿华智慧云
