
```js

input {
  file {
    path => "/usr/local/nginx/logs/access.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
    type => "nginx_access"
  }
}

filter {
  grok {
    match => { 
      "message" => '%{IPORHOST:remote_addr} - %{DATA:remote_user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATH:request_path}(?:\?%{URIPARAM:query_params})? HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{DATA:http_referer}" "%{DATA:user_agent}"'
    }
  }

  date {
    match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
    target => "@timestamp"
  }
}

output {
  elasticsearch {
    hosts => [
      "192.168.201.112:9200",
      "192.168.201.113:9200",
      "192.168.201.114:9200"
    ]
    index => "nginx-access-%{+YYYY.MM.dd}"
  }
}


```

6.ELK架构采集nginx

# 一、介绍和安装


```js
核心功能：数据收集、转换和传输管道。支持从多种来源（文件、数据库等）提取数据，过滤（如解析JSON、正则匹
配）后发送到Elasticsearch。

特点：功能强大但资源消耗较高，适合复杂的数据处理场景。

相比Logstash，Filebeat占用资源极少，适合部署在边缘节点（如服务器、容器）持续采集日志文件。

专一性：仅负责日志收集和转发（到Logstash或Elasticsearch），不处理数据，避免资源浪费。

场景互补：Filebeat + Logstash组合中，Filebeat负责采集，Logstash负责复杂处理，分工明确提升效率。
```
下载网址：https://artifacts.elastic.co/downloads/logstash/logstash-7.17.29-x86_64.rpm

```js
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.17.29-x86_64.rpm
```

```js
rpm -ivh logstash-7.17.29-x86_64.rpm
```

```js
ln  -sv /usr/share/logstash/bin/logstash  /usr/local/bin/
```
# 二、收集TCP端口日志
配置目录
```js
/etc/logstash
```

```js
vi /etc/logstash/logstash-sample.conf
```
```js
input {
  file {
    path => ["/tmp/*.txt"]  # 监听 /tmp 目录下的 .txt 文件
    start_position => "beginning"  # 从文件开头读取
  }
}

output {
  stdout {}  # 输出到控制台（测试用）
}
```
知道配置启动
```js
logstash -f /etc/logstash/logstash-sample.conf
```

![image.png](/static/img/7832b32a36879835035d7bbd7b380320.image.webp)

# 三、收集TCP端口日志

```js
input {
  tcp {
    port => 8888
    codec => "json"  # 可选：如果接收JSON格式数据
  }
}

output {
  stdout {
    codec => rubydebug  # 更好的格式化输出
  }
}
```
启动后去另一台服务器telnet该服务器端口测试

![image.png](/static/img/52909a85a3cfb1d68491594572abffce.image.webp)

成功收集日志
![image.png](/static/img/de6912db1d0f4a52d97dcd11374a70e2.image.webp)

# 三、收集http协议端口日志

```js
input {
  http {
    port => 8888
  }
}

output {
  stdout {
    codec => rubydebug  # 更好的格式化输出
  }
}
```
测试

```js
curl "http://192.168.201.104:8888?greeting=你好啊"
```

![image.png](/static/img/8edd571194859b7120eba60d2548379b.image.webp)

# 三、发送数据输出到redis环境

```js

input {
  tcp {
    port => 9999
  }
}

output {
  stdout {}
  
  redis {
    host => "192.168.201.104"
    port => 6379
    db => 0
    password => "12345678"
    data_type => "list"
    key => "hello word"
  }
}

```
我往该IP端口发送消息，就会往redis写入数据
![image.png](/static/img/848854c44de8221511db0c4112b02912.image.webp)


![image.png](/static/img/1dd28b1b703b4402fde651cd1fc620b6.image.webp)

# 四、发送数据输出到本地文件

```js
input {
  tcp {
    port => 9999
  }
}

output {
  stdout {}
  
 file {
    path => ["/tmp/1.txt"]  # 监听 /tmp 目录下的 .txt 文件
  }
}
```
发送消息写入该服务器下的路径文件
![image.png](/static/img/5b2138ab4135fa3067d18950466a900f.image.webp)

![image.png](/static/img/71299cac68c2794994dcd442267949ba.image.webp)


# 五、发送数据输出到ES

```js
input {
  tcp {
    port => 9999
  }
}

output {
  stdout {}
  

  elasticsearch {
    hosts => ["192.168.201.112:9200", "192.168.201.113:9200", "192.168.201.114:9200"]
    index => "logstash-%{+YYYY.MM.dd}"
  }
}

```


![image.png](/static/img/a874ae8e7ee48a1faf5ea6133f303bc7.image.webp)


![image.png](/static/img/73b9b05ddfe640293ae7e70ac0704a43.image.webp)

5.logstash

# 一、在Filebeat写好配置文件

```yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /usr/local/nginx/logs/access.log*
  tags: ["access"]
  fields:
    name: "access"

- type: log
  enabled: true
  paths:
    -  /usr/local/nginx/logs/error.log*
  tags: ["error"]
  fields:
    name: "error"
    
  fields_under_root: true

output.elasticsearch:
  enabled: true
  hosts: ["http://192.168.201.112:9200", "http://192.168.201.113:9200", "http://192.168.201.114:9200"]
  index: "nginx-elk-%{+yyyy.MM.dd}"
  indices:
    - index: "nginx-elk-access-%{+yyyy.MM.dd}"
      when.contains:
        tags: "access"
    - index: "nginx-elk-error-%{+yyyy.MM.dd}"
      when.contains:
        tags: "error"

# 禁用索引生命周期管理
setup.ilm.enabled: false
# 设置索引模板
setup.template.name: "mysongxuan-elk"
setup.template.pattern: "mysongxuan-elk*"
#设置已有索引模版直接覆盖
setup.template.overwrite: true

#设置索引模版分片
setup.template.settings:
    #设置分片数
    index.number_of_shards: 3
    #设置副本数
    index.number_of_replicas: 0
```
在nginx服务器上启动Filebeat
```js
filebeat -e -c /etc/filebeat/filebeat.yml
```

![image.png](/static/img/610238ba4ef2d5e3ce5e65988ccc39be.image.webp)

然后去索引管理创建好索引模式
![image.png](/static/img/11a99c9e86e975cd9f8ba2dcc2a2d09e.image.webp)
最后查看已经完成配置
![image.png](/static/img/432876178efecccc020a5e1d67a82f15.image.webp)


# 二、这种方式查看，不会把字段分出来

![image.png](/static/img/17c900146c150991c9b791fc9991971e.image.webp)

修改nginx.conf
```js
worker_processes  1;

events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    sendfile        on;

    include   songxuan.conf; #配置外置文件

    # 定义JSON格式日志
    log_format json_combined escape=json
    '{'
        '"timestamp":"$time_iso8601",'
        '"remote_addr":"$remote_addr",'
        '"remote_user":"$remote_user",'
        '"request":"$request",'
        '"status": "$status",'
        '"body_bytes_sent":"$body_bytes_sent",'
        '"request_time":"$request_time",'
        '"http_referer":"$http_referer",'
        '"http_user_agent":"$http_user_agent",'
        '"http_x_forwarded_for":"$http_x_forwarded_for"'
    '}';
    # 指定访问日志路径和格式
    access_log /usr/local/nginx/logs/access.log json_combined;
    error_log  /usr/local/nginx/logs/error.log;

}

```
```yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /usr/local/nginx/logs/access.log*
  tags: ["access"]
  
  #添加json格式
  json.keys_under_root: true
  json.add_error_key: true
  
  fields:
    name: "access"

- type: log
  enabled: true
  paths:
    -  /usr/local/nginx/logs/error.log*
  tags: ["error"]
  fields:
    name: "error"
    
  fields_under_root: true

output.elasticsearch:
  enabled: true
  hosts: ["http://192.168.201.112:9200", "http://192.168.201.113:9200", "http://192.168.201.114:9200"]
  index: "nginx-elk-%{+yyyy.MM.dd}"
  indices:
    - index: "nginx-elk-access-%{+yyyy.MM.dd}"
      when.contains:
        tags: "access"
    - index: "nginx-elk-error-%{+yyyy.MM.dd}"
      when.contains:
        tags: "error"

# 禁用索引生命周期管理
setup.ilm.enabled: false
# 设置索引模板
setup.template.name: "mysongxuan-elk"
setup.template.pattern: "mysongxuan-elk*"
#设置已有索引模版直接覆盖
setup.template.overwrite: true

#设置索引模版分片
setup.template.settings:
    #设置分片数
    index.number_of_shards: 3
    #设置副本数
    index.number_of_replicas: 0
```

这样配置查看能，能区别更清楚字段

![image.png](/static/img/7cccab639f232d2fc4860573d2934156.image.webp)


4.ES+kibana+Filebeat收集Nginx日志

# 一、配置自定义索引

```yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /tmp/test.log
    - /tmp/*.txt
  tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"]
  fields:
    school: "北京昌平区沙河镇"
    class: "linux80"

- type: log
  enabled: true
  paths:
    - /tmp/test/*/*.log
  tags: ["oldboyedu-python","云原生开发"]
  fields:
    name: "oldboy"
    hobby: "linux,抖音"
  fields_under_root: true

output.elasticsearch:
  enabled: true
  hosts: ["http://192.168.201.112:9200", "http://192.168.201.113:9200", "http://192.168.201.114:9200"]
  index: "mysongxuan-elk-%{+yyyy.MM.dd}"
  
# 禁用索引生命周期管理
setup.ilm.enabled: false
# 设置索引名字
setup.template.name: "mysongxuan-elk"
setup.template.pattern: "mysongxuan-elk*"

```

/var/lib/filebeat/ 存储Filebeat的注册表文件（registry），记录日志文件的采集状态（如offset、inode等）
删除该目录会‌重置采集状态‌，但不会直接影响模板加载
```js
rm -rf /var/lib/filebeat/*
```

重新配置文件后启动，我的自定义索引模版就出来了

![image.png](/static/img/8ad91fe68e640ce4058c3f4457bde1f9.image.webp)
然后创建索引模式，就可以看到我自定义的索引模式了
![image.png](/static/img/a88a6b65f559c435363fc3b48bae9410.image.webp)

![image.png](/static/img/3415397d784cd47e894f0ee3ed9a9f54.image.webp)






# 二、根据不同的tags进行索引查询

```yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /tmp/*.txt
  tags: ["songxuan","xuan"]
  fields:
    name: "松轩"
    class: "linux"

- type: log
  enabled: true
  paths:
    - /tmp/*.log
  tags: ["xiaoming","xiaoqiang"]
  fields:
    name: "小明"
    hobby: "python"
  fields_under_root: true

output.elasticsearch:
  enabled: true
  hosts: ["http://192.168.201.112:9200", "http://192.168.201.113:9200", "http://192.168.201.114:9200"]
  index: "mysongxuan-elk-%{+yyyy.MM.dd}"
  indices:
    - index: "mysongxuan-elk-%{+yyyy.MM.dd}"
      when.contains:
        tags: "songxuan"
    - index: "mysongxuan-python-%{+yyyy.MM.dd}"
      when.contains:
        tags: "xiaoming"

# 禁用索引生命周期管理
setup.ilm.enabled: false
# 设置索引模板
setup.template.name: "mysongxuan-elk"
setup.template.pattern: "mysongxuan-elk*"
```
删除后启动
```js
rm -rf /var/lib/filebeat/*
```

```js
filebeat -e -c /etc/filebeat/filebeat.yml
```
重新查看后有就出现两条索引了，
![image.png](/static/img/8d7d3548745e6656089372121608136b.image.webp)

最后去索引模式创建索引

![image.png](/static/img/4c0249b73fb3dc5646b0713b9d6b91c5.image.webp)

然后查看标签是songxuan的只有对应文本的数据

![image.png](/static/img/44624a62306a5147602c98578dcdbd2e.image.webp)

![image.png](/static/img/1ed457164c2579b9b81d34a7d615ffaa.image.webp)



3.自定义索引

#  一、log日志配置

```js
filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /tmp/test.log

output.console:
  pretty: true
```
```js
echo 123 > test.log
```

![image.png](/static/img/7fa4344ce5bdc86b1e21cb14bd24b952.image.webp)


这是他的数据原文件目录，中断会跟局该目录下的log.json进行读取。
```js
cd /var/lib/filebeat/registry/filebeat/
```

也可以用通配符去匹配

```js
filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /tmp/test.log
      - /tmp/*.txt

output.console:
  pretty: true
```


加标签
```yml

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /tmp/test.log
    - /tmp/*.txt
  tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"]
  fields:
    school: "北京昌平区沙河镇"
    class: "linux80"

- type: log
  enabled: true
  paths:
    - /tmp/test/*/*.log
  tags: ["oldboyedu-python","云原生开发"]
  fields:
    name: "oldboy"
    hobby: "linux,抖音"
  fields_under_root: true

output.console:
  pretty: true

```

运行命令
```js
filebeat -e -c /etc/filebeat/filebeat.yml
```



# 二、把收集的数据发送到集群上
写上Elasticsearch集群的地址配置上去。
```yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /tmp/test.log
    - /tmp/*.txt
  tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"]
  fields:
    school: "北京昌平区沙河镇"
    class: "linux80"

- type: log
  enabled: true
  paths:
    - /tmp/test/*/*.log
  tags: ["oldboyedu-python","云原生开发"]
  fields:
    name: "oldboy"
    hobby: "linux,抖音"
  fields_under_root: true

output.elasticsearch:
  hosts: ["http://192.168.201.112:9200", "http://192.168.201.113:9200", "http://192.168.201.114:9200"]
```

启动后去Kibana页面查看
![image.png](/static/img/e70a427525165e9743f802a156cb4c5c.image.webp)

![image.png](/static/img/0cb1401e7cdfcb92e85b054cfc542e7f.image.webp)


# 三、创建索引模式

![image.png](/static/img/f1524b861db3b10fc4cedb79db41174b.image.webp)

可以通过别名＋通配符的方式去匹配

![image.png](/static/img/d2b8acac3364e172b1d7b60ae195afff.image.webp)

然后创建
![image.png](/static/img/faaf8b6c441a43d965c58636828ba407.image.webp)


最后点击这里可以查看采集到日志的数据

![image.png](/static/img/eceba1bf91ee8a64bad6a5021e2f6fbf.image.webp)


# 四、筛选字段查看

![image.png](/static/img/c2547960429ae363ce79a91cd5f1de37.image.webp)

当我去给文本添加内容时，也是能收集到的
![image.png](/static/img/8b09332bc4d041e2f20c70935750e362.image.webp)


![image.png](/static/img/aa4a188c7ece2dd59f7ced60e2eda371.image.webp)