I have been struggling for quite some time with my filebeat setup.
I have installed filebeat 7.10 on an ubuntu instance. Somehow part of the logs were sent to my cluster, but now when I check the systemctl status it always says failed, regardless of how many things I tried.
root@ip-172-31-35-75:/var/log/filebeat# systemctl status filebeat
× filebeat.service - Filebeat sends log files to Logstash or directly to Elasticsearch.
Loaded: loaded (/lib/systemd/system/filebeat.service; disabled; vendor preset: enabled)
Active: failed (Result: exit-code) since Fri 2023-06-09 09:13:30 UTC; 839ms ago
Docs: https://www.elastic.co/products/beats/filebeat
Process: 267824 ExecStart=/usr/share/filebeat/bin/filebeat --environment systemd $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS (code=exited, status=2)
Main PID: 267824 (code=exited, status=2)
CPU: 141ms
Jun 09 09:13:30 ip-172-31-35-75 systemd[1]: filebeat.service: Main process exited, code=exited, status=2/INVALIDARGUMENT
Jun 09 09:13:30 ip-172-31-35-75 systemd[1]: filebeat.service: Failed with result 'exit-code'.
Jun 09 09:13:30 ip-172-31-35-75 systemd[1]: filebeat.service: Scheduled restart job, restart counter is at 5.
Jun 09 09:13:30 ip-172-31-35-75 systemd[1]: Stopped Filebeat sends log files to Logstash or directly to Elasticsearch..
Jun 09 09:13:30 ip-172-31-35-75 systemd[1]: filebeat.service: Start request repeated too quickly.
Jun 09 09:13:30 ip-172-31-35-75 systemd[1]: filebeat.service: Failed with result 'exit-code'.
Jun 09 09:13:30 ip-172-31-35-75 systemd[1]: Failed to start Filebeat sends log files to Logstash or directly to Elasticsearch..
root@ip-172-31-35-75:/var/log/filebeat#
This is my filebeat.yml configuration
filebeat.inputs:
- type: log
enabled: true
paths:
- /home/ubuntu/.pm2/logs/*.log
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
<!-- setup.template.settings:
index.number_of_shards: 1
#index.codec: best_compression
#_source.enabled: false -->
setup.kibana:
host: "http://*****5601"
output.elasticsearch:
hosts: ["http://****9200"]
#protocol: "https"
username: ""
password: ""
processors:
- decode_json_fields:
fields: ["message"]
target: ""
overwrite_keys: true
setup.ilm.enabled: false
setup.pack.security.enabled: false
setup.xpack.graph.enabled: false
setup.xpack.watcher.enabled: false
setup.xpack.monitoring.enabled: false
setup.xpack.reporting.enabled: false
logging.level: debug
logging.to_files: true
logging.files:
path: /var/log/filebeat
name: filebeat
keepfiles: 7
permissions: 0644
Additionally, I do not see any errors in /var/log/filebeat
root@ip-172-31-35-75:/var/log/filebeat# cat filebeat
2023-06-09T09:13:30.293Z INFO instance/beat.go:645 Home path: [/usr/share/filebeat] Config path: [/etc/filebeat] Data path: [/var/lib/filebeat] Logs path: [/var/log/filebeat]
2023-06-09T09:13:30.293Z DEBUG [beat] instance/beat.go:697 Beat metadata path: /var/lib/filebeat/meta.json
2023-06-09T09:13:30.293Z INFO instance/beat.go:653 Beat ID: 4cee4800-191d-4e30-88ae-c4a79cdc579d
2023-06-09T09:13:30.294Z DEBUG [processors] processors/processor.go:120 Generated new processors: decode_json_fields=message
2023-06-09T09:13:30.294Z DEBUG [seccomp] seccomp/seccomp.go:117 Loading syscall filter {"seccomp_filter": {"no_new_privs":true,"flag":"tsync","policy":{"default_action":"errno","syscalls":[{"names":["accept","accept4","access","arch_prctl","bind","brk","chmod","chown","clock_gettime","clone","close","connect","dup","dup2","epoll_create","epoll_create1","epoll_ctl","epoll_pwait","epoll_wait","exit","exit_group","fchdir","fchmod","fchmodat","fchown","fchownat","fcntl","fdatasync","flock","fstat","fstatfs","fsync","ftruncate","futex","getcwd","getdents","getdents64","geteuid","getgid","getpeername","getpid","getppid","getrandom","getrlimit","getrusage","getsockname","getsockopt","gettid","gettimeofday","getuid","inotify_add_watch","inotify_init1","inotify_rm_watch","ioctl","kill","listen","lseek","lstat","madvise","mincore","mkdirat","mmap","mprotect","munmap","nanosleep","newfstatat","open","openat","pipe","pipe2","poll","ppoll","pread64","pselect6","pwrite64","read","readlink","readlinkat","recvfrom","recvmmsg","recvmsg","rename","renameat","rt_sigaction","rt_sigprocmask","rt_sigreturn","sched_getaffinity","sched_yield","sendfile","sendmmsg","sendmsg","sendto","set_robust_list","setitimer","setsockopt","shutdown","sigaltstack","socket","splice","stat","statfs","sysinfo","tgkill","time","tkill","uname","unlink","unlinkat","wait4","waitid","write","writev"],"action":"allow"}]}}}
2023-06-09T09:13:30.294Z INFO [seccomp] seccomp/seccomp.go:124 Syscall filter successfully installed
2023-06-09T09:13:30.294Z INFO [beat] instance/beat.go:981 Beat info {"system_info": {"beat": {"path": {"config": "/etc/filebeat", "data": "/var/lib/filebeat", "home": "/usr/share/filebeat", "logs": "/var/log/filebeat"}, "type": "filebeat", "uuid": "4cee4800-191d-4e30-88ae-c4a79cdc579d"}}}
2023-06-09T09:13:30.294Z INFO [beat] instance/beat.go:990 Build info {"system_info": {"build": {"commit": "1428d58cf2ed945441fb2ed03961cafa9e4ad3eb", "libbeat": "7.10.0", "time": "2020-11-09T19:57:04.000Z", "version": "7.10.0"}}}
2023-06-09T09:13:30.294Z INFO [beat] instance/beat.go:993 Go runtime info {"system_info": {"go": {"os":"linux","arch":"amd64","max_procs":2,"version":"go1.14.7"}}}
2023-06-09T09:13:30.294Z INFO [beat] instance/beat.go:997 Host info {"system_info": {"host": {"architecture":"x86_64","boot_time":"2023-04-24T15:51:50Z","containerized":false,"name":"ip-172-31-35-75","ip":["127.0.0.1/8","::1/128","172.31.35.75/20","fe80::8bf:3fff:fe77:62c/64"],"kernel_version":"5.19.0-1023-aws","mac":["0a:bf:3f:77:06:2c"],"os":{"family":"debian","platform":"ubuntu","name":"Ubuntu","version":"22.04.2 LTS (Jammy Jellyfish)","major":22,"minor":4,"patch":2,"codename":"jammy"},"timezone":"UTC","timezone_offset_sec":0,"id":"ec2d83d9dbbd59eb7af1e043bc0e6dc5"}}}
2023-06-09T09:13:30.295Z INFO [beat] instance/beat.go:1026 Process info {"system_info": {"process": {"capabilities": {"inheritable":null,"permitted":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40"],"effective":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40"],"bounding":["chown","dac_override","dac_read_search","fowner","fsetid","kill","setgid","setuid","setpcap","linux_immutable","net_bind_service","net_broadcast","net_admin","net_raw","ipc_lock","ipc_owner","sys_module","sys_rawio","sys_chroot","sys_ptrace","sys_pacct","sys_admin","sys_boot","sys_nice","sys_resource","sys_time","sys_tty_config","mknod","lease","audit_write","audit_control","setfcap","mac_override","mac_admin","syslog","wake_alarm","block_suspend","audit_read","38","39","40"],"ambient":null}, "cwd": "/", "exe": "/usr/share/filebeat/bin/filebeat", "name": "filebeat", "pid": 267824, "ppid": 1, "seccomp": {"mode":"filter","no_new_privs":true}, "start_time": "2023-06-09T09:13:29.780Z"}}}
2023-06-09T09:13:30.295Z INFO instance/beat.go:299 Setup Beat: filebeat; Version: 7.10.0
2023-06-09T09:13:30.295Z DEBUG [beat] instance/beat.go:325 Initializing output plugins
2023-06-09T09:13:30.295Z INFO eslegclient/connection.go:99 elasticsearch url: http://35.178.63.225:9200
2023-06-09T09:13:30.295Z DEBUG [publisher] pipeline/consumer.go:148 start pipeline event consumer
2023-06-09T09:13:30.295Z INFO [publisher] pipeline/module.go:113 Beat name: ip-172-31-35-75
2023-06-09T09:13:30.296Z INFO [monitoring] log/log.go:118 Starting metrics logging every 30s
2023-06-09T09:13:30.296Z INFO instance/beat.go:455 filebeat start running.
2023-06-09T09:13:30.296Z DEBUG [test] registrar/migrate.go:304 isFile(/var/lib/filebeat/registry) -> false
2023-06-09T09:13:30.296Z DEBUG [test] registrar/migrate.go:304 isFile() -> false
2023-06-09T09:13:30.296Z DEBUG [test] registrar/migrate.go:297 isDir(/var/lib/filebeat/registry/filebeat) -> true
Please let me know if you have any idea what could be wrong and where