logstash with loki, grafana not picking all the kubernetes pod logs

Question

I have setup running some log generator with loki and logstash. Grafana is able to identify the datasource and labels are picking, but the log generator logs are coming under grafana labels. What iam doing wrong here.

---
# Source: logstash/templates/poddisruptionbudget.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: "logstash-logstash-pdb"
  labels:
    app: "logstash-logstash"
    chart: "logstash"
    heritage: "Helm"
    release: "logstash"
spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      app: "logstash-logstash"
---
# Source: logstash/templates/configmap-pipeline.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-logstash-pipeline
  labels:
    app: "logstash-logstash"
    chart: "logstash"
    heritage: "Helm"
    release: "logstash"
data:
  logstash.yml: |
    http.host: "0.0.0.0"
    path.config: /usr/share/logstash/pipeline
  logstash.conf: |
    input {
      file {
         path => ["/var/log/*.log"]
         start_position => "beginning"
         ignore_older => 0
         sincedb_path => "/dev/null"
       }
    }
    filter {
         if [kubernetes] {
           mutate {
             add_field => {
               "container_name" => "%{[kubernetes][container][name]}"
               "namespace" => "%{[kubernetes][namespace]}"
               "pod" => "%{[kubernetes][pod][name]}"
             }
             replace => { "host" => "%{[kubernetes][node][name]}"}
           }
         }
         mutate {
           remove_field => ["tags"]
         }
       }
       output {
         stdout { codec => rubydebug}
         loki {
           url => "http://loki-loki-distributed-distributor.loki-benchmark.svc.cluster.local:3100/loki/api/v1/push"
         }
       }
---
# Source: logstash/templates/service-headless.yaml
kind: Service
apiVersion: v1
metadata:
  name: "logstash-logstash-headless"
  labels:
    app: "logstash-logstash"
    chart: "logstash"
    heritage: "Helm"
    release: "logstash"
spec:
  clusterIP: None
  selector:
    app: "logstash-logstash"
  ports:
    - name: http
      port: 9600
---
# Source: logstash/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: logstash-logstash
  labels:
    app: "logstash-logstash"
    chart: "logstash"
    heritage: "Helm"
    release: "logstash"
spec:
  serviceName: logstash-logstash-headless
  selector:
    matchLabels:
      app: "logstash-logstash"
      release: "logstash"
  replicas: 1
  podManagementPolicy: Parallel
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      name: "logstash-logstash"
      labels:
        app: "logstash-logstash"
        chart: "logstash"
        heritage: "Helm"
        release: "logstash"
      annotations:
        pipelinechecksum: e5576a55d691ae22c1da1204f1e548e8aa936dc6415af52eb65699f5a155bb8
    spec:
      securityContext:
        fsGroup: 1000
        runAsUser: 1000
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - "logstash-logstash"
            topologyKey: kubernetes.io/hostname
      terminationGracePeriodSeconds: 120
      volumes:
        - name: logstashpipeline
          configMap:
            name: logstash-logstash-pipeline
      containers:
      - name: "logstash"
        securityContext:
          capabilities:
            drop:
            - ALL
          runAsNonRoot: true
          runAsUser: 1000
        image: "grafana/logstash-output-loki:1.0.1"
        imagePullPolicy: "IfNotPresent"
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /
            port: http
          initialDelaySeconds: 300
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /
            port: http
          initialDelaySeconds: 60
          periodSeconds: 10
          successThreshold: 3
          timeoutSeconds: 5
        ports:
        - name: http
          containerPort: 9600
        resources:
          limits:
            cpu: 1000m
            memory: 1536Mi
          requests:
            cpu: 100m
            memory: 1536Mi
        env:
          - name: LS_JAVA_OPTS
            value: "-Xmx1g -Xms1g"
          - name: XPACK_MONITORING_ENABLED
            value: "false"
        volumeMounts:
          - name: logstashpipeline
            mountPath: /usr/share/logstash/pipeline/logstash.conf
            subPath: logstash.conf

Answer 1

You can try adding these include fields in logtash configuration which should help you reslove the issue.

     output {
             stdout { codec => rubydebug}
             loki {
               url => "http://loki-loki-distributed-distributor.loki-benchmark.svc.cluster.local:3100/loki/api/v1/push"
include_fields => ["container_name","namespace","pod","host"]

 }
           }