kubernetes Multiline logs for Elasticsearch (Kibana)

If you’re having issues with Kubernetes Multiline logs here is the solution for you.

The Kubernetes autodiscover provider watches for Kubernetes pods to start, update, and stop. These are the available fields on every event:

If the include_annotations config is added to the provider config, then the list of annotations present in the config are added to the event.

By using below Yaml file you can post all K8s logs to ElasticSearch(hosted or in-house).

---

apiVersion: v1

kind: ConfigMap

metadata:

  name: filebeat-config

  namespace: kube-system

  labels:

    k8s-app: filebeat

data:

  filebeat.yml: |-

    filebeat.config:

      inputs:

        # Mounted `filebeat-inputs` configmap:

        path: ${path.config}/inputs.d/*.yml

        # Reload inputs configs as they change:

        reload.enabled: false

      modules:

        path: ${path.config}/modules.d/*.yml

        # Reload module configs as they change:

        reload.enabled: false

    # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:

    #          hints.enabled: true

    filebeat.autodiscover:

      providers:

        - type: kubernetes

          templates:

            - condition:

                or:

                  - equals:

                      kubernetes.namespace: default
              config:

                - type: docker

                  containers.ids:

                    - "${data.kubernetes.container.id}"

                  multiline.pattern: '^[[:space:]]'

                  multiline.negate: false

                  multiline.match: after

    processors:

      - add_cloud_metadata:

    cloud.id: ${ELASTIC_CLOUD_ID}

    cloud.auth: ${ELASTIC_CLOUD_AUTH}

    output.elasticsearch:

      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9243}']
#use https:// if it is hosted ES

      username: ${ELASTICSEARCH_USERNAME}

      password: ${ELASTICSEARCH_PASSWORD}

---

apiVersion: v1

kind: ConfigMap

metadata:

  name: filebeat-inputs

  namespace: kube-system

  labels:

    k8s-app: filebeat

data:

  kubernetes.yml: |-

    - type: docker

      containers.path: "/var/lib/docker/containers"

      symlinks: true

      containers.ids:

      - "*"

      multiline.pattern: '^DEBUG'

      multiline.negate: true

      multiline.match: 'after'

      processors:

        - add_kubernetes_metadata:

            in_cluster: true

---

apiVersion: extensions/v1beta1

kind: DaemonSet

metadata:

  name: filebeat

  namespace: kube-system

  labels:

    k8s-app: filebeat

spec:

  template:

    metadata:

      labels:

        k8s-app: filebeat

    spec:

      serviceAccountName: filebeat

      terminationGracePeriodSeconds: 30

      containers:

      - name: filebeat

        image: docker.elastic.co/beats/filebeat:6.4.0

        args: [

          "-c", "/etc/filebeat.yml",

          "-e",

        ]

        env:

        - name: ELASTICSEARCH_HOST

          value: "elasticsearch"    #use https:// if it is hosted ES

        - name: ELASTICSEARCH_PORT

          value: "9243"

        - name: ELASTICSEARCH_USERNAME

          value: "elasticsearch"

        - name: ELASTICSEARCH_PASSWORD

          value: "changeme"

        - name: ELASTIC_CLOUD_ID

          value:

        - name: ELASTIC_CLOUD_AUTH

          value:

        securityContext:

          runAsUser: 0

        resources:

          limits:

            memory: 200Mi

          requests:

            cpu: 100m

            memory: 100Mi

        volumeMounts:

        - name: config

          mountPath: /etc/filebeat.yml

          readOnly: true

          subPath: filebeat.yml

        - name: inputs

          mountPath: /usr/share/filebeat/inputs.d

          readOnly: true

        - name: data

          mountPath: /usr/share/filebeat/data

        - name: varlogpods

          mountPath: /var/log/pods

          readOnly: true

        - name: varlogcontainers

          mountPath: /var/log/containers

        - name: varlibdockercontainers

          mountPath: /var/lib/docker/containers

          readOnly: true

      volumes:

      - name: config

        configMap:

          defaultMode: 0600

          name: filebeat-config

      - name: varlogpods

        hostPath:

          path: /var/log/pods

      - name: varlogcontainers

        hostPath:

          path: /var/log/containers

      - name: varlibdockercontainers

        hostPath:

          path: /var/lib/docker/containers

      - name: inputs

        configMap:

          defaultMode: 0600

          name: filebeat-inputs

      # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart

      - name: data

        hostPath:

          path: /var/lib/filebeat-data

          type: DirectoryOrCreate

---

apiVersion: rbac.authorization.k8s.io/v1beta1

kind: ClusterRoleBinding

metadata:

  name: filebeat

subjects:

- kind: ServiceAccount

  name: filebeat

  namespace: kube-system

roleRef:

  kind: ClusterRole

  name: filebeat

  apiGroup: rbac.authorization.k8s.io

---

apiVersion: rbac.authorization.k8s.io/v1beta1

kind: ClusterRole

metadata:

  name: filebeat

  labels:

    k8s-app: filebeat

rules:

- apiGroups: [""] # "" indicates the core API group

  resources:

  - namespaces

  - pods

  verbs:

  - get

  - watch

  - list

---

apiVersion: v1

kind: ServiceAccount

metadata:

  name: filebeat

  namespace: kube-system

  labels:

    k8s-app: filebeat

---