DEVHIDE
Login Or Sign up

Passing Env Values to Helm to create metricbeat

344 Views Asked by At

I am using managed ElasticCloud in Azure. I need to install metricbeats using helm charts on AKS. To configure these beats I need to give elastic cloud details like -- cloud.id and cloud.auth. I want pass this values from github environment.

To do so, I added this variables as secrets in Github environment.

This is how my values.yaml looks like:

daemonset:
  annotations: {}
  labels: {}
  affinity: {}
  enabled: true
  envFrom: []
  extraEnvs: []
  extraVolumes: []
  extraVolumeMounts: []
  hostAliases: []
  hostNetworking: false
  
  metricbeatConfig:
    metricbeat.yml: |
      metricbeat.modules:
      - module: kubernetes
        metricsets:
          - container
          - node
          - pod
          - system
          - volume
        period: 10s
        host: "${NODE_NAME}"
        hosts: ["https://${NODE_NAME}:10250"]
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        ssl.verification_mode: "none"
        
        processors:
        - add_kubernetes_metadata: ~
      - module: kubernetes
        enabled: true
        metricsets:
          - event
      - module: system
        period: 10s
        metricsets:
          - cpu
          - load
          - memory
          - network
          - process
          - process_summary
        processes: ['.*']
        process.include_top_n:
          by_cpu: 5
          by_memory: 5
      - module: system
        period: 1m
        metricsets:
          - filesystem
          - fsstat
        processors:
        - drop_event.when.regexp:
            system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
      cloud.id: elk:ZWFzdHVzLmF6dXJlLmVsYXN0aWMtY2xvdWQuY29tOjQ0MyRmZjI5MjNjYTNkNTI0Yzg1ODBiMDk0ZGQxNTRmODQ5ZSQ3MmEyYjY5OTdjMWQ0ODFjYTViYjlkZTY3OGIxMzAzZA==
      cloud.auth: elastic:jixuoLFqs0vneFNaByIQdtL9
      output.elasticsearch:
        protocol: https
        ssl.verification_mode: "none"

  nodeSelector: {}
  
  securityContext:
    runAsUser: 0
    privileged: false
  resources:
    requests:
      cpu: "100m"
      memory: "100Mi"
    limits:
      cpu: "1000m"
      memory: "200Mi"
  tolerations: []

deployment:
  annotations: {}
  labels: {}
  affinity: {}
  enabled: true
  envFrom: []
  
  extraEnvs: []
  
  extraVolumes: []
 
  extraVolumeMounts: []
  
  hostAliases: []
  
  metricbeatConfig:
    metricbeat.yml: |
      metricbeat.modules:
      - module: kubernetes
        enabled: true
        metricsets:
          - state_node
          - state_deployment
          - state_replicaset
          - state_pod
          - state_container
        period: 10s
        hosts: ["${KUBE_STATE_METRICS_HOSTS}"]
      cloud.id: elk:ZWFzdHVzLmF6dXJlLmVsYXN0aWMtY2xvdWQuY29tOjQ0MyRmZjI5MjNjYTNkNTI0Yzg1ODBiMDk0ZGQxNTRmODQ5ZSQ3MmEyYjY5OTdjMWQ0ODFjYTViYjlkZTY3OGIxMzAzZA==
      cloud.auth: elastic:jixuoLFqs0vneFNaByIQdtL9
      output.elasticsearch:
        protocol: https
        ssl.verification_mode: "none"

  nodeSelector: {}
  secretMounts: []
  securityContext:
    runAsUser: 0
    privileged: false
  resources:
    requests:
      cpu: "100m"
      memory: "100Mi"
    limits:
      cpu: "1000m"
      memory: "200Mi"
  tolerations: []

replicas: 1

extraContainers: ""

extraInitContainers: ""

hostPathRoot: /var/lib

image: "docker.elastic.co/beats/metricbeat"
imageTag: "7.17.3"
imagePullPolicy: "IfNotPresent"
imagePullSecrets: []

livenessProbe:
  exec:
    command:
      - sh
      - -c
      - |
        #!/usr/bin/env bash -e
        curl --fail 127.0.0.1:5066
  failureThreshold: 3
  initialDelaySeconds: 10
  periodSeconds: 10
  timeoutSeconds: 5

readinessProbe:
  exec:
    command:
      - sh
      - -c
      - |
        #!/usr/bin/env bash -e
        metricbeat test output
  failureThreshold: 3
  initialDelaySeconds: 10
  periodSeconds: 10
  timeoutSeconds: 5

managedServiceAccount: true

clusterRoleRules:
  - apiGroups: [""]
    resources:
      - nodes
      - namespaces
      - events
      - pods
      - services
    verbs: ["get", "list", "watch"]
  - apiGroups: ["extensions"]
    resources:
      - replicasets
    verbs: ["get", "list", "watch"]
  - apiGroups: ["apps"]
    resources:
      - statefulsets
      - deployments
      - replicasets
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources:
      - nodes/stats
    verbs: ["get"]
  - apiGroups:
      - ""
    resources:
      - nodes/stats
    verbs:
      - get
  - nonResourceURLs:
      - "/metrics"
    verbs:
      - get

podAnnotations:
  {}
serviceAccount: ""

serviceAccountAnnotations:
  {}

terminationGracePeriod: 30

priorityClassName: ""

updateStrategy: RollingUpdate

nameOverride: ""
fullnameOverride: ""

kube_state_metrics:
  enabled: true
  host: ""

secrets: []

secret:
  name: elastic
data:
  cloud.id: elk:ZWFzdHVzLmF6dXJlLmVsYXN0aWMtY2xvdWQuY29tOjQ0MyRmZjI5MjNjYTNkNTI0Yzg1ODBiMDk0ZGQxNTRmODQ5ZSQ3MmEyYjY5OTdjMWQ0ODFjYTViYjlkZTY3OGIxMzAzZA==
  cloud.auth: elastic:jixuoLFqs0vneFNaByIQdtL9

under templates folder - secret.yaml

apiVersion: v1
kind: Secret
metadata:
  name: {{ .Values.secret.name }}
type: Opaque
data:
  CLOUD_ID: {{ .Values.data.cloud.id }}
  CLOUD_AUTH: {{ .Values.data.cloud.auth }}

and passing it in deployment.yaml and daemonset.yaml as --

apiVersion: apps/v1
kind: Deployment
metadata:
  name: '{{ template "metricbeat.fullname" . }}-metrics'
  labels:
    app: '{{ template "metricbeat.fullname" . }}-metrics'
    chart: '{{ .Chart.Name }}-{{ .Chart.Version }}'
    heritage: '{{ .Release.Service }}'
    release: '{{ .Release.Name }}'
    {{- if .Values.deployment.labels }}
    {{- range $key, $value := .Values.deployment.labels }}
    {{ $key }}: {{ $value | quote }}
    {{- end }}
    {{- else }}
    {{- range $key, $value := .Values.labels }}
    {{ $key }}: {{ $value | quote }}
    {{- end }}
    {{- end }}
  {{- if .Values.deployment.annotations}}
  annotations:
    {{- range $key, $value := .Values.deployment.annotations }}
    {{ $key }}: {{ $value | quote }}
    {{- end }}
  {{- end }}
spec:
  replicas: {{ .Values.replicas }}
  selector:
    matchLabels:
      app: '{{ template "metricbeat.fullname" . }}-metrics'
      release: '{{ .Release.Name }}'
  template:
    metadata:
      annotations:
        {{- range $key, $value := .Values.podAnnotations }}
        {{ $key }}: {{ $value | quote }}
        {{- end }}
        {{/* This forces a restart if the configmap has changed */}}
        {{- if or  .Values.metricbeatConfig .Values.deployment.metricbeatConfig }}
        configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }}
        {{- end }}
      labels:
        app: '{{ template "metricbeat.fullname" . }}-metrics'
        chart: '{{ .Chart.Name }}-{{ .Chart.Version }}'
        release: '{{ .Release.Name }}'
        {{- if .Values.deployment.labels }}
        {{- range $key, $value := .Values.deployment.labels }}
        {{ $key }}: {{ $value | quote }}
        {{- end }}
        {{- else }}
        {{- range $key, $value := .Values.labels }}
        {{ $key }}: {{ $value | quote }}
        {{- end }}
        {{- end }}
    spec:
      affinity: {{ toYaml .Values.deployment.affinity | nindent 8 }}
      nodeSelector: {{ toYaml .Values.deployment.nodeSelector | nindent 8 }}
      tolerations: {{ toYaml ( .Values.tolerations | default .Values.deployment.tolerations ) | nindent 8 }}
      {{- if .Values.priorityClassName }}
      priorityClassName: {{ .Values.priorityClassName  }}
      {{- end }}
      serviceAccountName: {{ template "metricbeat.serviceAccount" . }}
      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }}
      {{- if .Values.deployment.hostAliases }}
      hostAliases: {{ toYaml .Values.deployment.hostAliases | nindent 6 }}
      {{- end }}
      volumes:
      {{- range .Values.secretMounts | default .Values.deployment.secretMounts }}
      - name: {{ .name }}
        secret:
          secretName: {{ .secretName }}
      {{- end }}
      {{- if .Values.metricbeatConfig }}
      - name: metricbeat-config
        configMap:
          defaultMode: 0600
          name: {{ template "metricbeat.fullname" . }}-config
      {{- else if .Values.deployment.metricbeatConfig }}
      - name: metricbeat-config
        configMap:
          defaultMode: 0600
          name: {{ template "metricbeat.fullname" . }}-deployment-config
      {{- end }}
      {{- if .Values.extraVolumes | default .Values.deployment.extraVolumes }}
{{ toYaml ( .Values.extraVolumes | default .Values.deployment.extraVolumes ) | indent 6 }}
      {{- end }}
      {{- if .Values.imagePullSecrets }}
      imagePullSecrets:
{{ toYaml .Values.imagePullSecrets | indent 8 }}
      {{- end }}
      {{- if .Values.extraInitContainers }}
      initContainers:
{{ tpl .Values.extraInitContainers . | indent 6 }}
      {{- end }}
      containers:
      - name: "metricbeat"
        image: "{{ .Values.image }}:{{ .Values.imageTag }}"
        imagePullPolicy: "{{ .Values.imagePullPolicy }}"
        args:
        {{- if index .Values "metricbeatConfig" "kube-state-metrics-metricbeat.yml" }}
          - "-c"
          - "/usr/share/metricbeat/kube-state-metrics-metricbeat.yml"
        {{- end }}
          - "-e"
          - "-E"
          - "http.enabled=true"
        livenessProbe:
{{ toYaml .Values.livenessProbe | indent 10 }}
        readinessProbe:
{{ toYaml .Values.readinessProbe | indent 10 }}
        resources: {{ toYaml ( .Values.resources | default .Values.deployment.resources ) | nindent 10 }}
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: KUBE_STATE_METRICS_HOSTS
          {{- if .Values.kube_state_metrics.enabled }}
          value: "$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_HOST):$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_PORT_HTTP)"
          {{- else }}
          value: {{ .Values.kube_state_metrics.host | default "kube-state-metrics:8080"}}
          {{- end }}
        - name: cloud.id
          valueFrom:
            secretKeyRef:
              name: {{ .Values.secret.name }}
              key: CLOUD_ID
        - name: cloud.auth
          valueFrom:
            secretKeyRef:
              name: {{ .Values.secret.name }}
              key: CLOUD_AUTH

{{- if .Values.extraEnvs | default .Values.deployment.extraEnvs }}
{{ toYaml ( .Values.extraEnvs | default .Values.deployment.extraEnvs ) | indent 8 }}
{{- end }}
        envFrom: {{ toYaml ( .Values.envFrom | default .Values.deployment.envFrom ) | nindent 10 }}
        securityContext: {{ toYaml ( .Values.podSecurityContext | default .Values.deployment.securityContext ) | nindent 10 }}
        volumeMounts:
        {{- range .Values.secretMounts | default .Values.deployment.secretMounts }}
        - name: {{ .name }}
          mountPath: {{ .path }}
          {{- if .subPath }}
          subPath: {{ .subPath }}
          {{- end }}
        {{- end }}
        {{- range $path, $config := .Values.metricbeatConfig }}
        - name: metricbeat-config
          mountPath: /usr/share/metricbeat/{{ $path }}
          readOnly: true
          subPath: {{ $path }}
        {{ else }}
        {{- range $path, $config := .Values.deployment.metricbeatConfig }}
        - name: metricbeat-config
          mountPath: /usr/share/metricbeat/{{ $path }}
          readOnly: true
          subPath: {{ $path }}
        {{- end }}
        {{- end }}
        {{- if .Values.extraVolumeMounts | default .Values.deployment.extraVolumeMounts }}
{{ toYaml ( .Values.extraVolumeMounts | default .Values.deployment.extraVolumeMounts ) | indent 8 }}
        {{- end }}
      {{- if .Values.extraContainers }}
{{ tpl .Values.extraContainers . | indent 6 }}
      {{- end }}
{{- end }}

and this is how I am executing helm:

helm upgrade --install mbeat elastic/metricbeat --version 7.17.3 -f ./values.yaml --set cloud.id=${{ secrets.CLOUD_ID }},cloud.auth=${{ secrets.CLOUD_AUTH }}

But it is still taking the value of cloud.id and cloud.auth given in values.yaml its not using what I setting through helm upgrade install command

enter image description here

kubernetes-helm elastic-stack metricbeat
Original Q&A
1

There are 1 best solutions below

11
Harsh Manvar On

You generate secret file will be something like

template/secret.yaml

apiVersion: v1
kind: Secret
metadata:
  name: {{ .Values.secret.name }}
data:
  CLOUD_ID: {{ .Values.data.cloud_id }}

this is your values.yaml

replicaCount: 1

secret:
  name: common

data:
  cloud_id: "xyz"
  username: "user"
  password: "admin123"

So values from values.yaml gets applied to the secret and the template got created.

The error mentioned in question: Exiting: error loading config file: yaml: invalid map key: map[interface {}]interface {}{".Values.cloud.id":interface {}(nil)}

I am not able to find the .Values.cloud.id that you have used which throws an error.

values.yaml

daemonset:
  metricbeatConfig:
    metricbeat.yml:
      metricbeat.modules:
        - module: kubernetes
          enabled: true
          metricsets:
            - state_node
            - state_deployment
            - state_replicaset
            - state_pod
            - state_container
          period: 10s
          hosts: ["${KUBE_STATE_METRICS_HOSTS}"]
        cloud.id: cloudId

secret.yaml

apiVersion: v1
kind: secret
metadata:
  name: secret-name
data:
{{- $root := . -}}
{{- range $key, $value := .Values.daemonset.metricbeatConfig }}
  {{- if $key | regexMatch ".*\\.ya?ml$" }}
  {{ $key }}: |
{{ toYaml $value | default "{}" | indent 4 }}
  {{- else }}
  {{ $key }}: {{ toYaml $value | indent 4 }}
  {{- end }}
{{- end -}}

it will create secret something like

apiVersion: v1
kind: secret
metadata:
  name: secret-name
data:
  metricbeat.yml: |
      metricbeat.modules:
        - module: kubernetes
          enabled: true
          metricsets:
            - state_node
            - state_deployment
            - state_replicaset
            - state_pod
            - state_container
          period: 10s
          hosts: ["${KUBE_STATE_METRICS_HOSTS}"]
        cloud.id: cloudId

if you trying to get the value of cloud.id from values.yaml you can not directly get that as it's part of metricbeat.yml file. Do let me know if you are looking for that.

Update :

values.yaml

secret:
  name: elastic

data:
  cloud_id: elk:ZWFzdHVzLmF6dXJlLmVsYXN0aWMtY2xvdWQuY29tOjQ0MyRmZjI5MjNjYTNkNTI0Yzg1ODBiMDk0ZGQxNTRmODQ5ZSQ3MmEyYjY5OTdjMWQ0ODFjYTViYjlkZTY3OGIxMzAzZA==
  cloud_auth: elastic:jixuoLFqs0vneFNaByIQdtL9

secret.yaml

apiVersion: v1
kind: Secret
metadata:
  name: {{ .Values.secret.name }}
type: Opaque
data:
  CLOUD_ID: {{ .Values.data.cloud_id }}
  CLOUD_AUTH: {{ .Values.data.cloud_auth }}

command

 helm template testschart --version 1.0.2 --output-dir ./yaml . --debug --set data.cloud_id="asdfasf"

enter image description here

Related Questions in KUBERNETES-HELM

Related Questions in ELASTIC-STACK

Related Questions in METRICBEAT

Trending Questions

Popular # Hahtags

javascript python java c# php android html jquery c++ css ios sql mysql r reactjs node.js arrays c asp.net json

Popular Questions

.

Copyright © 2021 Jogjafile Inc.