I have Horizontal Pod Autoscaler with max 4 pods and when there are more than 1 pods running, i see application server log in all the pods are exact replica, Main issue is logs are written from one pod only, nothing is printed from other 2 Pods (auto scaled one's)
Example:
My Application server is Jboss 5.1 When there are 3 pods, like Pod 1, Pod 2 & Pod 3 If i access jboss server log file through the terminal in each of these pod's, content of server.log file is exact replica in all 3 pods and i am not able to find the logs from Pod 2 & Pod3
Is there a config i need to update to print logs from all the pods with in the container (Note HPA is enabled to scale up to 4 pods)
Below is my Deployment yaml:
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "32"
creationTimestamp: "2022-04-06T07:02:40Z"
generation: 149
labels:
customerid: 4000-sss-prod
environment: prod
managedFields:
- apiVersion: apps/v1
fieldsType: FieldsV1
fieldsV1:
f:spec:
f:replicas: {}
manager: vpa-recommender
operation: Update
subresource: scale
- apiVersion: apps/v1
fieldsType: FieldsV1
fieldsV1:
f:spec:
f:progressDeadlineSeconds: {}
f:revisionHistoryLimit: {}
f:selector: {}
f:strategy:
f:rollingUpdate:
.: {}
f:maxSurge: {}
f:maxUnavailable: {}
f:type: {}
f:template:
f:metadata:
f:labels:
.: {}
f:app: {}
f:version: {}
f:spec:
f:containers:
k:{"name":"ecommerce"}:
.: {}
f:env:
.: {}
k:{"name":"JAVA_OPTS"}:
.: {}
f:name: {}
k:{"name":"JIRA_KEY"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"SF_ACCOUNT_NUMBER"}:
.: {}
f:name: {}
f:value: {}
f:imagePullPolicy: {}
f:name: {}
f:ports:
.: {}
k:{"containerPort":9443,"protocol":"TCP"}:
.: {}
f:containerPort: {}
f:protocol: {}
f:resources:
.: {}
f:limits: {}
f:requests: {}
f:securityContext:
.: {}
f:capabilities:
.: {}
f:add: {}
f:privileged: {}
f:terminationMessagePath: {}
f:terminationMessagePolicy: {}
f:volumeMounts:
.: {}
k:{"mountPath":"/var/persistent/"}:
.: {}
f:mountPath: {}
f:name: {}
f:dnsPolicy: {}
f:restartPolicy: {}
f:schedulerName: {}
f:securityContext: {}
f:terminationGracePeriodSeconds: {}
f:volumes:
.: {}
k:{"name":"XXXX3-persistent-pv"}:
.: {}
f:name: {}
f:persistentVolumeClaim: {}
manager: Go-http-client
operation: Update
time: "2022-04-06T07:02:40Z"
- apiVersion: apps/v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations: {}
f:status:
f:conditions:
.: {}
k:{"type":"Available"}:
.: {}
f:type: {}
k:{"type":"Progressing"}:
.: {}
f:lastTransitionTime: {}
f:status: {}
f:type: {}
manager: kube-controller-manager
operation: Update
time: "2022-06-25T06:11:17Z"
- apiVersion: apps/v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:labels:
.: {}
f:customerid: {}
f:environment: {}
f:spec:
f:template:
f:metadata:
f:labels:
f:customerid: {}
f:environment: {}
f:spec:
f:containers:
k:{"name":"ecommerce"}:
f:env:
k:{"name":"JAVA_OPTS"}:
f:value: {}
f:image: {}
f:resources:
f:limits:
f:cpu: {}
f:memory: {}
f:requests:
f:cpu: {}
f:memory: {}
f:nodeSelector: {}
f:volumes:
k:{"name":"XXXX3-persistent-pv"}:
f:persistentVolumeClaim:
f:claimName: {}
manager: GoogleCloudConsole
operation: Update
time: "2023-04-03T07:00:21Z"
- apiVersion: apps/v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
f:deployment.kubernetes.io/revision: {}
f:status:
f:availableReplicas: {}
f:conditions:
k:{"type":"Available"}:
f:lastTransitionTime: {}
f:lastUpdateTime: {}
f:message: {}
f:reason: {}
f:status: {}
k:{"type":"Progressing"}:
f:lastUpdateTime: {}
f:message: {}
f:reason: {}
f:observedGeneration: {}
f:readyReplicas: {}
f:replicas: {}
f:updatedReplicas: {}
manager: kube-controller-manager
operation: Update
subresource: status
time: "2023-04-13T11:58:40Z"
name: ecommerce
namespace: 4000-sss-prod
resourceVersion: "7896582563"
uid: fghyytu-k258-4jb4-77z0-XXXXXXXXXXXX
spec:
progressDeadlineSeconds: 600
replicas: 3
revisionHistoryLimit: 10
selector:
matchLabels:
app: ecommerce
version: beta
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
app: ecommerce
customerid: 4000-sss-prod
environment: prod
version: beta
spec:
containers:
- env:
- name: SF_ACCOUNT_NUMBER
value: "4000"
- name: JIRA_KEY
value: sss-prod
- name: JAVA_OPTS
value: -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/persistent/auto-heap-dumps
-Dxxx4.host.address=www.gosss.com -Dxxx4.db.connection=jdbc:oracle:thin:@4000db.xxxx.net:1521:stage
-Dxxx4.db.user=DRY_RUN_BOND -Dxxx4.db.access.token=dry_run_bond -Djboss.server.log.dir=/var/XXXXXXXXXXXX/XXXXXXXXXXXX/XXXXXXXXXXXX/logs/
image: gcr.io/XXXXXXXXXXXX-core/XXXXXXXXXXXX/XXXXXXXXXXXX:XXXXXXXXXXXX_40
imagePullPolicy: IfNotPresent
name: ecommerce
ports:
- containerPort: 9443
protocol: TCP
resources:
limits:
cpu: "4"
memory: 18Gi
requests:
cpu: "1"
memory: 8Gi
securityContext:
capabilities:
add:
- SYS_ADMIN
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/persistent/
name: XXXX3-persistent-pv
dnsPolicy: ClusterFirst
nodeSelector:
cloud.google.com/gke-nodepool: stable-prod-e2
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 150
volumes:
- name: XXXX3-persistent-pv
persistentVolumeClaim:
claimName: 4000-sss-fs
status:
availableReplicas: 3
conditions:
- lastTransitionTime: "2022-04-06T07:42:00Z"
lastUpdateTime: "2023-04-03T06:58:35Z"
message: ReplicaSet "ecommerce-7cd6dd596c" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
- lastTransitionTime: "2023-04-13T11:58:40Z"
lastUpdateTime: "2023-04-13T11:58:40Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 149
readyReplicas: 3
replicas: 3
updatedReplicas: 3