Fighting with getting splunk to run in docker on photonOS system.
I'm mounting /opt/splunk/etc and /opt/splunk/var as nfs volumes and the ansible installation process built into the docker image fails when it tries to change the ownership of the /opt/splunk/etc folder.
2023-06-07T22:50:23.959798220Z TASK [splunk_common : Update Splunk directory owner] ***************************
2023-06-07T22:50:23.960519384Z fatal: [localhost]: FAILED! => {
2023-06-07T22:50:23.960528648Z "changed": false,
2023-06-07T22:50:23.960534086Z "gid": 999,
2023-06-07T22:50:23.960539096Z "group": "ansible",
2023-06-07T22:50:23.960544285Z "mode": "0777",
2023-06-07T22:50:23.960562227Z "owner": "ansible",
2023-06-07T22:50:23.960567851Z "path": "/opt/splunk/etc",
2023-06-07T22:50:23.960572882Z "size": 0,
2023-06-07T22:50:23.960577824Z "state": "directory",
2023-06-07T22:50:23.960582803Z "uid": 999
2023-06-07T22:50:23.960587710Z }
2023-06-07T22:50:23.960597501Z MSG:
2023-06-07T22:50:23.960602388Z
2023-06-07T22:50:23.960620964Z chown failed: [Errno 1] Operation not permitted: b'/opt/splunk/etc'
By default the volume mounts as root so I've tried to chown'd the nfs source folders to the UID:GID to match the splunk:splunk and ansible:ansible users and I can see in the container that the volumes mount correctly with these user permissions. However the above error persists.
jeff@photon-machine [ ~ ]$ docker container exec splunk ls -l /opt/splunk
total 3052
...snip...
drwxrwxrwx 1 splunk splunk 0 Jun 7 21:58 etc
...snip...
drwxrwxrwx 1 splunk splunk 0 Jun 7 21:58 var
Ugh...
Jeff
Here's the details of the setup: Docker running in PhotonOS, leveraging NFSv4 and CIFv3 shares hosted on Synology NAS.
Jeff@diskstation:~$ uname -a
Linux diskstation 3.10.108 #42962 SMP Fri Mar 24 00:28:41 CST 2023 x86_64 GNU/Linux synology_avoton_415+
jeff@photon-machine [ ~ ]$ uname -a
Linux photon-machine 5.10.168-2.ph4-esx #1-photon SMP Thu Mar 2 03:44:00 UTC 2023 x86_64 GNU/Linux
jeff@photon-machine [ ~ ]$ docker -v
Docker version 20.10.14, build a224086
jeff@photon-machine [ ~ ]$ docker inspect splunk
[
{
"Id": "0c30a3224f353ea998d3e756175712b3e7b9969ad3daa76bf3cd7f3565c0ded1",
"Created": "2023-06-07T01:28:19.766822084Z",
"Path": "/sbin/entrypoint.sh",
"Args": [
"start-service"
],
"State": {
"Status": "exited",
"Running": false,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 0,
"ExitCode": 255,
"Error": "",
"StartedAt": "2023-06-07T01:51:04.731635988Z",
"FinishedAt": "2023-06-07T04:49:44.961296555Z",
"Health": {
"Status": "unhealthy",
"FailingStreak": 20,
"Log": [
{
"Start": "2023-06-07T04:40:17.518101926Z",
"End": "2023-06-07T04:40:48.394343685Z",
"ExitCode": -1,
"Output": "Health check exceeded timeout (30s)"
},
{
"Start": "2023-06-07T04:41:27.585164745Z",
"End": "2023-06-07T04:42:03.530697274Z",
"ExitCode": -1,
"Output": "Health check exceeded timeout (30s)"
},
{
"Start": "2023-06-07T04:42:57.966912236Z",
"End": "2023-06-07T04:44:38.672315024Z",
"ExitCode": -1,
"Output": "Health check exceeded timeout (30s)"
},
{
"Start": "2023-06-07T04:45:14.428014408Z",
"End": "2023-06-07T04:45:45.98340735Z",
"ExitCode": -1,
"Output": "Health check exceeded timeout (30s)"
},
{
"Start": "2023-06-07T04:46:33.825896157Z",
"End": "2023-06-07T04:46:34.426481881Z",
"ExitCode": 137,
"Output": ""
}
]
}
},
"Image": "sha256:04b6d9d4490c5af5b901896920b73fb33537bb82a3ee16927f479456199b19b1",
"ResolvConfPath": "/var/lib/docker/containers/0c30a3224f353ea998d3e756175712b3e7b9969ad3daa76bf3cd7f3565c0ded1/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/0c30a3224f353ea998d3e756175712b3e7b9969ad3daa76bf3cd7f3565c0ded1/hostname",
"HostsPath": "/var/lib/docker/containers/0c30a3224f353ea998d3e756175712b3e7b9969ad3daa76bf3cd7f3565c0ded1/hosts",
"LogPath": "/var/lib/docker/containers/0c30a3224f353ea998d3e756175712b3e7b9969ad3daa76bf3cd7f3565c0ded1/0c30a3224f353ea998d3e756175712b3e7b9969ad3daa76bf3cd7f3565c0ded1-json.log",
"Name": "/splunk",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "docker-default",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"splunk_var_cif:/opt/splunk/var",
"splunk_etc_cif:/opt/splunk/etc"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "bridge",
"PortBindings": {},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": [
"AUDIT_WRITE",
"CHOWN",
"DAC_OVERRIDE",
"FOWNER",
"FSETID",
"KILL",
"MKNOD",
"NET_BIND_SERVICE",
"NET_RAW",
"SETFCAP",
"SETGID",
"SETPCAP",
"SETUID",
"SYS_CHROOT"
],
"CapDrop": [
"AUDIT_CONTROL",
"BLOCK_SUSPEND",
"DAC_READ_SEARCH",
"IPC_LOCK",
"IPC_OWNER",
"LEASE",
"LINUX_IMMUTABLE",
"MAC_ADMIN",
"MAC_OVERRIDE",
"NET_ADMIN",
"NET_BROADCAST",
"SYSLOG",
"SYS_ADMIN",
"SYS_BOOT",
"SYS_MODULE",
"SYS_NICE",
"SYS_PACCT",
"SYS_PTRACE",
"SYS_RAWIO",
"SYS_RESOURCE",
"SYS_TIME",
"SYS_TTY_CONFIG",
"WAKE_ALARM"
],
"CgroupnsMode": "host",
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": [],
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": true,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DeviceRequests": [],
"KernelMemory": 0,
"KernelMemoryTCP": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
],
"Init": false
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/1c87ea8979fed2b10ca39669fdb7e9c528c08c73cca41ae98aec7a16ee0171c9-init/diff:/var/lib/docker/overlay2/8f05a00f757925e42612483444171bc331f29a2b16dfbd7dd24d77da90a239b3/diff:/var/lib/docker/overlay2/22ef39100ae8b649da3861d3436bfc13242bef6b40015fe3f04ec4577447fe69/diff:/var/lib/docker/overlay2/9ee62e22b056d50c75ea1216f08b8b47ed51a03976e44aea05af305d0f9587f6/diff:/var/lib/docker/overlay2/dab8f18053315fddc60ddb28f67ed33cdbc39095a3e9179fc7a1e42d0ca4ddc2/diff:/var/lib/docker/overlay2/1fdcff861cd8066ffad875ffa4c062369d2d053b41aca1d4506695448a4508a0/diff:/var/lib/docker/overlay2/61de3e442a14b7495e31aa260df899781f451706173aae9fec28b0121db0c7cf/diff:/var/lib/docker/overlay2/022df818fa54be0ff80efe6aeb9a6afb8439f9ec5d68bc58a3a9d4480fdd5755/diff:/var/lib/docker/overlay2/5cdd113b3166943d0a6e75b78c633e9358c8e9d078e3dc98ad38114f8e7af7d8/diff:/var/lib/docker/overlay2/0d9a5942db333802c68aff909b9fb275ed884a61741fd487feebbe5494bd711a/diff:/var/lib/docker/overlay2/581f3da760a2700c6de684784dfc93335533fdd771bc46cf26f4ddafcf620365/diff:/var/lib/docker/overlay2/9bd76717063684dbd16bfbb23461b9859087d3115ac06659bd157b391f926fbd/diff",
"MergedDir": "/var/lib/docker/overlay2/1c87ea8979fed2b10ca39669fdb7e9c528c08c73cca41ae98aec7a16ee0171c9/merged",
"UpperDir": "/var/lib/docker/overlay2/1c87ea8979fed2b10ca39669fdb7e9c528c08c73cca41ae98aec7a16ee0171c9/diff",
"WorkDir": "/var/lib/docker/overlay2/1c87ea8979fed2b10ca39669fdb7e9c528c08c73cca41ae98aec7a16ee0171c9/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "volume",
"Name": "splunk_var_cif",
"Source": "/var/lib/docker/volumes/splunk_var_cif/_data",
"Destination": "/opt/splunk/var",
"Driver": "local",
"Mode": "z",
"RW": true,
"Propagation": ""
},
{
"Type": "volume",
"Name": "splunk_etc_cif",
"Source": "/var/lib/docker/volumes/splunk_etc_cif/_data",
"Destination": "/opt/splunk/etc",
"Driver": "local",
"Mode": "z",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "99127cc0ce43",
"Domainname": "",
"User": "ansible",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"8000/tcp": {},
"8065/tcp": {},
"8088/tcp": {},
"8089/tcp": {},
"8191/tcp": {},
"9887/tcp": {},
"9997/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"SPLUNK_PASSWORD=REDACTED",
"SPLUNK_START_ARGS=--accept-license",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"container=oci",
"BUSYBOX_URL=",
"PYTHON_VERSION=3.7.16",
"PYTHON_GPG_KEY_ID=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D",
"SPLUNK_HOME=/opt/splunk",
"SPLUNK_GROUP=splunk",
"SPLUNK_USER=splunk",
"TMPSPLUNKDIR=/opt/splunk/tmp",
"TMPETCDIR=/opt/splunk/tmp/etc",
"SPLUNK_ROLE=splunk_standalone",
"SPLUNK_DEFAULTS_URL=",
"SPLUNK_ANSIBLE_HOME=/opt/ansible",
"ANSIBLE_USER=ansible",
"ANSIBLE_GROUP=ansible",
"CONTAINER_ARTIFACT_DIR=/opt/container_artifact"
],
"Cmd": [
"start-service"
],
"Healthcheck": {
"Test": [
"CMD-SHELL",
"/sbin/checkstate.sh || exit 1"
],
"Interval": 30000000000,
"Timeout": 30000000000,
"StartPeriod": 180000000000,
"Retries": 5
},
"Image": "splunk/splunk:latest",
"Volumes": {
"/opt/splunk/etc": {},
"/opt/splunk/var": {}
},
"WorkingDir": "/opt/splunk",
"Entrypoint": [
"/sbin/entrypoint.sh"
],
"OnBuild": null,
"Labels": {
"architecture": "x86_64",
"build-date": "2023-05-03T15:02:09",
"com.redhat.component": "ubi8-minimal-container",
"com.redhat.license_terms": "https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI",
"description": "Splunk Enterprise is a platform for operational intelligence. Our software lets you collect, analyze, and act upon the untapped value of big data that your technology infrastructure, security systems, and business applications generate. It gives you insights to drive operational performance and business results.",
"distribution-scope": "public",
"io.buildah.version": "1.27.3",
"io.k8s.description": "The Universal Base Image Minimal is a stripped down image that uses microdnf as a package manager. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.",
"io.k8s.display-name": "Red Hat Universal Base Image 8 Minimal",
"io.openshift.expose-services": "",
"io.openshift.tags": "minimal rhel8",
"maintainer": "[email protected]",
"name": "splunk",
"release": "1",
"summary": "UBI 8 Docker image of Splunk Enterprise",
"url": "https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8-minimal/images/8.8-860",
"vcs-ref": "dee8029ddcc7ecbfbebb0905d2b15e134338616c",
"vcs-type": "git",
"vendor": "splunk",
"version": "9.0.5"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "9b657058a56ad136dcd067cb165beab924d7ff88a4e29951b51cc9602d917997",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"8000/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "49810"
},
{
"HostIp": "::",
"HostPort": "49810"
}
],
"8065/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "49809"
},
{
"HostIp": "::",
"HostPort": "49809"
}
],
"8088/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "49808"
},
{
"HostIp": "::",
"HostPort": "49808"
}
],
"8089/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "49807"
},
{
"HostIp": "::",
"HostPort": "49807"
}
],
"8191/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "49806"
},
{
"HostIp": "::",
"HostPort": "49806"
}
],
"9887/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "49805"
},
{
"HostIp": "::",
"HostPort": "49805"
}
],
"9997/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "49804"
},
{
"HostIp": "::",
"HostPort": "49804"
}
]
},
"SandboxKey": "/var/run/docker/netns/9b657058a56a",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "06508cd155fb478ea8c644db3620285695c5da0c350b1f8500f7b4510b01a4ec",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.5",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "02:42:ac:11:00:05",
"Networks": {
"bridge": {
"IPAMConfig": {},
"Links": null,
"Aliases": null,
"NetworkID": "60b3d4ca9cf390ae7f2d59922000b8233be3a1baa1987301b040b76751113ba3",
"EndpointID": "06508cd155fb478ea8c644db3620285695c5da0c350b1f8500f7b4510b01a4ec",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.5",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:11:00:05",
"DriverOpts": null
}
}
}
}
]