In centos8/rhel8, containers created with systemd as cgroup-managers are stored like this under /sys/fs/cgroup [1]
In centos9 they are all stored under /sys/fs/cgroup/machine.slice.
When nova_virtlogd_wrapper tries to spin nova_virtlogd, it gets an error by default:
Error: create directory `/sys/fs/cgroup/../../libpod-366360a13cc787666d27753ec730ea082d838b9bfe013d62eea6a8ad10b055ac.scope`: No such file or directory: OCI runtime attempted to invoke a command that was not found
If we add --cgroup-manager=cgroupfs to the wrapper, we have something like this:
Error: writing file `/sys/fs/cgroup/libpod_parent/libpod-4718f186840ee64b1ea3e4440dbbdcdc3a04e12405d5f890a1ab65df1c31707a/cgroup.procs`: No such file or directory: OCI runtime attempted to invoke a command that was not found
If we add --cgroup-manager=cgroupfs --cgroup-parent=machine.slice, we get this:
Error: systemd slice received as cgroup parent when using cgroupfs: invalid argument
If we try this --cgroup-manager=cgroupfs --cgroup-parent=test/dvd, we get this:
Error: writing file `/sys/fs/cgroup/test/dvd/libpod-6a6e41e75024c10b4b458a3e53ffc757f0ba6abbcede98f13711884887ab7529/cgroup.procs`: No such file or directory: OCI runtime attempted to invoke a command that was not found
So it looks like the virtlogd_wrapper container is not using the same cgroup format as the host itself.
What's interesting is that we can confirm that it's the same version of podman called in both the host and the container [2] when it's called with nsenter but when we don't call nsenter, we see the difference in version [3].
Workaround for this issue would be to NovaEnableVirtlogdContainerWrapper: false.
[1]
~~~
[root@undercloud-0 ~]# find /sys/fs/cgroup/ -maxdepth 3 -type d -name libpod\* | cut -f 1-5 -d "/" | sort | uniq
/sys/fs/cgroup/blkio
/sys/fs/cgroup/cpu,cpuacct
/sys/fs/cgroup/cpuset
/sys/fs/cgroup/devices
/sys/fs/cgroup/freezer
/sys/fs/cgroup/hugetlb
/sys/fs/cgroup/memory
/sys/fs/cgroup/net_cls,net_prio
/sys/fs/cgroup/perf_event
/sys/fs/cgroup/pids
/sys/fs/cgroup/systemd
~~~
[2]
~~~
[root@standalone stdouts]# diff -u <(rpm -qa | grep pod) <(podman exec -u0 nova_virtlogd_wrapper nsenter --preserve-credentials -m -t 1 rpm -qa | grep pod)
[root@standalone stdouts]# diff -u <(podman info) <(podman exec -u0 nova_virtlogd_wrapper nsenter --preserve-credentials -m -t 1 podman info)
--- /dev/fd/63 2021-07-13 22:13:06.223105268 +0000
+++ /dev/fd/62 2021-07-13 22:13:06.225105271 +0000
@@ -27,7 +27,7 @@
uidmap: null
kernel: 5.13.0-1.el9.x86_64
linkmode: dynamic
- memFree: 8744153088
+ memFree: 8710406144
memTotal: 15712301056
ociRuntime:
name: crun
@@ -55,7 +55,7 @@
version: ""
swapFree: 1073737728
swapTotal: 1073737728
- uptime: 25m 7.44s
+ uptime: 25m 7.53s
registries:
192.168.24.2:8787:
Blocked: false
~~~
[3]
~~~
[root@standalone stdouts]# diff -u <(rpm -qa | grep pod) <(podman exec -u0 nova_virtlogd_wrapper rpm -qa | grep pod)
--- /dev/fd/63 2021-07-13 22:14:32.258239798 +0000
+++ /dev/fd/62 2021-07-13 22:14:32.258239798 +0000
@@ -1,4 +1,3 @@
-perl-podlators-4.14-459.el9.noarch
-ansible-collection-containers-podman-1.4.1-2.el9.noarch
-podman-catatonit-3.3.0-0.6.el9.x86_64
-podman-3.3.0-0.6.el9.x86_64
+podman-3.1.0-0.13.module_el8.5.0+733+9bb5dffa.x86_64
+perl-podlators-4.11-1.el8.noarch
+podman-catatonit-3.1.0-0.13.module_el8.5.0+733+9bb5dffa.x86_64
~~~
Related fix proposed to branch: master
Review: https:/