Re: [vserver] File System Full

From: Herbert Poetzl <herbert_at_13thfloor.at>
Date: Fri 10 Feb 2012 - 15:41:29 GMT
Message-ID: <20120210154128.GL13512@MAIL.13thfloor.at>

On Fri, Feb 10, 2012 at 03:10:48PM +0000, Phil Daws wrote:
> 2.6.28.7-vs2.3.0.36.7
> util-vserver-0.30.216-1.pre2793.el5.centos

quite interesting choice ... and quite brave also :)

> Host is using ext4 for /vservers

ext4 was doing strange things up till recently (read
up to 3.x), combined with that quite old Linux-VServer
patch (must be from early 2009) it isn't _that_
unexpected that you encounter one or the other unusual
behaviour ...

my suggestions would be to update immediately, not
just because of the disk full issue, but because of
several kernel exploits and various other issues found
and successfully fixed in mainline and Linux-VServer

HTH,
Herbert

> Guest is:
> cat /etc/mtab
> /dev/hdv1 / ufs defaults 0 0
> none /proc proc defaults 0 0
> none /tmp tmpfs size=128m,mode=1777,noexec,nosuid,nodev 0 0

> --
> Thanks, Phil

> ----- Original Message -----
>> On Fri, Feb 10, 2012 at 10:45:00AM +0000, Phil Daws wrote:
>>> Hello all,

>>> we are seeing a strange issue where one of our vserver guests
>>> is reporting that its file system is full yet there is plenty
>>> of space:

>>> usera:/var/run# df -h
>>> Filesystem Size Used Avail Use% Mounted on
>>> /dev/hdv1 1008G 187G 812G 19% /
>>> none 128M 0 128M 0% /tmp
>>> usera:/var/run# touch testfile
>>> touch: cannot touch `testfile': No space left on device

>> as usual, kernel/patch/util-vserver version please.

>> an in particular, what filesystem, mount options and
>> tag setup?

>> TIA,
>> Herbert

>>> Below is a strace -f when running the touch testfile command:

>>> usera:/var/run# strace -f touch testfile
>>> execve("/usr/bin/touch", ["touch", "testfile"], [/* 28 vars */]) =
>>> 0
>>> brk(0) = 0x60d000
>>> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
>>> directory)
>>> mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
>>> -1, 0) = 0x7f3429ad3000
>>> access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or
>>> directory)
>>> open("/etc/ld.so.cache", O_RDONLY) = 3
>>> fstat(3, {st_mode=S_IFREG|0644, st_size=153901, ...}) = 0
>>> mmap(NULL, 153901, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3429aad000
>>> close(3) = 0
>>> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
>>> directory)
>>> open("/lib/librt.so.1", O_RDONLY) = 3
>>> read(3,
>>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220!\0\0\0\0\0\0@"...,
>>> 832) = 832
>>> fstat(3, {st_mode=S_IFREG|0644, st_size=31744, ...}) = 0
>>> mmap(NULL, 2128848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE,
>>> 3, 0) = 0x7f34296b0000
>>> mprotect(0x7f34296b7000, 2093056, PROT_NONE) = 0
>>> mmap(0x7f34298b6000, 8192, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f34298b6000
>>> close(3) = 0
>>> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
>>> directory)
>>> open("/lib/libc.so.6", O_RDONLY) = 3
>>> read(3,
>>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\355\1\0\0\0\0\0@"...,
>>> 832) = 832
>>> fstat(3, {st_mode=S_IFREG|0755, st_size=1432968, ...}) = 0
>>> mmap(NULL, 3541032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE,
>>> 3, 0) = 0x7f342934f000
>>> mprotect(0x7f34294a7000, 2093056, PROT_NONE) = 0
>>> mmap(0x7f34296a6000, 20480, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x157000) = 0x7f34296a6000
>>> mmap(0x7f34296ab000, 18472, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f34296ab000
>>> close(3) = 0
>>> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
>>> directory)
>>> open("/lib/libpthread.so.0", O_RDONLY) = 3
>>> read(3,
>>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340Y\0\0\0\0\0\0@"...,
>>> 832) = 832
>>> fstat(3, {st_mode=S_IFREG|0755, st_size=131260, ...}) = 0
>>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
>>> -1, 0) = 0x7f3429aac000
>>> mmap(NULL, 2208640, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE,
>>> 3, 0) = 0x7f3429133000
>>> mprotect(0x7f342914a000, 2093056, PROT_NONE) = 0
>>> mmap(0x7f3429349000, 8192, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f3429349000
>>> mmap(0x7f342934b000, 13184, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f342934b000
>>> close(3) = 0
>>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
>>> -1, 0) = 0x7f3429aab000
>>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
>>> -1, 0) = 0x7f3429aaa000
>>> arch_prctl(ARCH_SET_FS, 0x7f3429aab700) = 0
>>> open("/dev/urandom", O_RDONLY) = 3
>>> read(3, "\270_f\37!\232;"..., 7) = 7
>>> close(3) = 0
>>> mprotect(0x7f3429349000, 4096, PROT_READ) = 0
>>> mprotect(0x7f34296a6000, 16384, PROT_READ) = 0
>>> mprotect(0x7f34298b6000, 4096, PROT_READ) = 0
>>> mprotect(0x7f3429ad5000, 4096, PROT_READ) = 0
>>> munmap(0x7f3429aad000, 153901) = 0
>>> set_tid_address(0x7f3429aab9d0) = 13524
>>> set_robust_list(0x7f3429aab9e0, 0x18) = 0
>>> futex(0x7fff31ad739c, FUTEX_WAKE_PRIVATE, 1) = 0
>>> futex(0x7fff31ad739c, 0x189 /* FUTEX_??? */, 1, NULL, 7f3429aab700)
>>> = -1 ENOSYS (Function not implemented)
>>> rt_sigaction(SIGRTMIN, {0x7f3429138860, [], SA_RESTORER|SA_SIGINFO,
>>> 0x7f3429141f60}, NULL, 8) = 0
>>> rt_sigaction(SIGRT_1, {0x7f34291388f0, [],
>>> SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f3429141f60}, NULL, 8) = 0
>>> rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
>>> getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024,
>>> rlim_max=RLIM_INFINITY}) = 0
>>> brk(0) = 0x60d000
>>> brk(0x62e000) = 0x62e000
>>> open("/usr/lib/locale/locale-archive", O_RDONLY) = 3
>>> fstat(3, {st_mode=S_IFREG|0644, st_size=85880592, ...}) = 0
>>> mmap(NULL, 85880592, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3423f4c000
>>> close(3) = 0
>>> open("testfile", O_WRONLY|O_CREAT|O_NOCTTY|O_NONBLOCK, 0666) = -1
>>> ENOSPC (No space left on device)
>>> futimesat(AT_FDCWD, "testfile", NULL) = -1 ENOENT (No such file or
>>> directory)
>>> open("/usr/share/locale/locale.alias", O_RDONLY) = 3
>>> fstat(3, {st_mode=S_IFREG|0644, st_size=2570, ...}) = 0
>>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
>>> -1, 0) = 0x7f3429ad2000
>>> read(3, "# Locale name alias data base.\n# "..., 4096) = 2570
>>> read(3, ""..., 4096) = 0
>>> close(3) = 0
>>> munmap(0x7f3429ad2000, 4096) = 0
>>> open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo",
>>> O_RDONLY) = -1 ENOENT (No such file or directory)
>>> open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo",
>>> O_RDONLY) = -1 ENOENT (No such file or directory)
>>> open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY)
>>> = -1 ENOENT (No such file or directory)
>>> open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo",
>>> O_RDONLY) = -1 ENOENT (No such file or directory)
>>> open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo",
>>> O_RDONLY) = -1 ENOENT (No such file or directory)
>>> open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) =
>>> -1 ENOENT (No such file or directory)
>>> write(2, "touch: "..., 7touch: ) = 7
>>> write(2, "cannot touch `testfile'"..., 23cannot touch `testfile') =
>>> 23
>>> open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY)
>>> = -1 ENOENT (No such file or directory)
>>> open("/usr/share/locale/en_US.utf8/LC_MESSAGES/libc.mo", O_RDONLY)
>>> = -1 ENOENT (No such file or directory)
>>> open("/usr/share/locale/en_US/LC_MESSAGES/libc.mo", O_RDONLY) = -1
>>> ENOENT (No such file or directory)
>>> open("/usr/share/locale/en.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) =
>>> -1 ENOENT (No such file or directory)
>>> open("/usr/share/locale/en.utf8/LC_MESSAGES/libc.mo", O_RDONLY) =
>>> -1 ENOENT (No such file or directory)
>>> open("/usr/share/locale/en/LC_MESSAGES/libc.mo", O_RDONLY) = -1
>>> ENOENT (No such file or directory)
>>> write(2, ": No space left on device"..., 25: No space left on
>>> device) = 25
>>> write(2, "\n"..., 1
>>> ) = 1
>>> close(1) = 0
>>> close(2) = 0
>>> exit_group(1) = ?

>>> Any thoughts as I have checked the host system and that looks
>>> absolutely fine; and so are the other vservers. Weird! :)
>>> --
>>> Thanks, Phil
Received on Fri Feb 10 15:41:37 2012

[Next/Previous Months] [Main vserver Project Homepage] [Howto Subscribe/Unsubscribe] [Paul Sladen's vserver stuff]
Generated on Fri 10 Feb 2012 - 15:41:37 GMT by hypermail 2.1.8