On Tue, Mar 13, 2007 at 12:43:30PM +1300, Peter wrote:
> Hi. We have a problem where a domU exits and cannot be restarted. It
> looks like the problem is because its file:/blah device is already
> mounted. And it cannot be unmounted.
>
> Any ideas on a fix? Or a way to umount that loop back device?
>
> The only way we seem to be able to recover is with a dom0/host server
> restart.
>
> [root@host104 ~]# xm list | grep sitesound
> (domU not running)
>
> [root@host104 ~]# looplist.sh | grep sitesound
> /dev/loop14: [0901]:89325577
> (/home/users/sitesoundstudios.com/debian.sarge-2.ext3.nobackup)
>
> (shows the domU file system is mounted as /dev/loop14)
>
> [root@host104 ~]# ps auxf | grep [l]oop14\]
> root 15438 0.0 0.0 0 0 ? S< Feb22 0:01 [loop14]
>
> [root@host104 ~]# losetup -d /dev/loop14
> ioctl: LOOP_CLR_FD: Device or resource busy
>
> # cat /home/users/sitesoundstudios.com/params
> rkernel = "/home/users/sitesoundstudios.com/linux"
> memory = 288
> swap = 128
> name = "sitesoundstudios.com"
> ipaddr = "72.249.38.171"
> disk = [
> 'file:/home/users/sitesoundstudios.com/debian.sarge-2.ext3.nobackup,xvda1,w',
> 'file:/home/users/sitesoundstudios.com/swapfs.swp,xvda9,w' ]
> root = "/dev/xvda1 ro"
> restart = 'onreboot'
> console = 50002
> vif = [ 'mac=aa:00:56:6d:26:14,bridge=xen-br0' ]
>
> xm version is 3.0.3-0
>
> Possibly related is the alloc_pages errors below.
>
> xend.log during the attempt to start the domU.
>
> [2007-03-08 21:53:47 xend.XendDomainInfo 19559] DEBUG (XendDomainInfo:1253)
> XendDomainInfo.construct: None
> [2007-03-08 21:53:47 xend.XendDomainInfo 19559] DEBUG (XendDomainInfo:1285)
> XendDomainInfo.initDomain: 201 1.0
> [2007-03-08 21:53:47 xend 19559] DEBUG (balloon:127) Balloon: 3743796
> KiB free;
> need 294912; done.
> [2007-03-08 21:53:47 xend 19559] INFO (image:136) buildDomain os=linux
> dom=201
> vcpus=1
> [2007-03-08 21:53:47 xend 19559] DEBUG (image:191) dom = 201
> [2007-03-08 21:53:47 xend 19559] DEBUG (image:192) image =
> /home/users/sitesoundstudios.com/linux
> [2007-03-08 21:53:47 xend 19559] DEBUG (image:193) store_evtchn = 1
> [2007-03-08 21:53:47 xend 19559] DEBUG (image:194) console_evtchn = 2
> [2007-03-08 21:53:47 xend 19559] DEBUG (image:195) cmdline =
> root=/dev/xvda1 ro
> [2007-03-08 21:53:47 xend 19559] DEBUG (image:196) ramdisk =
> [2007-03-08 21:53:47 xend 19559] DEBUG (image:197) vcpus = 1
> [2007-03-08 21:53:47 xend 19559] DEBUG (image:198) features =
> [2007-03-08 21:53:47 xend 19559] DEBUG (blkif:24) exception looking up
> device
> number for xvda1: [Errno 2] No such file or directory: '/dev/xvda1'
> [2007-03-08 21:53:47 xend 19559] DEBUG (DevController:110) DevController:
> writing {'backend-id': '0', 'virtual-device': '51713', 'device-type':
> 'disk',
> 'state': '1', 'backend': '/local/domain/0/backend/vbd/201/51713'} to
> /local/domain/201/device/vbd/51713.
> [2007-03-08 21:53:47 xend 19559] DEBUG (DevController:112) DevController:
> writing {'domain': 'sitesoundstudios.com', 'frontend':
> '/local/domain/201/device/vbd/51713', 'dev': 'xvda1', 'state': '1',
> 'params':
> '/home/users/sitesoundstudios.com/debian.sarge-2.ext3.nobackup', 'mode':
> 'w',
> 'online': '1', 'frontend-id': '201', 'type': 'file'} to
> /local/domain/0/backend/vbd/201/51713.
> [2007-03-08 21:53:47 xend 19559] DEBUG (blkif:24) exception looking up
> device
> number for xvda9: [Errno 2] No such file or directory: '/dev/xvda9'
> [2007-03-08 21:53:47 xend 19559] DEBUG (DevController:110) DevController:
> writing {'backend-id': '0', 'virtual-device': '51721', 'device-type':
> 'disk',
> 'state': '1', 'backend': '/local/domain/0/backend/vbd/201/51721'} to
> /local/domain/201/device/vbd/51721.
> [2007-03-08 21:53:47 xend 19559] DEBUG (DevController:112) DevController:
> writing {'domain': 'sitesoundstudios.com', 'frontend':
> '/local/domain/201/device/vbd/51721', 'dev': 'xvda9', 'state': '1',
> 'params':
> '/home/users/sitesoundstudios.com/swapfs.swp', 'mode': 'w', 'online': '1',
> 'frontend-id': '201', 'type': 'file'} to
> /local/domain/0/backend/vbd/201/51721.
> [2007-03-08 21:53:47 xend 19559] DEBUG (DevController:110) DevController:
> writing {'backend-id': '0', 'mac': 'aa:00:56:6d:26:14', 'handle': '0',
> 'state':
> '1', 'backend': '/local/domain/0/backend/vif/201/0'} to
> /local/domain/201/device/vif/0.
> [2007-03-08 21:53:47 xend 19559] DEBUG (DevController:112) DevController:
> writing {'bridge': 'xen-br0', 'domain': 'sitesoundstudios.com',
> 'handle': '0',
> 'script': '/etc/xen/scripts/vif-bridge', 'state': '1', 'frontend':
> '/local/domain/201/device/vif/0', 'mac': 'aa:00:56:6d:26:14', 'online': '1',
> 'frontend-id': '201'} to /local/domain/0/backend/vif/201/0.
> [2007-03-08 21:53:47 xend.XendDomainInfo 19559] DEBUG (XendDomainInfo:712)
> Storing VM details: {'shadow_memory': '0', 'uuid':
> '9cfb0c48-9bb2-c0bf-b410-d760bcfd24e1', 'on_reboot': 'restart',
> 'start_time':
> '1173390827.91', 'on_poweroff': 'destroy', 'name': 'sitesoundstudios.com',
> 'xend/restart_count': '0', 'vcpus': '1', 'vcpu_avail': '1', 'memory': '288',
> 'on_crash': 'destroy', 'image': "(linux (kernel
> /home/users/sitesoundstudios.com/linux) (root '/dev/xvda1 ro'))", 'maxmem':
> '288'}
> [2007-03-08 21:53:47 xend.XendDomainInfo 19559] DEBUG (XendDomainInfo:747)
> Storing domain details: {'console/ring-ref': '2511515', 'console/port': '2',
> 'name': 'sitesoundstudios.com', 'console/limit': '1048576', 'vm':
> '/vm/9cfb0c48-9bb2-c0bf-b410-d760bcfd24e1', 'domid': '201',
> 'cpu/0/availability': 'online', 'memory/target': '294912', 'store/ring-ref':
> '1570138', 'store/port': '1'}
> [2007-03-08 21:53:47 xend 19559] DEBUG (XendDomain:198) Cannot recreate
> information for dying domain 163. Xend will ignore this domain from now on.
> [2007-03-08 21:53:47 xend.XendDomainInfo 19559] DEBUG (XendDomainInfo:933)
> XendDomainInfo.handleShutdownWatch
> [2007-03-08 21:53:48 xend 19559] DEBUG (XendDomain:198) Cannot recreate
> information for dying domain 163. Xend will ignore this domain from now on.
This looks suspicious too. It looks like you have a VM that has not been
properly cleaned up -- presumably it's the one that's holding your loopback
device open. Do you have the logs for what happened to domain 163?
Ewan.
> [2007-03-08 21:53:48 xend 19559] DEBUG (DevController:143) Waiting for
> devices
> vif.
> [2007-03-08 21:53:48 xend 19559] DEBUG (DevController:149) Waiting for 0.
> [2007-03-08 21:53:48 xend 19559] DEBUG (DevController:464)
> hotplugStatusCallback /local/domain/0/backend/vif/201/0/hotplug-status.
>
>
>
> Mar 8 17:46:44 host104 kernel: swapper: page allocation failure. order:0,
> mode:0x20
> Mar 8 17:46:44 host104 kernel: [<c013ac94>] __alloc_pages+0x215/0x30d
> Mar 8 17:46:44 host104 kernel: [<c0153835>] kmem_getpages+0x35/0x96
> Mar 8 17:46:44 host104 kernel: [<c0154514>] cache_grow+0xa8/0x173
> Mar 8 17:46:44 host104 kernel: [<c01547a5>] cache_alloc_refill+0x1c6/0x1f6
> Mar 8 17:46:44 host104 kernel: [<c01549ad>] kmem_cache_alloc+0x64/0x68
> Mar 8 17:46:44 host104 kernel: [<c02c606e>]
> alloc_skb_from_cache+0x50/0x103
> Mar 8 17:46:44 host104 kernel: [<c0256597>] __dev_alloc_skb+0x4d/0x79
> Mar 8 17:46:44 host104 kernel: [<dd0c1602>]
> tg3_alloc_rx_skb+0x7e/0x141 [tg3]
> Mar 8 17:46:44 host104 kernel: [<dd0c1b39>] tg3_rx+0x3e3/0x4ac [tg3]
> Mar 8 17:46:44 host104 kernel: [<dd0c1c61>] tg3_poll+0x5f/0x170 [tg3]
> Mar 8 17:46:44 host104 kernel: [<c02cc9ac>] net_rx_action+0x83/0x12f
> Mar 8 17:46:44 host104 kernel: [<c011e292>] __do_softirq+0x52/0xb2
> Mar 8 17:46:44 host104 kernel: [<c011e335>] do_softirq+0x43/0x4b
> Mar 8 17:46:44 host104 kernel: [<c010637f>] do_IRQ+0x1f/0x25
> Mar 8 17:46:44 host104 kernel: [<c02549c7>] evtchn_do_upcall+0x49/0x86
> Mar 8 17:46:44 host104 kernel: [<c032a573>] schedule+0x393/0x6ae
> Mar 8 17:46:44 host104 kernel: [<c0104bd1>] hypervisor_callback+0x3d/0x48
> Mar 8 17:46:44 host104 kernel: [<c01089b5>] safe_halt+0x1a/0x36
> Mar 8 17:46:44 host104 kernel: [<c0108993>] start_hz_timer+0x2/0xa
> Mar 8 17:46:44 host104 kernel: [<c01029c0>] xen_idle+0x22/0x3d
> Mar 8 17:46:44 host104 kernel: [<c0102a11>] cpu_idle+0x36/0x4b
> Mar 8 17:46:44 host104 kernel: [<c03e675a>] start_kernel+0x194/0x1dd
> Mar 8 17:46:44 host104 kernel: [<c03e62e1>] unknown_bootoption+0x0/0x1bc
> Mar 8 17:46:44 host104 kernel: Mem-info:
> Mar 8 17:46:44 host104 kernel: DMA per-cpu:
> Mar 8 17:46:44 host104 kernel: cpu 0 hot: high 186, batch 31 used:30
> Mar 8 17:46:44 host104 kernel: cpu 0 cold: high 62, batch 15 used:58
> Mar 8 17:46:44 host104 kernel: DMA32 per-cpu: empty
> Mar 8 17:46:44 host104 kernel: Normal per-cpu: empty
> Mar 8 17:46:44 host104 kernel: HighMem per-cpu: empty
> Mar 8 17:46:44 host104 kernel: Free pages: 968kB (0kB HighMem)
> Mar 8 17:46:44 host104 kernel: Active:11096 inactive:75176 dirty:1968
> writeback:0 unstable:0 free:242 slab:6302 mapped:7591 pagetables:360
> Mar 8 17:46:44 host104 kernel: DMA free:968kB min:2724kB low:3404kB
> high:4084kB active:44384kB inactive:300704kB present:464800kB
> pages_scanned:0
> all_unreclaimable? no
> Mar 8 17:46:44 host104 kernel: lowmem_reserve[]: 0 0 0 0
> Mar 8 17:46:44 host104 kernel: DMA32 free:0kB min:0kB low:0kB high:0kB
> active:0kB inactive:0kB present:0kB pages_scanned:0 all_unreclaimable? no
> Mar 8 17:46:44 host104 kernel: lowmem_reserve[]: 0 0 0 0
> Mar 8 17:46:44 host104 kernel: Normal free:0kB min:0kB low:0kB high:0kB
> active:0kB inactive:0kB present:0kB pages_scanned:0 all_unreclaimable? no
> Mar 8 17:46:44 host104 kernel: lowmem_reserve[]: 0 0 0 0
> Mar 8 17:46:44 host104 kernel: HighMem free:0kB min:128kB low:128kB
> high:128kB
> active:0kB inactive:0kB present:0kB pages_scanned:0 all_unreclaimable? no
> Mar 8 17:46:44 host104 kernel: lowmem_reserve[]: 0 0 0 0
> Mar 8 17:46:44 host104 kernel: DMA: 0*4kB 1*8kB 0*16kB 0*32kB 1*64kB
> 1*128kB
> 1*256kB 1*512kB 0*1024kB 0*2048kB 0*4096kB = 968kB
> Mar 8 17:46:44 host104 kernel: DMA32: empty
> Mar 8 17:46:44 host104 kernel: Normal: empty
> Mar 8 17:46:44 host104 kernel: HighMem: empty
> Mar 8 17:46:44 host104 kernel: Swap cache: add 142243, delete 137647, find
> 876198/903026, race 0+1
> Mar 8 17:46:44 host104 kernel: Free swap = 4672848kB
> Mar 8 17:46:44 host104 kernel: Total swap = 4723080kB
> Mar 8 17:46:44 host104 kernel: Free swap: 4672848kB
> Mar 8 17:46:44 host104 kernel: 116200 pages of RAM
> Mar 8 17:46:44 host104 kernel: 0 pages of HIGHMEM
> Mar 8 17:46:44 host104 kernel: 18762 reserved pages
> Mar 8 17:46:44 host104 kernel: 67516 pages shared
> Mar 8 17:46:44 host104 kernel: 4596 pages swap cached
> Mar 8 17:46:44 host104 kernel: 1968 pages dirty
> Mar 8 17:46:44 host104 kernel: 0 pages writeback
> Mar 8 17:46:44 host104 kernel: 7591 pages mapped
> Mar 8 17:46:44 host104 kernel: 6302 pages slab
> Mar 8 17:46:44 host104 kernel: 360 pages pagetables
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@xxxxxxxxxxxxxxxxxxx
> http://lists.xensource.com/xen-devel
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
Home