|
|
conductor 日志：
2 d& e" {, u1 V; r: r! T2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager [req-50fa7432-2666-4cda-9712-151321519600 4e5dbd37d7d34992bac0217f5f6dc462 bd230134367943099e27b872e8b1ec53 - default default] Failed to schedule instances: nova.exception_Remote.NoValidHost_Remote: No valid host was found. * }) {6 p8 z) k# _' g/ @9 g
Traceback (most recent call last):/ a8 s* q& ^( Y1 Z( S! H1 I
* [. d7 H) V. @2 @ File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/rpc/server.py", line 241, in inner9 W2 N0 ~+ L7 U5 `- U2 S
return func(*args, **kwargs)
! a1 ^' f+ ~5 G3 M# L9 }
# J: f% y$ L+ R File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/scheduler/manager.py", line 209, in select_destinations
$ y. v3 P Z0 C, y0 }" h raise exception.NoValidHost(reason="")
( Q/ e( g& s# J4 h! E
* A! [- X7 P% k2 X tnova.exception.NoValidHost: No valid host was found.
* r/ ^% Z" k, B U2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager Traceback (most recent call last):% [0 Z$ r7 E ~- I8 r( f
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/conductor/manager.py", line 1547, in schedule_and_build_instances
& ^: k6 m) G! K `: l0 j2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager instance_uuids, return_alternates=True)& P6 A1 V$ ~2 c9 `5 \0 d/ `& @
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/conductor/manager.py", line 908, in _schedule_instances
6 \9 y! j# q$ z7 Y' p7 f- G2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager return_alternates=return_alternates)
- [4 P+ ?9 ~ k2 K/ J' ~: a$ N2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/scheduler/client/query.py", line 42, in select_destinations
) d# h. W! z% o" A1 l2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager instance_uuids, return_objects, return_alternates)
. N# ? j6 K* q6 `. r+ W" g9 d6 W2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/scheduler/rpcapi.py", line 160, in select_destinations
! z# v1 T9 k* d2 Z: m# b) u2 o# m2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager return cctxt.call(ctxt, 'select_destinations', **msg_args)- x+ R1 g g5 N; K0 E7 S% i; c5 n3 z
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/rpc/client.py", line 192, in call3 A: Q1 @6 C- j8 w& V
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager retry=self.retry, transport_options=self.transport_options)% `+ _: `# `- X8 ~2 ^9 K
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/transport.py", line 128, in _send
8 z0 b6 j& p" k X! p/ [& R2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager transport_options=transport_options) e* P* W$ {' c& ?
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 691, in send' M2 j( b! \6 t: `
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager transport_options=transport_options)
! r" u% W; [% z3 u2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 681, in _send
0 H6 y" _3 |' o! y& I. o0 S; b. \2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager raise result1 i, O0 X! \0 `8 j( l
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager nova.exception_Remote.NoValidHost_Remote: No valid host was found.
( e4 G: e1 S8 V$ E5 p( _9 Y1 N |2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager Traceback (most recent call last):
; ? V! p' O' n2 N2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager
7 l y4 |9 N0 ?: d& Q2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/rpc/server.py", line 241, in inner1 A0 b4 b" C; J: P
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager return func(*args, **kwargs)
( F* `8 |1 l* ?* N2 v( ^. b$ E1 W2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager & I: g) u) H4 r3 S5 c/ O+ C+ N
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/scheduler/manager.py", line 209, in select_destinations
* J' y9 J5 [9 W* N2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager raise exception.NoValidHost(reason="")
2 `' w; Z! ?1 _9 [$ n# |5 M4 l* D2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager 3 ]( _/ X: x( q# w- L; Z- E
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager nova.exception.NoValidHost: No valid host was found. / I# v* S0 y' T7 m
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager : f, ^+ B" d9 Y( b
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager 1 t5 `! e8 T5 \/ x' ^4 d
2023-04-09 11:33:26.745 21 WARNING nova.scheduler.utils [req-50fa7432-2666-4cda-9712-151321519600 4e5dbd37d7d34992bac0217f5f6dc462 bd230134367943099e27b872e8b1ec53 - default default] Failed to compute_task_build_instances: No valid host was found.
; V0 d" v$ H) L& @& f- l! U) KTraceback (most recent call last):; m' E9 S% ?# C8 n
6 [) L4 w! J# R& e$ q
File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/rpc/server.py", line 241, in inner
4 U* a+ q9 ?2 J/ E; X return func(*args, **kwargs) ~1 t3 c8 K4 Q
- q, S' z$ T% K4 \1 t0 K File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/scheduler/manager.py", line 209, in select_destinations# P- l, G. P7 y: \( o0 }, H
raise exception.NoValidHost(reason="")) v, `2 y$ o; S. a$ h
5 V+ [! I3 e4 c3 n3 b3 u" ~
nova.exception.NoValidHost: No valid host was found.
2 r/ e/ d0 m# w% I6 }+ h0 \: nova.exception_Remote.NoValidHost_Remote: No valid host was found.
5 z4 {8 E2 S# J# Q2023-04-09 11:33:26.747 21 WARNING nova.scheduler.utils [req-50fa7432-2666-4cda-9712-151321519600 4e5dbd37d7d34992bac0217f5f6dc462 bd230134367943099e27b872e8b1ec53 - default default] [instance: 1b944501-f52c-42e1-8c75-4c54b9238b1e] Setting instance to ERROR state.: nova.exception_Remote.NoValidHost_Remote: No valid host was found. $ C0 |" E% O, k/ I
8 Z d# K8 K5 E' ^& ], D
- ]0 L, c% ?8 f; N% O; Z
- N4 ^, d- E& _' B5 Y, l [compute日志：
4 V. E) B: l. |4 S) k% ^! P7 q$ }" m$ q) d% K3 M
2023-04-09 11:33:23.351 6 ERROR nova.compute.manager [req-d046ac01-9bb6-43a4-97a7-b6510f3c3de7 - - - - -] No compute node record for host compute01: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host compute01 could not be found.
4 U; X& |- U6 L: f, w3 c2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils [req-d046ac01-9bb6-43a4-97a7-b6510f3c3de7 - - - - -] Could not determine disk usage: oslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command.
* H8 R5 V. h6 ]6 t9 GCommand: ceph df --format=json --id nova --conf /etc/ceph/ceph.conf$ V/ P$ s G O3 U* g" l
Exit code: 1$ K4 P/ T/ Q1 k0 \3 S1 X
Stdout: ''3 ` e3 @/ j6 b/ p
Stderr: '2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc05bcc8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc060bf8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cc2ebbe90) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n[errno 2] RADOS object not found (error connecting to the cluster)\n', i9 d2 r1 w9 O6 B7 n
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils Traceback (most recent call last):* b/ T) J' r( {3 r2 V
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/storage/rbd_utils.py", line 430, in get_pool_info( p& e9 r4 Y- E' b
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils out, _ = processutils.execute(*args)/ o4 V0 ?* D8 I. B6 C8 P
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_concurrency/processutils.py", line 441, in execute
% y4 K4 p/ J6 A/ t/ e6 }2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils cmd=sanitized_cmd)! M$ c4 t4 ?% ]# n
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils oslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command.
) \" @( X8 l# |* K a* y2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils Command: ceph df --format=json --id nova --conf /etc/ceph/ceph.conf
! o2 \4 X! \0 }* ^2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils Exit code: 17 T9 [: d' a2 Q' N; T ~, ]
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils Stdout: ''1 P% w: U* @$ q7 [
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils Stderr: '2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc05bcc8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc060bf8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cc2ebbe90) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n[errno 2] RADOS object not found (error connecting to the cluster)\n'
6 f- z: @# B1 p* l/ G( h2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils / W; q9 m4 m/ w7 p7 C1 K
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager [req-d046ac01-9bb6-43a4-97a7-b6510f3c3de7 - - - - -] Error updating resources for node compute01.: nova.exception.StorageError: Storage error: Could not determine disk usage- o2 R* t' r* @$ ~2 ^ c, F# _
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Traceback (most recent call last):5 u7 L8 v+ M: S' [) x7 m5 Z% _. g; X
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/storage/rbd_utils.py", line 430, in get_pool_info
9 z& ?: s. [# L& S2 D! X" c2023-04-09 11:33:23.770 6 ERROR nova.compute.manager out, _ = processutils.execute(*args)
" Y4 i" g1 H& _6 V: F P. _2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_concurrency/processutils.py", line 441, in execute
- x# F) w8 F# \+ {. ^8 ~' c' ]. t2023-04-09 11:33:23.770 6 ERROR nova.compute.manager cmd=sanitized_cmd)
1 d3 D! J" \( r" v9 B2023-04-09 11:33:23.770 6 ERROR nova.compute.manager oslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command.+ `/ K0 i. c3 C+ O2 w
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Command: ceph df --format=json --id nova --conf /etc/ceph/ceph.conf
+ D$ ~* F, y" s& U; W2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Exit code: 1# P5 X1 O, S) L
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Stdout: ''
5 O' a. k7 E* g' r8 }% f( k2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Stderr: '2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc05bcc8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc060bf8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cc2ebbe90) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n[errno 2] RADOS object not found (error connecting to the cluster)\n'2 E b9 b% o3 T. r8 N4 U
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager
7 k# |) b- y" N0 D" J/ J2023-04-09 11:33:23.770 6 ERROR nova.compute.manager During handling of the above exception, another exception occurred:
& p! X( ~: j2 F1 N$ m# ^2023-04-09 11:33:23.770 6 ERROR nova.compute.manager
# `' T5 z" S/ P' s5 ^+ Z* f* b* s2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Traceback (most recent call last):+ A' K" x- a4 ?) g" \9 Y2 ?
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/compute/manager.py", line 10057, in _update_available_resource_for_node
: g; D& R) D3 X- T2023-04-09 11:33:23.770 6 ERROR nova.compute.manager startup=startup)7 C+ u. A5 f% h$ k
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/compute/resource_tracker.py", line 884, in update_available_resource- K6 e* O8 w" v: y
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager resources = self.driver.get_available_resource(nodename)
+ ]. E+ j; |3 Z0 a; o; `2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 9021, in get_available_resource
2 S; {- t, ~4 W8 C+ T$ j0 Q2023-04-09 11:33:23.770 6 ERROR nova.compute.manager disk_info_dict = self._get_local_gb_info()- }. @# o+ g( J; G7 [
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 7385, in _get_local_gb_info
- {9 g5 V* i; {2023-04-09 11:33:23.770 6 ERROR nova.compute.manager info = rbd_utils.RBDDriver().get_pool_info()5 G0 ^ z0 p- \
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/storage/rbd_utils.py", line 434, in get_pool_info
, p! ?1 j% |# A" p6 u2023-04-09 11:33:23.770 6 ERROR nova.compute.manager reason='Could not determine disk usage')- v2 M# h2 \3 {- U% Z; s
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager nova.exception.StorageError: Storage error: Could not determine disk usage; y1 s9 C' |+ [/ ^7 }0 Q+ Y; [; a: q
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager ' p6 U* ]3 P D I3 Z7 L
8 y* E% {! R. R! p* N5 W
9 r+ f6 B$ m( s
- j. ]2 _" C" w
, l$ k6 g0 O$ u: z D& R+ ~ |
|
发表于 2023-4-9 11:36:41