|
|
conductor 日志：
/ h% r$ ?( Z$ \) a I2 |* Z2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager [req-50fa7432-2666-4cda-9712-151321519600 4e5dbd37d7d34992bac0217f5f6dc462 bd230134367943099e27b872e8b1ec53 - default default] Failed to schedule instances: nova.exception_Remote.NoValidHost_Remote: No valid host was found. 3 w' t5 [: G) v- [
Traceback (most recent call last):
% S- \. F' Q2 i/ A2 C) L _; Y$ t( x+ N5 Q j( ?- F. B& v
File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/rpc/server.py", line 241, in inner
% \7 n) I- @3 s/ N return func(*args, **kwargs)8 c L6 v8 V* q/ d1 [
B5 h1 b; ], @2 f4 X7 h8 L9 y
File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/scheduler/manager.py", line 209, in select_destinations/ r4 s7 d j& |; \ g
raise exception.NoValidHost(reason="") V% k3 U9 Y2 `; X3 g
; }' u9 x0 @- ^! E$ E# Snova.exception.NoValidHost: No valid host was found.
: [9 l$ ~* J5 ^8 O5 _1 \' h2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager Traceback (most recent call last):
8 h Z, s4 G/ S- }" U2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/conductor/manager.py", line 1547, in schedule_and_build_instances# O3 {/ F1 p& G6 ^
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager instance_uuids, return_alternates=True)
3 X" q2 p5 ^# U" \* Z6 [! o2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/conductor/manager.py", line 908, in _schedule_instances3 G. o8 T/ G) i5 u2 r. s- G1 k
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager return_alternates=return_alternates)
+ {- S' C+ B8 U" x% I2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/scheduler/client/query.py", line 42, in select_destinations5 x" L4 j. p$ w7 j( n8 G
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager instance_uuids, return_objects, return_alternates)
- Y/ j7 f! Y# T& k9 U! F2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/scheduler/rpcapi.py", line 160, in select_destinations
; r# h2 `1 B. ?+ u6 D$ G2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager return cctxt.call(ctxt, 'select_destinations', **msg_args)' ]4 L6 r+ J! {( g9 U2 r7 }( P, u0 l! r# y
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/rpc/client.py", line 192, in call# h; N0 z- {. u+ u' r. p; V/ \
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager retry=self.retry, transport_options=self.transport_options)
8 C# w1 H0 Q4 Z1 E+ k7 ]- q7 N2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/transport.py", line 128, in _send
, l% k! W5 ?$ R4 P$ W3 j7 b# f: ?2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager transport_options=transport_options)% h6 P' v( K( _
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 691, in send
. g4 M7 N4 L; V$ N X2 R* W2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager transport_options=transport_options) V+ T$ w0 _4 a
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 681, in _send( u9 f, K: C- d8 Z' h
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager raise result7 m0 S- b$ p( |& @9 [& v+ H! M5 l
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager nova.exception_Remote.NoValidHost_Remote: No valid host was found. 3 T3 ]' ?8 q: B4 b! {0 {2 _
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager Traceback (most recent call last): ]2 S' _2 g/ ~, Z
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager
6 g! t8 j4 ~* n" _2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/rpc/server.py", line 241, in inner
# Y' \. M/ Q' z4 @3 f! q Q2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager return func(*args, **kwargs)
+ i9 L; P) t9 ~ R) @) {" S( d2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager ' f: ^+ V& |' @6 ^0 a
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/scheduler/manager.py", line 209, in select_destinations& P; v ?: _1 q0 K* f/ v
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager raise exception.NoValidHost(reason="")5 p# ~$ H. `, Q! ~. O1 }' Y% b
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager
9 H9 R7 c' G- B) x+ X2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager nova.exception.NoValidHost: No valid host was found. ; L& c# y) G: |6 e9 z
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager . Y# a5 F: L0 [4 {9 Z& ?( w6 Z* E
2023-04-09 11:33:26.301 21 ERROR nova.conductor.manager ; K4 S e0 X7 ~1 {* L- G
2023-04-09 11:33:26.745 21 WARNING nova.scheduler.utils [req-50fa7432-2666-4cda-9712-151321519600 4e5dbd37d7d34992bac0217f5f6dc462 bd230134367943099e27b872e8b1ec53 - default default] Failed to compute_task_build_instances: No valid host was found. ' @, N; c# g; R# Z7 a# C8 `/ Z7 N0 K0 ]
Traceback (most recent call last):# q* [& r; p" S
" G, G0 g) g. ?; O/ U; H; i
File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_messaging/rpc/server.py", line 241, in inner
( l( b4 E- G- X: {8 N( k0 T return func(*args, **kwargs)
4 w" o0 z Z4 X7 X7 _/ l4 s; s7 _% ]; R: \, G" a. {# M2 K5 {
File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/scheduler/manager.py", line 209, in select_destinations
B# x& B4 L+ {9 \8 R. i, ~ raise exception.NoValidHost(reason="")- ]* J+ {/ ~7 K: \! ]* y% A0 |/ b# R
v7 K0 m5 G8 C% S& _% H/ znova.exception.NoValidHost: No valid host was found. + b4 z0 K% f2 G4 Z
: nova.exception_Remote.NoValidHost_Remote: No valid host was found.
9 B A; s# ]: `2023-04-09 11:33:26.747 21 WARNING nova.scheduler.utils [req-50fa7432-2666-4cda-9712-151321519600 4e5dbd37d7d34992bac0217f5f6dc462 bd230134367943099e27b872e8b1ec53 - default default] [instance: 1b944501-f52c-42e1-8c75-4c54b9238b1e] Setting instance to ERROR state.: nova.exception_Remote.NoValidHost_Remote: No valid host was found. . h( X0 o* k+ I- w' _8 q) `
% L o) D1 f& ], H" }9 b
# @+ r% ?5 U7 n0 f0 ?+ ? l' p3 k9 S( f% b" o- e
compute日志：
4 l. ^* B" ?7 Q9 o1 [0 }
: X6 G% H% y0 S" p2023-04-09 11:33:23.351 6 ERROR nova.compute.manager [req-d046ac01-9bb6-43a4-97a7-b6510f3c3de7 - - - - -] No compute node record for host compute01: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host compute01 could not be found. q% o5 t4 M1 b) ~
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils [req-d046ac01-9bb6-43a4-97a7-b6510f3c3de7 - - - - -] Could not determine disk usage: oslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command.
1 J+ Z* |4 a7 C6 u; o. ~Command: ceph df --format=json --id nova --conf /etc/ceph/ceph.conf( s& s2 C/ [8 @9 [" x% U/ `7 N9 p
Exit code: 18 y$ V, r8 R$ C; j2 z' g5 i
Stdout: ''" d5 [* H3 g( @& |: p( t0 i9 u: E) K
Stderr: '2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc05bcc8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc060bf8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cc2ebbe90) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n[errno 2] RADOS object not found (error connecting to the cluster)\n'! r6 r: O2 H3 i) n
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils Traceback (most recent call last):
0 k4 \4 _; l+ u8 {6 M8 `2 ?! `2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/storage/rbd_utils.py", line 430, in get_pool_info
) o0 B# k- D: G& b9 u8 }& N2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils out, _ = processutils.execute(*args)0 @' |8 N& y( F! ?! \8 F
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_concurrency/processutils.py", line 441, in execute
& S) e% |& r4 T% X8 ~% R+ a V5 ^) `2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils cmd=sanitized_cmd)
0 N' m* |: I! f' z& l& {) I* x2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils oslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command.+ K% |3 e! e1 h4 D1 Z# M
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils Command: ceph df --format=json --id nova --conf /etc/ceph/ceph.conf
. w7 U1 N6 a# }" ?. D& e2 u) ^4 `2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils Exit code: 1
; U3 \# e3 ?8 f8 |2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils Stdout: ''
" L$ I3 y4 E* G6 s, x& e: D2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils Stderr: '2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc05bcc8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc060bf8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cc2ebbe90) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n[errno 2] RADOS object not found (error connecting to the cluster)\n'" w) z" C. m2 g* D* {2 a, y# t
2023-04-09 11:33:23.766 6 ERROR nova.storage.rbd_utils / {- s5 f+ S6 @+ p3 v8 k* v' D
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager [req-d046ac01-9bb6-43a4-97a7-b6510f3c3de7 - - - - -] Error updating resources for node compute01.: nova.exception.StorageError: Storage error: Could not determine disk usage$ f! n5 J6 {+ x) V
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Traceback (most recent call last):
/ e! c: q. s; A+ v( ^2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/storage/rbd_utils.py", line 430, in get_pool_info. k. D4 A2 W1 ?
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager out, _ = processutils.execute(*args)
9 L" F2 s4 R/ e, Q* s% ]2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/oslo_concurrency/processutils.py", line 441, in execute6 s) a' T7 m, k8 M! t+ q5 K. j
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager cmd=sanitized_cmd)
- v5 L. Q& H Q2023-04-09 11:33:23.770 6 ERROR nova.compute.manager oslo_concurrency.processutils.ProcessExecutionError: Unexpected error while running command.
' c4 n0 {8 u: {: b7 S5 v3 y7 d2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Command: ceph df --format=json --id nova --conf /etc/ceph/ceph.conf/ O8 S6 H* r2 C5 X: q# G5 g1 |
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Exit code: 1+ d/ j/ a& q s
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Stdout: ''
7 Y, J6 d# M) l- V5 R) ]: w5 I2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Stderr: '2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.743+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc05bcc8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.751+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cbc060bf8) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory\n2023-04-09T11:33:23.756+0800 7f8cc2ebd700 -1 AuthRegistry(0x7f8cc2ebbe90) no keyring found at /etc/ceph/ceph.client.nova.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx\n[errno 2] RADOS object not found (error connecting to the cluster)\n'+ F: J' l# {, X# @/ I- a
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager 9 A( I; x( a: @, M* p& l
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager During handling of the above exception, another exception occurred:
/ v2 X4 O& x! n& [" o0 C2023-04-09 11:33:23.770 6 ERROR nova.compute.manager
, J8 y; t( m3 O" U+ c2023-04-09 11:33:23.770 6 ERROR nova.compute.manager Traceback (most recent call last):2 \$ S6 \2 {/ j4 w
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/compute/manager.py", line 10057, in _update_available_resource_for_node
/ ?) y% P0 `2 Y. ]2023-04-09 11:33:23.770 6 ERROR nova.compute.manager startup=startup)
$ M% s' r+ a4 V0 {3 F1 V9 [2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/compute/resource_tracker.py", line 884, in update_available_resource2 S) L' `+ m. j! L+ w5 @- Z W. C
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager resources = self.driver.get_available_resource(nodename)9 U9 A+ C6 L# q. k4 m9 \
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 9021, in get_available_resource6 x( f/ A- d1 b3 P9 a5 C1 d! C
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager disk_info_dict = self._get_local_gb_info()
4 D- j! X+ e; I, [& w* b2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/virt/libvirt/driver.py", line 7385, in _get_local_gb_info6 X4 T, a& A v8 ?. X1 L2 L! {( j; ~
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager info = rbd_utils.RBDDriver().get_pool_info()
6 f! d& V8 Z% Y3 y4 M) T0 Q Y$ i) e2023-04-09 11:33:23.770 6 ERROR nova.compute.manager File "/var/lib/kolla/venv/lib/python3.6/site-packages/nova/storage/rbd_utils.py", line 434, in get_pool_info
3 M6 ], g' [$ W2023-04-09 11:33:23.770 6 ERROR nova.compute.manager reason='Could not determine disk usage')
, w3 l1 b" j) I' ~* `) W. ~" A; s2023-04-09 11:33:23.770 6 ERROR nova.compute.manager nova.exception.StorageError: Storage error: Could not determine disk usage! l l- I5 {: e8 A
2023-04-09 11:33:23.770 6 ERROR nova.compute.manager
" R) W: u6 d/ w; [" ]; A- L
( N- c/ J* v, v6 z4 i% `! _1 p b9 r; v: q1 _# g' ?5 K& c% I- o9 z$ C
3 q7 u# d3 [8 I, K
; O; A, U- r& T$ A |
|
发表于 2023-4-9 11:36:41