Comment 3 for bug 1284431

this starts to happen despite the oslo.messaging apparently being reconnected:

oslo.messaging._drivers.impl_rabbit [-] Failed to publish message to topic 'conductor': Socket closed
oslo.messaging._drivers.impl_rabbit Traceback (most recent call last):
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 622, in ensure
oslo.messaging._drivers.impl_rabbit return method(*args, **kwargs)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 718, in _publish
oslo.messaging._drivers.impl_rabbit publisher = cls(self.conf, self.channel, topic, **kwargs)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 379, in __init__
oslo.messaging._drivers.impl_rabbit **options)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 326, in __init__
oslo.messaging._drivers.impl_rabbit self.reconnect(channel)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 334, in reconnect
oslo.messaging._drivers.impl_rabbit routing_key=self.routing_key)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/kombu/messaging.py", line 85, in __init__
oslo.messaging._drivers.impl_rabbit self.revive(self._channel)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/kombu/messaging.py", line 218, in revive
oslo.messaging._drivers.impl_rabbit self.declare()
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/kombu/messaging.py", line 105, in declare
oslo.messaging._drivers.impl_rabbit self.exchange.declare()
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/kombu/entity.py", line 166, in declare
oslo.messaging._drivers.impl_rabbit nowait=nowait, passive=passive,
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/amqp/channel.py", line 620, in exchange_declare
oslo.messaging._drivers.impl_rabbit (40, 11), # Channel.exchange_declare_ok
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/amqp/abstract_channel.py", line 67, in wait
oslo.messaging._drivers.impl_rabbit self.channel_id, allowed_methods)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/amqp/connection.py", line 237, in _wait_method
oslo.messaging._drivers.impl_rabbit self.method_reader.read_method()
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/nova/lib/python2.7/site-packages/amqp/method_framing.py", line 189, in read_method
oslo.messaging._drivers.impl_rabbit raise m
oslo.messaging._drivers.impl_rabbit IOError: Socket closed

oslo.messaging._drivers.impl_rabbit [-] Reconnecting to AMQP server on 192.0.2.29:5672
oslo.messaging._drivers.impl_rabbit [-] Delaying reconnect for 1.0 seconds...
oslo.messaging._drivers.impl_rabbit [-] Connected to AMQP server on 192.0.2.29:5672
oslo.messaging._drivers.impl_rabbit [-] Failed to publish message to topic 'q-plugin': Socket closed
oslo.messaging._drivers.impl_rabbit Traceback (most recent call last):
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 622, in ensure
oslo.messaging._drivers.impl_rabbit return method(*args, **kwargs)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 718, in _publish
oslo.messaging._drivers.impl_rabbit publisher = cls(self.conf, self.channel, topic, **kwargs)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 379, in __init__
oslo.messaging._drivers.impl_rabbit **options)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 326, in __init__
oslo.messaging._drivers.impl_rabbit self.reconnect(channel)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 334, in reconnect
oslo.messaging._drivers.impl_rabbit routing_key=self.routing_key)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/kombu/messaging.py", line 85, in __init__
oslo.messaging._drivers.impl_rabbit self.revive(self._channel)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/kombu/messaging.py", line 218, in revive
oslo.messaging._drivers.impl_rabbit self.declare()
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/kombu/messaging.py", line 105, in declare
oslo.messaging._drivers.impl_rabbit self.exchange.declare()
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/kombu/entity.py", line 166, in declare
oslo.messaging._drivers.impl_rabbit nowait=nowait, passive=passive,
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/amqp/channel.py", line 620, in exchange_declare
oslo.messaging._drivers.impl_rabbit (40, 11), # Channel.exchange_declare_ok
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/amqp/abstract_channel.py", line 67, in wait
oslo.messaging._drivers.impl_rabbit self.channel_id, allowed_methods)
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/amqp/connection.py", line 237, in _wait_method
oslo.messaging._drivers.impl_rabbit self.method_reader.read_method()
oslo.messaging._drivers.impl_rabbit File "/opt/stack/venvs/openstack/lib/python2.7/site-packages/amqp/method_framing.py", line 189, in read_method
oslo.messaging._drivers.impl_rabbit raise m
oslo.messaging._drivers.impl_rabbit IOError: Socket closed
oslo.messaging._drivers.impl_rabbit
oslo.messaging._drivers.impl_rabbit [-] Reconnecting to AMQP server on 192.0.2.29:5672
oslo.messaging._drivers.impl_rabbit [-] Delaying reconnect for 1.0 seconds...
oslo.messaging._drivers.impl_rabbit [-] Connected to AMQP server on 192.0.2.29:5672