2016-03-20 05:05:09 WARNING keystonemiddleware.auth_token [-] Authorization failed for token 2016-03-20 05:05:09 WARNING keystonemiddleware.auth_token [-] Identity response: {"error": {"message": "Failed to validate token", "code": 404, "title": "Not Found"}} 2016-03-20 05:05:09 WARNING keystonemiddleware.auth_token [-] Authorization failed for token 2016-03-19 20:59:54 WARNING keystonemiddleware.auth_token [-] Authorization failed for token 2016-03-19 20:59:54 WARNING keystonemiddleware.auth_token [-] Identity response: {"error": {"message": "Failed to validate token", "code": 404, "title": "Not Found"}} 2016-03-19 20:59:54 WARNING keystonemiddleware.auth_token [-] Authorization failed for token 2016-03-19 17:57:58 WARNING keystonemiddleware.auth_token [-] Authorization failed for token 2016-03-19 17:57:58 WARNING keystonemiddleware.auth_token [-] Identity response: {"error": {"message": "Failed to validate token", "code": 404, "title": "Not Found"}} 2016-03-19 17:57:58 WARNING keystonemiddleware.auth_token [-] Authorization failed for token 2016-03-19 06:37:37 ERR wsme.api [req-95ca35fd-bc97-411d-b800-f88c0da649c8 ] Server-side error: "(_mysql_exceptions.OperationalError) (2003, "Can't connect to MySQL server on '10.14.68.3' (111)") [SQL: u'SELECT 1']". Detail: Traceback (most recent call last): File "/usr/lib/python2.7/dist-packages/wsmeext/pecan.py", line 84, in callfunction result = f(self, *args, **kwargs) File "/usr/lib/python2.7/dist-packages/ironic/api/controllers/v1/node.py", line 1065, in detail resource_url) File "/usr/lib/python2.7/dist-packages/ironic/api/controllers/v1/node.py", line 905, in _get_nodes_collection nodes = self._get_nodes_by_instance(instance_uuid) File "/usr/lib/python2.7/dist-packages/ironic/api/controllers/v1/node.py", line 938, in _get_nodes_by_instance instance_uuid) File "/usr/lib/python2.7/dist-packages/ironic/objects/node.py", line 183, in get_by_instance_uuid db_node = cls.dbapi.get_node_by_instance(instance_uuid) File "/usr/lib/python2.7/dist-packages/ironic/db/sqlalchemy/api.py", line 303, in get_node_by_instance result = query.one() File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2473, in one ret = list(self) File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2516, in __iter__ return self._execute_and_instances(context) File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2529, in _execute_and_instances close_with_result=True) File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2520, in _connection_from_session **kw) File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 882, in connection execution_options=execution_options) File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 889, in _connection_for_bind conn = engine.contextual_connect(**kw) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2041, in contextual_connect **kwargs) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 92, in __init__ self.dispatch.engine_connect(self, self.__branch) File "/usr/lib/python2.7/dist-packages/sqlalchemy/event/attr.py", line 256, in __call__ fn(*args, **kw) File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/engines.py", line 80, in _connect_ping_listener connection.scalar(select([1])) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 844, in scalar return self.execute(object, *multiparams, **params).scalar() File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 914, in execute return meth(self, multiparams, params) File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection return connection._execute_clauseelement(self, multiparams, params) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement compiled_sql, distilled_params File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1078, in _execute_context None, None) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1337, in _handle_dbapi_exception util.raise_from_cause(newraise, exc_info) File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause reraise(type(exception), exception, tb=exc_tb) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1071, in _execute_context conn = self._revalidate_connection() File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 393, in _revalidate_connection self.__connection = self.engine.raw_connection(_connection=self) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2104, in raw_connection self.pool.unique_connection, _connection) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2080, in _wrap_pool_connect util.reraise(*sys.exc_info()) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2074, in _wrap_pool_connect return fn() File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 318, in unique_connection return _ConnectionFairy._checkout(self) File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 713, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 485, in checkout rec.checkin() File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__ compat.reraise(exc_type, exc_value, exc_tb) File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 482, in checkout dbapi_connection = rec.get_connection() File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 563, in get_connection self.connection = self.__connect() File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 607, in __connect connection = self.__pool._invoke_creator(self) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/strategies.py", line 97, in connect return dialect.connect(*cargs, **cparams) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 385, in connect return self.dbapi.connect(*cargs, **cparams) File "/usr/lib/python2.7/dist-packages/MySQLdb/__init__.py", line 81, in Connect return Connection(*args, **kwargs) File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 206, in __init__ super(Connection, self).__init__(*args, **kwargs2) DBConnectionError: (_mysql_exceptions.OperationalError) (2003, "Can't connect to MySQL server on '10.14.68.3' (111)") [SQL: u'SELECT 1'] 2016-03-19 02:48:06 WARNING keystonemiddleware.auth_token [-] Authorization failed for token 2016-03-19 02:48:06 WARNING keystonemiddleware.auth_token [-] Identity response: {"error": {"message": "Failed to validate token", "code": 404, "title": "Not Found"}} 2016-03-19 02:48:06 WARNING keystonemiddleware.auth_token [-] Authorization failed for token 2016-03-19 02:47:49 WARNING oslo_config.cfg [-] Option "amqp_durable_queues" from group "DEFAULT" is deprecated. Use option "amqp_durable_queues" from group "oslo_messaging_rabbit". 2016-03-19 02:47:49 WARNING keystonemiddleware.auth_token [-] Configuring admin URI using auth fragments. This is deprecated, use 'identity_uri' instead. 2016-03-19 02:47:49 WARNING keystonemiddleware.auth_token [-] Use of the auth_admin_prefix, auth_host, auth_port, auth_protocol, identity_uri, admin_token, admin_user, admin_password, and admin_tenant_name configuration options is deprecated in favor of auth_plugin and related options and may be removed in a future release. 2016-03-19 00:45:44 ERR wsme.api [req-b571e65c-66ff-42e7-a50f-66092d8c0a60 ] Server-side error: "(_mysql_exceptions.OperationalError) (2003, "Can't connect to MySQL server on '10.14.68.3' (111)") [SQL: u'SELECT 1']". Detail: Traceback (most recent call last): File "/usr/lib/python2.7/dist-packages/wsmeext/pecan.py", line 84, in callfunction result = f(self, *args, **kwargs) File "/usr/lib/python2.7/dist-packages/ironic/api/controllers/v1/node.py", line 1027, in get_all fields=fields) File "/usr/lib/python2.7/dist-packages/ironic/api/controllers/v1/node.py", line 919, in _get_nodes_collection filters=filters) File "/usr/lib/python2.7/dist-packages/ironic/objects/node.py", line 207, in list sort_dir=sort_dir) File "/usr/lib/python2.7/dist-packages/ironic/db/sqlalchemy/api.py", line 211, in get_node_list sort_key, sort_dir, query) File "/usr/lib/python2.7/dist-packages/ironic/db/sqlalchemy/api.py", line 143, in _paginate_query return query.all() File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2399, in all return list(self) File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2516, in __iter__ return self._execute_and_instances(context) File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2529, in _execute_and_instances close_with_result=True) File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2520, in _connection_from_session **kw) File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 882, in connection execution_options=execution_options) File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 889, in _connection_for_bind conn = engine.contextual_connect(**kw) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2041, in contextual_connect **kwargs) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 92, in __init__ self.dispatch.engine_connect(self, self.__branch) File "/usr/lib/python2.7/dist-packages/sqlalchemy/event/attr.py", line 256, in __call__ fn(*args, **kw) File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/engines.py", line 80, in _connect_ping_listener connection.scalar(select([1])) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 844, in scalar return self.execute(object, *multiparams, **params).scalar() File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 914, in execute return meth(self, multiparams, params) File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection return connection._execute_clauseelement(self, multiparams, params) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement compiled_sql, distilled_params File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1078, in _execute_context None, None) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1337, in _handle_dbapi_exception util.raise_from_cause(newraise, exc_info) File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause reraise(type(exception), exception, tb=exc_tb) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1071, in _execute_context conn = self._revalidate_connection() File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 393, in _revalidate_connection self.__connection = self.engine.raw_connection(_connection=self) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2104, in raw_connection self.pool.unique_connection, _connection) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2080, in _wrap_pool_connect util.reraise(*sys.exc_info()) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2074, in _wrap_pool_connect return fn() File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 318, in unique_connection return _ConnectionFairy._checkout(self) File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 713, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 485, in checkout rec.checkin() File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__ compat.reraise(exc_type, exc_value, exc_tb) File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 482, in checkout dbapi_connection = rec.get_connection() File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 563, in get_connection self.connection = self.__connect() File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 607, in __connect connection = self.__pool._invoke_creator(self) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/strategies.py", line 97, in connect return dialect.connect(*cargs, **cparams) File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 385, in connect return self.dbapi.connect(*cargs, **cparams) File "/usr/lib/python2.7/dist-packages/MySQLdb/__init__.py", line 81, in Connect return Connection(*args, **kwargs) File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 206, in __init__ super(Connection, self).__init__(*args, **kwargs2) DBConnectionError: (_mysql_exceptions.OperationalError) (2003, "Can't connect to MySQL server on '10.14.68.3' (111)") [SQL: u'SELECT 1'] 2016-03-18 23:27:00 CRIT keystonemiddleware.auth_token [-] Unable to validate token: Failed to fetch token data from identity server 2016-03-18 23:27:00 WARNING keystonemiddleware.auth_token [-] Identity response: {"error": {"message": "An unexpected error prevented the server from fulfilling your request.", "code": 500, "title": "Internal Server Error"}} 2016-03-18 23:27:00 ERR keystonemiddleware.auth_token [-] Bad response code while validating token: 500 2016-03-18 20:39:09 WARNING keystonemiddleware.auth_token [-] Authorization failed for token