keystone endpoint-list was being stuck because of mysql. mysql was stuck for some reason. on killing mysql and starting it back, system recovered. many of other mysql commands were stuck as well.
root@nodec10:/var/log/mysql# ps ax| grep mysql
609 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
5059 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
8319 ? S 0:00 /bin/sh /usr/bin/mysqld_safe
9683 ? Sl 67:41 /usr/sbin/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/usr/lib/mysql/plugin --user=mysql --log-error=/var/log/mysql/error.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/run/mysqld/mysqld.sock --port=3306 --wsrep_start_position=00000000-0000-0000-0000-000000000000:-1
14121 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
15248 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
26339 pts/16 S+ 0:00 grep --color=auto mysql
26677 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
27280 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
27396 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
28155 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
28204 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
28501 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
28681 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
28942 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
29165 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
29380 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
30772 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
31788 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
32297 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
keystone endpoint-list was being stuck because of mysql. mysql was stuck for some reason. on killing mysql and starting it back, system recovered. many of other mysql commands were stuck as well.
root@nodec10: /var/log/ mysql# ps ax| grep mysql mysqld_ safe /var/lib/ mysql --plugin- dir=/usr/ lib/mysql/ plugin --user=mysql --log-error= /var/log/ mysql/error. log --pid-file= /var/run/ mysqld/ mysqld. pid --socket= /var/run/ mysqld/ mysqld. sock --port=3306 --wsrep_ start_position= 00000000- 0000-0000- 0000-0000000000 00:-1
609 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
5059 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
8319 ? S 0:00 /bin/sh /usr/bin/
9683 ? Sl 67:41 /usr/sbin/mysqld --basedir=/usr --datadir=
14121 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
15248 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
26339 pts/16 S+ 0:00 grep --color=auto mysql
26677 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
27280 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
27396 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
28155 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
28204 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
28501 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
28681 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
28942 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
29165 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
29380 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
30772 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
31788 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
32297 ? S 0:00 mysql -ucmon -px xx -hlocalhost -P3306 -e use cmon; truncate table cmon_log;
keystone-all.log reported following errors: sqlalchemy. engines [req-898fff2f- fc93-42ce- a707-5dfcd8935a a7 - - - - -] SQL connection failed. -14 attempts left. sqlalchemy. engines [req-f386c607- f517-4a83- 83eb-281f984527 b2 - - - - -] SQL connection failed. -14 attempts left. sqlalchemy. engines [req-f66b6d86- 872c-4bcd- 9506-afea2aaf7a 49 - - - - -] SQL connection failed. -15 attempts left. sqlalchemy. engines [req-cb3a0244- 836e-4490- ada0-4dd7edaba1 10 - - - - -] SQL connection failed. -15 attempts left. sqlalchemy. engines [req-67023e89- 6033-4bbc- a7e7-eb658fbadd 13 - - - - -] SQL connection failed. -15 attempts left. sqlalchemy. engines [req-898fff2f- fc93-42ce- a707-5dfcd8935a a7 - - - - -] SQL connection failed. -15 attempts left. sqlalchemy. engines [req-f386c607- f517-4a83- 83eb-281f984527 b2 - - - - -] SQL connection failed. -15 attempts left. sqlalchemy. engines [req-f66b6d86- 872c-4bcd- 9506-afea2aaf7a 49 - - - - -] SQL connection failed. -16 attempts left. sqlalchemy. engines [req-cb3a0244- 836e-4490- ada0-4dd7edaba1 10 - - - - -] SQL connection failed. -16 attempts left. sqlalchemy. engines [req-67023e89- 6033-4bbc- a7e7-eb658fbadd 13 - - - - -] SQL connection failed. -16 attempts left. sqlalchemy. engines [req-898fff2f- fc93-42ce- a707-5dfcd8935a a7 - - - - -] SQL connection failed. -16 attempts left. sqlalchemy. engines [req-f386c607- f517-4a83- 83eb-281f984527 b2 - - - - -] SQL connection failed. -16 attempts left.
2016-11-17 11:46:05.224 30182 WARNING oslo_db.
2016-11-17 11:46:06.231 30179 WARNING oslo_db.
2016-11-17 11:46:08.245 30181 WARNING oslo_db.
2016-11-17 11:46:08.246 30180 WARNING oslo_db.
2016-11-17 11:46:09.688 30183 WARNING oslo_db.
2016-11-17 11:46:10.225 30182 WARNING oslo_db.
2016-11-17 11:46:11.233 30179 WARNING oslo_db.
2016-11-17 11:46:13.249 30181 WARNING oslo_db.
2016-11-17 11:46:13.251 30180 WARNING oslo_db.
2016-11-17 11:46:14.691 30183 WARNING oslo_db.
2016-11-17 11:46:15.230 30182 WARNING oslo_db.
2016-11-17 11:46:16.237 30179 WARNING oslo_db.