Hi,

sorry for asking to general help / please forward this message to the correct support team/thanks.

I manage Genoa node Fiware Lab node, that's up & running from a long time.

We experience now a problem just submitting nova or neutron client commands like:

root@controller01:~# nova --debug list
root@controller01:~# neutron --debug agent-list

we got the error:
InternalServerError: An unexpected error prevented the server from fulfilling your request. (HTTP 500)

that do not allow to proceed anymore, making the Genoa node administration no more possible.

Looking at the command output with --debug mode (see below dumps), it seems that a possible issue arise when calling keystone federated authentication at http://130.206.84.8:4730/v2.0/

There is any new known issue on keystone authentication?

Please let us know if you need any further log/info.

Thanks for your help
Giorgio Robino

—

root@controller01:~# nova --debug list
DEBUG (session:196) REQ: curl -g -i -X GET http://130.206.84.8:4730/v2.0/ -H "Accept: application/json" -H "User-Agent: python-keystoneclient"
INFO (connectionpool:187) Starting new HTTP connection (1): 130.206.84.8
DEBUG (connectionpool:353) "GET /v2.0/ HTTP/1.1" 200 422
DEBUG (session:226) RESP: [200] date: Thu, 07 Sep 2017 07:39:42 GMT vary: X-Auth-Token content-length: 422 content-type: application/json
RESP BODY: {"version": {"status": "stable", "updated": "2014-04-17T00:00:00Z", "media-types": [

,

], "id": "v2.0", "links": [

,

]}}

DEBUG (v2:76) Making authentication request to http://130.206.84.8:4730/v2.0/tokens
DEBUG (connectionpool:353) "POST /v2.0/tokens HTTP/1.1" 500 143
DEBUG (session:398) Request returned failure status: 500
DEBUG (shell:911) An unexpected error prevented the server from fulfilling your request. (HTTP 500)
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/novaclient/shell.py", line 908, in main
OpenStackComputeShell().main(argv)
File "/usr/lib/python2.7/dist-packages/novaclient/shell.py", line 835, in main
args.func(self.cs, args)
File "/usr/lib/python2.7/dist-packages/novaclient/v2/shell.py", line 1377, in do_list
sort_dirs=sort_dirs)
File "/usr/lib/python2.7/dist-packages/novaclient/v2/servers.py", line 594, in list
return self._list("/servers%s%s" % (detail, query_string), "servers")
File "/usr/lib/python2.7/dist-packages/novaclient/base.py", line 64, in _list
_resp, body = self.api.client.get(url)
File "/usr/lib/python2.7/dist-packages/keystoneclient/adapter.py", line 170, in get
return self.request(url, 'GET', **kwargs)
File "/usr/lib/python2.7/dist-packages/novaclient/client.py", line 90, in request
**kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/adapter.py", line 206, in request
resp = super(LegacyJsonAdapter, self).request(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/adapter.py", line 95, in request
return self.session.request(url, method, **kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/utils.py", line 318, in inner
return func(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 315, in request
auth_headers = self.get_auth_headers(auth)
File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 600, in get_auth_headers
return auth.get_headers(self, **kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/base.py", line 114, in get_headers
token = self.get_token(session)
File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/base.py", line 104, in get_token
return self.get_access(session).auth_token
File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/base.py", line 144, in get_access
self.auth_ref = self.get_auth_ref(session)
File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/generic/base.py", line 176, in get_auth_ref
return self._plugin.get_auth_ref(session, **kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/v2.py", line 78, in get_auth_ref
authenticated=False, log=False)
File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 499, in post
return self.request(url, 'POST', **kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/utils.py", line 318, in inner
return func(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 399, in request
raise exceptions.from_response(resp, method, url)
InternalServerError: An unexpected error prevented the server from fulfilling your request. (HTTP 500)
ERROR (InternalServerError): An unexpected error prevented the server from fulfilling your request. (HTTP 500)

—

root@controller01:~# neutron --debug agent-list
DEBUG: keystoneclient.session REQ: curl -g -i -X GET http://130.206.84.8:4730/v2.0/ -H "Accept: application/json" -H "User-Agent: python-keystoneclient"
DEBUG: keystoneclient.session RESP: [200] date: Thu, 07 Sep 2017 07:40:32 GMT vary: X-Auth-Token content-length: 422 content-type: application/json
RESP BODY: {"version": {"status": "stable", "updated": "2014-04-17T00:00:00Z", "media-types": [

,

], "id": "v2.0", "links": [

,

]}}

DEBUG: stevedore.extension found extension EntryPoint.parse('table = cliff.formatters.table:TableFormatter')
DEBUG: stevedore.extension found extension EntryPoint.parse('csv = cliff.formatters.commaseparated:CSVLister')
DEBUG: stevedore.extension found extension EntryPoint.parse('yaml = clifftablib.formatters:YamlFormatter')
DEBUG: stevedore.extension found extension EntryPoint.parse('json = clifftablib.formatters:JsonFormatter')
DEBUG: stevedore.extension found extension EntryPoint.parse('html = clifftablib.formatters:HtmlFormatter')
DEBUG: neutronclient.neutron.v2_0.agent.ListAgent get_data(Namespace(columns=[], fields=[], formatter='table', max_width=0, quote_mode='nonnumeric', request_format='json', show_details=False, sort_dir=[], sort_key=[]))
DEBUG: keystoneclient.auth.identity.v2 Making authentication request to http://130.206.84.8:4730/v2.0/tokens
DEBUG: keystoneclient.session Request returned failure status: 500
ERROR: neutronclient.shell An unexpected error prevented the server from fulfilling your request. (HTTP 500)
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/neutronclient/shell.py", line 766, in run_subcommand
return run_command(cmd, cmd_parser, sub_argv)
File "/usr/lib/python2.7/dist-packages/neutronclient/shell.py", line 101, in run_command
return cmd.run(known_args)
File "/usr/lib/python2.7/dist-packages/neutronclient/common/command.py", line 29, in run
return super(OpenStackCommand, self).run(parsed_args)
File "/usr/lib/python2.7/dist-packages/cliff/display.py", line 91, in run
column_names, data = self.take_action(parsed_args)
File "/usr/lib/python2.7/dist-packages/neutronclient/common/command.py", line 35, in take_action
return self.get_data(parsed_args)
File "/usr/lib/python2.7/dist-packages/neutronclient/neutron/v2_0/_init_.py", line 691, in get_data
data = self.retrieve_list(parsed_args)
File "/usr/lib/python2.7/dist-packages/neutronclient/neutron/v2_0/_init_.py", line 631, in retrieve_list
neutron_client = self.get_client()
File "/usr/lib/python2.7/dist-packages/neutronclient/neutron/v2_0/_init_.py", line 398, in get_client
return self.app.client_manager.neutron
File "/usr/lib/python2.7/dist-packages/neutronclient/common/clientmanager.py", line 39, in _get_
self._handle = self.factory(instance)
File "/usr/lib/python2.7/dist-packages/neutronclient/neutron/client.py", line 35, in make_client
instance.initialize()
File "/usr/lib/python2.7/dist-packages/neutronclient/common/clientmanager.py", line 113, in initialize
httpclient.authenticate()
File "/usr/lib/python2.7/dist-packages/neutronclient/client.py", line 323, in authenticate
self.get_token()
File "/usr/lib/python2.7/dist-packages/keystoneclient/adapter.py", line 110, in get_token
return self.session.get_token(auth or self.auth)
File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 621, in get_token
return (self.get_auth_headers(auth) or {}).get('X-Auth-Token')
File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 600, in get_auth_headers
return auth.get_headers(self, **kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/base.py", line 114, in get_headers
token = self.get_token(session)
File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/base.py", line 104, in get_token
return self.get_access(session).auth_token
File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/base.py", line 144, in get_access
self.auth_ref = self.get_auth_ref(session)
File "/usr/lib/python2.7/dist-packages/keystoneclient/auth/identity/v2.py", line 78, in get_auth_ref
authenticated=False, log=False)
File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 499, in post
return self.request(url, 'POST', **kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/utils.py", line 318, in inner
return func(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/keystoneclient/session.py", line 399, in request
raise exceptions.from_response(resp, method, url)
InternalServerError: An unexpected error prevented the server from fulfilling your request. (HTTP 500)
An unexpected error prevented the server from fulfilling your request. (HTTP 500)

—