While checking for failed jobs in AWX I found the following at the beginning of the output of a job today:
Traceback (most recent call last):
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/backends/base/base.py", line 289, in ensure_connection
self.connect()
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/backends/base/base.py", line 270, in connect
self.connection = self.get_new_connection(conn_params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/backends/postgresql/base.py", line 275, in get_new_connection
connection = self.Database.connect(**conn_params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/psycopg/connection.py", line 748, in connect
raise last_ex.with_traceback(None)
psycopg.OperationalError: connection failed: Success
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/awx/conf/settings.py", line 83, in _ctit_db_wrapper
yield
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/awx/conf/settings.py", line 429, in _get_local_with_cache
return self._get_local(name)
^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/awx/conf/settings.py", line 348, in _get_local
self._preload_cache()
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/awx/conf/settings.py", line 317, in _preload_cache
for setting in Setting.objects.filter(key__in=settings_to_cache.keys(), user__isnull=True).order_by('pk'):
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/models/query.py", line 398, in __iter__
self._fetch_all()
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/models/query.py", line 1881, in _fetch_all
self._result_cache = list(self._iterable_class(self))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/models/query.py", line 91, in __iter__
results = compiler.execute_sql(
^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/models/sql/compiler.py", line 1560, in execute_sql
cursor = self.connection.cursor()
^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/backends/base/base.py", line 330, in cursor
return self._cursor()
^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/backends/base/base.py", line 306, in _cursor
self.ensure_connection()
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/backends/base/base.py", line 288, in ensure_connection
with self.wrap_database_errors:
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/utils.py", line 91, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/backends/base/base.py", line 289, in ensure_connection
self.connect()
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/backends/base/base.py", line 270, in connect
self.connection = self.get_new_connection(conn_params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/django/db/backends/postgresql/base.py", line 275, in get_new_connection
connection = self.Database.connect(**conn_params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/psycopg/connection.py", line 748, in connect
raise last_ex.with_traceback(None)
django.db.utils.OperationalError: connection failed: Success
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/awx/main/tasks/jobs.py", line 605, in run
res = receptor_job.run()
^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/awx/main/tasks/receptor.py", line 407, in run
if self.unit_id is not None and settings.RECEPTOR_RELEASE_WORK:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/awx/conf/settings.py", line 540, in __getattr_without_cache__
return getattr(self._wrapped, name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/awx/conf/settings.py", line 438, in __getattr__
value = self._get_local_with_cache(name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/cachetools/__init__.py", line 823, in wrapper
v = method(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/awx/conf/settings.py", line 428, in _get_local_with_cache
with _ctit_db_wrapper(trans_safe=True):
File "/usr/lib64/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/awx/conf/settings.py", line 102, in _ctit_db_wrapper
sqlstate_str = psycopg.errors.lookup(sqlstate)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/awx/venv/awx/lib64/python3.11/site-packages/psycopg/errors.py", line 513, in lookup
return _sqlcodes[sqlstate.upper()]
^^^^^^^^^^^^^^
AttributeError: 'NoneType' object has no attribute 'upper'
The strange thing is that the job continued and succeeded, but the status was ‘Error’. None of the other jobs that were started yesterday show this error.
Anyone else ever seen this?