In extreme rare cases, I have seen some missing submissions. What happened today is we lost 6 submissions from 2 different users. They have been definitely submitted from Kobocollect, but somehow there’s no trace of them on the server.
Going through the kobocat uwsgi logs, I notice these exceptions:
First:
ERROR 2022-09-20 02:24:32,069 log 12966 140655028680576 Internal Server Error: /api/v1/data/66
Traceback (most recent call last):
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 1184, in _get_socket_no_auth
sock_info = self.sockets.popleft()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 1089, in connect
sock = _configured_socket(self.address, self.opts)
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 895, in _configured_socket
sock = _create_connection(address, options)
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 876, in _create_connection
raise err
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 869, in _create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/venv/lib/python3.8/site-packages/django/core/handlers/exception.py", line 34, in inner
response = get_response(request)
File "/opt/venv/lib/python3.8/site-packages/django/core/handlers/base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/opt/venv/lib/python3.8/site-packages/django/core/handlers/base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/usr/local/lib/python3.8/contextlib.py", line 75, in inner
return func(*args, **kwds)
File "/opt/venv/lib/python3.8/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/viewsets.py", line 114, in view
return self.dispatch(request, *args, **kwargs)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/views.py", line 505, in dispatch
response = self.handle_exception(exc)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/views.py", line 465, in handle_exception
self.raise_uncaught_exception(exc)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
raise exc
File "/opt/venv/lib/python3.8/site-packages/rest_framework/views.py", line 502, in dispatch
response = handler(request, *args, **kwargs)
File "./onadata/apps/api/viewsets/data_viewset.py", line 686, in list
res = super().list(request, *args, **kwargs)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/mixins.py", line 46, in list
return Response(serializer.data)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/serializers.py", line 760, in data
ret = super().data
File "/opt/venv/lib/python3.8/site-packages/rest_framework/serializers.py", line 260, in data
self._data = self.to_representation(self.instance)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/serializers.py", line 677, in to_representation
return [
File "/opt/venv/lib/python3.8/site-packages/rest_framework/serializers.py", line 678, in <listcomp>
self.child.to_representation(item) for item in iterable
File "./onadata/libs/serializers/data_serializer.py", line 73, in to_representation
return [MongoHelper.to_readable_dict(record) for record in cursor]
File "./onadata/libs/serializers/data_serializer.py", line 73, in <listcomp>
return [MongoHelper.to_readable_dict(record) for record in cursor]
File "/opt/venv/lib/python3.8/site-packages/pymongo/cursor.py", line 1156, in next
if len(self.__data) or self._refresh():
File "/opt/venv/lib/python3.8/site-packages/pymongo/cursor.py", line 1073, in _refresh
self.__send_message(q)
File "/opt/venv/lib/python3.8/site-packages/pymongo/cursor.py", line 953, in __send_message
response = client._run_operation_with_response(
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1342, in _run_operation_with_response
return self._retryable_read(
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1457, in _retryable_read
with self._slaveok_for_server(read_pref, server, session,
File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
return next(self.gen)
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1282, in _slaveok_for_server
with self._get_socket(server, session, exhaust=exhaust) as sock_info:
File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
return next(self.gen)
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1221, in _get_socket
with server.get_socket(
File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
return next(self.gen)
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 1135, in get_socket
sock_info = self._get_socket_no_auth()
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 1187, in _get_socket_no_auth
sock_info = self.connect()
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 1098, in connect
_raise_connection_failure(self.address, error)
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 290, in _raise_connection_failure
raise AutoReconnect(msg)
pymongo.errors.AutoReconnect: mongo.mydomain.private:27017: [Errno 111] Connection refused
ERROR 2022-09-20 02:24:32,636 log 12697 140655028680576 Internal Server Error: /api/v1/data/56
Traceback (most recent call last):
File "/opt/venv/lib/python3.8/site-packages/django/core/handlers/exception.py", line 34, in inner
response = get_response(request)
File "/opt/venv/lib/python3.8/site-packages/django/core/handlers/base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/opt/venv/lib/python3.8/site-packages/django/core/handlers/base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/usr/local/lib/python3.8/contextlib.py", line 75, in inner
return func(*args, **kwds)
File "/opt/venv/lib/python3.8/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/viewsets.py", line 114, in view
return self.dispatch(request, *args, **kwargs)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/views.py", line 505, in dispatch
response = self.handle_exception(exc)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/views.py", line 465, in handle_exception
self.raise_uncaught_exception(exc)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
raise exc
File "/opt/venv/lib/python3.8/site-packages/rest_framework/views.py", line 502, in dispatch
response = handler(request, *args, **kwargs)
File "./onadata/apps/api/viewsets/data_viewset.py", line 686, in list
res = super().list(request, *args, **kwargs)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/mixins.py", line 46, in list
return Response(serializer.data)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/serializers.py", line 760, in data
ret = super().data
File "/opt/venv/lib/python3.8/site-packages/rest_framework/serializers.py", line 260, in data
self._data = self.to_representation(self.instance)
File "/opt/venv/lib/python3.8/site-packages/rest_framework/serializers.py", line 677, in to_representation
return [
File "/opt/venv/lib/python3.8/site-packages/rest_framework/serializers.py", line 678, in <listcomp>
self.child.to_representation(item) for item in iterable
File "./onadata/libs/serializers/data_serializer.py", line 73, in to_representation
return [MongoHelper.to_readable_dict(record) for record in cursor]
File "./onadata/libs/serializers/data_serializer.py", line 73, in <listcomp>
return [MongoHelper.to_readable_dict(record) for record in cursor]
File "/opt/venv/lib/python3.8/site-packages/pymongo/cursor.py", line 1156, in next
if len(self.__data) or self._refresh():
File "/opt/venv/lib/python3.8/site-packages/pymongo/cursor.py", line 1093, in _refresh
self.__send_message(g)
File "/opt/venv/lib/python3.8/site-packages/pymongo/cursor.py", line 953, in __send_message
response = client._run_operation_with_response(
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1342, in _run_operation_with_response
return self._retryable_read(
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1464, in _retryable_read
return func(session, server, sock_info, slave_ok)
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1334, in _cmd
return server.run_operation_with_response(
File "/opt/venv/lib/python3.8/site-packages/pymongo/server.py", line 117, in run_operation_with_response
reply = sock_info.receive_message(request_id)
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 646, in receive_message
self._raise_connection_failure(error)
File "/opt/venv/lib/python3.8/site-packages/pymongo/pool.py", line 643, in receive_message
return receive_message(self.sock, request_id,
File "/opt/venv/lib/python3.8/site-packages/pymongo/network.py", line 196, in receive_message
_receive_data_on_socket(sock, 16))
File "/opt/venv/lib/python3.8/site-packages/pymongo/network.py", line 261, in _receive_data_on_socket
raise AutoReconnect("connection closed")
pymongo.errors.AutoReconnect: connection closed
Second:
Traceback (most recent call last):
File "/opt/venv/lib/python3.8/site-packages/django/db/models/query.py", line 538, in get_or_create
return self.get(**kwargs), False
File "/opt/venv/lib/python3.8/site-packages/django/db/models/query.py", line 406, in get
raise self.model.DoesNotExist(
onadata.apps.viewer.models.parsed_instance.ParsedInstance.DoesNotExist: ParsedInstance matching query does not exist.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./onadata/apps/viewer/models/parsed_instance.py", line 60, in update_mongo_instance
xform_instances.replace_one({'_id': record['_id']}, record, upsert=True)
File "/opt/venv/lib/python3.8/site-packages/pymongo/collection.py", line 926, in replace_one
self._update_retryable(
File "/opt/venv/lib/python3.8/site-packages/pymongo/collection.py", line 854, in _update_retryable
return self.__database.client._retryable_write(
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1490, in _retryable_write
with self._tmp_session(session) as s:
File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
return next(self.gen)
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1823, in _tmp_session
s = self._ensure_session(session)
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1810, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1763, in __start_session
server_session = self._get_server_session()
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1796, in _get_server_session
return self._topology.get_server_session()
File "/opt/venv/lib/python3.8/site-packages/pymongo/topology.py", line 482, in get_server_session
self._select_servers_loop(
File "/opt/venv/lib/python3.8/site-packages/pymongo/topology.py", line 208, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: No servers found yet
Submission could not be saved to Mongo.
Traceback (most recent call last):
File "/opt/venv/lib/python3.8/site-packages/django/db/models/query.py", line 538, in get_or_create
return self.get(**kwargs), False
File "/opt/venv/lib/python3.8/site-packages/django/db/models/query.py", line 406, in get
raise self.model.DoesNotExist(
onadata.apps.viewer.models.parsed_instance.ParsedInstance.DoesNotExist: ParsedInstance matching query does not exist.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./onadata/apps/viewer/models/parsed_instance.py", line 60, in update_mongo_instance
xform_instances.replace_one({'_id': record['_id']}, record, upsert=True)
File "/opt/venv/lib/python3.8/site-packages/pymongo/collection.py", line 926, in replace_one
self._update_retryable(
File "/opt/venv/lib/python3.8/site-packages/pymongo/collection.py", line 854, in _update_retryable
return self.__database.client._retryable_write(
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1490, in _retryable_write
with self._tmp_session(session) as s:
File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
return next(self.gen)
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1823, in _tmp_session
s = self._ensure_session(session)
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1810, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1763, in __start_session
server_session = self._get_server_session()
File "/opt/venv/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1796, in _get_server_session
return self._topology.get_server_session()
File "/opt/venv/lib/python3.8/site-packages/pymongo/topology.py", line 482, in get_server_session
self._select_servers_loop(
File "/opt/venv/lib/python3.8/site-packages/pymongo/topology.py", line 208, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: No servers found yet
I found hundreds of instances of the second error.
As a first step, I ran python3 run.py to restart all containers, and all submissions since then seem fine and no exceptions have been noted.
@jnm sorry for tagging you, but it’s a bit urgent and critical as the data for this was collected in a remote location and submitted later, so there’s no option to go back and get the data.
- Is there any way to retrieve these submissions? The user had set the option in Kobocollect to delete the submission from the device after sending due to storage space issues.
- Am I right in assuming that this is a server load issue and such future issues could be prevented by adding more workers? If so, isn’t there a fallback to not accept submissions when the server is overloaded?
Thanks a lot.
