fix AssertError for newer but lower message idx (#194)

Author: snubba

pysyncobj/config.py

@@ -5,8 +5,9 @@ class FAIL_REASON:
     MISSING_LEADER = 2      #: Leader is currently missing (leader election in progress, or no connection)
     DISCARDED = 3           #: Command discarded (cause of new leader elected and another command was applied instead)
     NOT_LEADER = 4          #: Leader has changed, old leader did not have time to commit command.
-    LEADER_CHANGED = 5      #: Simmilar to NOT_LEADER - leader has changed without command commit.
+    LEADER_CHANGED = 5      #: Similar to NOT_LEADER - leader has changed without command commit.
     REQUEST_DENIED = 6      #: Command denied
+    UNKNOWN_OUTCOME = 7     #: Unknown outcome - Unknown outcome, might happen if a older command is tried to apply
 
 class SERIALIZER_STATE:
     NOT_SERIALIZING = 0     #: Serialization not started or already finished.

pysyncobj/syncobj.py

@@ -974,8 +974,15 @@ def __onMessageReceived(self, node, message):
                 else:
                     idx = message['log_idx']
                     term = message['log_term']
-                    assert idx > self.__raftLastApplied
-                    self.__commandsWaitingCommit[idx].append((term, callback))
+                    if idx <= self.__raftLastApplied:
+                        # Stale apply_command_response:
+                        # The log position was already applied (e.g. via snapshot installed after leader reconnect)
+                        # before this response was processed. Or the leader was restarted without notice (and remains leader).
+                        # The command outcome cannot be tracked; signal the caller so it can retry rather than hanging
+                        # forever.
+                        callback(None, FAIL_REASON.UNKNOWN_OUTCOME)
+                    else:
+                        self.__commandsWaitingCommit[idx].append((term, callback))
 
         if self.__raftState == _RAFT_STATE.CANDIDATE:
             if message['type'] == 'response_vote' and message['term'] == self.__raftCurrentTerm: