fix and test parsing of incomplete packets

parse.nim

@@ -89,6 +89,7 @@ proc parseModifierName(input: string,
     return 
   (result.complete, result.name) = parseKeyword(input, packet)
   if not result.complete:
+    packet.cursor.dec()
     return
   # the following separator must either be ' ' (binary-arg) or '\t' (simple-arg)
   if input[packet.cursor - 1] == ' ':
@@ -111,6 +112,9 @@ proc parseModifierSimpleValue(input: string,
   result = input.getUntil(['\n'], packet)
   if result.complete:
     packet.cursor.inc()
+  else:
+    result.value = input[packet.cursor .. input.high()]
+    packet.cursor += result.value.len()
 
 proc parseModifierValue(input: string,
                         packet: var PsycPacket): tuple[complete: bool,
@@ -118,13 +122,17 @@ proc parseModifierValue(input: string,
   assert(packet.cursor < input.len())
   if packet.remainingPartLen < 0:
     return parseModifierSimpleValue(input, packet)
-  result.complete = packet.remainingPartLen < input.high() - packet.cursor
+  result.complete = packet.remainingPartLen <= input.high() + "\n".len() - packet.cursor
   if result.complete:
     result.value = input[packet.cursor .. packet.cursor + packet.remainingPartLen - 1]
     if input[packet.cursor + packet.remainingPartLen] != '\n':
       raise new(ValueError) # missing '\n' after binary-arg
-    packet.cursor += packet.remainingPartLen + 1
+    packet.cursor += result.value.len() + 1
     packet.remainingPartLen = -1
+  else:
+    result.value = input[packet.cursor .. input.high()]
+    packet.cursor += result.value.len()
+    packet.remainingPartLen -= result.value.len()
 
 proc parseContentLength(input: string,
                         packet: var PsycPacket): tuple[complete: bool, value: int] =
@@ -143,22 +151,36 @@ proc parseContentLength(input: string,
 proc parseData(input: string,
                packet: var PsycPacket): tuple[complete: bool, value: string] =
   assert(packet.cursor < input.len())
-  if packet.remainingPartLen < 0:
+  if packet.contentLength < 0:
-    if packet.contentLength < 0:
+    if input.continuesWith("|\n", packet.cursor):
-      if input.continuesWith("|\n", packet.cursor):
+      packet.cursor += 2
-        return (true, "")
+      return (true, "")
-      let findResult = input.find("\n|\n")
+    let findResult = input.find("\n|\n")
-      if findResult < 0:
+    if findResult < 0:
-        return (false, input[packet.cursor .. input.high()])
+      result.complete = false
-      return (true, input[packet.cursor .. findResult - 1])
+      result.value = input[packet.cursor .. input.high()]
-  packet.remainingPartLen = packet.contentLength -
+      packet.cursor = input.high() + 1
-                            packet.entityHeaderLen -
+      return
-                            packet.methodName.len() -
+    else:
-                            sizeof('\n')
+      result.complete = true
-  result.complete = packet.remainingPartLen <= input.high() - packet.cursor
+      result.value = input[packet.cursor .. findResult - 1]
+      packet.cursor = findResult + 3
+      return
+  elif packet.remainingPartLen < 0:
+    packet.remainingPartLen = packet.contentLength -
+                              packet.entityHeaderLen -
+                              packet.methodName.len() -
+                              sizeof('\n')
+  result.complete =
+    packet.remainingPartLen <= input.high() + "\n|\n".len() - packet.cursor
   if result.complete:
     result.value = input[packet.cursor .. packet.cursor + packet.remainingPartLen - 1]
+    packet.cursor += packet.remainingPartLen + "\n|\n".len()
     packet.remainingPartLen = -1
+  else:
+    result.value = input[packet.cursor .. input.high()]
+    packet.cursor += result.value.len()
+    packet.remainingPartLen -= result.value.len()
 
 proc isModifierName(input: string, cursor: int): bool =
   const operators = ['=', ':', '+', '-', '?']
@@ -168,6 +190,7 @@ proc newPacket*(): PsycPacket =
   PsycPacket(routingHeader: newSeq[Modifier](),
              entityHeader: newSeq[Modifier](),
              state: ParseState.RoutingModifierName,
+             contentLength: -1,
              remainingPartLen: -1)
 
 proc consumePart*(packet: var PsycPacket, slice: Slice[int]): string =
@@ -186,9 +209,6 @@ proc consumePart*(packet: var PsycPacket, slice: Slice[int]): string =
 proc parse*(input: string,
             packet: var PsycPacket): tuple[needMoreInput: bool,
                                            unparsed: Slice[int]] =
-  #proc addedPart(packet: var PsycPacket, slice: Slice[int]) =
-  #  packet.remainingPartLen -= slice.len()
-  #  packet.cursor += slice.len()
   result.needMoreInput = true
   case packet.state:
     of ParseState.RoutingModifierName:
@@ -204,13 +224,14 @@ proc parse*(input: string,
           packet.routingHeader.add(modifier)
     of ParseState.RoutingModifierValue:
       let (complete, value) = parseModifierSimpleValue(input, packet)
+      echo "parseModifierSimpleValue returned ", value
       result.needMoreInput = not complete
-      if complete and value.len() > 0:
+      if value.len() > 0:
-        packet.state = ParseState.RoutingModifierName
         if packet.routingHeader[^1].value.isNil():
           packet.routingHeader[^1].value = ""
         packet.routingHeader[^1].value.add(value)
-        #packet.addedPart(slice)
+      if complete:
+        packet.state = ParseState.RoutingModifierName
     of ParseState.ContentLength:
       let (complete, value) = parseContentLength(input, packet)
       if complete:
@@ -235,13 +256,12 @@ proc parse*(input: string,
       let (complete, value) = parseModifierValue(input, packet)
       result.needMoreInput = not complete
       packet.entityHeaderLen += (packet.cursor - oldCursor)
+      if value.len() > 0:
+        if packet.entityHeader[^1].value.isNil():
+          packet.entityHeader[^1].value = ""
+        packet.entityHeader[^1].value.add(value)
       if complete:
         packet.state = ParseState.EntityModifierName
-        if not value.isNil():
-          if packet.entityHeader[^1].value.isNil():
-            packet.entityHeader[^1].value = ""
-          packet.entityHeader[^1].value.add(value)
-        #packet.addedPart(slice)
     of ParseState.Method:
       let (complete, value) = parseMethod(input, packet)
       if complete:
@@ -252,13 +272,15 @@ proc parse*(input: string,
     of ParseState.Data:
       let (complete, value) = parseData(input, packet)
       result.needMoreInput = not complete
+      if value.len() > 0:
+        packet.data.add(value)
       if complete:
         packet.state = ParseState.Complete
-        packet.data.add(value)
-        #packet.addedPart(slice)
     of ParseState.Complete:
       assert(false)
   result.unparsed = packet.cursor .. input.high()
+  if packet.state != ParseState.Complete:
+    result.needMoreInput = result.needMoreInput or packet.cursor > input.high()
   if result.needMoreInput:
     packet.cursor = 0
 
@@ -269,7 +291,8 @@ suite "parser tests":
   test "state sync":
     let input = ":_target\talice\n\n?\n|\n"
     while packet.state != ParseState.Complete:
-      discard parse(input, packet)
+      let (needMore, _) = parse(input, packet)
+      check(not needMore)
     check(packet.routingHeader.len() == 1)
     check(packet.routingHeader[0] == Modifier(op: ':',
                                               name: "_target",
@@ -280,7 +303,8 @@ suite "parser tests":
   test "simple-arg":
     let input = ":_target\talice\n\n:_hello\tworld\n:_hallo\twelt\n|\n"
     while packet.state != ParseState.Complete:
-      discard parse(input, packet)
+      let (needMore, _) = parse(input, packet)
+      check(not needMore)
     check(packet.entityHeader.len() == 2)
     check(packet.entityHeader[0] == Modifier(op: ':', name: "_hello", value: "world"))
     check(packet.entityHeader[1] == Modifier(op: ':', name: "_hallo", value: "welt"))
@@ -288,7 +312,8 @@ suite "parser tests":
   test "binary-arg":
     let input = ":_target\talice\n\n:_hello 5\tworld\n:_hallo 4\twelt\n|\n"
     while packet.state != ParseState.Complete:
-      discard parse(input, packet)
+      let (needMore, _) = parse(input, packet)
+      check(not needMore)
     check(packet.entityHeader.len() == 2)
     check(packet.entityHeader[0] == Modifier(op: ':', name: "_hello", value: "world"))
     check(packet.entityHeader[1] == Modifier(op: ':', name: "_hallo", value: "welt"))
@@ -296,15 +321,123 @@ suite "parser tests":
   test "method/data":
     let input = ":_target\talice\n\n_hello_world\nHello Alice!\n|\n"
     while packet.state != ParseState.Complete:
-      discard parse(input, packet)
+      let (needMore, _) = parse(input, packet)
+      check(not needMore)
     check(packet.methodName == "_hello_world")
     check(packet.data == "Hello Alice!")
 
   test "content length":
     let input = ":_target\talice\n39\n:_hello\tworld\n_hello_world\nHello Alice!\n|\n"
     while packet.state != ParseState.Complete:
-     discard parse(input, packet)
+      let (needMore, _) = parse(input, packet)
+      check(not needMore)
     check(packet.contentLength == 39)
     check(packet.data == "Hello Alice!")
 
+  test "incomplete":
+    let
+      input1 = ":_target\t"
+      input2 = "alice\n"
+      input3 = "39\n"
+      input4 = ":_hello\t"
+      input5 = "world\n"
+      input6 = "_hello_world\n"
+      input7 = "Hello Alice!\n|\n"
+    var
+      needMore: bool
+      unparsed: Slice[int]
+
+    (needMore, unparsed) = parse(input1[0..4], packet)
+    check(needMore)
+    check(unparsed == 0..4)
+    check(packet.state == ParseState.RoutingModifierName)
+    check(packet.routingHeader.len() == 0)
+
+    (needMore, unparsed) = parse(input1[unparsed] & input1[5..8], packet)
+    check(needMore)
+    check(unparsed == 9..8)
+    check(packet.state == ParseState.RoutingModifierValue)
+    check(packet.routingHeader[0] == Modifier(op: ':', name: "_target", value: nil))
+
+    (needMore, unparsed) = parse(input2[0..2], packet)
+    check(needMore)
+    check(unparsed == 3..2)
+    check(packet.state == ParseState.RoutingModifierValue)
+    check(packet.routingHeader[0] == Modifier(op: ':', name: "_target", value: "ali"))
+
+    (needMore, unparsed) = parse(input2[3..5], packet)
+    check(needMore)
+    check(unparsed == 3..2)
+    check(packet.state == ParseState.RoutingModifierName)
+    check(packet.routingHeader[0] == Modifier(op: ':', name: "_target", value: "alice"))
+
+    (needMore, unparsed) = parse(input3[0..1], packet)
+    check(not needMore)
+    check(unparsed == 0..1)
+    check(packet.state == ParseState.ContentLength)
+
+    (needMore, unparsed) = parse(input3[0..1], packet)
+    check(needMore)
+    check(unparsed == 0..1)
+    check(packet.state == ParseState.ContentLength)
+    check(packet.contentLength == -1)
+
+    (needMore, unparsed) = parse(input3[unparsed] & input3[2], packet)
+    check(needMore)
+    check(unparsed == 3..2)
+    check(packet.state == ParseState.EntityModifierName)
+    check(packet.contentLength == 39)
+
+    (needMore, unparsed) = parse(input4[0..3], packet)
+    check(needMore)
+    check(unparsed == 0..3)
+    check(packet.state == ParseState.EntityModifierName)
+    check(packet.entityHeader.len() == 0)
+   
+    (needMore, unparsed) = parse(input4[unparsed] & input4[4..7], packet)
+    check(needMore)
+    check(unparsed == 8..7)
+    check(packet.state == ParseState.EntityModifierValue)
+    check(packet.entityHeader[0] == Modifier(op: ':', name: "_hello", value: nil))
+    
+    (needMore, unparsed) = parse(input5[0..2], packet)
+    check(needMore)
+    check(unparsed == 3..2)
+    check(packet.state == ParseState.EntityModifierValue)
+    check(packet.entityHeader[0] == Modifier(op: ':', name: "_hello", value: "wor"))
+
+    (needMore, unparsed) = parse(input5[unparsed] & input5[3..5], packet)
+    check(needMore)
+    check(unparsed == 3..2)
+    check(packet.state == ParseState.EntityModifierName)
+    check(packet.entityHeader[0] == Modifier(op: ':', name: "_hello", value: "world"))
+
+    (needMore, unparsed) = parse(input5[unparsed] & input6[0..6], packet)
+    check(not needMore)
+    check(unparsed == 0..6)
+    check(packet.state == ParseState.Method)
+
+    (needMore, unparsed) = parse(input6[0..6], packet)
+    check(needMore)
+    check(unparsed == 0..6)
+    check(packet.state == ParseState.Method)
+
+    (needMore, unparsed) = parse(input6[unparsed] & input6[7..12], packet)
+    check(needMore)
+    check(unparsed == 13..12)
+    check(packet.state == ParseState.Data)
+    check(packet.methodName == "_hello_world")
+
+    (needMore, unparsed) = parse(input7[0..5], packet)
+    check(needMore)
+    check(unparsed == 6..5)
+    check(packet.state == ParseState.Data)
+    check(packet.data == "Hello ")
+
+    (needMore, unparsed) = parse(input7[6..14], packet)
+    check(not needMore)
+    check(unparsed == 9..8)
+    check(packet.state == ParseState.Complete)
+    check(packet.data == "Hello Alice!")
+
   echo "parser tests completed"