verify separator also with TOKEN_ATTR_FIXED_LENGTH (part 2)

src/shared.c

@@ -1173,6 +1173,8 @@ int input_tokenizer (const u8 *input_buf, const int input_len, hc_token_t *token
       }
     }
 
+    if (token->sep[token_idx] != 0x00)
+    {
       const u8 *next_pos = NULL;
 
       if (token->attr[token_idx] & TOKEN_ATTR_SEPARATOR_FARTHEST)
@@ -1194,6 +1196,22 @@ int input_tokenizer (const u8 *input_buf, const int input_len, hc_token_t *token
 
       len_left -= len + 1; // +1 = separator
     }
+    else
+    {
+      const int len = token->len[token_idx];
+
+      token->buf[token_idx + 1] = token->buf[token_idx] + len;
+
+      len_left -= len;
+
+      if (token->sep[token_idx] != 0)
+      {
+        token->buf[token_idx + 1]++; // +1 = separator
+
+        len_left--; // -1 = separator
+      }
+    }
+  }
 
   if (token->attr[token_idx] & TOKEN_ATTR_FIXED_LENGTH)
   {