Merge pull request #3726 from matrix/fix_tokenizer_ATTR_FIXED_LEN

Added verification of token buffer length when using TOKEN_ATTR_FIXED_LENGTH

docs/changes.txt

@@ -58,6 +58,7 @@
 - Fixed bug in 29600 module OPTS_TYPE setting
 - Fixed bug in grep out-of-memory workaround on Unit Test
 - Fixed bug in input_tokenizer when TOKEN_ATTR_FIXED_LENGTH is used and refactor modules
+- Added verification of token buffer length when using TOKEN_ATTR_FIXED_LENGTH
 - Fixed build failed for 4410 with vector width > 1
 - Fixed build failed for 18400 with Apple Metal
 - Fixed build failed for 18600 with Apple Metal

src/shared.c

@@ -1190,6 +1190,11 @@ int input_tokenizer (const u8 *input_buf, const int input_len, hc_token_t *token
 
       const int len = next_pos - token->buf[token_idx];
 
+      if (token->attr[token_idx] & TOKEN_ATTR_FIXED_LENGTH)
+      {
+        if (len != token->len[token_idx]) return (PARSER_TOKEN_LENGTH);
+      }
+
       token->len[token_idx] = len;
 
       token->buf[token_idx + 1] = next_pos + 1; // +1 = separator