|
|
|
@ -1173,6 +1173,8 @@ int input_tokenizer (const u8 *input_buf, const int input_len, hc_token_t *token
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (token->sep[token_idx] != 0x00)
|
|
|
|
|
{
|
|
|
|
|
const u8 *next_pos = NULL;
|
|
|
|
|
|
|
|
|
|
if (token->attr[token_idx] & TOKEN_ATTR_SEPARATOR_FARTHEST)
|
|
|
|
@ -1194,6 +1196,22 @@ int input_tokenizer (const u8 *input_buf, const int input_len, hc_token_t *token
|
|
|
|
|
|
|
|
|
|
len_left -= len + 1; // +1 = separator
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
const int len = token->len[token_idx];
|
|
|
|
|
|
|
|
|
|
token->buf[token_idx + 1] = token->buf[token_idx] + len;
|
|
|
|
|
|
|
|
|
|
len_left -= len;
|
|
|
|
|
|
|
|
|
|
if (token->sep[token_idx] != 0)
|
|
|
|
|
{
|
|
|
|
|
token->buf[token_idx + 1]++; // +1 = separator
|
|
|
|
|
|
|
|
|
|
len_left--; // -1 = separator
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (token->attr[token_idx] & TOKEN_ATTR_FIXED_LENGTH)
|
|
|
|
|
{
|
|
|
|
|