Use $HEX[...] in case the password includes the separater character, increases potfile reading performance

docs/changes.txt

@@ -1,5 +1,11 @@
 * changes v3.20 -> v3.21:
 
+##
+## Features
+##
+
+- Use $HEX[...] in case the password includes the separater character. This makes the MAX_CUT_TRIES logic no longer required which then increase potfile reading
+
 ##
 ## Bugs
 ##

include/common.h

@@ -79,7 +79,6 @@ but this is nededed for VS compiler which doesn't have inline keyword but has __
 #define SPEED_MAXAGE        4096
 #define BLOCK_SIZE          64
 #define EXPECTED_ITERATIONS 10000
-#define MAX_CUT_TRIES       4
 
 #if defined (_WIN)
 #define EOL "\r\n"

include/convert.h

@@ -8,7 +8,7 @@
 
 #include <ctype.h>
 
-bool need_hexify (const u8 *buf, const int len, bool always_ascii);
+bool need_hexify (const u8 *buf, const int len, const char separator, bool always_ascii);
 void exec_hexify (const u8 *buf, const int len, u8 *out);
 
 bool is_valid_hex_char (const u8 c);

src/convert.c

@@ -75,34 +75,48 @@ static bool printable_ascii (const u8 *buf, const int len)
   return true;
 }
 
-bool need_hexify (const u8 *buf, const int len, bool always_ascii)
+static bool matches_separator (const u8 *buf, const int len, const char separator)
 {
-  if (always_ascii == true)
+  for (int i = 0; i < len; i++)
   {
-    if (printable_ascii (buf, len) == true)
+    const char c = (char) buf[i];
-    {
+
-      return false;
+    if (c == separator) return true;
-    }
-    else
-    {
-      return true;
-    }
-  }
-  else
-  {
-    if (printable_utf8 (buf, len) == true)
-    {
-      return false;
-    }
-    else
-    {
-      return true;
-    }
   }
 
   return false;
 }
 
+bool need_hexify (const u8 *buf, const int len, const char separator, bool always_ascii)
+{
+  bool rc = false;
+
+  if (always_ascii == true)
+  {
+    if (printable_ascii (buf, len) == false)
+    {
+      rc = true;
+    }
+  }
+  else
+  {
+    if (printable_utf8 (buf, len) == false)
+    {
+      rc = true;
+    }
+  }
+
+  if (rc == false)
+  {
+    if (matches_separator (buf, len, separator) == true)
+    {
+      rc = true;
+    }
+  }
+
+  return rc;
+}
+
 void exec_hexify (const u8 *buf, const int len, u8 *out)
 {
   const int max_len = (len >= 31) ? 31 : len;

src/outfile.c

@@ -383,7 +383,7 @@ int outfile_write (hashcat_ctx_t *hashcat_ctx, const char *out_buf, const unsign
   {
     const bool always_ascii = (hashconfig->hash_type & OPTS_TYPE_PT_ALWAYS_ASCII) ? true : false;
 
-    if ((user_options->outfile_autohex == true) && (need_hexify (plain_ptr, plain_len, always_ascii) == true))
+    if ((user_options->outfile_autohex == true) && (need_hexify (plain_ptr, plain_len, hashconfig->separator, always_ascii) == true))
     {
       tmp_buf[tmp_len++] = '$';
       tmp_buf[tmp_len++] = 'H';

src/outfile_check.c

@@ -152,7 +152,7 @@ static int outfile_remove (hashcat_ctx_t *hashcat_ctx)
 
                 if (line_len == 0) continue;
 
-                size_t iter = MAX_CUT_TRIES;
+                size_t iter = 1;
 
                 for (size_t i = line_len - 1; i && iter; i--, line_len--)
                 {

src/potfile.c

@@ -218,7 +218,7 @@ void potfile_write_append (hashcat_ctx_t *hashcat_ctx, const char *out_buf, u8 *
 
     tmp_len += out_len;
 
-    tmp_buf[tmp_len] = ':';
+    tmp_buf[tmp_len] = hashconfig->separator;
 
     tmp_len += 1;
   }
@@ -227,7 +227,7 @@ void potfile_write_append (hashcat_ctx_t *hashcat_ctx, const char *out_buf, u8 *
   {
     const bool always_ascii = (hashconfig->hash_type & OPTS_TYPE_PT_ALWAYS_ASCII) ? true : false;
 
-    if ((user_options->outfile_autohex == true) && (need_hexify (plain_ptr, plain_len, always_ascii) == true))
+    if ((user_options->outfile_autohex == true) && (need_hexify (plain_ptr, plain_len, hashconfig->separator, always_ascii) == true))
     {
       tmp_buf[tmp_len++] = '$';
       tmp_buf[tmp_len++] = 'H';
@@ -262,9 +262,9 @@ void potfile_write_append (hashcat_ctx_t *hashcat_ctx, const char *out_buf, u8 *
 
 int potfile_remove_parse (hashcat_ctx_t *hashcat_ctx)
 {
-  hashconfig_t  *hashconfig  = hashcat_ctx->hashconfig;
+  const hashconfig_t  *hashconfig  = hashcat_ctx->hashconfig;
-  hashes_t      *hashes      = hashcat_ctx->hashes;
+  const hashes_t      *hashes      = hashcat_ctx->hashes;
-  potfile_ctx_t *potfile_ctx = hashcat_ctx->potfile_ctx;
+  const potfile_ctx_t *potfile_ctx = hashcat_ctx->potfile_ctx;
 
   if (potfile_ctx->enabled == false) return 0;
 
@@ -325,27 +325,29 @@ int potfile_remove_parse (hashcat_ctx_t *hashcat_ctx)
 
   char *line_buf = (char *) hcmalloc (HCBUFSIZ_LARGE);
 
-  // to be safe work with a copy (because of line_len loop, i etc)
-  // moved up here because it's easier to handle continue case
-  // it's just 64kb
-
-  char *line_buf_cpy = (char *) hcmalloc (HCBUFSIZ_LARGE);
-
   while (!feof (potfile_ctx->fp))
   {
     int line_len = fgetl (potfile_ctx->fp, line_buf);
 
     if (line_len == 0) continue;
 
-    const int line_len_orig = line_len;
+    char *last_separator = strrchr (line_buf, hashconfig->separator);
 
-    int iter = MAX_CUT_TRIES;
+    if (last_separator == NULL) continue; // ??
 
-    for (int i = line_len - 1; i && iter; i--, line_len--)
+    char *line_pw_buf = last_separator + 1;
-    {
-      if (line_buf[i] != ':') continue;
 
-      iter--;
+    int line_pw_len = line_buf + line_len - line_pw_buf;
+
+    char *line_hash_buf = line_buf;
+
+    int line_hash_len = last_separator - line_buf;
+
+    line_hash_buf[line_hash_len] = 0;
+
+    if (line_hash_len == 0) continue;
+
+    if (line_pw_len == 0) continue;
 
     if (hashconfig->is_salted)
     {
@@ -361,28 +363,23 @@ int potfile_remove_parse (hashcat_ctx_t *hashcat_ctx)
 
     if (hashconfig->hash_mode == 6800)
     {
-        if (i < 64) // 64 = 16 * u32 in salt_buf[]
+      if (line_hash_len < 64) // 64 = 16 * u32 in salt_buf[]
       {
         // manipulate salt_buf
-          memcpy (hash_buf.salt->salt_buf, line_buf, i);
+        memcpy (hash_buf.salt->salt_buf, line_hash_buf, line_hash_len);
 
-          hash_buf.salt->salt_len = i;
+        hash_buf.salt->salt_len = line_hash_len;
 
         found = (hash_t *) bsearch (&hash_buf, hashes_buf, hashes_cnt, sizeof (hash_t), sort_by_hash_t_salt);
       }
     }
     else if (hashconfig->hash_mode == 2500)
     {
-        if (i < 64) // 64 = 16 * u32 in salt_buf[]
+      if (line_hash_len < 64) // 64 = 16 * u32 in salt_buf[]
       {
-          // here we have in line_buf: ESSID:MAC1:MAC2   (without the plain)
+        // here we have in line_hash_buf: ESSID:MAC1:MAC2   (without the plain)
-          // manipulate salt_buf
 
-          memcpy (line_buf_cpy, line_buf, i);
+        char *mac2_pos = strrchr (line_hash_buf, ':');
-
-          line_buf_cpy[i] = 0;
-
-          char *mac2_pos = strrchr (line_buf_cpy, ':');
 
         if (mac2_pos == NULL) continue;
 
@@ -391,7 +388,7 @@ int potfile_remove_parse (hashcat_ctx_t *hashcat_ctx)
 
         if (strlen (mac2_pos) != 12) continue;
 
-          char *mac1_pos = strrchr (line_buf_cpy, ':');
+        char *mac1_pos = strrchr (line_hash_buf, ':');
 
         if (mac1_pos == NULL) continue;
 
@@ -400,13 +397,13 @@ int potfile_remove_parse (hashcat_ctx_t *hashcat_ctx)
 
         if (strlen (mac1_pos) != 12) continue;
 
-          u32 essid_length = mac1_pos - line_buf_cpy - 1;
+        u32 essid_length = mac1_pos - line_hash_buf - 1;
 
         if (hashconfig->is_salted)
         {
           // this should be always true, but we need it to make scan-build happy
 
-            memcpy (hash_buf.salt->salt_buf, line_buf_cpy, essid_length);
+          memcpy (hash_buf.salt->salt_buf, line_hash_buf, essid_length);
 
           hash_buf.salt->salt_len = essid_length;
         }
@@ -427,13 +424,7 @@ int potfile_remove_parse (hashcat_ctx_t *hashcat_ctx)
 
               break;
             }
-            }
 
-            // early skip ;)
-            if (!found) continue;
-
-            for (u32 mac_idx = 0, orig_mac_idx = 0; mac_idx < 6; mac_idx += 1, orig_mac_idx += 2)
-            {
             if (wpa->orig_mac2[mac_idx] != hex_to_u8 ((const u8 *) &mac2_pos[orig_mac_idx]))
             {
               found = NULL;
@@ -446,7 +437,7 @@ int potfile_remove_parse (hashcat_ctx_t *hashcat_ctx)
     }
     else
     {
-        int parser_status = hashconfig->parse_func ((u8 *) line_buf, line_len - 1, &hash_buf, hashconfig);
+      int parser_status = hashconfig->parse_func ((u8 *) line_hash_buf, line_hash_len, &hash_buf, hashconfig);
 
       if (parser_status == PARSER_OK)
       {
@@ -463,8 +454,8 @@ int potfile_remove_parse (hashcat_ctx_t *hashcat_ctx)
 
     if (found == NULL) continue;
 
-      char *pw_buf = line_buf + line_len;
+    char *pw_buf = line_pw_buf;
-      int   pw_len = line_len_orig - line_len;
+    int   pw_len = line_pw_len;
 
     found->pw_buf = (char *) hcmalloc (pw_len + 1);
     found->pw_len = pw_len;
@@ -474,12 +465,7 @@ int potfile_remove_parse (hashcat_ctx_t *hashcat_ctx)
     found->pw_buf[found->pw_len] = 0;
 
     found->cracked = 1;
-
-      break;
   }
-  }
-
-  hcfree (line_buf_cpy);
 
   hcfree (line_buf);
 

src/status.c

@@ -737,8 +737,8 @@ char *status_get_input_candidates_dev (const hashcat_ctx_t *hashcat_ctx, const i
 
   const bool always_ascii = (hashconfig->hash_type & OPTS_TYPE_PT_ALWAYS_ASCII) ? true : false;
 
-  const bool need_hex1 = need_hexify (plain_ptr1, plain_len1, always_ascii);
+  const bool need_hex1 = need_hexify (plain_ptr1, plain_len1, 0, always_ascii);
-  const bool need_hex2 = need_hexify (plain_ptr2, plain_len2, always_ascii);
+  const bool need_hex2 = need_hexify (plain_ptr2, plain_len2, 0, always_ascii);
 
   if ((need_hex1 == true) || (need_hex2 == true))
   {