mirror of
https://github.com/hashcat/hashcat.git
synced 2024-11-23 08:38:09 +00:00
Merge pull request #1676 from philsmd/master
outfile check: performance improvement by using binary instead of linear search
This commit is contained in:
Jens Steube
GitHub
commit
0339b3bca0
@ -11,6 +11,7 @@
|
||||
- Add option --markov-hcstat2 to make it clear that the new hcstat2 format (compressed hcstat2gen output) must be used
|
||||
- Do not allocate memory segments for bitmap tables if we dont need it, for example in benchmark mode
|
||||
- Removed duplicate words in the dictionary file example.dict
|
||||
- Improved the speed of the outfile folder scan when using lots of hashes/salts
|
||||
|
||||
##
|
||||
## Bugs
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
|
||||
int sort_by_digest_p0p1 (const void *v1, const void *v2, void *v3);
|
||||
int sort_by_salt (const void *v1, const void *v2);
|
||||
int sort_by_salt_buf (const void *v1, const void *v2, MAYBE_UNUSED void * v3);
|
||||
int sort_by_hash (const void *v1, const void *v2, void *v3);
|
||||
int sort_by_hash_no_salt (const void *v1, const void *v2, void *v3);
|
||||
|
||||
|
||||
@ -79,6 +79,11 @@ int sort_by_salt (const void *v1, const void *v2)
|
||||
return 0;
|
||||
}
|
||||
|
||||
int sort_by_salt_buf (const void *v1, const void *v2, MAYBE_UNUSED void * v3)
|
||||
{
|
||||
return sort_by_salt (v1, v2);
|
||||
}
|
||||
|
||||
int sort_by_hash (const void *v1, const void *v2, void *v3)
|
||||
{
|
||||
const hash_t *h1 = (const hash_t *) v1;
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
#include "interface.h"
|
||||
#include "shared.h"
|
||||
#include "thread.h"
|
||||
#include "bitops.h"
|
||||
|
||||
static int outfile_remove (hashcat_ctx_t *hashcat_ctx)
|
||||
{
|
||||
@ -33,6 +34,11 @@ static int outfile_remove (hashcat_ctx_t *hashcat_ctx)
|
||||
u32 hash_mode = hashconfig->hash_mode;
|
||||
char separator = hashconfig->separator;
|
||||
|
||||
salt_t *salts_buf = hashes->salts_buf;
|
||||
const u32 salts_cnt = hashes->salts_cnt;
|
||||
|
||||
char *digests_buf = hashes->digests_buf;
|
||||
|
||||
char *root_directory = outcheck_ctx->root_directory;
|
||||
u32 outfile_check_timer = user_options->outfile_check_timer;
|
||||
|
||||
@ -45,8 +51,6 @@ static int outfile_remove (hashcat_ctx_t *hashcat_ctx)
|
||||
if (esalt_size > 0) hash_buf.esalt = hcmalloc (esalt_size);
|
||||
if (hook_salt_size > 0) hash_buf.hook_salt = hcmalloc (hook_salt_size);
|
||||
|
||||
u32 digest_buf[64] = { 0 };
|
||||
|
||||
outfile_data_t *out_info = NULL;
|
||||
|
||||
char **out_files = NULL;
|
||||
@ -65,220 +69,245 @@ static int outfile_remove (hashcat_ctx_t *hashcat_ctx)
|
||||
|
||||
check_left--;
|
||||
|
||||
if (check_left == 0)
|
||||
if (check_left != 0) continue;
|
||||
|
||||
check_left = outfile_check_timer;
|
||||
|
||||
if (hc_path_exist (root_directory) == false) continue;
|
||||
|
||||
const bool is_dir = hc_path_is_directory (root_directory);
|
||||
|
||||
if (is_dir == false) continue;
|
||||
|
||||
struct stat outfile_check_stat;
|
||||
|
||||
if (stat (root_directory, &outfile_check_stat) == -1)
|
||||
{
|
||||
if (hc_path_exist (root_directory) == true)
|
||||
event_log_error (hashcat_ctx, "%s: %s", root_directory, strerror (errno));
|
||||
|
||||
hcfree (out_files);
|
||||
hcfree (out_info);
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (outfile_check_stat.st_mtime > folder_mtime)
|
||||
{
|
||||
char **out_files_new = scan_directory (root_directory);
|
||||
|
||||
int out_cnt_new = count_dictionaries (out_files_new);
|
||||
|
||||
outfile_data_t *out_info_new = NULL;
|
||||
|
||||
if (out_cnt_new > 0)
|
||||
{
|
||||
const bool is_dir = hc_path_is_directory (root_directory);
|
||||
out_info_new = (outfile_data_t *) hccalloc (out_cnt_new, sizeof (outfile_data_t));
|
||||
|
||||
if (is_dir == true)
|
||||
for (int i = 0; i < out_cnt_new; i++)
|
||||
{
|
||||
struct stat outfile_check_stat;
|
||||
out_info_new[i].file_name = out_files_new[i];
|
||||
|
||||
if (stat (root_directory, &outfile_check_stat) == -1)
|
||||
{
|
||||
event_log_error (hashcat_ctx, "%s: %s", root_directory, strerror (errno));
|
||||
|
||||
hcfree (out_files);
|
||||
hcfree (out_info);
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (outfile_check_stat.st_mtime > folder_mtime)
|
||||
{
|
||||
char **out_files_new = scan_directory (root_directory);
|
||||
|
||||
int out_cnt_new = count_dictionaries (out_files_new);
|
||||
|
||||
outfile_data_t *out_info_new = NULL;
|
||||
|
||||
if (out_cnt_new > 0)
|
||||
{
|
||||
out_info_new = (outfile_data_t *) hccalloc (out_cnt_new, sizeof (outfile_data_t));
|
||||
|
||||
for (int i = 0; i < out_cnt_new; i++)
|
||||
{
|
||||
out_info_new[i].file_name = out_files_new[i];
|
||||
|
||||
// check if there are files that we have seen/checked before (and not changed)
|
||||
|
||||
for (int j = 0; j < out_cnt; j++)
|
||||
{
|
||||
if (strcmp (out_info[j].file_name, out_info_new[i].file_name) == 0)
|
||||
{
|
||||
struct stat outfile_stat;
|
||||
|
||||
if (stat (out_info_new[i].file_name, &outfile_stat) == 0)
|
||||
{
|
||||
if (outfile_stat.st_ctime == out_info[j].ctime)
|
||||
{
|
||||
out_info_new[i].ctime = out_info[j].ctime;
|
||||
out_info_new[i].seek = out_info[j].seek;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hcfree (out_info);
|
||||
hcfree (out_files);
|
||||
|
||||
out_files = out_files_new;
|
||||
out_cnt = out_cnt_new;
|
||||
out_info = out_info_new;
|
||||
|
||||
folder_mtime = outfile_check_stat.st_mtime;
|
||||
}
|
||||
// check if there are files that we have seen/checked before (and not changed)
|
||||
|
||||
for (int j = 0; j < out_cnt; j++)
|
||||
{
|
||||
FILE *fp = fopen (out_info[j].file_name, "rb");
|
||||
if (strcmp (out_info[j].file_name, out_info_new[i].file_name) != 0) continue;
|
||||
|
||||
if (fp != NULL)
|
||||
{
|
||||
//hc_thread_mutex_lock (status_ctx->mux_display);
|
||||
struct stat outfile_stat;
|
||||
|
||||
struct stat outfile_stat;
|
||||
if (stat (out_info_new[i].file_name, &outfile_stat) != 0) continue;
|
||||
|
||||
if (fstat (fileno (fp), &outfile_stat))
|
||||
{
|
||||
fclose (fp);
|
||||
if (outfile_stat.st_ctime != out_info[j].ctime) continue;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (outfile_stat.st_ctime > out_info[j].ctime)
|
||||
{
|
||||
out_info[j].ctime = outfile_stat.st_ctime;
|
||||
out_info[j].seek = 0;
|
||||
}
|
||||
|
||||
fseeko (fp, out_info[j].seek, SEEK_SET);
|
||||
|
||||
char *line_buf = (char *) hcmalloc (HCBUFSIZ_LARGE);
|
||||
|
||||
while (!feof (fp))
|
||||
{
|
||||
char *ptr = fgets (line_buf, HCBUFSIZ_LARGE - 1, fp);
|
||||
|
||||
if (ptr == NULL) break;
|
||||
|
||||
size_t line_len = strlen (line_buf);
|
||||
|
||||
if (line_len == 0) continue;
|
||||
|
||||
size_t iter = 1;
|
||||
|
||||
for (size_t i = line_len - 1; i && iter; i--, line_len--)
|
||||
{
|
||||
if (line_buf[i] != separator) continue;
|
||||
|
||||
iter--;
|
||||
|
||||
int parser_status = PARSER_OK;
|
||||
|
||||
if ((hash_mode != 2500) && (hash_mode != 2501) && (hash_mode != 6800))
|
||||
{
|
||||
parser_status = hashconfig->parse_func ((u8 *) line_buf, (u32) line_len - 1, &hash_buf, hashconfig);
|
||||
}
|
||||
|
||||
u32 found = 0;
|
||||
|
||||
if (parser_status == PARSER_OK)
|
||||
{
|
||||
for (u32 salt_pos = 0; (found == 0) && (salt_pos < hashes->salts_cnt); salt_pos++)
|
||||
{
|
||||
if (hashes->salts_shown[salt_pos] == 1) continue;
|
||||
|
||||
salt_t *salt_buf = &hashes->salts_buf[salt_pos];
|
||||
|
||||
for (u32 digest_pos = 0; (found == 0) && (digest_pos < salt_buf->digests_cnt); digest_pos++)
|
||||
{
|
||||
u32 idx = salt_buf->digests_offset + digest_pos;
|
||||
|
||||
if (hashes->digests_shown[idx] == 1) continue;
|
||||
|
||||
u32 cracked = 0;
|
||||
|
||||
if (hash_mode == 6800)
|
||||
{
|
||||
if (i == salt_buf->salt_len)
|
||||
{
|
||||
cracked = (memcmp (line_buf, salt_buf->salt_buf, salt_buf->salt_len) == 0);
|
||||
}
|
||||
}
|
||||
else if ((hash_mode == 2500) || (hash_mode == 2501))
|
||||
{
|
||||
// this comparison is a bit inaccurate as we compare only ESSID
|
||||
// call it a bug, but it's good enough for a special case used in a special case
|
||||
// in this case all essid will be marked as cracked that match the essid
|
||||
|
||||
if (i == salt_buf->salt_len)
|
||||
{
|
||||
cracked = (memcmp (line_buf, salt_buf->salt_buf, salt_buf->salt_len) == 0);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
char *digests_buf_ptr = (char *) hashes->digests_buf;
|
||||
|
||||
memcpy (digest_buf, digests_buf_ptr + (hashes->salts_buf[salt_pos].digests_offset * dgst_size) + (digest_pos * dgst_size), dgst_size);
|
||||
|
||||
cracked = (sort_by_digest_p0p1 (digest_buf, hash_buf.digest, hashconfig) == 0);
|
||||
}
|
||||
|
||||
if (cracked == 1)
|
||||
{
|
||||
found = 1;
|
||||
|
||||
hashes->digests_shown[idx] = 1;
|
||||
|
||||
hashes->digests_done++;
|
||||
|
||||
salt_buf->digests_done++;
|
||||
|
||||
if (salt_buf->digests_done == salt_buf->digests_cnt)
|
||||
{
|
||||
hashes->salts_shown[salt_pos] = 1;
|
||||
|
||||
hashes->salts_done++;
|
||||
|
||||
if (hashes->salts_done == hashes->salts_cnt) mycracked (hashcat_ctx);
|
||||
}
|
||||
}
|
||||
|
||||
if (status_ctx->shutdown_inner == true) break;
|
||||
}
|
||||
|
||||
if (status_ctx->shutdown_inner == true) break;
|
||||
}
|
||||
}
|
||||
|
||||
if (found) break;
|
||||
|
||||
if (status_ctx->shutdown_inner == true) break;
|
||||
}
|
||||
|
||||
if (status_ctx->shutdown_inner == true) break;
|
||||
}
|
||||
|
||||
hcfree (line_buf);
|
||||
|
||||
out_info[j].seek = ftello (fp);
|
||||
|
||||
//hc_thread_mutex_unlock (status_ctx->mux_display);
|
||||
|
||||
fclose (fp);
|
||||
}
|
||||
|
||||
if (status_ctx->shutdown_inner == true) break;
|
||||
out_info_new[i].ctime = out_info[j].ctime;
|
||||
out_info_new[i].seek = out_info[j].seek;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
check_left = outfile_check_timer;
|
||||
hcfree (out_info);
|
||||
hcfree (out_files);
|
||||
|
||||
out_files = out_files_new;
|
||||
out_cnt = out_cnt_new;
|
||||
out_info = out_info_new;
|
||||
|
||||
folder_mtime = outfile_check_stat.st_mtime;
|
||||
}
|
||||
|
||||
for (int j = 0; j < out_cnt; j++)
|
||||
{
|
||||
FILE *fp = fopen (out_info[j].file_name, "rb");
|
||||
|
||||
if (fp == NULL) continue;
|
||||
|
||||
//hc_thread_mutex_lock (status_ctx->mux_display);
|
||||
|
||||
struct stat outfile_stat;
|
||||
|
||||
if (fstat (fileno (fp), &outfile_stat))
|
||||
{
|
||||
fclose (fp);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (outfile_stat.st_ctime > out_info[j].ctime)
|
||||
{
|
||||
out_info[j].ctime = outfile_stat.st_ctime;
|
||||
out_info[j].seek = 0;
|
||||
}
|
||||
|
||||
fseeko (fp, out_info[j].seek, SEEK_SET);
|
||||
|
||||
char *line_buf = (char *) hcmalloc (HCBUFSIZ_LARGE);
|
||||
|
||||
while (!feof (fp))
|
||||
{
|
||||
char *ptr = fgets (line_buf, HCBUFSIZ_LARGE - 1, fp);
|
||||
|
||||
if (ptr == NULL) break;
|
||||
|
||||
size_t line_len = strlen (line_buf);
|
||||
|
||||
if (line_len == 0) continue;
|
||||
|
||||
size_t cut_tries = 5;
|
||||
|
||||
for (size_t i = line_len - 1; i && cut_tries; i--, line_len--)
|
||||
{
|
||||
if (line_buf[i] != separator) continue;
|
||||
|
||||
cut_tries--;
|
||||
|
||||
if (is_salted == true) memset (hash_buf.salt, 0, sizeof (salt_t)); // needed ? (let's play it safe!)
|
||||
|
||||
int parser_status = PARSER_HASH_LENGTH;
|
||||
|
||||
if ((hash_mode == 2500) || (hash_mode == 2501)) // special case WPA/WPA2
|
||||
{
|
||||
// fake the parsing of the salt
|
||||
|
||||
u32 identifier_len = 32 + 1 + 12 + 1 + 12 + 1; // format is [ID_MD5]:[MAC1]:[MAC2]:$salt:$pass
|
||||
|
||||
if ((line_len - 1) < identifier_len) continue;
|
||||
|
||||
hash_buf.salt->salt_len = line_len - 1 - identifier_len;
|
||||
|
||||
memcpy (hash_buf.salt->salt_buf, line_buf + identifier_len, hash_buf.salt->salt_len);
|
||||
|
||||
// fake the parsing of the digest
|
||||
|
||||
if (is_valid_hex_string ((u8 *) line_buf, 32) == false) break;
|
||||
|
||||
u32 *digest = (u32 *) hash_buf.digest;
|
||||
|
||||
digest[0] = hex_to_u32 ((u8 *) line_buf + 0);
|
||||
digest[1] = hex_to_u32 ((u8 *) line_buf + 8);
|
||||
digest[2] = hex_to_u32 ((u8 *) line_buf + 16);
|
||||
digest[3] = hex_to_u32 ((u8 *) line_buf + 24);
|
||||
|
||||
digest[0] = byte_swap_32 (digest[0]);
|
||||
digest[1] = byte_swap_32 (digest[1]);
|
||||
digest[2] = byte_swap_32 (digest[2]);
|
||||
digest[3] = byte_swap_32 (digest[3]);
|
||||
|
||||
parser_status = PARSER_OK;
|
||||
}
|
||||
else if (hash_mode == 6800) // special case LastPass (only email address in outfile/potfile)
|
||||
{
|
||||
// fake the parsing of the hash/salt
|
||||
|
||||
hash_buf.salt->salt_len = line_len - 1;
|
||||
|
||||
memcpy (hash_buf.salt->salt_buf, line_buf, line_len - 1);
|
||||
|
||||
parser_status = PARSER_OK;
|
||||
}
|
||||
else // "normal" case: hash in the outfile is the same as the hash in the original hash file
|
||||
{
|
||||
parser_status = hashconfig->parse_func ((u8 *) line_buf, (u32) line_len - 1, &hash_buf, hashconfig);
|
||||
}
|
||||
|
||||
if (parser_status != PARSER_OK) continue;
|
||||
|
||||
|
||||
salt_t *salt_buf = salts_buf;
|
||||
|
||||
if (is_salted == true)
|
||||
{
|
||||
salt_buf = (salt_t *) hc_bsearch_r (hash_buf.salt, salts_buf, salts_cnt, sizeof (salt_t), sort_by_salt_buf, (void *) hashconfig);
|
||||
}
|
||||
|
||||
if (salt_buf == NULL) continue;
|
||||
|
||||
u32 salt_pos = salt_buf - salts_buf; // the offset from the start of the array (unit: sizeof (salt_t))
|
||||
|
||||
if (hashes->salts_shown[salt_pos] == 1) break; // already marked as cracked (no action needed)
|
||||
|
||||
u32 idx = salt_buf->digests_offset;
|
||||
|
||||
bool cracked = false;
|
||||
|
||||
if (hash_mode == 6800)
|
||||
{
|
||||
// the comparison with only matching salt is a bit inaccurate
|
||||
// call it a bug, but it's good enough for a special case used in a special case
|
||||
|
||||
cracked = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
char *digests_buf_ptr = digests_buf + (salt_buf->digests_offset * dgst_size);
|
||||
u32 digests_buf_cnt = salt_buf->digests_cnt;
|
||||
|
||||
char *digest_buf = (char *) hc_bsearch_r (hash_buf.digest, digests_buf_ptr, digests_buf_cnt, dgst_size, sort_by_digest_p0p1, (void *) hashconfig);
|
||||
|
||||
if (digest_buf != NULL)
|
||||
{
|
||||
idx += (digest_buf - digests_buf_ptr) / dgst_size;
|
||||
|
||||
if (hashes->digests_shown[idx] == 1) break;
|
||||
|
||||
cracked = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (cracked == true)
|
||||
{
|
||||
hashes->digests_shown[idx] = 1;
|
||||
|
||||
hashes->digests_done++;
|
||||
|
||||
salt_buf->digests_done++;
|
||||
|
||||
if (salt_buf->digests_done == salt_buf->digests_cnt)
|
||||
{
|
||||
hashes->salts_shown[salt_pos] = 1;
|
||||
|
||||
hashes->salts_done++;
|
||||
|
||||
if (hashes->salts_done == salts_cnt) mycracked (hashcat_ctx);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
if (status_ctx->shutdown_inner == true) break;
|
||||
}
|
||||
|
||||
if (status_ctx->shutdown_inner == true) break;
|
||||
}
|
||||
|
||||
hcfree (line_buf);
|
||||
|
||||
out_info[j].seek = ftello (fp);
|
||||
|
||||
//hc_thread_mutex_unlock (status_ctx->mux_display);
|
||||
|
||||
fclose (fp);
|
||||
|
||||
if (status_ctx->shutdown_inner == true) break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -326,8 +355,9 @@ int outcheck_ctx_init (hashcat_ctx_t *hashcat_ctx)
|
||||
|
||||
if ((user_options->hash_mode == 5200) ||
|
||||
((user_options->hash_mode >= 6200) && (user_options->hash_mode <= 6299)) ||
|
||||
(user_options->hash_mode == 9000) ||
|
||||
((user_options->hash_mode >= 13700) && (user_options->hash_mode <= 13799)) ||
|
||||
(user_options->hash_mode == 9000)) return 0;
|
||||
(user_options->hash_mode == 14600)) return 0;
|
||||
|
||||
if (user_options->outfile_check_dir == NULL)
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user