From 2933f3320a62093846c4b48740b45155ec7e9b8d Mon Sep 17 00:00:00 2001 From: philsmd Date: Fri, 31 Aug 2018 12:55:05 +0200 Subject: [PATCH] outfile check: performance improvement by using binary instead of linear search --- docs/changes.txt | 1 + include/hashes.h | 1 + src/hashes.c | 5 + src/outfile_check.c | 334 ++++++++++++++++++++++++-------------------- 4 files changed, 189 insertions(+), 152 deletions(-) diff --git a/docs/changes.txt b/docs/changes.txt index 49b98cd91..b5821b764 100644 --- a/docs/changes.txt +++ b/docs/changes.txt @@ -11,6 +11,7 @@ - Add option --markov-hcstat2 to make it clear that the new hcstat2 format (compressed hcstat2gen output) must be used - Do not allocate memory segments for bitmap tables if we dont need it, for example in benchmark mode - Removed duplicate words in the dictionary file example.dict +- Improved the speed of the outfile folder scan when using lots of hashes/salts ## ## Bugs diff --git a/include/hashes.h b/include/hashes.h index 8af3e3a7b..90f5dbea5 100644 --- a/include/hashes.h +++ b/include/hashes.h @@ -8,6 +8,7 @@ int sort_by_digest_p0p1 (const void *v1, const void *v2, void *v3); int sort_by_salt (const void *v1, const void *v2); +int sort_by_salt_buf (const void *v1, const void *v2, MAYBE_UNUSED void * v3); int sort_by_hash (const void *v1, const void *v2, void *v3); int sort_by_hash_no_salt (const void *v1, const void *v2, void *v3); diff --git a/src/hashes.c b/src/hashes.c index cd026f149..6ec376dea 100644 --- a/src/hashes.c +++ b/src/hashes.c @@ -79,6 +79,11 @@ int sort_by_salt (const void *v1, const void *v2) return 0; } +int sort_by_salt_buf (const void *v1, const void *v2, MAYBE_UNUSED void * v3) +{ + return sort_by_salt (v1, v2); +} + int sort_by_hash (const void *v1, const void *v2, void *v3) { const hash_t *h1 = (const hash_t *) v1; diff --git a/src/outfile_check.c b/src/outfile_check.c index 680c7cde5..ab85d9299 100644 --- a/src/outfile_check.c +++ b/src/outfile_check.c @@ -15,6 +15,7 @@ #include "interface.h" #include "shared.h" #include "thread.h" +#include "bitops.h" static int outfile_remove (hashcat_ctx_t *hashcat_ctx) { @@ -33,6 +34,11 @@ static int outfile_remove (hashcat_ctx_t *hashcat_ctx) u32 hash_mode = hashconfig->hash_mode; char separator = hashconfig->separator; + salt_t *salts_buf = hashes->salts_buf; + const u32 salts_cnt = hashes->salts_cnt; + + char *digests_buf = hashes->digests_buf; + char *root_directory = outcheck_ctx->root_directory; u32 outfile_check_timer = user_options->outfile_check_timer; @@ -45,8 +51,6 @@ static int outfile_remove (hashcat_ctx_t *hashcat_ctx) if (esalt_size > 0) hash_buf.esalt = hcmalloc (esalt_size); if (hook_salt_size > 0) hash_buf.hook_salt = hcmalloc (hook_salt_size); - u32 digest_buf[64] = { 0 }; - outfile_data_t *out_info = NULL; char **out_files = NULL; @@ -65,220 +69,245 @@ static int outfile_remove (hashcat_ctx_t *hashcat_ctx) check_left--; - if (check_left == 0) + if (check_left != 0) continue; + + check_left = outfile_check_timer; + + if (hc_path_exist (root_directory) == false) continue; + + const bool is_dir = hc_path_is_directory (root_directory); + + if (is_dir == false) continue; + + struct stat outfile_check_stat; + + if (stat (root_directory, &outfile_check_stat) == -1) { - if (hc_path_exist (root_directory) == true) + event_log_error (hashcat_ctx, "%s: %s", root_directory, strerror (errno)); + + hcfree (out_files); + hcfree (out_info); + + return -1; + } + + if (outfile_check_stat.st_mtime > folder_mtime) + { + char **out_files_new = scan_directory (root_directory); + + int out_cnt_new = count_dictionaries (out_files_new); + + outfile_data_t *out_info_new = NULL; + + if (out_cnt_new > 0) { - const bool is_dir = hc_path_is_directory (root_directory); + out_info_new = (outfile_data_t *) hccalloc (out_cnt_new, sizeof (outfile_data_t)); - if (is_dir == true) + for (int i = 0; i < out_cnt_new; i++) { - struct stat outfile_check_stat; + out_info_new[i].file_name = out_files_new[i]; + + // check if there are files that we have seen/checked before (and not changed) - if (stat (root_directory, &outfile_check_stat) == -1) + for (int j = 0; j < out_cnt; j++) { - event_log_error (hashcat_ctx, "%s: %s", root_directory, strerror (errno)); + if (strcmp (out_info[j].file_name, out_info_new[i].file_name) != 0) continue; + + struct stat outfile_stat; + + if (stat (out_info_new[i].file_name, &outfile_stat) != 0) continue; - hcfree (out_files); - hcfree (out_info); + if (outfile_stat.st_ctime != out_info[j].ctime) continue; - return -1; + out_info_new[i].ctime = out_info[j].ctime; + out_info_new[i].seek = out_info[j].seek; } + } + } - if (outfile_check_stat.st_mtime > folder_mtime) - { - char **out_files_new = scan_directory (root_directory); + hcfree (out_info); + hcfree (out_files); - int out_cnt_new = count_dictionaries (out_files_new); + out_files = out_files_new; + out_cnt = out_cnt_new; + out_info = out_info_new; - outfile_data_t *out_info_new = NULL; + folder_mtime = outfile_check_stat.st_mtime; + } - if (out_cnt_new > 0) - { - out_info_new = (outfile_data_t *) hccalloc (out_cnt_new, sizeof (outfile_data_t)); - - for (int i = 0; i < out_cnt_new; i++) - { - out_info_new[i].file_name = out_files_new[i]; - - // check if there are files that we have seen/checked before (and not changed) - - for (int j = 0; j < out_cnt; j++) - { - if (strcmp (out_info[j].file_name, out_info_new[i].file_name) == 0) - { - struct stat outfile_stat; - - if (stat (out_info_new[i].file_name, &outfile_stat) == 0) - { - if (outfile_stat.st_ctime == out_info[j].ctime) - { - out_info_new[i].ctime = out_info[j].ctime; - out_info_new[i].seek = out_info[j].seek; - } - } - } - } - } - } + for (int j = 0; j < out_cnt; j++) + { + FILE *fp = fopen (out_info[j].file_name, "rb"); - hcfree (out_info); - hcfree (out_files); + if (fp == NULL) continue; - out_files = out_files_new; - out_cnt = out_cnt_new; - out_info = out_info_new; + //hc_thread_mutex_lock (status_ctx->mux_display); - folder_mtime = outfile_check_stat.st_mtime; - } + struct stat outfile_stat; - for (int j = 0; j < out_cnt; j++) - { - FILE *fp = fopen (out_info[j].file_name, "rb"); + if (fstat (fileno (fp), &outfile_stat)) + { + fclose (fp); - if (fp != NULL) - { - //hc_thread_mutex_lock (status_ctx->mux_display); + continue; + } - struct stat outfile_stat; + if (outfile_stat.st_ctime > out_info[j].ctime) + { + out_info[j].ctime = outfile_stat.st_ctime; + out_info[j].seek = 0; + } - if (fstat (fileno (fp), &outfile_stat)) - { - fclose (fp); + fseeko (fp, out_info[j].seek, SEEK_SET); - continue; - } + char *line_buf = (char *) hcmalloc (HCBUFSIZ_LARGE); - if (outfile_stat.st_ctime > out_info[j].ctime) - { - out_info[j].ctime = outfile_stat.st_ctime; - out_info[j].seek = 0; - } + while (!feof (fp)) + { + char *ptr = fgets (line_buf, HCBUFSIZ_LARGE - 1, fp); - fseeko (fp, out_info[j].seek, SEEK_SET); + if (ptr == NULL) break; - char *line_buf = (char *) hcmalloc (HCBUFSIZ_LARGE); + size_t line_len = strlen (line_buf); - while (!feof (fp)) - { - char *ptr = fgets (line_buf, HCBUFSIZ_LARGE - 1, fp); + if (line_len == 0) continue; - if (ptr == NULL) break; + size_t cut_tries = 5; - size_t line_len = strlen (line_buf); + for (size_t i = line_len - 1; i && cut_tries; i--, line_len--) + { + if (line_buf[i] != separator) continue; - if (line_len == 0) continue; + cut_tries--; - size_t iter = 1; + if (is_salted == true) memset (hash_buf.salt, 0, sizeof (salt_t)); // needed ? (let's play it safe!) - for (size_t i = line_len - 1; i && iter; i--, line_len--) - { - if (line_buf[i] != separator) continue; + int parser_status = PARSER_HASH_LENGTH; - iter--; + if ((hash_mode == 2500) || (hash_mode == 2501)) // special case WPA/WPA2 + { + // fake the parsing of the salt - int parser_status = PARSER_OK; + u32 identifier_len = 32 + 1 + 12 + 1 + 12 + 1; // format is [ID_MD5]:[MAC1]:[MAC2]:$salt:$pass - if ((hash_mode != 2500) && (hash_mode != 2501) && (hash_mode != 6800)) - { - parser_status = hashconfig->parse_func ((u8 *) line_buf, (u32) line_len - 1, &hash_buf, hashconfig); - } + if ((line_len - 1) < identifier_len) continue; - u32 found = 0; + hash_buf.salt->salt_len = line_len - 1 - identifier_len; - if (parser_status == PARSER_OK) - { - for (u32 salt_pos = 0; (found == 0) && (salt_pos < hashes->salts_cnt); salt_pos++) - { - if (hashes->salts_shown[salt_pos] == 1) continue; + memcpy (hash_buf.salt->salt_buf, line_buf + identifier_len, hash_buf.salt->salt_len); - salt_t *salt_buf = &hashes->salts_buf[salt_pos]; + // fake the parsing of the digest - for (u32 digest_pos = 0; (found == 0) && (digest_pos < salt_buf->digests_cnt); digest_pos++) - { - u32 idx = salt_buf->digests_offset + digest_pos; + if (is_valid_hex_string ((u8 *) line_buf, 32) == false) break; - if (hashes->digests_shown[idx] == 1) continue; + u32 *digest = (u32 *) hash_buf.digest; - u32 cracked = 0; + digest[0] = hex_to_u32 ((u8 *) line_buf + 0); + digest[1] = hex_to_u32 ((u8 *) line_buf + 8); + digest[2] = hex_to_u32 ((u8 *) line_buf + 16); + digest[3] = hex_to_u32 ((u8 *) line_buf + 24); - if (hash_mode == 6800) - { - if (i == salt_buf->salt_len) - { - cracked = (memcmp (line_buf, salt_buf->salt_buf, salt_buf->salt_len) == 0); - } - } - else if ((hash_mode == 2500) || (hash_mode == 2501)) - { - // this comparison is a bit inaccurate as we compare only ESSID - // call it a bug, but it's good enough for a special case used in a special case - // in this case all essid will be marked as cracked that match the essid + digest[0] = byte_swap_32 (digest[0]); + digest[1] = byte_swap_32 (digest[1]); + digest[2] = byte_swap_32 (digest[2]); + digest[3] = byte_swap_32 (digest[3]); - if (i == salt_buf->salt_len) - { - cracked = (memcmp (line_buf, salt_buf->salt_buf, salt_buf->salt_len) == 0); - } - } - else - { - char *digests_buf_ptr = (char *) hashes->digests_buf; + parser_status = PARSER_OK; + } + else if (hash_mode == 6800) // special case LastPass (only email address in outfile/potfile) + { + // fake the parsing of the hash/salt - memcpy (digest_buf, digests_buf_ptr + (hashes->salts_buf[salt_pos].digests_offset * dgst_size) + (digest_pos * dgst_size), dgst_size); + hash_buf.salt->salt_len = line_len - 1; - cracked = (sort_by_digest_p0p1 (digest_buf, hash_buf.digest, hashconfig) == 0); - } + memcpy (hash_buf.salt->salt_buf, line_buf, line_len - 1); - if (cracked == 1) - { - found = 1; + parser_status = PARSER_OK; + } + else // "normal" case: hash in the outfile is the same as the hash in the original hash file + { + parser_status = hashconfig->parse_func ((u8 *) line_buf, (u32) line_len - 1, &hash_buf, hashconfig); + } - hashes->digests_shown[idx] = 1; + if (parser_status != PARSER_OK) continue; - hashes->digests_done++; - salt_buf->digests_done++; + salt_t *salt_buf = salts_buf; - if (salt_buf->digests_done == salt_buf->digests_cnt) - { - hashes->salts_shown[salt_pos] = 1; + if (is_salted == true) + { + salt_buf = (salt_t *) hc_bsearch_r (hash_buf.salt, salts_buf, salts_cnt, sizeof (salt_t), sort_by_salt_buf, (void *) hashconfig); + } - hashes->salts_done++; + if (salt_buf == NULL) continue; - if (hashes->salts_done == hashes->salts_cnt) mycracked (hashcat_ctx); - } - } + u32 salt_pos = salt_buf - salts_buf; // the offset from the start of the array (unit: sizeof (salt_t)) - if (status_ctx->shutdown_inner == true) break; - } + if (hashes->salts_shown[salt_pos] == 1) break; // already marked as cracked (no action needed) - if (status_ctx->shutdown_inner == true) break; - } - } + u32 idx = salt_buf->digests_offset; - if (found) break; + bool cracked = false; - if (status_ctx->shutdown_inner == true) break; - } + if (hash_mode == 6800) + { + // the comparison with only matching salt is a bit inaccurate + // call it a bug, but it's good enough for a special case used in a special case - if (status_ctx->shutdown_inner == true) break; - } + cracked = true; + } + else + { + char *digests_buf_ptr = digests_buf + (salt_buf->digests_offset * dgst_size); + u32 digests_buf_cnt = salt_buf->digests_cnt; - hcfree (line_buf); + char *digest_buf = (char *) hc_bsearch_r (hash_buf.digest, digests_buf_ptr, digests_buf_cnt, dgst_size, sort_by_digest_p0p1, (void *) hashconfig); - out_info[j].seek = ftello (fp); + if (digest_buf != NULL) + { + idx += (digest_buf - digests_buf_ptr) / dgst_size; - //hc_thread_mutex_unlock (status_ctx->mux_display); + if (hashes->digests_shown[idx] == 1) break; - fclose (fp); + cracked = true; } + } + + if (cracked == true) + { + hashes->digests_shown[idx] = 1; + + hashes->digests_done++; + + salt_buf->digests_done++; + + if (salt_buf->digests_done == salt_buf->digests_cnt) + { + hashes->salts_shown[salt_pos] = 1; + + hashes->salts_done++; - if (status_ctx->shutdown_inner == true) break; + if (hashes->salts_done == salts_cnt) mycracked (hashcat_ctx); + } + + break; } + + if (status_ctx->shutdown_inner == true) break; } + + if (status_ctx->shutdown_inner == true) break; } - check_left = outfile_check_timer; + hcfree (line_buf); + + out_info[j].seek = ftello (fp); + + //hc_thread_mutex_unlock (status_ctx->mux_display); + + fclose (fp); + + if (status_ctx->shutdown_inner == true) break; } } @@ -326,8 +355,9 @@ int outcheck_ctx_init (hashcat_ctx_t *hashcat_ctx) if ((user_options->hash_mode == 5200) || ((user_options->hash_mode >= 6200) && (user_options->hash_mode <= 6299)) || + (user_options->hash_mode == 9000) || ((user_options->hash_mode >= 13700) && (user_options->hash_mode <= 13799)) || - (user_options->hash_mode == 9000)) return 0; + (user_options->hash_mode == 14600)) return 0; if (user_options->outfile_check_dir == NULL) {