arno
/
hashcat
mirror of https://github.com/hashcat/hashcat.git
1
0
Fork 0
Code Issues Releases Wiki Activity

outfile check: performance improvement by using binary instead of linear search

Browse Source
pull/1676/head
philsmd 6 years ago
parent a134ea52b3
commit 2933f3320a
No known key found for this signature in database
GPG Key ID: 4F25D016D9D6A8AF
4 changed files with 189 additions and 152 deletions
Show Stats Download Patch File Download Diff File

1
docs/changes.txt
Unescape View File

@ -11,6 +11,7 @@
- Add option --markov-hcstat2 to make it clear that the new hcstat2 format (compressed hcstat2gen output) must be used
- Do not allocate memory segments for bitmap tables if we dont need it, for example in benchmark mode
- Removed duplicate words in the dictionary file example.dict
- Improved the speed of the outfile folder scan when using lots of hashes/salts
##
## Bugs

1
include/hashes.h
Unescape View File

@ -8,6 +8,7 @@
int sort_by_digest_p0p1 (const void *v1, const void *v2, void *v3);
int sort_by_salt (const void *v1, const void *v2);
int sort_by_salt_buf (const void *v1, const void *v2, MAYBE_UNUSED void * v3);
int sort_by_hash (const void *v1, const void *v2, void *v3);
int sort_by_hash_no_salt (const void *v1, const void *v2, void *v3);

5
src/hashes.c
Unescape View File

@ -79,6 +79,11 @@ int sort_by_salt (const void *v1, const void *v2)
return 0;
}
int sort_by_salt_buf (const void *v1, const void *v2, MAYBE_UNUSED void * v3)
{
return sort_by_salt (v1, v2);
}
int sort_by_hash (const void *v1, const void *v2, void *v3)
{
const hash_t *h1 = (const hash_t *) v1;

334
src/outfile_check.c
Unescape View File

@ -15,6 +15,7 @@
#include "interface.h"
#include "shared.h"
#include "thread.h"
#include "bitops.h"
static int outfile_remove (hashcat_ctx_t *hashcat_ctx)
{
@ -33,6 +34,11 @@ static int outfile_remove (hashcat_ctx_t *hashcat_ctx)
u32 hash_mode = hashconfig->hash_mode;
char separator = hashconfig->separator;
salt_t *salts_buf = hashes->salts_buf;
const u32 salts_cnt = hashes->salts_cnt;
char *digests_buf = hashes->digests_buf;
char *root_directory = outcheck_ctx->root_directory;
u32 outfile_check_timer = user_options->outfile_check_timer;
@ -45,8 +51,6 @@ static int outfile_remove (hashcat_ctx_t *hashcat_ctx)
if (esalt_size > 0) hash_buf.esalt = hcmalloc (esalt_size);
if (hook_salt_size > 0) hash_buf.hook_salt = hcmalloc (hook_salt_size);
u32 digest_buf[64] = { 0 };
outfile_data_t *out_info = NULL;
char **out_files = NULL;
@ -65,220 +69,245 @@ static int outfile_remove (hashcat_ctx_t *hashcat_ctx)
check_left--;
if (check_left == 0)
if (check_left != 0) continue;
check_left = outfile_check_timer;
if (hc_path_exist (root_directory) == false) continue;
const bool is_dir = hc_path_is_directory (root_directory);
if (is_dir == false) continue;
struct stat outfile_check_stat;
if (stat (root_directory, &outfile_check_stat) == -1)
{
if (hc_path_exist (root_directory) == true)
event_log_error (hashcat_ctx, "%s: %s", root_directory, strerror (errno));
hcfree (out_files);
hcfree (out_info);
return -1;
}
if (outfile_check_stat.st_mtime > folder_mtime)
{
char **out_files_new = scan_directory (root_directory);
int out_cnt_new = count_dictionaries (out_files_new);
outfile_data_t *out_info_new = NULL;
if (out_cnt_new > 0)
{
const bool is_dir = hc_path_is_directory (root_directory);
out_info_new = (outfile_data_t *) hccalloc (out_cnt_new, sizeof (outfile_data_t));
if (is_dir == true)
for (int i = 0; i < out_cnt_new; i++)
{
struct stat outfile_check_stat;
out_info_new[i].file_name = out_files_new[i];
// check if there are files that we have seen/checked before (and not changed)
if (stat (root_directory, &outfile_check_stat) == -1)
for (int j = 0; j < out_cnt; j++)
{
event_log_error (hashcat_ctx, "%s: %s", root_directory, strerror (errno));
if (strcmp (out_info[j].file_name, out_info_new[i].file_name) != 0) continue;
struct stat outfile_stat;
if (stat (out_info_new[i].file_name, &outfile_stat) != 0) continue;
hcfree (out_files);
hcfree (out_info);
if (outfile_stat.st_ctime != out_info[j].ctime) continue;
return -1;
out_info_new[i].ctime = out_info[j].ctime;
out_info_new[i].seek = out_info[j].seek;
}
}
}
if (outfile_check_stat.st_mtime > folder_mtime)
{
char **out_files_new = scan_directory (root_directory);
hcfree (out_info);
hcfree (out_files);
int out_cnt_new = count_dictionaries (out_files_new);
out_files = out_files_new;
out_cnt = out_cnt_new;
out_info = out_info_new;
outfile_data_t *out_info_new = NULL;
folder_mtime = outfile_check_stat.st_mtime;
}
if (out_cnt_new > 0)
{
out_info_new = (outfile_data_t *) hccalloc (out_cnt_new, sizeof (outfile_data_t));
for (int i = 0; i < out_cnt_new; i++)
{
out_info_new[i].file_name = out_files_new[i];
// check if there are files that we have seen/checked before (and not changed)
for (int j = 0; j < out_cnt; j++)
{
if (strcmp (out_info[j].file_name, out_info_new[i].file_name) == 0)
{
struct stat outfile_stat;
if (stat (out_info_new[i].file_name, &outfile_stat) == 0)
{
if (outfile_stat.st_ctime == out_info[j].ctime)
{
out_info_new[i].ctime = out_info[j].ctime;
out_info_new[i].seek = out_info[j].seek;
}
}
}
}
}
}
for (int j = 0; j < out_cnt; j++)
{
FILE *fp = fopen (out_info[j].file_name, "rb");
hcfree (out_info);
hcfree (out_files);
if (fp == NULL) continue;
out_files = out_files_new;
out_cnt = out_cnt_new;
out_info = out_info_new;
//hc_thread_mutex_lock (status_ctx->mux_display);
folder_mtime = outfile_check_stat.st_mtime;
}
struct stat outfile_stat;
for (int j = 0; j < out_cnt; j++)
{
FILE *fp = fopen (out_info[j].file_name, "rb");
if (fstat (fileno (fp), &outfile_stat))
{
fclose (fp);
if (fp != NULL)
{
//hc_thread_mutex_lock (status_ctx->mux_display);
continue;
}
struct stat outfile_stat;
if (outfile_stat.st_ctime > out_info[j].ctime)
{
out_info[j].ctime = outfile_stat.st_ctime;
out_info[j].seek = 0;
}
if (fstat (fileno (fp), &outfile_stat))
{
fclose (fp);
fseeko (fp, out_info[j].seek, SEEK_SET);
continue;
}
char *line_buf = (char *) hcmalloc (HCBUFSIZ_LARGE);
if (outfile_stat.st_ctime > out_info[j].ctime)
{
out_info[j].ctime = outfile_stat.st_ctime;
out_info[j].seek = 0;
}
while (!feof (fp))
{
char *ptr = fgets (line_buf, HCBUFSIZ_LARGE - 1, fp);
fseeko (fp, out_info[j].seek, SEEK_SET);
if (ptr == NULL) break;
char *line_buf = (char *) hcmalloc (HCBUFSIZ_LARGE);
size_t line_len = strlen (line_buf);
while (!feof (fp))
{
char *ptr = fgets (line_buf, HCBUFSIZ_LARGE - 1, fp);
if (line_len == 0) continue;
if (ptr == NULL) break;
size_t cut_tries = 5;
size_t line_len = strlen (line_buf);
for (size_t i = line_len - 1; i && cut_tries; i--, line_len--)
{
if (line_buf[i] != separator) continue;
if (line_len == 0) continue;
cut_tries--;
size_t iter = 1;
if (is_salted == true) memset (hash_buf.salt, 0, sizeof (salt_t)); // needed ? (let's play it safe!)
for (size_t i = line_len - 1; i && iter; i--, line_len--)
{
if (line_buf[i] != separator) continue;
int parser_status = PARSER_HASH_LENGTH;
iter--;
if ((hash_mode == 2500) || (hash_mode == 2501)) // special case WPA/WPA2
{
// fake the parsing of the salt
int parser_status = PARSER_OK;
u32 identifier_len = 32 + 1 + 12 + 1 + 12 + 1; // format is [ID_MD5]:[MAC1]:[MAC2]:$salt:$pass
if ((hash_mode != 2500) && (hash_mode != 2501) && (hash_mode != 6800))
{
parser_status = hashconfig->parse_func ((u8 *) line_buf, (u32) line_len - 1, &hash_buf, hashconfig);
}
if ((line_len - 1) < identifier_len) continue;
u32 found = 0;
hash_buf.salt->salt_len = line_len - 1 - identifier_len;
if (parser_status == PARSER_OK)
{
for (u32 salt_pos = 0; (found == 0) && (salt_pos < hashes->salts_cnt); salt_pos++)
{
if (hashes->salts_shown[salt_pos] == 1) continue;
memcpy (hash_buf.salt->salt_buf, line_buf + identifier_len, hash_buf.salt->salt_len);
salt_t *salt_buf = &hashes->salts_buf[salt_pos];
// fake the parsing of the digest
for (u32 digest_pos = 0; (found == 0) && (digest_pos < salt_buf->digests_cnt); digest_pos++)
{
u32 idx = salt_buf->digests_offset + digest_pos;
if (is_valid_hex_string ((u8 *) line_buf, 32) == false) break;
if (hashes->digests_shown[idx] == 1) continue;
u32 *digest = (u32 *) hash_buf.digest;
u32 cracked = 0;
digest[0] = hex_to_u32 ((u8 *) line_buf + 0);
digest[1] = hex_to_u32 ((u8 *) line_buf + 8);
digest[2] = hex_to_u32 ((u8 *) line_buf + 16);
digest[3] = hex_to_u32 ((u8 *) line_buf + 24);
if (hash_mode == 6800)
{
if (i == salt_buf->salt_len)
{
cracked = (memcmp (line_buf, salt_buf->salt_buf, salt_buf->salt_len) == 0);
}
}
else if ((hash_mode == 2500) || (hash_mode == 2501))
{
// this comparison is a bit inaccurate as we compare only ESSID
// call it a bug, but it's good enough for a special case used in a special case
// in this case all essid will be marked as cracked that match the essid
digest[0] = byte_swap_32 (digest[0]);
digest[1] = byte_swap_32 (digest[1]);
digest[2] = byte_swap_32 (digest[2]);
digest[3] = byte_swap_32 (digest[3]);
if (i == salt_buf->salt_len)
{
cracked = (memcmp (line_buf, salt_buf->salt_buf, salt_buf->salt_len) == 0);
}
}
else
{
char *digests_buf_ptr = (char *) hashes->digests_buf;
parser_status = PARSER_OK;
}
else if (hash_mode == 6800) // special case LastPass (only email address in outfile/potfile)
{
// fake the parsing of the hash/salt
memcpy (digest_buf, digests_buf_ptr + (hashes->salts_buf[salt_pos].digests_offset * dgst_size) + (digest_pos * dgst_size), dgst_size);
hash_buf.salt->salt_len = line_len - 1;
cracked = (sort_by_digest_p0p1 (digest_buf, hash_buf.digest, hashconfig) == 0);
}
memcpy (hash_buf.salt->salt_buf, line_buf, line_len - 1);
if (cracked == 1)
{
found = 1;
parser_status = PARSER_OK;
}
else // "normal" case: hash in the outfile is the same as the hash in the original hash file
{
parser_status = hashconfig->parse_func ((u8 *) line_buf, (u32) line_len - 1, &hash_buf, hashconfig);
}
hashes->digests_shown[idx] = 1;
if (parser_status != PARSER_OK) continue;
hashes->digests_done++;
salt_buf->digests_done++;
salt_t *salt_buf = salts_buf;
if (salt_buf->digests_done == salt_buf->digests_cnt)
{
hashes->salts_shown[salt_pos] = 1;
if (is_salted == true)
{
salt_buf = (salt_t *) hc_bsearch_r (hash_buf.salt, salts_buf, salts_cnt, sizeof (salt_t), sort_by_salt_buf, (void *) hashconfig);
}
hashes->salts_done++;
if (salt_buf == NULL) continue;
if (hashes->salts_done == hashes->salts_cnt) mycracked (hashcat_ctx);
}
}
u32 salt_pos = salt_buf - salts_buf; // the offset from the start of the array (unit: sizeof (salt_t))
if (status_ctx->shutdown_inner == true) break;
}
if (hashes->salts_shown[salt_pos] == 1) break; // already marked as cracked (no action needed)
if (status_ctx->shutdown_inner == true) break;
}
}
u32 idx = salt_buf->digests_offset;
if (found) break;
bool cracked = false;
if (status_ctx->shutdown_inner == true) break;
}
if (hash_mode == 6800)
{
// the comparison with only matching salt is a bit inaccurate
// call it a bug, but it's good enough for a special case used in a special case
if (status_ctx->shutdown_inner == true) break;
}
cracked = true;
}
else
{
char *digests_buf_ptr = digests_buf + (salt_buf->digests_offset * dgst_size);
u32 digests_buf_cnt = salt_buf->digests_cnt;
hcfree (line_buf);
char *digest_buf = (char *) hc_bsearch_r (hash_buf.digest, digests_buf_ptr, digests_buf_cnt, dgst_size, sort_by_digest_p0p1, (void *) hashconfig);
out_info[j].seek = ftello (fp);
if (digest_buf != NULL)
{
idx += (digest_buf - digests_buf_ptr) / dgst_size;
//hc_thread_mutex_unlock (status_ctx->mux_display);
if (hashes->digests_shown[idx] == 1) break;
fclose (fp);
cracked = true;
}
}
if (cracked == true)
{
hashes->digests_shown[idx] = 1;
hashes->digests_done++;
salt_buf->digests_done++;
if (salt_buf->digests_done == salt_buf->digests_cnt)
{
hashes->salts_shown[salt_pos] = 1;
hashes->salts_done++;
if (status_ctx->shutdown_inner == true) break;
if (hashes->salts_done == salts_cnt) mycracked (hashcat_ctx);
}
break;
}
if (status_ctx->shutdown_inner == true) break;
}
if (status_ctx->shutdown_inner == true) break;
}
check_left = outfile_check_timer;
hcfree (line_buf);
out_info[j].seek = ftello (fp);
//hc_thread_mutex_unlock (status_ctx->mux_display);
fclose (fp);
if (status_ctx->shutdown_inner == true) break;
}
}
@ -326,8 +355,9 @@ int outcheck_ctx_init (hashcat_ctx_t *hashcat_ctx)
if ((user_options->hash_mode == 5200) ||
((user_options->hash_mode >= 6200) && (user_options->hash_mode <= 6299)) ||
(user_options->hash_mode == 9000) ||
((user_options->hash_mode >= 13700) && (user_options->hash_mode <= 13799)) ||
(user_options->hash_mode == 9000)) return 0;
(user_options->hash_mode == 14600)) return 0;
if (user_options->outfile_check_dir == NULL)
{

Write Preview
Loading…
Cancel
Save