diff --git a/docs/changes.txt b/docs/changes.txt index 45c0dbce8..7cf7a6248 100644 --- a/docs/changes.txt +++ b/docs/changes.txt @@ -11,7 +11,6 @@ - Added hash-mode: AES-192-ECB NOKDF (PT = $salt, key = $pass) - Added hash-mode: AES-256-ECB NOKDF (PT = $salt, key = $pass) - ## ## Bugs ## @@ -42,6 +41,7 @@ ## Technical ## +- File handling: Do not abort on seeing a BOM in input files, just warn and ignore the BOM - Brain: Add brain_ctx_t to hashcat_ctx_t to enable runtime check if hashcat was compiled with brain support - Autodetect: Limit the number of errors per hash-mode try to 100 to avoid long startup time - Folders: Do not escape the variable cpath_real to prevent certain OpenCL runtimes from running into an error which do not support escape characters diff --git a/include/shared.h b/include/shared.h index fafc36ee0..496b611fd 100644 --- a/include/shared.h +++ b/include/shared.h @@ -63,6 +63,7 @@ bool hc_path_create (const char *path); bool hc_path_has_bom (const char *path); bool hc_string_is_digit (const char *s); +int hc_string_bom_size (const u8 *s); void hc_string_trim_trailing (char *s); void hc_string_trim_leading (char *s); diff --git a/include/types.h b/include/types.h index b8565ac0f..302180055 100644 --- a/include/types.h +++ b/include/types.h @@ -1067,9 +1067,11 @@ typedef struct hc_fp bool is_gzip; bool is_zip; + int bom_size; char *mode; const char *path; + } HCFILE; #include "ext_nvrtc.h" diff --git a/src/filehandling.c b/src/filehandling.c index 50cc87016..54ba73369 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -60,11 +60,12 @@ bool hc_fopen (HCFILE *fp, const char *path, char *mode) return false; } - fp->pfp = NULL; - fp->is_gzip = false; - fp->is_zip = false; + fp->pfp = NULL; + fp->is_gzip = false; + fp->is_zip = false; + fp->bom_size = 0; - unsigned char check[4] = { 0 }; + unsigned char check[8] = { 0 }; int fd_tmp = open (path, O_RDONLY); @@ -74,8 +75,15 @@ bool hc_fopen (HCFILE *fp, const char *path, char *mode) if (read (fd_tmp, check, sizeof (check)) > 0) { - if (check[0] == 0x1f && check[1] == 0x8b && check[2] == 0x08) fp->is_gzip = true; - if (check[0] == 0x50 && check[1] == 0x4b && check[2] == 0x03 && check[3] == 0x04) fp->is_zip = true; + if (check[0] == 0x1f && check[1] == 0x8b && check[2] == 0x08) fp->is_gzip = true; + if (check[0] == 0x50 && check[1] == 0x4b && check[2] == 0x03 && check[3] == 0x04) fp->is_zip = true; + + // compressed files with BOM will be undetected! + + if ((fp->is_gzip == false) && (fp->is_zip == false)) + { + fp->bom_size = hc_string_bom_size (check); + } } close (fd_tmp); @@ -104,7 +112,16 @@ bool hc_fopen (HCFILE *fp, const char *path, char *mode) } else { - if ((fp->pfp = fdopen (fp->fd, mode)) == NULL) return false; + if ((fp->pfp = fdopen (fp->fd, mode)) == NULL) return false; + + if (fp->bom_size) + { + // atm just skip bom + + const int nread = fread (check, sizeof (char), fp->bom_size, fp->pfp); + + if (nread != fp->bom_size) return false; + } } fp->path = path; @@ -152,9 +169,10 @@ bool hc_fopen_raw (HCFILE *fp, const char *path, char *mode) return false; } - fp->pfp = NULL; - fp->is_gzip = false; - fp->is_zip = false; + fp->pfp = NULL; + fp->is_gzip = false; + fp->is_zip = false; + fp->bom_size = 0; if (fmode == -1) { diff --git a/src/shared.c b/src/shared.c index 46046346d..b101cc01f 100644 --- a/src/shared.c +++ b/src/shared.c @@ -388,7 +388,7 @@ bool hc_path_has_bom (const char *path) HCFILE fp; - if (hc_fopen (&fp, path, "rb") == false) return false; + if (hc_fopen_raw (&fp, path, "rb") == false) return false; const size_t nread = hc_fread (buf, 1, sizeof (buf), &fp); @@ -396,95 +396,104 @@ bool hc_path_has_bom (const char *path) if (nread < 1) return false; + const int bom_size = hc_string_bom_size (buf); + + const bool has_bom = bom_size > 0; + + return has_bom; +} + +int hc_string_bom_size (const u8 *s) +{ /* signatures from https://en.wikipedia.org/wiki/Byte_order_mark#Byte_order_marks_by_encoding */ // utf-8 - if ((buf[0] == 0xef) - && (buf[1] == 0xbb) - && (buf[2] == 0xbf)) return true; + if ((s[0] == 0xef) + && (s[1] == 0xbb) + && (s[2] == 0xbf)) return 3; // utf-16 - if ((buf[0] == 0xfe) - && (buf[1] == 0xff)) return true; + if ((s[0] == 0xfe) + && (s[1] == 0xff)) return 2; - if ((buf[0] == 0xff) - && (buf[1] == 0xfe)) return true; + if ((s[0] == 0xff) + && (s[1] == 0xfe)) return 2; // utf-32 - if ((buf[0] == 0x00) - && (buf[1] == 0x00) - && (buf[2] == 0xfe) - && (buf[3] == 0xff)) return true; + if ((s[0] == 0x00) + && (s[1] == 0x00) + && (s[2] == 0xfe) + && (s[3] == 0xff)) return 4; - if ((buf[0] == 0xff) - && (buf[1] == 0xfe) - && (buf[2] == 0x00) - && (buf[3] == 0x00)) return true; + if ((s[0] == 0xff) + && (s[1] == 0xfe) + && (s[2] == 0x00) + && (s[3] == 0x00)) return 4; // utf-7 - if ((buf[0] == 0x2b) - && (buf[1] == 0x2f) - && (buf[2] == 0x76) - && (buf[3] == 0x38)) return true; + if ((s[0] == 0x2b) + && (s[1] == 0x2f) + && (s[2] == 0x76) + && (s[3] == 0x38)) return 4; - if ((buf[0] == 0x2b) - && (buf[1] == 0x2f) - && (buf[2] == 0x76) - && (buf[3] == 0x39)) return true; + if ((s[0] == 0x2b) + && (s[1] == 0x2f) + && (s[2] == 0x76) + && (s[3] == 0x39)) return 4; - if ((buf[0] == 0x2b) - && (buf[1] == 0x2f) - && (buf[2] == 0x76) - && (buf[3] == 0x2b)) return true; + if ((s[0] == 0x2b) + && (s[1] == 0x2f) + && (s[2] == 0x76) + && (s[3] == 0x2b)) return 4; - if ((buf[0] == 0x2b) - && (buf[1] == 0x2f) - && (buf[2] == 0x76) - && (buf[3] == 0x2f)) return true; + if ((s[0] == 0x2b) + && (s[1] == 0x2f) + && (s[2] == 0x76) + && (s[3] == 0x2f)) return 4; - if ((buf[0] == 0x2b) - && (buf[1] == 0x2f) - && (buf[2] == 0x76) - && (buf[3] == 0x38) - && (buf[4] == 0x2d)) return true; + if ((s[0] == 0x2b) + && (s[1] == 0x2f) + && (s[2] == 0x76) + && (s[3] == 0x38) + && (s[4] == 0x2d)) return 5; // utf-1 - if ((buf[0] == 0xf7) - && (buf[1] == 0x64) - && (buf[2] == 0x4c)) return true; + if ((s[0] == 0xf7) + && (s[1] == 0x64) + && (s[2] == 0x4c)) return 3; // utf-ebcdic - if ((buf[0] == 0xdd) - && (buf[1] == 0x73) - && (buf[2] == 0x66) - && (buf[3] == 0x73)) return true; + if ((s[0] == 0xdd) + && (s[1] == 0x73) + && (s[2] == 0x66) + && (s[3] == 0x73)) return 4; // scsu - if ((buf[0] == 0x0e) - && (buf[1] == 0xfe) - && (buf[2] == 0xff)) return true; + if ((s[0] == 0x0e) + && (s[1] == 0xfe) + && (s[2] == 0xff)) return 3; // bocu-1 - if ((buf[0] == 0xfb) - && (buf[1] == 0xee) - && (buf[2] == 0x28)) return true; + if ((s[0] == 0xfb) + && (s[1] == 0xee) + && (s[2] == 0x28)) return 3; // gb-18030 - if ((buf[0] == 0x84) - && (buf[1] == 0x31) - && (buf[2] == 0x95) - && (buf[3] == 0x33)) return true; + if ((s[0] == 0x84) + && (s[1] == 0x31) + && (s[2] == 0x95) + && (s[3] == 0x33)) return 4; - return false; + return 0; } bool hc_string_is_digit (const char *s) diff --git a/src/straight.c b/src/straight.c index d9cc2b3c3..2fe206d0b 100644 --- a/src/straight.c +++ b/src/straight.c @@ -18,9 +18,9 @@ static int straight_ctx_add_wl (hashcat_ctx_t *hashcat_ctx, const char *dict) { if (hc_path_has_bom (dict) == true) { - event_log_error (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", dict); + event_log_warning (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", dict); - return -1; + //return -1; } straight_ctx_t *straight_ctx = hashcat_ctx->straight_ctx; diff --git a/src/user_options.c b/src/user_options.c index 43bbe05c4..0adc6b546 100644 --- a/src/user_options.c +++ b/src/user_options.c @@ -2297,9 +2297,9 @@ int user_options_check_files (hashcat_ctx_t *hashcat_ctx) if (hc_path_has_bom (user_options_extra->hc_hash) == true) { - event_log_error (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", user_options_extra->hc_hash); + event_log_warning (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", user_options_extra->hc_hash); - return -1; + //return -1; } } } @@ -2347,9 +2347,9 @@ int user_options_check_files (hashcat_ctx_t *hashcat_ctx) if (hc_path_has_bom (rp_file) == true) { - event_log_error (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", rp_file); + event_log_warning (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", rp_file); - return -1; + //return -1; } } } @@ -2385,9 +2385,9 @@ int user_options_check_files (hashcat_ctx_t *hashcat_ctx) if (hc_path_has_bom (dictfile1) == true) { - event_log_error (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", dictfile1); + event_log_warning (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", dictfile1); - return -1; + //return -1; } if (hc_path_exist (dictfile2) == false) @@ -2413,9 +2413,9 @@ int user_options_check_files (hashcat_ctx_t *hashcat_ctx) if (hc_path_has_bom (dictfile2) == true) { - event_log_error (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", dictfile2); + event_log_warning (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", dictfile2); - return -1; + //return -1; } } } @@ -2445,9 +2445,9 @@ int user_options_check_files (hashcat_ctx_t *hashcat_ctx) if (hc_path_has_bom (maskfile) == true) { - event_log_error (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", maskfile); + event_log_warning (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", maskfile); - return -1; + //return -1; } } } @@ -2489,9 +2489,9 @@ int user_options_check_files (hashcat_ctx_t *hashcat_ctx) if (hc_path_has_bom (maskfile) == true) { - event_log_error (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", maskfile); + event_log_warning (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", maskfile); - return -1; + //return -1; } } } @@ -2533,9 +2533,9 @@ int user_options_check_files (hashcat_ctx_t *hashcat_ctx) if (hc_path_has_bom (maskfile) == true) { - event_log_error (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", maskfile); + event_log_warning (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", maskfile); - return -1; + //return -1; } } } @@ -2581,9 +2581,9 @@ int user_options_check_files (hashcat_ctx_t *hashcat_ctx) if (hc_path_has_bom (rp_file) == true) { - event_log_error (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", rp_file); + event_log_warning (hashcat_ctx, "%s: Byte Order Mark (BOM) was detected", rp_file); - return -1; + //return -1; } } }