From 3802adff7e13ddb9f6a99b49b308946f1912da3c Mon Sep 17 00:00:00 2001 From: Gabriele Gristina Date: Tue, 4 Jan 2022 01:14:39 +0100 Subject: [PATCH] Autotune: Added error handling. By default skipping device on error, with --force using accel/loops/threads min values instead --- docs/changes.txt | 1 + include/types.h | 13 ++++++++++- src/autotune.c | 60 ++++++++++++++++++++++++++++++------------------ src/hashcat.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 23 deletions(-) diff --git a/docs/changes.txt b/docs/changes.txt index 94f7ef326..0ae515eb4 100644 --- a/docs/changes.txt +++ b/docs/changes.txt @@ -39,6 +39,7 @@ - Unit tests: Updated test.sh to set default device-type to CPU with Apple Intel and added -f (--force) option - OpenCL Backend: moved functions to ext_OpenCL.c and includes to ext_OpenCL.h - HIP Backend: moved functions to ext_hip.c/ext_hiprtc.c and includes to ext_hip.h/ext_hiprtc.h +- Autotune: Added error handling. By default skipping device on error, with --force using accel/loops/threads min values instead * changes v6.2.4 -> v6.2.5 diff --git a/include/types.h b/include/types.h index 57a3029f9..7bf675f77 100644 --- a/include/types.h +++ b/include/types.h @@ -197,6 +197,13 @@ typedef enum st_status_rc } st_status_t; +typedef enum at_status_rc +{ + AT_STATUS_PASSED = 0, + AT_STATUS_FAILED = 1, + +} at_status_t; + typedef enum status_rc { STATUS_INIT = 0, @@ -1127,7 +1134,11 @@ typedef struct hc_device_param u32 kernel_preferred_wgs_multiple; - st_status_t st_status; + st_status_t st_status; // selftest status + + at_status_t at_status; // autotune status + + int at_rc; // autotune rc int vector_width; diff --git a/src/autotune.c b/src/autotune.c index 4a61b5fa5..912a65b4a 100644 --- a/src/autotune.c +++ b/src/autotune.c @@ -104,10 +104,10 @@ static u32 previous_power_of_two (const u32 x) static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) { - const hashconfig_t *hashconfig = hashcat_ctx->hashconfig; - const backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; - const straight_ctx_t *straight_ctx = hashcat_ctx->straight_ctx; - const user_options_t *user_options = hashcat_ctx->user_options; + const hashconfig_t *hashconfig = hashcat_ctx->hashconfig; + const backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; + const straight_ctx_t *straight_ctx = hashcat_ctx->straight_ctx; + const user_options_t *user_options = hashcat_ctx->user_options; const double target_msec = backend_ctx->target_msec; @@ -120,6 +120,20 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param const u32 kernel_threads_min = device_param->kernel_threads_min; const u32 kernel_threads_max = device_param->kernel_threads_max; + // stores the minimum values + // they could be used if the autotune fails and user specify --force + + if (user_options->force == true) + { + device_param->kernel_accel = kernel_accel_min; + device_param->kernel_loops = kernel_loops_min; + device_param->kernel_threads = kernel_threads_min; + device_param->hardware_power = ((hashconfig->opts_type & OPTS_TYPE_MP_MULTI_DISABLE) ? 1 : device_param->device_processors) * kernel_threads_min; + device_param->kernel_power = device_param->hardware_power * kernel_accel_min; + } + + // start engine + u32 kernel_accel = kernel_accel_min; u32 kernel_loops = kernel_loops_min; @@ -212,6 +226,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param } } + device_param->at_rc = -2; + if (device_param->is_cuda == true) { if (run_cuda_kernel_atinit (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, kernel_power_max) == -1) return -1; @@ -236,6 +252,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param { if (straight_ctx->kernel_rules_cnt > 1) { + device_param->at_rc = -3; + if (device_param->is_cuda == true) { if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_rules_c, device_param->cuda_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), device_param->cuda_stream) == -1) return -1; @@ -290,6 +308,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param { event_log_error (hashcat_ctx, "Kernel minimum runtime larger than default TDR"); + device_param->at_rc = -4; + return -1; } @@ -437,44 +457,36 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param // reset them fake words // reset other buffers in case autotune cracked something + device_param->at_rc = -5; + if (device_param->is_cuda == true) { if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, device_param->size_pws) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_digests_shown, device_param->size_shown) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_result, device_param->size_results) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_tmps, device_param->size_tmps) == -1) return -1; } if (device_param->is_hip == true) { if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_pws_buf, device_param->size_pws) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_digests_shown, device_param->size_shown) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_result, device_param->size_results) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_tmps, device_param->size_tmps) == -1) return -1; } if (device_param->is_opencl == true) { if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_pws_buf, device_param->size_pws) == -1) return -1; - if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_digests_shown, device_param->size_shown) == -1) return -1; - if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_result, device_param->size_results) == -1) return -1; - if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_tmps, device_param->size_tmps) == -1) return -1; + device_param->at_rc = -6; + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } @@ -482,8 +494,7 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param device_param->exec_pos = 0; - memset (device_param->exec_msec, 0, EXEC_CACHE * sizeof (double)); - + memset (device_param->exec_msec, 0, EXEC_CACHE * sizeof (double)); memset (device_param->exec_us_prev1, 0, EXPECTED_ITERATIONS * sizeof (double)); memset (device_param->exec_us_prev2, 0, EXPECTED_ITERATIONS * sizeof (double)); memset (device_param->exec_us_prev3, 0, EXPECTED_ITERATIONS * sizeof (double)); @@ -517,7 +528,6 @@ HC_API_CALL void *thread_autotune (void *p) thread_param_t *thread_param = (thread_param_t *) p; hashcat_ctx_t *hashcat_ctx = thread_param->hashcat_ctx; - backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; if (backend_ctx->enabled == false) return NULL; @@ -528,6 +538,11 @@ HC_API_CALL void *thread_autotune (void *p) if (device_param->skipped_warning == true) return NULL; + // init autotunes status and rc + + device_param->at_status = AT_STATUS_FAILED; + device_param->at_rc = -1; // generic error + if (device_param->is_cuda == true) { if (hc_cuCtxPushCurrent (hashcat_ctx, device_param->cuda_context) == -1) return NULL; @@ -538,11 +553,12 @@ HC_API_CALL void *thread_autotune (void *p) if (hc_hipCtxPushCurrent (hashcat_ctx, device_param->hip_context) == -1) return NULL; } - const int rc_autotune = autotune (hashcat_ctx, device_param); + // check for autotune failure - if (rc_autotune == -1) + if (autotune (hashcat_ctx, device_param) == 0) { - // we should do something here, tell hashcat main that autotune failed to abort + device_param->at_status = AT_STATUS_PASSED; + device_param->at_rc = 0; } if (device_param->is_cuda == true) diff --git a/src/hashcat.c b/src/hashcat.c index bf936ccc0..774986514 100644 --- a/src/hashcat.c +++ b/src/hashcat.c @@ -211,6 +211,66 @@ static int inner2_loop (hashcat_ctx_t *hashcat_ctx) hc_thread_wait (backend_ctx->backend_devices_cnt, c_threads); + // check for any autotune failures + // by default, skipping device on error + // using --force, accel/loops/threads min values are used instead of skipping + + int at_err = 0; + + for (int backend_devices_idx = 0; backend_devices_idx < backend_ctx->backend_devices_cnt; backend_devices_idx++) + { + if (backend_ctx->enabled == false) continue; + + hc_device_param_t *device_param = backend_ctx->devices_param + backend_devices_idx; + + if (device_param->skipped == true) continue; + + if (device_param->skipped_warning == true) continue; + + if (device_param->at_status == AT_STATUS_FAILED) + { + at_err++; + + if (user_options->force == false) + { + event_log_warning (hashcat_ctx, "* Device #%u: skipped, due to kernel autotune failure (%d).", device_param->device_id + 1, device_param->at_rc); + + device_param->skipped = true; + + // update counters + + if (device_param->is_hip == true) backend_ctx->hip_devices_active--; + if (device_param->is_cuda == true) backend_ctx->cuda_devices_active--; + if (device_param->is_opencl == true) backend_ctx->opencl_devices_active--; + + backend_ctx->backend_devices_active--; + } + else + { + event_log_warning (hashcat_ctx, "* Device #%u: detected kernel autotune failure (%d), min values will be used", device_param->device_id + 1, device_param->at_rc); + } + } + } + + if (at_err > 0) + { + event_log_warning (hashcat_ctx, NULL); + + if (user_options->force == false) + { + // if all enabled devices fail, abort session + if (backend_ctx->backend_devices_active <= 0) + { + event_log_error (hashcat_ctx, "Aborting session due to kernel autotune failures, for all active devices."); + + event_log_warning (hashcat_ctx, "You can use --force to override this, but do not report related errors."); + event_log_warning (hashcat_ctx, NULL); + + return -1; + } + } + } + EVENT (EVENT_AUTOTUNE_FINISHED); /**