diff --git a/docs/changes.txt b/docs/changes.txt index 9d1a7b8b0..93a4d9d6c 100644 --- a/docs/changes.txt +++ b/docs/changes.txt @@ -66,6 +66,7 @@ - OpenCL Backend: moved functions to ext_OpenCL.c and includes to ext_OpenCL.h - HIP Backend: moved functions to ext_hip.c/ext_hiprtc.c and includes to ext_hip.h/ext_hiprtc.h - CUDA Backend: moved functions to ext_cuda.c/ext_nvrtc.c and includes to ext_cuda.h/ext_nvrtc.h +- Autotune: Added error handling. By default skipping device on error, with --force using accel/loops/threads min values instead - Makefile: updated MACOSX_DEPLOYMENT_TARGET to 10.15 and removed OpenCL framework from LFLAGS_NATIVE on MacOS - Status code: updated negative status code, usefull in Unit tests engine (test.sh) - Unit tests: added -r (--runtime) option diff --git a/docs/status_codes.txt b/docs/status_codes.txt index b7889ff2e..33f44d0a6 100644 --- a/docs/status_codes.txt +++ b/docs/status_codes.txt @@ -1,16 +1,17 @@ status codes on exit: ===================== --8 = mixed backend errors (combo of -3, -4, -5, -6, -7 errors type) --7 = backend error: Invalid module_extra_buffer_size --6 = backend error: Too many compute units to keep minimum kernel accel limit --5 = backend error: main kernel build error --4 = backend error: memory hit --3 = backend error: skipping hash-type due to module_unstable_warning settings --2 = gpu-watchdog alarm --1 = error - 0 = OK/cracked - 1 = exhausted - 2 = aborted - 3 = aborted by checkpoint - 4 = aborted by runtime +-10 = autotune failure + -8 = mixed backend errors (combo of -3, -4, -5, -6, -7 errors type) + -7 = backend error: Invalid module_extra_buffer_size + -6 = backend error: Too many compute units to keep minimum kernel accel limit + -5 = backend error: main kernel build error + -4 = backend error: memory hit + -3 = backend error: skipping hash-type due to module_unstable_warning settings + -2 = gpu-watchdog alarm + -1 = error + 0 = OK/cracked + 1 = exhausted + 2 = aborted + 3 = aborted by checkpoint + 4 = aborted by runtime diff --git a/include/types.h b/include/types.h index d6b6428a3..94d2f1505 100644 --- a/include/types.h +++ b/include/types.h @@ -197,6 +197,13 @@ typedef enum st_status_rc } st_status_t; +typedef enum at_status_rc +{ + AT_STATUS_PASSED = 0, + AT_STATUS_FAILED = 1, + +} at_status_t; + typedef enum status_rc { STATUS_INIT = 0, @@ -1127,7 +1134,11 @@ typedef struct hc_device_param u32 kernel_preferred_wgs_multiple; - st_status_t st_status; + st_status_t st_status; // selftest status + + at_status_t at_status; // autotune status + + int at_rc; // autotune rc int vector_width; diff --git a/src/autotune.c b/src/autotune.c index c89373126..bcbdd0eb3 100644 --- a/src/autotune.c +++ b/src/autotune.c @@ -104,10 +104,10 @@ static u32 previous_power_of_two (const u32 x) static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) { - const hashconfig_t *hashconfig = hashcat_ctx->hashconfig; - const backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; - const straight_ctx_t *straight_ctx = hashcat_ctx->straight_ctx; - const user_options_t *user_options = hashcat_ctx->user_options; + const hashconfig_t *hashconfig = hashcat_ctx->hashconfig; + const backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; + const straight_ctx_t *straight_ctx = hashcat_ctx->straight_ctx; + const user_options_t *user_options = hashcat_ctx->user_options; const double target_msec = backend_ctx->target_msec; @@ -120,6 +120,20 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param const u32 kernel_threads_min = device_param->kernel_threads_min; const u32 kernel_threads_max = device_param->kernel_threads_max; + // stores the minimum values + // they could be used if the autotune fails and user specify --force + + if (user_options->force == true) + { + device_param->kernel_accel = kernel_accel_min; + device_param->kernel_loops = kernel_loops_min; + device_param->kernel_threads = kernel_threads_min; + device_param->hardware_power = ((hashconfig->opts_type & OPTS_TYPE_MP_MULTI_DISABLE) ? 1 : device_param->device_processors) * kernel_threads_min; + device_param->kernel_power = device_param->hardware_power * kernel_accel_min; + } + + // start engine + u32 kernel_accel = kernel_accel_min; u32 kernel_loops = kernel_loops_min; @@ -212,6 +226,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param } } + device_param->at_rc = -2; + if (device_param->is_cuda == true) { if (run_cuda_kernel_atinit (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, kernel_power_max) == -1) return -1; @@ -236,6 +252,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param { if (straight_ctx->kernel_rules_cnt > 1) { + device_param->at_rc = -3; + if (device_param->is_cuda == true) { if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_rules_c, device_param->cuda_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), device_param->cuda_stream) == -1) return -1; @@ -290,6 +308,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param { event_log_error (hashcat_ctx, "Kernel minimum runtime larger than default TDR"); + device_param->at_rc = -4; + return -1; } @@ -437,44 +457,36 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param // reset them fake words // reset other buffers in case autotune cracked something + device_param->at_rc = -5; + if (device_param->is_cuda == true) { if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, device_param->size_pws) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_digests_shown, device_param->size_shown) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_result, device_param->size_results) == -1) return -1; - if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_tmps, device_param->size_tmps) == -1) return -1; } if (device_param->is_hip == true) { if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_pws_buf, device_param->size_pws) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_digests_shown, device_param->size_shown) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_result, device_param->size_results) == -1) return -1; - if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_tmps, device_param->size_tmps) == -1) return -1; } if (device_param->is_opencl == true) { if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_pws_buf, device_param->size_pws) == -1) return -1; - if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_plain_bufs, device_param->size_plains) == -1) return -1; - if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_digests_shown, device_param->size_shown) == -1) return -1; - if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_result, device_param->size_results) == -1) return -1; - if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_tmps, device_param->size_tmps) == -1) return -1; + device_param->at_rc = -6; + if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1; } @@ -482,8 +494,7 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param device_param->exec_pos = 0; - memset (device_param->exec_msec, 0, EXEC_CACHE * sizeof (double)); - + memset (device_param->exec_msec, 0, EXEC_CACHE * sizeof (double)); memset (device_param->exec_us_prev1, 0, EXPECTED_ITERATIONS * sizeof (double)); memset (device_param->exec_us_prev2, 0, EXPECTED_ITERATIONS * sizeof (double)); memset (device_param->exec_us_prev3, 0, EXPECTED_ITERATIONS * sizeof (double)); @@ -517,7 +528,6 @@ HC_API_CALL void *thread_autotune (void *p) thread_param_t *thread_param = (thread_param_t *) p; hashcat_ctx_t *hashcat_ctx = thread_param->hashcat_ctx; - backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx; if (backend_ctx->enabled == false) return NULL; @@ -527,6 +537,11 @@ HC_API_CALL void *thread_autotune (void *p) if (device_param->skipped == true) return NULL; if (device_param->skipped_warning == true) return NULL; + // init autotunes status and rc + + device_param->at_status = AT_STATUS_FAILED; + device_param->at_rc = -1; // generic error + if (device_param->is_cuda == true) { if (hc_cuCtxPushCurrent (hashcat_ctx, device_param->cuda_context) == -1) return NULL; @@ -537,11 +552,12 @@ HC_API_CALL void *thread_autotune (void *p) if (hc_hipCtxPushCurrent (hashcat_ctx, device_param->hip_context) == -1) return NULL; } - const int rc_autotune = autotune (hashcat_ctx, device_param); + // check for autotune failure - if (rc_autotune == -1) + if (autotune (hashcat_ctx, device_param) == 0) { - // we should do something here, tell hashcat main that autotune failed to abort + device_param->at_status = AT_STATUS_PASSED; + device_param->at_rc = 0; } if (device_param->is_cuda == true) diff --git a/src/hashcat.c b/src/hashcat.c index d3e70a874..fabce3002 100644 --- a/src/hashcat.c +++ b/src/hashcat.c @@ -211,6 +211,66 @@ static int inner2_loop (hashcat_ctx_t *hashcat_ctx) hc_thread_wait (backend_ctx->backend_devices_cnt, c_threads); + // check for any autotune failures + // by default, skipping device on error + // using --force, accel/loops/threads min values are used instead of skipping + + int at_err = 0; + + for (int backend_devices_idx = 0; backend_devices_idx < backend_ctx->backend_devices_cnt; backend_devices_idx++) + { + if (backend_ctx->enabled == false) continue; + + hc_device_param_t *device_param = backend_ctx->devices_param + backend_devices_idx; + + if (device_param->skipped == true) continue; + + if (device_param->skipped_warning == true) continue; + + if (device_param->at_status == AT_STATUS_FAILED) + { + at_err++; + + if (user_options->force == false) + { + event_log_warning (hashcat_ctx, "* Device #%u: skipped, due to kernel autotune failure (%d).", device_param->device_id + 1, device_param->at_rc); + + device_param->skipped = true; + + // update counters + + if (device_param->is_hip == true) backend_ctx->hip_devices_active--; + if (device_param->is_cuda == true) backend_ctx->cuda_devices_active--; + if (device_param->is_opencl == true) backend_ctx->opencl_devices_active--; + + backend_ctx->backend_devices_active--; + } + else + { + event_log_warning (hashcat_ctx, "* Device #%u: detected kernel autotune failure (%d), min values will be used", device_param->device_id + 1, device_param->at_rc); + } + } + } + + if (at_err > 0) + { + event_log_warning (hashcat_ctx, NULL); + + if (user_options->force == false) + { + // if all enabled devices fail, abort session + if (backend_ctx->backend_devices_active <= 0) + { + event_log_error (hashcat_ctx, "Aborting session due to kernel autotune failures, for all active devices."); + + event_log_warning (hashcat_ctx, "You can use --force to override this, but do not report related errors."); + event_log_warning (hashcat_ctx, NULL); + + return -10; + } + } + } + EVENT (EVENT_AUTOTUNE_FINISHED); /** diff --git a/tools/test.sh b/tools/test.sh index e47c2d734..e1d924f5e 100755 --- a/tools/test.sh +++ b/tools/test.sh @@ -403,6 +403,12 @@ function status() if [ "${RET}" -ne 0 ]; then case ${RET} in + 246) + echo "autotune failure, cmdline : ${CMD}" >> "${OUTD}/logfull.txt" 2>> "${OUTD}/logfull.txt" + + e_rs=$((e_rs + 1)) + ;; + 248) echo "skipped by runtime (mixed backend errors detected), cmdline : ${CMD}" >> "${OUTD}/logfull.txt" 2>> "${OUTD}/logfull.txt"