Autotune: Added error handling. By default skipping device on error, with --force using accel/loops/threads min values instead

docs/changes.txt

@@ -39,6 +39,7 @@
 - Unit tests: Updated test.sh to set default device-type to CPU with Apple Intel and added -f (--force) option
 - OpenCL Backend: moved functions to ext_OpenCL.c and includes to ext_OpenCL.h
 - HIP Backend: moved functions to ext_hip.c/ext_hiprtc.c and includes to ext_hip.h/ext_hiprtc.h
+- Autotune: Added error handling. By default skipping device on error, with --force using accel/loops/threads min values instead
 
 * changes v6.2.4 -> v6.2.5
 

include/types.h

@@ -197,6 +197,13 @@ typedef enum st_status_rc
 
 } st_status_t;
 
+typedef enum at_status_rc
+{
+  AT_STATUS_PASSED        = 0,
+  AT_STATUS_FAILED        = 1,
+
+} at_status_t;
+
 typedef enum status_rc
 {
   STATUS_INIT               = 0,
@@ -1127,7 +1134,11 @@ typedef struct hc_device_param
 
   u32     kernel_preferred_wgs_multiple;
 
-  st_status_t st_status;
+  st_status_t st_status;        // selftest status
+
+  at_status_t at_status;        // autotune status
+
+  int     at_rc;                // autotune rc
 
   int     vector_width;
 

src/autotune.c

@@ -104,10 +104,10 @@ static u32 previous_power_of_two (const u32 x)
 
 static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param)
 {
-  const hashconfig_t    *hashconfig   = hashcat_ctx->hashconfig;
+  const hashconfig_t   *hashconfig   = hashcat_ctx->hashconfig;
-  const backend_ctx_t   *backend_ctx  = hashcat_ctx->backend_ctx;
+  const backend_ctx_t  *backend_ctx  = hashcat_ctx->backend_ctx;
-  const straight_ctx_t  *straight_ctx = hashcat_ctx->straight_ctx;
+  const straight_ctx_t *straight_ctx = hashcat_ctx->straight_ctx;
-  const user_options_t  *user_options = hashcat_ctx->user_options;
+  const user_options_t *user_options = hashcat_ctx->user_options;
 
   const double target_msec = backend_ctx->target_msec;
 
@@ -120,6 +120,20 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
   const u32 kernel_threads_min = device_param->kernel_threads_min;
   const u32 kernel_threads_max = device_param->kernel_threads_max;
 
+  // stores the minimum values
+  // they could be used if the autotune fails and user specify --force
+
+  if (user_options->force == true)
+  {
+    device_param->kernel_accel   = kernel_accel_min;
+    device_param->kernel_loops   = kernel_loops_min;
+    device_param->kernel_threads = kernel_threads_min;
+    device_param->hardware_power = ((hashconfig->opts_type & OPTS_TYPE_MP_MULTI_DISABLE) ? 1 : device_param->device_processors) * kernel_threads_min;
+    device_param->kernel_power   = device_param->hardware_power * kernel_accel_min;
+  }
+
+  // start engine
+
   u32 kernel_accel = kernel_accel_min;
   u32 kernel_loops = kernel_loops_min;
 
@@ -212,6 +226,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
       }
     }
 
+    device_param->at_rc = -2;
+
     if (device_param->is_cuda == true)
     {
       if (run_cuda_kernel_atinit (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, kernel_power_max) == -1) return -1;
@@ -236,6 +252,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
       {
         if (straight_ctx->kernel_rules_cnt > 1)
         {
+          device_param->at_rc = -3;
+
           if (device_param->is_cuda == true)
           {
             if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_rules_c, device_param->cuda_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), device_param->cuda_stream) == -1) return -1;
@@ -290,6 +308,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
       {
         event_log_error (hashcat_ctx, "Kernel minimum runtime larger than default TDR");
 
+        device_param->at_rc = -4;
+
         return -1;
       }
 
@@ -437,44 +457,36 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
   // reset them fake words
   // reset other buffers in case autotune cracked something
 
+  device_param->at_rc = -5;
+
   if (device_param->is_cuda == true)
   {
     if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, device_param->size_pws) == -1) return -1;
-
     if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_plain_bufs, device_param->size_plains) == -1) return -1;
-
     if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_digests_shown, device_param->size_shown) == -1) return -1;
-
     if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_result, device_param->size_results) == -1) return -1;
-
     if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_tmps, device_param->size_tmps) == -1) return -1;
   }
 
   if (device_param->is_hip == true)
   {
     if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_pws_buf, device_param->size_pws) == -1) return -1;
-
     if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_plain_bufs, device_param->size_plains) == -1) return -1;
-
     if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_digests_shown, device_param->size_shown) == -1) return -1;
-
     if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_result, device_param->size_results) == -1) return -1;
-
     if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_tmps, device_param->size_tmps) == -1) return -1;
   }
 
   if (device_param->is_opencl == true)
   {
     if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_pws_buf, device_param->size_pws) == -1) return -1;
-
     if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_plain_bufs, device_param->size_plains) == -1) return -1;
-
     if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_digests_shown, device_param->size_shown) == -1) return -1;
-
     if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_result, device_param->size_results) == -1) return -1;
-
     if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_tmps, device_param->size_tmps) == -1) return -1;
 
+    device_param->at_rc = -6;
+
     if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;
   }
 
@@ -482,8 +494,7 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
 
   device_param->exec_pos = 0;
 
-  memset (device_param->exec_msec, 0, EXEC_CACHE * sizeof (double));
+  memset (device_param->exec_msec,          0,          EXEC_CACHE * sizeof (double));
-
   memset (device_param->exec_us_prev1,      0, EXPECTED_ITERATIONS * sizeof (double));
   memset (device_param->exec_us_prev2,      0, EXPECTED_ITERATIONS * sizeof (double));
   memset (device_param->exec_us_prev3,      0, EXPECTED_ITERATIONS * sizeof (double));
@@ -517,7 +528,6 @@ HC_API_CALL void *thread_autotune (void *p)
   thread_param_t *thread_param = (thread_param_t *) p;
 
   hashcat_ctx_t *hashcat_ctx = thread_param->hashcat_ctx;
-
   backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx;
 
   if (backend_ctx->enabled == false) return NULL;
@@ -528,6 +538,11 @@ HC_API_CALL void *thread_autotune (void *p)
 
   if (device_param->skipped_warning == true) return NULL;
 
+  // init autotunes status and rc
+
+  device_param->at_status = AT_STATUS_FAILED;
+  device_param->at_rc = -1; // generic error
+
   if (device_param->is_cuda == true)
   {
     if (hc_cuCtxPushCurrent (hashcat_ctx, device_param->cuda_context) == -1) return NULL;
@@ -538,11 +553,12 @@ HC_API_CALL void *thread_autotune (void *p)
     if (hc_hipCtxPushCurrent (hashcat_ctx, device_param->hip_context) == -1) return NULL;
   }
 
-  const int rc_autotune = autotune (hashcat_ctx, device_param);
+  // check for autotune failure
 
-  if (rc_autotune == -1)
+  if (autotune (hashcat_ctx, device_param) == 0)
   {
-    // we should do something here, tell hashcat main that autotune failed to abort
+    device_param->at_status = AT_STATUS_PASSED;
+    device_param->at_rc = 0;
   }
 
   if (device_param->is_cuda == true)

src/hashcat.c

@@ -211,6 +211,66 @@ static int inner2_loop (hashcat_ctx_t *hashcat_ctx)
 
   hc_thread_wait (backend_ctx->backend_devices_cnt, c_threads);
 
+  // check for any autotune failures
+  // by default, skipping device on error
+  // using --force, accel/loops/threads min values are used instead of skipping
+
+  int at_err = 0;
+
+  for (int backend_devices_idx = 0; backend_devices_idx < backend_ctx->backend_devices_cnt; backend_devices_idx++)
+  {
+    if (backend_ctx->enabled == false) continue;
+
+    hc_device_param_t *device_param = backend_ctx->devices_param + backend_devices_idx;
+
+    if (device_param->skipped == true) continue;
+
+    if (device_param->skipped_warning == true) continue;
+
+    if (device_param->at_status == AT_STATUS_FAILED)
+    {
+      at_err++;
+
+      if (user_options->force == false)
+      {
+        event_log_warning (hashcat_ctx, "* Device #%u: skipped, due to kernel autotune failure (%d).", device_param->device_id + 1, device_param->at_rc);
+
+        device_param->skipped = true;
+
+        // update counters
+
+        if (device_param->is_hip == true)    backend_ctx->hip_devices_active--;
+        if (device_param->is_cuda == true)   backend_ctx->cuda_devices_active--;
+        if (device_param->is_opencl == true) backend_ctx->opencl_devices_active--;
+
+        backend_ctx->backend_devices_active--;
+      }
+      else
+      {
+        event_log_warning (hashcat_ctx, "* Device #%u: detected kernel autotune failure (%d), min values will be used", device_param->device_id + 1, device_param->at_rc);
+      }
+    }
+  }
+
+  if (at_err > 0)
+  {
+    event_log_warning (hashcat_ctx, NULL);
+
+    if (user_options->force == false)
+    {
+      // if all enabled devices fail, abort session
+      if (backend_ctx->backend_devices_active <= 0)
+      {
+        event_log_error (hashcat_ctx, "Aborting session due to kernel autotune failures, for all active devices.");
+
+        event_log_warning (hashcat_ctx, "You can use --force to override this, but do not report related errors.");
+        event_log_warning (hashcat_ctx, NULL);
+
+        return -1;
+      }
+    }
+  }
+
   EVENT (EVENT_AUTOTUNE_FINISHED);
 
   /**