Change out-of-boundary fix in order to re-enable password length 256 with rules in pure kernel mode

OpenCL/inc_rp.cl

@@ -770,7 +770,7 @@ DECLSPEC int apply_rules (CONSTANT_AS const u32 *cmds, u32 *buf, const int in_le
     const u8 p1   = (cmd >> 16) & 0xff;
 
     // we need to guarantee input length < 256 otherwise functions like rule_op_mangle_switch_last() and others will read out of boundary
-    out_len = apply_rule (name, p0, p1, buf, out_len & 255);
+    out_len = apply_rule (name, p0, p1, buf, out_len);
   }
 
   return out_len;

OpenCL/inc_rp_optimized.cl

@@ -2350,7 +2350,7 @@ DECLSPEC u32 apply_rules_optimized (CONSTANT_AS const u32 *cmds, u32 *buf0, u32
     const u32 p1   = (cmd >> 16) & 0xff;
 
     // we need to guarantee input length < 32 otherwise functions like rule_op_mangle_switch_last() and others will read out of boundary
-    out_len = apply_rule_optimized (name, p0, p1, buf0, buf1, out_len & 31);
+    out_len = apply_rule_optimized (name, p0, p1, buf0, buf1, out_len);
   }
 
   return out_len;

src/slow_candidates.c

@@ -218,10 +218,14 @@ void slow_candidates_next (hashcat_ctx_t *hashcat_ctx, void *extra_info)
 
     if (hashconfig->opti_type & OPTI_TYPE_OPTIMIZED_KERNEL)
     {
+      extra_info_straight->out_len = MIN (extra_info_straight->out_len, 31); // max length supported by apply_rules_optimized()
+
       extra_info_straight->out_len = apply_rules_optimized (straight_ctx->kernel_rules_buf[extra_info_straight->rule_pos].cmds, &out_ptr[0], &out_ptr[4], extra_info_straight->out_len);
     }
     else
     {
+      extra_info_straight->out_len = MIN (extra_info_straight->out_len, 256); // max length supported by apply_rules()
+
       extra_info_straight->out_len = apply_rules (straight_ctx->kernel_rules_buf[extra_info_straight->rule_pos].cmds, out_ptr, extra_info_straight->out_len);
     }