/** * Author......: See docs/credits.txt * License.....: MIT */ #define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" #include "inc_rp_optimized.h" #include "inc_rp_optimized.cl" #include "inc_simd.cl" #include "inc_hash_md5.cl" void gen336 (u32x digest_pre[4], u32 salt_buf[4], u32x digest[4]) { u32x digest_t0[2]; u32x digest_t1[2]; u32x digest_t2[2]; u32x digest_t3[2]; digest_t0[0] = digest_pre[0]; digest_t0[1] = digest_pre[1] & 0xff; digest_t1[0] = digest_pre[0] << 8; digest_t1[1] = digest_pre[0] >> 24 | digest_pre[1] << 8; digest_t2[0] = digest_pre[0] << 16; digest_t2[1] = digest_pre[0] >> 16 | digest_pre[1] << 16; digest_t3[0] = digest_pre[0] << 24; digest_t3[1] = digest_pre[0] >> 8 | digest_pre[1] << 24; u32x salt_buf_t0[4]; u32x salt_buf_t1[5]; u32x salt_buf_t2[5]; u32x salt_buf_t3[5]; salt_buf_t0[0] = salt_buf[0]; salt_buf_t0[1] = salt_buf[1]; salt_buf_t0[2] = salt_buf[2]; salt_buf_t0[3] = salt_buf[3]; salt_buf_t1[0] = salt_buf[0] << 8; salt_buf_t1[1] = salt_buf[0] >> 24 | salt_buf[1] << 8; salt_buf_t1[2] = salt_buf[1] >> 24 | salt_buf[2] << 8; salt_buf_t1[3] = salt_buf[2] >> 24 | salt_buf[3] << 8; salt_buf_t1[4] = salt_buf[3] >> 24; salt_buf_t2[0] = salt_buf[0] << 16; salt_buf_t2[1] = salt_buf[0] >> 16 | salt_buf[1] << 16; salt_buf_t2[2] = salt_buf[1] >> 16 | salt_buf[2] << 16; salt_buf_t2[3] = salt_buf[2] >> 16 | salt_buf[3] << 16; salt_buf_t2[4] = salt_buf[3] >> 16; salt_buf_t3[0] = salt_buf[0] << 24; salt_buf_t3[1] = salt_buf[0] >> 8 | salt_buf[1] << 24; salt_buf_t3[2] = salt_buf[1] >> 8 | salt_buf[2] << 24; salt_buf_t3[3] = salt_buf[2] >> 8 | salt_buf[3] << 24; salt_buf_t3[4] = salt_buf[3] >> 8; u32x w0_t[4]; u32x w1_t[4]; u32x w2_t[4]; u32x w3_t[4]; // generate the 16 * 21 buffer w0_t[0] = 0; w0_t[1] = 0; w0_t[2] = 0; w0_t[3] = 0; w1_t[0] = 0; w1_t[1] = 0; w1_t[2] = 0; w1_t[3] = 0; w2_t[0] = 0; w2_t[1] = 0; w2_t[2] = 0; w2_t[3] = 0; w3_t[0] = 0; w3_t[1] = 0; w3_t[2] = 0; w3_t[3] = 0; // 0..5 w0_t[0] = digest_t0[0]; w0_t[1] = digest_t0[1]; // 5..21 w0_t[1] |= salt_buf_t1[0]; w0_t[2] = salt_buf_t1[1]; w0_t[3] = salt_buf_t1[2]; w1_t[0] = salt_buf_t1[3]; w1_t[1] = salt_buf_t1[4]; // 21..26 w1_t[1] |= digest_t1[0]; w1_t[2] = digest_t1[1]; // 26..42 w1_t[2] |= salt_buf_t2[0]; w1_t[3] = salt_buf_t2[1]; w2_t[0] = salt_buf_t2[2]; w2_t[1] = salt_buf_t2[3]; w2_t[2] = salt_buf_t2[4]; // 42..47 w2_t[2] |= digest_t2[0]; w2_t[3] = digest_t2[1]; // 47..63 w2_t[3] |= salt_buf_t3[0]; w3_t[0] = salt_buf_t3[1]; w3_t[1] = salt_buf_t3[2]; w3_t[2] = salt_buf_t3[3]; w3_t[3] = salt_buf_t3[4]; // 63.. w3_t[3] |= digest_t3[0]; md5_transform_vector (w0_t, w1_t, w2_t, w3_t, digest); w0_t[0] = 0; w0_t[1] = 0; w0_t[2] = 0; w0_t[3] = 0; w1_t[0] = 0; w1_t[1] = 0; w1_t[2] = 0; w1_t[3] = 0; w2_t[0] = 0; w2_t[1] = 0; w2_t[2] = 0; w2_t[3] = 0; w3_t[0] = 0; w3_t[1] = 0; w3_t[2] = 0; w3_t[3] = 0; // 0..4 w0_t[0] = digest_t3[1]; // 4..20 w0_t[1] = salt_buf_t0[0]; w0_t[2] = salt_buf_t0[1]; w0_t[3] = salt_buf_t0[2]; w1_t[0] = salt_buf_t0[3]; // 20..25 w1_t[1] = digest_t0[0]; w1_t[2] = digest_t0[1]; // 25..41 w1_t[2] |= salt_buf_t1[0]; w1_t[3] = salt_buf_t1[1]; w2_t[0] = salt_buf_t1[2]; w2_t[1] = salt_buf_t1[3]; w2_t[2] = salt_buf_t1[4]; // 41..46 w2_t[2] |= digest_t1[0]; w2_t[3] = digest_t1[1]; // 46..62 w2_t[3] |= salt_buf_t2[0]; w3_t[0] = salt_buf_t2[1]; w3_t[1] = salt_buf_t2[2]; w3_t[2] = salt_buf_t2[3]; w3_t[3] = salt_buf_t2[4]; // 62.. w3_t[3] |= digest_t2[0]; md5_transform_vector (w0_t, w1_t, w2_t, w3_t, digest); w0_t[0] = 0; w0_t[1] = 0; w0_t[2] = 0; w0_t[3] = 0; w1_t[0] = 0; w1_t[1] = 0; w1_t[2] = 0; w1_t[3] = 0; w2_t[0] = 0; w2_t[1] = 0; w2_t[2] = 0; w2_t[3] = 0; w3_t[0] = 0; w3_t[1] = 0; w3_t[2] = 0; w3_t[3] = 0; // 0..3 w0_t[0] = digest_t2[1]; // 3..19 w0_t[0] |= salt_buf_t3[0]; w0_t[1] = salt_buf_t3[1]; w0_t[2] = salt_buf_t3[2]; w0_t[3] = salt_buf_t3[3]; w1_t[0] = salt_buf_t3[4]; // 19..24 w1_t[0] |= digest_t3[0]; w1_t[1] = digest_t3[1]; // 24..40 w1_t[2] = salt_buf_t0[0]; w1_t[3] = salt_buf_t0[1]; w2_t[0] = salt_buf_t0[2]; w2_t[1] = salt_buf_t0[3]; // 40..45 w2_t[2] = digest_t0[0]; w2_t[3] = digest_t0[1]; // 45..61 w2_t[3] |= salt_buf_t1[0]; w3_t[0] = salt_buf_t1[1]; w3_t[1] = salt_buf_t1[2]; w3_t[2] = salt_buf_t1[3]; w3_t[3] = salt_buf_t1[4]; // 61.. w3_t[3] |= digest_t1[0]; md5_transform_vector (w0_t, w1_t, w2_t, w3_t, digest); w0_t[0] = 0; w0_t[1] = 0; w0_t[2] = 0; w0_t[3] = 0; w1_t[0] = 0; w1_t[1] = 0; w1_t[2] = 0; w1_t[3] = 0; w2_t[0] = 0; w2_t[1] = 0; w2_t[2] = 0; w2_t[3] = 0; w3_t[0] = 0; w3_t[1] = 0; w3_t[2] = 0; w3_t[3] = 0; // 0..2 w0_t[0] = digest_t1[1]; // 2..18 w0_t[0] |= salt_buf_t2[0]; w0_t[1] = salt_buf_t2[1]; w0_t[2] = salt_buf_t2[2]; w0_t[3] = salt_buf_t2[3]; w1_t[0] = salt_buf_t2[4]; // 18..23 w1_t[0] |= digest_t2[0]; w1_t[1] = digest_t2[1]; // 23..39 w1_t[1] |= salt_buf_t3[0]; w1_t[2] = salt_buf_t3[1]; w1_t[3] = salt_buf_t3[2]; w2_t[0] = salt_buf_t3[3]; w2_t[1] = salt_buf_t3[4]; // 39..44 w2_t[1] |= digest_t3[0]; w2_t[2] = digest_t3[1]; // 44..60 w2_t[3] = salt_buf_t0[0]; w3_t[0] = salt_buf_t0[1]; w3_t[1] = salt_buf_t0[2]; w3_t[2] = salt_buf_t0[3]; // 60.. w3_t[3] = digest_t0[0]; md5_transform_vector (w0_t, w1_t, w2_t, w3_t, digest); w0_t[0] = 0; w0_t[1] = 0; w0_t[2] = 0; w0_t[3] = 0; w1_t[0] = 0; w1_t[1] = 0; w1_t[2] = 0; w1_t[3] = 0; w2_t[0] = 0; w2_t[1] = 0; w2_t[2] = 0; w2_t[3] = 0; w3_t[0] = 0; w3_t[1] = 0; w3_t[2] = 0; w3_t[3] = 0; // 0..1 w0_t[0] = digest_t0[1]; // 1..17 w0_t[0] |= salt_buf_t1[0]; w0_t[1] = salt_buf_t1[1]; w0_t[2] = salt_buf_t1[2]; w0_t[3] = salt_buf_t1[3]; w1_t[0] = salt_buf_t1[4]; // 17..22 w1_t[0] |= digest_t1[0]; w1_t[1] = digest_t1[1]; // 22..38 w1_t[1] |= salt_buf_t2[0]; w1_t[2] = salt_buf_t2[1]; w1_t[3] = salt_buf_t2[2]; w2_t[0] = salt_buf_t2[3]; w2_t[1] = salt_buf_t2[4]; // 38..43 w2_t[1] |= digest_t2[0]; w2_t[2] = digest_t2[1]; // 43..59 w2_t[2] |= salt_buf_t3[0]; w2_t[3] = salt_buf_t3[1]; w3_t[0] = salt_buf_t3[2]; w3_t[1] = salt_buf_t3[3]; w3_t[2] = salt_buf_t3[4]; // 59.. w3_t[2] |= digest_t3[0]; w3_t[3] = digest_t3[1]; md5_transform_vector (w0_t, w1_t, w2_t, w3_t, digest); w0_t[0] = salt_buf_t0[0]; w0_t[1] = salt_buf_t0[1]; w0_t[2] = salt_buf_t0[2]; w0_t[3] = salt_buf_t0[3]; w1_t[0] = 0x80; w1_t[1] = 0; w1_t[2] = 0; w1_t[3] = 0; w2_t[0] = 0; w2_t[1] = 0; w2_t[2] = 0; w2_t[3] = 0; w3_t[0] = 0; w3_t[1] = 0; w3_t[2] = 21 * 16 * 8; w3_t[3] = 0; md5_transform_vector (w0_t, w1_t, w2_t, w3_t, digest); } __kernel void m09720_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global oldoffice01_t *oldoffice01_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u64 gid_max) { /** * modifier */ const u64 lid = get_local_id (0); /** * base */ const u64 gid = get_global_id (0); if (gid >= gid_max) return; u32 pw_buf0[4]; u32 pw_buf1[4]; pw_buf0[0] = pws[gid].i[ 0]; pw_buf0[1] = pws[gid].i[ 1]; pw_buf0[2] = pws[gid].i[ 2]; pw_buf0[3] = pws[gid].i[ 3]; pw_buf1[0] = pws[gid].i[ 4]; pw_buf1[1] = pws[gid].i[ 5]; pw_buf1[2] = pws[gid].i[ 6]; pw_buf1[3] = pws[gid].i[ 7]; const u32 pw_len = pws[gid].pw_len; /** * salt */ u32 salt_buf[4]; salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; salt_buf[2] = salt_bufs[salt_pos].salt_buf[2]; salt_buf[3] = salt_bufs[salt_pos].salt_buf[3]; /** * loop */ for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { u32x w0[4] = { 0 }; u32x w1[4] = { 0 }; u32x w2[4] = { 0 }; u32x w3[4] = { 0 }; const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); append_0x80_2x4_VV (w0, w1, out_len); /** * md5 */ make_utf16le (w1, w2, w3); make_utf16le (w0, w0, w1); w3[2] = out_len * 8 * 2; w3[3] = 0; u32x digest_pre[4]; digest_pre[0] = MD5M_A; digest_pre[1] = MD5M_B; digest_pre[2] = MD5M_C; digest_pre[3] = MD5M_D; md5_transform_vector (w0, w1, w2, w3, digest_pre); digest_pre[0] &= 0xffffffff; digest_pre[1] &= 0x000000ff; digest_pre[2] &= 0x00000000; digest_pre[3] &= 0x00000000; u32x digest[4]; digest[0] = MD5M_A; digest[1] = MD5M_B; digest[2] = MD5M_C; digest[3] = MD5M_D; gen336 (digest_pre, salt_buf, digest); u32x a = digest[0]; u32x b = digest[1] & 0xff; u32x c = 0; u32x d = 0; COMPARE_M_SIMD (a, b, c, d); } } __kernel void m09720_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global oldoffice01_t *oldoffice01_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u64 gid_max) { } __kernel void m09720_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global oldoffice01_t *oldoffice01_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u64 gid_max) { } __kernel void m09720_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global oldoffice01_t *oldoffice01_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u64 gid_max) { /** * modifier */ const u64 lid = get_local_id (0); /** * base */ const u64 gid = get_global_id (0); if (gid >= gid_max) return; u32 pw_buf0[4]; u32 pw_buf1[4]; pw_buf0[0] = pws[gid].i[ 0]; pw_buf0[1] = pws[gid].i[ 1]; pw_buf0[2] = pws[gid].i[ 2]; pw_buf0[3] = pws[gid].i[ 3]; pw_buf1[0] = pws[gid].i[ 4]; pw_buf1[1] = pws[gid].i[ 5]; pw_buf1[2] = pws[gid].i[ 6]; pw_buf1[3] = pws[gid].i[ 7]; const u32 pw_len = pws[gid].pw_len; /** * salt */ u32 salt_buf[4]; salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; salt_buf[2] = salt_bufs[salt_pos].salt_buf[2]; salt_buf[3] = salt_bufs[salt_pos].salt_buf[3]; /** * digest */ const u32 search[4] = { digests_buf[digests_offset].digest_buf[DGST_R0], digests_buf[digests_offset].digest_buf[DGST_R1], digests_buf[digests_offset].digest_buf[DGST_R2], digests_buf[digests_offset].digest_buf[DGST_R3] }; /** * loop */ for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { u32x w0[4] = { 0 }; u32x w1[4] = { 0 }; u32x w2[4] = { 0 }; u32x w3[4] = { 0 }; const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); append_0x80_2x4_VV (w0, w1, out_len); /** * md5 */ make_utf16le (w1, w2, w3); make_utf16le (w0, w0, w1); w3[2] = out_len * 8 * 2; w3[3] = 0; u32x digest_pre[4]; digest_pre[0] = MD5M_A; digest_pre[1] = MD5M_B; digest_pre[2] = MD5M_C; digest_pre[3] = MD5M_D; md5_transform_vector (w0, w1, w2, w3, digest_pre); digest_pre[0] &= 0xffffffff; digest_pre[1] &= 0x000000ff; digest_pre[2] &= 0x00000000; digest_pre[3] &= 0x00000000; u32x digest[4]; digest[0] = MD5M_A; digest[1] = MD5M_B; digest[2] = MD5M_C; digest[3] = MD5M_D; gen336 (digest_pre, salt_buf, digest); u32x a = digest[0]; u32x b = digest[1] & 0xff; u32x c = 0; u32x d = 0; COMPARE_S_SIMD (a, b, c, d); } } __kernel void m09720_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global oldoffice01_t *oldoffice01_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u64 gid_max) { } __kernel void m09720_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global oldoffice01_t *oldoffice01_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u64 gid_max) { }