2015-12-04 14:47:52 +00:00
|
|
|
/**
|
2016-09-11 20:20:15 +00:00
|
|
|
* Author......: See docs/credits.txt
|
2015-12-04 14:47:52 +00:00
|
|
|
* License.....: MIT
|
|
|
|
*/
|
|
|
|
|
2019-03-22 14:16:25 +00:00
|
|
|
#ifdef KERNEL_STATIC
|
2019-03-21 22:00:38 +00:00
|
|
|
#include "inc_vendor.h"
|
|
|
|
#include "inc_types.h"
|
2016-05-25 21:04:26 +00:00
|
|
|
#include "inc_common.cl"
|
2017-07-18 11:23:42 +00:00
|
|
|
#include "inc_hash_sha256.cl"
|
2019-03-22 14:16:25 +00:00
|
|
|
#endif
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2016-05-25 21:04:26 +00:00
|
|
|
#define COMPARE_S "inc_comp_single.cl"
|
|
|
|
#define COMPARE_M "inc_comp_multi.cl"
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2019-03-08 10:14:33 +00:00
|
|
|
typedef struct sha256crypt_tmp
|
|
|
|
{
|
|
|
|
// pure version
|
|
|
|
|
|
|
|
u32 alt_result[8];
|
|
|
|
u32 p_bytes[64];
|
|
|
|
u32 s_bytes[64];
|
|
|
|
|
|
|
|
} sha256crypt_tmp_t;
|
|
|
|
|
2019-03-22 21:27:58 +00:00
|
|
|
KERNEL_FQ void m07400_init (KERN_ATTR_TMPS (sha256crypt_tmp_t))
|
2015-12-04 14:47:52 +00:00
|
|
|
{
|
|
|
|
/**
|
|
|
|
* base
|
|
|
|
*/
|
|
|
|
|
2017-08-19 14:39:22 +00:00
|
|
|
const u64 gid = get_global_id (0);
|
2015-12-04 14:47:52 +00:00
|
|
|
|
|
|
|
if (gid >= gid_max) return;
|
|
|
|
|
|
|
|
/**
|
2017-07-18 11:23:42 +00:00
|
|
|
* init
|
2015-12-04 14:47:52 +00:00
|
|
|
*/
|
|
|
|
|
2019-02-26 20:20:07 +00:00
|
|
|
const u32 pw_len = pws[gid].pw_len;
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
u32 w[64] = { 0 };
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-08-17 11:43:35 +00:00
|
|
|
for (int i = 0, idx = 0; i < pw_len; i += 4, idx += 1)
|
2015-12-04 14:47:52 +00:00
|
|
|
{
|
2017-07-18 11:23:42 +00:00
|
|
|
w[idx] = pws[gid].i[idx];
|
2015-12-04 14:47:52 +00:00
|
|
|
}
|
|
|
|
|
2017-08-17 11:43:35 +00:00
|
|
|
for (int i = 0, idx = 0; i < pw_len; i += 4, idx += 1)
|
2015-12-04 14:47:52 +00:00
|
|
|
{
|
2019-03-23 21:15:38 +00:00
|
|
|
w[idx] = hc_swap32_S (w[idx]);
|
2015-12-04 14:47:52 +00:00
|
|
|
}
|
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
const u32 salt_len = salt_bufs[salt_pos].salt_len;
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
u32 s[64] = { 0 };
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-08-17 11:43:35 +00:00
|
|
|
for (int i = 0, idx = 0; i < salt_len; i += 4, idx += 1)
|
2015-12-04 14:47:52 +00:00
|
|
|
{
|
2017-07-18 11:23:42 +00:00
|
|
|
s[idx] = salt_bufs[salt_pos].salt_buf[idx];
|
2015-12-04 14:47:52 +00:00
|
|
|
}
|
|
|
|
|
2017-08-17 11:43:35 +00:00
|
|
|
for (int i = 0, idx = 0; i < salt_len; i += 4, idx += 1)
|
2015-12-04 14:47:52 +00:00
|
|
|
{
|
2019-03-23 21:15:38 +00:00
|
|
|
s[idx] = hc_swap32_S (s[idx]);
|
2015-12-04 14:47:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2017-07-18 11:23:42 +00:00
|
|
|
* prepare
|
2015-12-04 14:47:52 +00:00
|
|
|
*/
|
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_ctx_t ctx;
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_init (&ctx);
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_update (&ctx, w, pw_len);
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_update (&ctx, s, salt_len);
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_update (&ctx, w, pw_len);
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_final (&ctx);
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
u32 final[16] = { 0 };
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
final[0] = ctx.h[0];
|
|
|
|
final[1] = ctx.h[1];
|
|
|
|
final[2] = ctx.h[2];
|
|
|
|
final[3] = ctx.h[3];
|
|
|
|
final[4] = ctx.h[4];
|
|
|
|
final[5] = ctx.h[5];
|
|
|
|
final[6] = ctx.h[6];
|
|
|
|
final[7] = ctx.h[7];
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
// alt_result
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_init (&ctx);
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_update (&ctx, w, pw_len);
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_update (&ctx, s, salt_len);
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
int pl;
|
2015-12-04 14:47:52 +00:00
|
|
|
|
2017-08-16 11:46:40 +00:00
|
|
|
for (pl = pw_len; pl > 32; pl -= 32)
|
2015-12-04 14:47:52 +00:00
|
|
|
{
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_update (&ctx, final, 32);
|
2015-12-04 14:47:52 +00:00
|
|
|
}
|
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
u32 t_final[16] = { 0 };
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2016-05-09 19:32:12 +00:00
|
|
|
#ifdef _unroll
|
|
|
|
#pragma unroll
|
|
|
|
#endif
|
2017-07-18 11:23:42 +00:00
|
|
|
for (int i = 0; i < 8; i++) t_final[i] = final[i];
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-08-24 18:07:43 +00:00
|
|
|
truncate_block_16x4_be_S (t_final + 0, t_final + 4, t_final + 8, t_final + 12, pl);
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_update (&ctx, t_final, pl);
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
for (int cnt = pw_len; cnt > 0; cnt >>= 1)
|
2016-05-08 20:56:36 +00:00
|
|
|
{
|
2017-07-18 11:23:42 +00:00
|
|
|
if ((cnt & 1) != 0)
|
2016-05-08 20:56:36 +00:00
|
|
|
{
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_update (&ctx, final, 32);
|
2016-05-08 20:56:36 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_update (&ctx, w, pw_len);
|
2016-05-08 20:56:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_final (&ctx);
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
tmps[gid].alt_result[0] = ctx.h[0];
|
|
|
|
tmps[gid].alt_result[1] = ctx.h[1];
|
|
|
|
tmps[gid].alt_result[2] = ctx.h[2];
|
|
|
|
tmps[gid].alt_result[3] = ctx.h[3];
|
|
|
|
tmps[gid].alt_result[4] = ctx.h[4];
|
|
|
|
tmps[gid].alt_result[5] = ctx.h[5];
|
|
|
|
tmps[gid].alt_result[6] = ctx.h[6];
|
|
|
|
tmps[gid].alt_result[7] = ctx.h[7];
|
2016-05-08 20:56:36 +00:00
|
|
|
|
|
|
|
// p_bytes
|
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_init (&ctx);
|
2016-05-08 20:56:36 +00:00
|
|
|
|
|
|
|
for (u32 j = 0; j < pw_len; j++)
|
|
|
|
{
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_update (&ctx, w, pw_len);
|
2016-05-08 20:56:36 +00:00
|
|
|
}
|
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_final (&ctx);
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
final[ 0] = ctx.h[0];
|
|
|
|
final[ 1] = ctx.h[1];
|
|
|
|
final[ 2] = ctx.h[2];
|
|
|
|
final[ 3] = ctx.h[3];
|
|
|
|
final[ 4] = ctx.h[4];
|
|
|
|
final[ 5] = ctx.h[5];
|
|
|
|
final[ 6] = ctx.h[6];
|
|
|
|
final[ 7] = ctx.h[7];
|
|
|
|
final[ 8] = 0;
|
|
|
|
final[ 9] = 0;
|
|
|
|
final[10] = 0;
|
|
|
|
final[11] = 0;
|
|
|
|
final[12] = 0;
|
|
|
|
final[13] = 0;
|
|
|
|
final[14] = 0;
|
|
|
|
final[15] = 0;
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
u32 p_final[64] = { 0 };
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
int idx;
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-08-16 11:46:40 +00:00
|
|
|
for (pl = pw_len, idx = 0; pl > 32; pl -= 32, idx += 8)
|
2016-05-08 20:56:36 +00:00
|
|
|
{
|
2017-07-18 11:23:42 +00:00
|
|
|
p_final[idx + 0] = final[0];
|
|
|
|
p_final[idx + 1] = final[1];
|
|
|
|
p_final[idx + 2] = final[2];
|
|
|
|
p_final[idx + 3] = final[3];
|
|
|
|
p_final[idx + 4] = final[4];
|
|
|
|
p_final[idx + 5] = final[5];
|
|
|
|
p_final[idx + 6] = final[6];
|
|
|
|
p_final[idx + 7] = final[7];
|
|
|
|
}
|
|
|
|
|
2017-08-24 18:07:43 +00:00
|
|
|
truncate_block_16x4_be_S (final + 0, final + 4, final + 8, final + 12, pl);
|
2017-07-18 11:23:42 +00:00
|
|
|
|
|
|
|
p_final[idx + 0] = final[0];
|
|
|
|
p_final[idx + 1] = final[1];
|
|
|
|
p_final[idx + 2] = final[2];
|
|
|
|
p_final[idx + 3] = final[3];
|
|
|
|
p_final[idx + 4] = final[4];
|
|
|
|
p_final[idx + 5] = final[5];
|
|
|
|
p_final[idx + 6] = final[6];
|
|
|
|
p_final[idx + 7] = final[7];
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
#ifdef _unroll
|
|
|
|
#pragma unroll
|
|
|
|
#endif
|
|
|
|
for (int i = 0; i < 64; i++) tmps[gid].p_bytes[i] = p_final[i];
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
// s_bytes
|
|
|
|
|
|
|
|
sha256_init (&ctx);
|
|
|
|
|
|
|
|
for (u32 j = 0; j < 16 + (tmps[gid].alt_result[0] >> 24); j++)
|
|
|
|
{
|
|
|
|
sha256_update (&ctx, s, salt_len);
|
|
|
|
}
|
|
|
|
|
|
|
|
sha256_final (&ctx);
|
|
|
|
|
|
|
|
final[ 0] = ctx.h[0];
|
|
|
|
final[ 1] = ctx.h[1];
|
|
|
|
final[ 2] = ctx.h[2];
|
|
|
|
final[ 3] = ctx.h[3];
|
|
|
|
final[ 4] = ctx.h[4];
|
|
|
|
final[ 5] = ctx.h[5];
|
|
|
|
final[ 6] = ctx.h[6];
|
|
|
|
final[ 7] = ctx.h[7];
|
|
|
|
final[ 8] = 0;
|
|
|
|
final[ 9] = 0;
|
|
|
|
final[10] = 0;
|
|
|
|
final[11] = 0;
|
|
|
|
final[12] = 0;
|
|
|
|
final[13] = 0;
|
|
|
|
final[14] = 0;
|
|
|
|
final[15] = 0;
|
|
|
|
|
|
|
|
u32 s_final[64] = { 0 };
|
|
|
|
|
2017-08-16 11:46:40 +00:00
|
|
|
for (pl = salt_len, idx = 0; pl > 32; pl -= 32, idx += 8)
|
2017-07-18 11:23:42 +00:00
|
|
|
{
|
|
|
|
s_final[idx + 0] = final[0];
|
|
|
|
s_final[idx + 1] = final[1];
|
|
|
|
s_final[idx + 2] = final[2];
|
|
|
|
s_final[idx + 3] = final[3];
|
|
|
|
s_final[idx + 4] = final[4];
|
|
|
|
s_final[idx + 5] = final[5];
|
|
|
|
s_final[idx + 6] = final[6];
|
|
|
|
s_final[idx + 7] = final[7];
|
|
|
|
}
|
|
|
|
|
2017-08-24 18:07:43 +00:00
|
|
|
truncate_block_16x4_be_S (final + 0, final + 4, final + 8, final + 12, pl);
|
2017-07-18 11:23:42 +00:00
|
|
|
|
|
|
|
s_final[idx + 0] = final[0];
|
|
|
|
s_final[idx + 1] = final[1];
|
|
|
|
s_final[idx + 2] = final[2];
|
|
|
|
s_final[idx + 3] = final[3];
|
|
|
|
s_final[idx + 4] = final[4];
|
|
|
|
s_final[idx + 5] = final[5];
|
|
|
|
s_final[idx + 6] = final[6];
|
|
|
|
s_final[idx + 7] = final[7];
|
|
|
|
|
|
|
|
#ifdef _unroll
|
|
|
|
#pragma unroll
|
|
|
|
#endif
|
|
|
|
for (int i = 0; i < 64; i++) tmps[gid].s_bytes[i] = s_final[i];
|
2016-05-08 20:56:36 +00:00
|
|
|
}
|
|
|
|
|
2019-03-22 21:27:58 +00:00
|
|
|
KERNEL_FQ void m07400_loop (KERN_ATTR_TMPS (sha256crypt_tmp_t))
|
2016-05-08 20:56:36 +00:00
|
|
|
{
|
|
|
|
/**
|
|
|
|
* base
|
|
|
|
*/
|
|
|
|
|
2017-08-19 14:39:22 +00:00
|
|
|
const u64 gid = get_global_id (0);
|
2016-05-08 20:56:36 +00:00
|
|
|
|
|
|
|
if (gid >= gid_max) return;
|
|
|
|
|
2019-02-26 20:20:07 +00:00
|
|
|
const u32 pw_len = pws[gid].pw_len;
|
2016-05-08 20:56:36 +00:00
|
|
|
|
|
|
|
const u32 salt_len = salt_bufs[salt_pos].salt_len;
|
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
u32 alt_result[16] = { 0 };
|
2016-05-08 20:56:36 +00:00
|
|
|
|
|
|
|
alt_result[0] = tmps[gid].alt_result[0];
|
|
|
|
alt_result[1] = tmps[gid].alt_result[1];
|
|
|
|
alt_result[2] = tmps[gid].alt_result[2];
|
|
|
|
alt_result[3] = tmps[gid].alt_result[3];
|
|
|
|
alt_result[4] = tmps[gid].alt_result[4];
|
|
|
|
alt_result[5] = tmps[gid].alt_result[5];
|
|
|
|
alt_result[6] = tmps[gid].alt_result[6];
|
|
|
|
alt_result[7] = tmps[gid].alt_result[7];
|
|
|
|
|
2017-07-18 11:23:42 +00:00
|
|
|
/* Repeatedly run the collected hash value through sha256 to burn
|
2016-05-08 20:56:36 +00:00
|
|
|
CPU cycles. */
|
|
|
|
|
|
|
|
for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++)
|
|
|
|
{
|
2017-07-18 11:23:42 +00:00
|
|
|
sha256_ctx_t ctx;
|
|
|
|
|
|
|
|
sha256_init (&ctx);
|
|
|
|
|
2017-08-16 11:46:40 +00:00
|
|
|
if (j & 1)
|
2016-05-08 20:56:36 +00:00
|
|
|
{
|
2017-08-16 11:46:40 +00:00
|
|
|
sha256_update_global (&ctx, tmps[gid].p_bytes, pw_len);
|
2016-05-08 20:56:36 +00:00
|
|
|
}
|
2017-08-16 11:46:40 +00:00
|
|
|
else
|
2016-05-08 20:56:36 +00:00
|
|
|
{
|
2017-08-16 11:46:40 +00:00
|
|
|
sha256_update (&ctx, alt_result, 32);
|
2016-05-08 20:56:36 +00:00
|
|
|
}
|
|
|
|
|
2017-08-16 11:46:40 +00:00
|
|
|
if (j % 3)
|
2017-07-18 11:23:42 +00:00
|
|
|
{
|
2017-08-16 11:46:40 +00:00
|
|
|
sha256_update_global (&ctx, tmps[gid].s_bytes, salt_len);
|
2017-07-18 11:23:42 +00:00
|
|
|
}
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-08-16 11:46:40 +00:00
|
|
|
if (j % 7)
|
2017-07-18 11:23:42 +00:00
|
|
|
{
|
2017-08-16 11:46:40 +00:00
|
|
|
sha256_update_global (&ctx, tmps[gid].p_bytes, pw_len);
|
2017-07-18 11:23:42 +00:00
|
|
|
}
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2017-08-16 11:46:40 +00:00
|
|
|
if (j & 1)
|
2017-07-18 11:23:42 +00:00
|
|
|
{
|
2017-08-16 11:46:40 +00:00
|
|
|
sha256_update (&ctx, alt_result, 32);
|
2017-07-18 11:23:42 +00:00
|
|
|
}
|
2017-08-16 11:46:40 +00:00
|
|
|
else
|
2016-05-08 20:56:36 +00:00
|
|
|
{
|
2017-08-16 11:46:40 +00:00
|
|
|
sha256_update_global (&ctx, tmps[gid].p_bytes, pw_len);
|
2016-05-08 20:56:36 +00:00
|
|
|
}
|
2017-07-18 11:23:42 +00:00
|
|
|
|
|
|
|
sha256_final (&ctx);
|
|
|
|
|
|
|
|
alt_result[0] = ctx.h[0];
|
|
|
|
alt_result[1] = ctx.h[1];
|
|
|
|
alt_result[2] = ctx.h[2];
|
|
|
|
alt_result[3] = ctx.h[3];
|
|
|
|
alt_result[4] = ctx.h[4];
|
|
|
|
alt_result[5] = ctx.h[5];
|
|
|
|
alt_result[6] = ctx.h[6];
|
|
|
|
alt_result[7] = ctx.h[7];
|
2016-05-08 20:56:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
tmps[gid].alt_result[0] = alt_result[0];
|
|
|
|
tmps[gid].alt_result[1] = alt_result[1];
|
|
|
|
tmps[gid].alt_result[2] = alt_result[2];
|
|
|
|
tmps[gid].alt_result[3] = alt_result[3];
|
|
|
|
tmps[gid].alt_result[4] = alt_result[4];
|
|
|
|
tmps[gid].alt_result[5] = alt_result[5];
|
|
|
|
tmps[gid].alt_result[6] = alt_result[6];
|
|
|
|
tmps[gid].alt_result[7] = alt_result[7];
|
|
|
|
}
|
|
|
|
|
2019-03-22 21:27:58 +00:00
|
|
|
KERNEL_FQ void m07400_comp (KERN_ATTR_TMPS (sha256crypt_tmp_t))
|
2016-05-08 20:56:36 +00:00
|
|
|
{
|
|
|
|
/**
|
|
|
|
* base
|
|
|
|
*/
|
|
|
|
|
2017-08-19 14:39:22 +00:00
|
|
|
const u64 gid = get_global_id (0);
|
2016-05-08 20:56:36 +00:00
|
|
|
|
|
|
|
if (gid >= gid_max) return;
|
|
|
|
|
2017-08-19 14:39:22 +00:00
|
|
|
const u64 lid = get_local_id (0);
|
2016-05-08 20:56:36 +00:00
|
|
|
|
2019-03-23 21:15:38 +00:00
|
|
|
const u32 r0 = hc_swap32_S (tmps[gid].alt_result[0]);
|
|
|
|
const u32 r1 = hc_swap32_S (tmps[gid].alt_result[1]);
|
|
|
|
const u32 r2 = hc_swap32_S (tmps[gid].alt_result[2]);
|
|
|
|
const u32 r3 = hc_swap32_S (tmps[gid].alt_result[3]);
|
2016-05-08 20:56:36 +00:00
|
|
|
|
|
|
|
#define il_pos 0
|
|
|
|
|
2019-03-22 14:16:25 +00:00
|
|
|
#ifdef KERNEL_STATIC
|
2016-05-08 20:56:36 +00:00
|
|
|
#include COMPARE_M
|
2019-03-22 14:16:25 +00:00
|
|
|
#endif
|
2016-05-08 20:56:36 +00:00
|
|
|
}
|