Merge pull request #4347 from matrix/perl_TextIconv

get rid of Text:Iconv, handle 31500 and 31600 in test_edge.sh

docs/changes.txt

@@ -168,6 +168,7 @@
 - Status Code: Add specific return code for self-test fail (-11)
 - Scrypt: Increase buffer sizes in module for hash mode 8900 to allow longer scrypt digests
 - Unicode: Update UTF-8 to UTF-16 conversion to match RFC 3629
+- Unit tests: Updated install_modules.sh, test.pl and some test modules (1000, 16000, 31300, 31500, 31600) by removing Text:Iconv and using Encode instead
 - Unit tests: Updated install_modules.sh with Crypt::Argon2
 - Unit tests: Updated install_modules.sh with Crypt::Passwd::XS, to test suite works also on Apple (ex: 1800)
 - Unit tests: Updated install_modules.sh to use an external module for Digest::BLAKE2, due to maintenance being discontinued

tools/install_modules.sh

@@ -94,7 +94,6 @@ cpanm Authen::Passphrase::LANManager   \
       Net::DNS::RR::NSEC3              \
       Net::DNS::SEC                    \
       POSIX                            \
-      Text::Iconv                      \
       ;
 
 ERRORS=$((ERRORS+$?))

tools/test.pl

@@ -12,7 +12,6 @@ use Data::Types qw (is_count is_whole);
 use File::Basename;
 use FindBin;
 use List::Util 'shuffle';
-use Text::Iconv;
 use Digest::MD4 qw (md4_hex);
 
 # allows require by filename
@@ -475,9 +474,9 @@ sub single
     {
       if ($MODE == 31600 || $MODE == 31500)
       {
-        my $converter = Text::Iconv->new('utf8', 'UTF-16LE');
+        my $utf16le = encode("UTF-16LE", decode("utf-8", $word));
 
-        $word = md4_hex ($converter->convert ($word));
+        $word = md4_hex ($utf16le);
       }
 
       my $hash = module_generate_hash ($word, $salt);

tools/test_edge.sh

@@ -101,7 +101,7 @@ SKIP_HASH_TYPES_METAL="21800"
 
 METAL_FORCE_KEEPFREE="8900 22700 27700 28200 29800"
 
-SKIP_OUT_MATCH_HASH_TYPES="14000 14100 22000"
+SKIP_OUT_MATCH_HASH_TYPES="14000 14100 22000 31500 31600"
 SKIP_SAME_SALT_HASH_TYPES="6600 7100 7200 8200 13200 13400 15300 15310 15900 15910 16900 18300 18900 20200 20300 20400 27000 27100 29700 29930 29940"
 #SKIP_SAME_SALT_HASH_TYPES="400 3200 5800 6400 6500 6600 6700 7100 7200 7401 7900 8200 9100 9200 9400 10500 10901 12001 12200 12300 12400 12500 12700 12800 12900 13000 13200 13400 13600 14700 14800 15100 15200 15300 15310 15400 15600 15900 15910 16200 16300 16700 16900 18300 18400 18800 18900 19000 19100 19600 19700 19800 19900 20011 20012 20013 20200 20300 20400 21501 22100 22400 22600 23100 23300 23500 23600 23700 23900 24100 24200 24410 24420 24500 25300 25400 25500 25600 25800 26100 26500 26600 27000 27100 27400 27500 27600 28100 28400 28600 28800 28900 29600 29700 29910 29920 29930 29940 30600 31200 31900"
 
@@ -642,6 +642,11 @@ for hash_type in $(ls tools/test_modules/*.pm | cut -d'm' -f3 | cut -d'.' -f1 |
         pt_base58=1
       fi
 
+      if [ $hash_type -eq 31500 ] || [ $hash_type -eq 31600 ]; then
+        # using ?a instead of ?d with masks
+        pt_base58=1
+      fi
+
       echo "[ ${OUTD} ] # Export tests for Hash-Type ${hash_type}, Attack-Type ${attack_type}, Kernel-Type ${kernel_type}" >> ${OUTD}/test_edge.details.log
 
       edge_out="${OUTD}/edge_${hash_type}_${kernel_type}_${attack_type}.out"

tools/test_modules/m01000.pm

@@ -9,7 +9,7 @@ use strict;
 use warnings;
 
 use Digest::MD4 qw (md4_hex);
-use Text::Iconv;
+use Encode;
 
 sub module_constraints { [[0, 256], [-1, -1], [0, 27], [-1, -1], [-1, -1]] }
 
@@ -17,9 +17,9 @@ sub module_generate_hash
 {
   my $word = shift;
 
-  my $converter = Text::Iconv->new('utf8', 'UTF-16LE');
+  my $utf16le = encode("UTF-16LE", decode("utf-8", $word));
 
-  my $digest = md4_hex ($converter->convert ($word));
+  my $digest = md4_hex ($utf16le);
 
   return $digest;
 }

tools/test_modules/m16000.pm

@@ -8,7 +8,7 @@
 use strict;
 use warnings;
 
-use Text::Iconv;
+use Encode;
 
 sub module_constraints { [[1, 8], [-1, -1], [-1, -1], [-1, -1], [-1, -1]] }
 
@@ -16,9 +16,7 @@ sub module_generate_hash
 {
   my $word = shift;
 
-  my $converter = Text::Iconv->new ("utf-8", "shift-jis");
-
-  $word = $converter->convert ($word);
+  $word = encode("shift_jis", $word);
 
   my $salt = substr ($word . '..', 1, 2);
 

tools/test_modules/m31300.pm

@@ -10,7 +10,7 @@ use warnings;
 
 use Digest::MD4 qw (md4 md4_hex);
 use Digest::MD5 qw (md5 md5_hex);
-use Text::Iconv;
+use Encode;
 
 sub module_constraints { [[0, 256], [96, 96], [0, 27], [96, 96], [-1, -1]] }
 
@@ -21,9 +21,9 @@ sub module_generate_hash
 
   my $salt_bin = pack ("H*", $salt);
 
-  my $converter = Text::Iconv->new('utf8', 'UTF-16LE');
+  my $utf16le = encode("UTF-16LE", $word);
 
-  my $digest = md5_hex (md4 ($converter->convert ($word)) . $salt_bin);
+  my $digest = md5_hex (md4 ($utf16le) . $salt_bin);
 
   my $hash = sprintf ('$sntp-ms$%s$%s', $digest, unpack ("H*", $salt_bin));
 

tools/test_modules/m31500.pm

@@ -9,7 +9,6 @@ use strict;
 use warnings;
 
 use Digest::MD4 qw (md4_hex);
-use Text::Iconv;
 use Encode;
 
 sub module_constraints { [[32, 32], [0, 256], [-1, -1], [-1, -1], [-1, -1]] }
@@ -18,9 +17,9 @@ sub module_get_random_password
 {
   my $word = shift;
 
-  my $converter = Text::Iconv->new('utf8', 'UTF-16LE');
+  my $utf16le = encode("UTF-16LE", $word);
 
-  $word = md4_hex ($converter->convert ($word));
+  $word = md4_hex ($utf16le);
 
   return $word;
 }

tools/test_modules/m31600.pm

@@ -10,7 +10,6 @@ use warnings;
 
 use Digest::MD4 qw (md4 md4_hex);
 use Crypt::PBKDF2;
-use Text::Iconv;
 use Encode;
 
 sub module_constraints { [[32, 32], [0, 256], [-1, -1], [-1, -1], [-1, -1]] }
@@ -19,9 +18,9 @@ sub module_get_random_password
 {
   my $word = shift;
 
-  my $converter = Text::Iconv->new('utf8', 'UTF-16LE');
+  my $utf16le = encode("UTF-16LE", $word);
 
-  $word = md4_hex ($converter->convert ($word));
+  $word = md4_hex ($utf16le);
 
   return $word;
 }