update repo and re-apply zip patch

.gitignore

@@ -18,6 +18,7 @@ kernels/**
 lib/*.a
 modules/*.dll
 modules/*.so
+obj/*/*/*.o
 obj/*.o
 obj/*.a
 include/CL

docs/changes.txt

@@ -7,6 +7,7 @@
 - Fully modularized hash-mode integration via plugin interface and converted all existing hash-modes
 - Support for inline VeraCrypt PIM Brute-Force
 - Support Deflate decompression for the 7-Zip hash type using zlib
+- Compressed wordlists, gzip and zip format, using zlib
 
 ##
 ## Algorithms

docs/credits.txt

@@ -18,7 +18,7 @@ Philipp "philsmd" Schmidt <philsmd@hashcat.net> (@philsmd)
 
 Gabriele "matrix" Gristina <matrix@hashcat.net> (@gm4tr1x)
 
-* gzip wordlists feature
+* Compressed wordlists feature (gzip/zip)
 * SHA-224 kernel module + optimizations
 * Some AES OpenCL kernel module optimizations
 * OpenCL Info feature

include/ext_lzma.h

@@ -5,10 +5,12 @@
 
 #ifndef _EXT_LZMA_H
 
-#include <Alloc.h>
 #include <LzmaDec.h>
 #include <Lzma2Dec.h>
 
+#include "contrib/minizip/ioapi.h"
+#include "contrib/minizip/unzip.h"
+
 int hc_lzma1_decompress (const unsigned char *in, SizeT *in_len, unsigned char *out, SizeT *out_len, const char *props);
 int hc_lzma2_decompress (const unsigned char *in, SizeT *in_len, unsigned char *out, SizeT *out_len, const char *props);
 

include/types.h

@@ -17,7 +17,18 @@
 #include <sys/time.h>
 #include <unistd.h>
 #include <math.h>
+
+// workaround to get the rid of "redefinition of typedef 'Byte'" build warning
+#if !defined (__APPLE__)
 #include "zlib.h"
+#endif
+
+#if !defined(__MACTYPES__)
+#define __MACTYPES__
+#include "ext_lzma.h"
+#undef __MACTYPES__
+#endif
+// end of workaround
 
 #if defined (_WIN)
 #define WINICONV_CONST
@@ -989,15 +1000,19 @@ typedef struct link_speed
 
 } link_speed_t;
 
-// handling gzip files
+// file handling
 
 typedef struct hc_fp
 {
   int         fd;
-  FILE       *pfp;
+
-  gzFile      gfp;
+  FILE       *pfp; // plain fp
+  gzFile      gfp; //  gzip fp
+  unzFile     ufp; //   zip fp
 
   bool        is_gzip;
+  bool        is_zip;
+
   char       *mode;
   const char *path;
 } HCFILE;

src/Makefile

@@ -180,6 +180,8 @@ endif
 ifeq ($(USE_SYSTEM_ZLIB),0)
 CFLAGS_ZLIB             += -Wno-implicit-fallthrough
 CFLAGS_ZLIB             += -Wno-implicit-function-declaration
+CFLAGS_ZLIB             += -Wno-unused-parameter
+CFLAGS_ZLIB             += -DIOAPI_NO_64
 endif
 
 ifeq ($(DEBUG),0)
@@ -319,7 +321,7 @@ WIN_OBJS                += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).WIN.o)
 endif
 
 ifeq ($(USE_SYSTEM_ZLIB),0)
-OBJS_ZLIB               := adler32 crc32 deflate inflate inffast inftrees trees gzread gzwrite gzclose zutil gzlib
+OBJS_ZLIB               := adler32 crc32 deflate inflate inffast inftrees trees gzread gzwrite gzclose zutil gzlib contrib/minizip/unzip contrib/minizip/ioapi
 
 NATIVE_OBJS             += $(foreach OBJ,$(OBJS_ZLIB),obj/$(OBJ).NATIVE.o)
 LINUX_OBJS              += $(foreach OBJ,$(OBJS_ZLIB),obj/$(OBJ).LINUX.o)
@@ -348,6 +350,7 @@ clean:
 	$(RM) -rf modules/*.dSYM
 	$(RM) -f modules/*.dll
 	$(RM) -f modules/*.so
+	$(RM) -f obj/*/*/*.o
 	$(RM) -f obj/*.o
 	$(RM) -f obj/*.a
 	$(RM) -f *.bin *.exe

src/filehandling.c

@@ -11,7 +11,7 @@
 
 #if defined (__CYGWIN__)
 // workaround for zlib with cygwin build
-int _wopen(const char *path, int oflag, ...)
+int _wopen (const char *path, int oflag, ...)
 {
   va_list ap;
   va_start (ap, oflag);
@@ -62,8 +62,9 @@ bool hc_fopen (HCFILE *fp, const char *path, char *mode)
 
   fp->pfp = NULL;
   fp->is_gzip = false;
+  fp->is_zip = false;
 
-  unsigned char check[3] = { 0 };
+  unsigned char check[4] = { 0 };
 
   int fd_tmp = open (path, O_RDONLY);
 
@@ -73,7 +74,8 @@ bool hc_fopen (HCFILE *fp, const char *path, char *mode)
 
     if (read (fd_tmp, check, sizeof(check)) > 0)
     {
-      if (check[0] == 0x1f && check[1] == 0x8b && check[2] == 0x08) fp->is_gzip = true;
+      if (check[0] == 0x1f && check[1] == 0x8b && check[2] == 0x08 && check[3] == 0x08) fp->is_gzip = true;
+      if (check[0] == 0x50 && check[1] == 0x4b && check[2] == 0x03 && check[3] == 0x04) fp->is_zip = true;
     }
 
     close (fd_tmp);
@@ -88,12 +90,18 @@ bool hc_fopen (HCFILE *fp, const char *path, char *mode)
     fp->fd = open (path, oflag, fmode);
   }
 
-  if (fp->fd == -1) return false;
+  if (fp->fd == -1 && fp->is_zip == false) return false;
 
   if (fp->is_gzip)
   {
     if ((fp->gfp = gzdopen (fp->fd, mode)) == NULL) return false;
   }
+  else if (fp->is_zip)
+  {
+    if ((fp->ufp = unzOpen64 (path)) == NULL) return false;
+
+    if (unzOpenCurrentFile (fp->ufp) != UNZ_OK) return false;
+  }
   else
   {
     if ((fp->pfp = fdopen (fp->fd, mode)) == NULL)  return false;
@@ -115,6 +123,12 @@ size_t hc_fread (void *ptr, size_t size, size_t nmemb, HCFILE *fp)
   {
     n = gzfread (ptr, size, nmemb, fp->gfp);
   }
+  else if (fp->is_zip)
+  {
+    unsigned s = size * nmemb;
+
+    n = unzReadCurrentFile (fp->ufp, ptr, s);
+  }
   else
   {
     n = fread (ptr, size, nmemb, fp->pfp);
@@ -133,6 +147,9 @@ size_t hc_fwrite (void *ptr, size_t size, size_t nmemb, HCFILE *fp)
   {
     n = gzfwrite (ptr, size, nmemb, fp->gfp);
   }
+  else if (fp->is_zip)
+  {
+  }
   else
   {
     n = fwrite (ptr, size, nmemb, fp->pfp);
@@ -153,6 +170,27 @@ int hc_fseek (HCFILE *fp, off_t offset, int whence)
   {
     r = gzseek (fp->gfp, (z_off_t) offset, whence);
   }
+  else if (fp->is_zip)
+  {
+    /*
+    // untested and not used in wordlist engine
+    zlib_filefunc64_32_def *d = NULL;
+    if (whence == SEEK_SET)
+    {
+      r = ZSEEK64(*d, fp->ufp, offset, ZLIB_FILEFUNC_SEEK_SET);
+    }
+    else if (whence == SEEK_CUR)
+    {
+      r = ZSEEK64(*d, fp->ufp, offset, ZLIB_FILEFUNC_SEEK_CUR);
+    }
+    else if (whence == SEEK_END)
+    {
+      r = ZSEEK64(*d, fp->ufp, offset, ZLIB_FILEFUNC_SEEK_END);
+    }
+    // or
+    // r = unzSetOffset (fp->ufp, offset);
+    */
+  }
   else
   {
     r = fseeko (fp->pfp, offset, whence);
@@ -169,6 +207,10 @@ void hc_rewind (HCFILE *fp)
   {
     gzrewind (fp->gfp);
   }
+  else if (fp->is_zip)
+  {
+    unzGoToFirstFile (fp->ufp);
+  }
   else
   {
     rewind (fp->pfp);
@@ -185,6 +227,10 @@ off_t hc_ftell (HCFILE *fp)
   {
     n = (off_t) gztell (fp->gfp);
   }
+  else if (fp->is_zip)
+  {
+    n = unztell (fp->ufp);
+  }
   else
   {
     n = ftello (fp->pfp);
@@ -203,6 +249,9 @@ int hc_fputc (int c, HCFILE *fp)
   {
     r = gzputc (fp->gfp, c);
   }
+  else if (fp->is_zip)
+  {
+  }
   else
   {
     r = fputc (c, fp->pfp);
@@ -221,6 +270,12 @@ int hc_fgetc (HCFILE *fp)
   {
     r = gzgetc (fp->gfp);
   }
+  else if (fp->is_zip)
+  {
+    unsigned char c = 0;
+
+    if (unzReadCurrentFile (fp->ufp, &c, 1) == 1) r = (int) c;
+  }
   else
   {
     r = fgetc (fp->pfp);
@@ -239,6 +294,10 @@ char *hc_fgets (char *buf, int len, HCFILE *fp)
   {
     r = gzgets (fp->gfp, buf, len);
   }
+  else if (fp->is_zip)
+  {
+    if (unzReadCurrentFile (fp->ufp, buf, len) > 0) r = buf;
+  }
   else
   {
     r = fgets (buf, len, fp->pfp);
@@ -257,6 +316,9 @@ int hc_vfprintf (HCFILE *fp, const char *format, va_list ap)
   {
     r = gzvprintf (fp->gfp, format, ap);
   }
+  else if (fp->is_zip)
+  {
+  }
   else
   {
     r = vfprintf (fp->pfp, format, ap);
@@ -279,6 +341,9 @@ int hc_fprintf (HCFILE *fp, const char *format, ...)
   {
     r = gzvprintf (fp->gfp, format, ap);
   }
+  else if (fp->is_zip)
+  {
+  }
   else
   {
     r = vfprintf (fp->pfp, format, ap);
@@ -330,6 +395,10 @@ int hc_feof (HCFILE *fp)
   {
     r = gzeof (fp->gfp);
   }
+  else if (fp->is_zip)
+  {
+    r = unzeof (fp->ufp);
+  }
   else
   {
     r = feof (fp->pfp);
@@ -346,6 +415,9 @@ void hc_fflush (HCFILE *fp)
   {
     gzflush (fp->gfp, Z_SYNC_FLUSH);
   }
+  else if (fp->is_zip)
+  {
+  }
   else
   {
     fflush (fp->pfp);
@@ -360,6 +432,12 @@ void hc_fclose (HCFILE *fp)
   {
     gzclose (fp->gfp);
   }
+  else if (fp->is_zip)
+  {
+    unzCloseCurrentFile (fp->ufp);
+
+    unzClose (fp->ufp);
+  }
   else
   {
     fclose (fp->pfp);
@@ -370,6 +448,7 @@ void hc_fclose (HCFILE *fp)
   fp->fd = -1;
   fp->pfp = NULL;
   fp->is_gzip = false;
+  fp->is_zip = false;
 
   fp->path = NULL;
   fp->mode = NULL;

src/shared.c

@@ -8,6 +8,7 @@
 #include "convert.h"
 #include "shared.h"
 #include "memory.h"
+#include "ext_lzma.h"
 #include <errno.h>
 
 #if defined (__CYGWIN__)