From 4e224dec8244736bea9510bbf7c2826dc59cd1a3 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Wed, 25 Aug 2021 16:17:37 +0300 Subject: [PATCH 01/11] Make object file naming unique --- src/Makefile | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/Makefile b/src/Makefile index 066c898af..96503483c 100644 --- a/src/Makefile +++ b/src/Makefile @@ -372,26 +372,26 @@ WIN_OBJS := $(foreach OBJ,$(OBJS_ALL),obj/$(OBJ).WIN.o) ifeq ($(USE_SYSTEM_LZMA),0) OBJS_LZMA := Alloc Lzma2Dec LzmaDec -NATIVE_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).NATIVE.o) -LINUX_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).LINUX.o) -WIN_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).WIN.o) +NATIVE_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).LZMA.NATIVE.o) +LINUX_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).LZMA.LINUX.o) +WIN_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).LZMA.WIN.o) endif ifeq ($(USE_SYSTEM_ZLIB),0) OBJS_ZLIB := adler32 crc32 deflate inflate inffast inftrees trees gzread gzwrite gzclose zutil gzlib contrib/minizip/unzip contrib/minizip/ioapi -NATIVE_OBJS += $(foreach OBJ,$(OBJS_ZLIB),obj/$(OBJ).NATIVE.o) -LINUX_OBJS += $(foreach OBJ,$(OBJS_ZLIB),obj/$(OBJ).LINUX.o) -WIN_OBJS += $(foreach OBJ,$(OBJS_ZLIB),obj/$(OBJ).WIN.o) +NATIVE_OBJS += $(foreach OBJ,$(OBJS_ZLIB),obj/$(OBJ).ZLIB.NATIVE.o) +LINUX_OBJS += $(foreach OBJ,$(OBJS_ZLIB),obj/$(OBJ).ZLIB.LINUX.o) +WIN_OBJS += $(foreach OBJ,$(OBJS_ZLIB),obj/$(OBJ).ZLIB.WIN.o) endif ifeq ($(USE_SYSTEM_XXHASH),0) ifeq ($(ENABLE_BRAIN),1) OBJS_XXHASH := xxhash -NATIVE_OBJS += $(foreach OBJ,$(OBJS_XXHASH),obj/$(OBJ).NATIVE.o) -LINUX_OBJS += $(foreach OBJ,$(OBJS_XXHASH),obj/$(OBJ).LINUX.o) -WIN_OBJS += $(foreach OBJ,$(OBJS_XXHASH),obj/$(OBJ).WIN.o) +NATIVE_OBJS += $(foreach OBJ,$(OBJS_XXHASH),obj/$(OBJ).XXHASH.NATIVE.o) +LINUX_OBJS += $(foreach OBJ,$(OBJS_XXHASH),obj/$(OBJ).XXHASH.LINUX.o) +WIN_OBJS += $(foreach OBJ,$(OBJS_XXHASH),obj/$(OBJ).XXHASH.WIN.o) endif endif @@ -399,9 +399,9 @@ ifeq ($(ENABLE_UNRAR),1) ifeq ($(USE_SYSTEM_UNRAR),0) OBJS_UNRAR := strlist strfn pathfn smallfn global file filefn filcreat archive arcread unicode system isnt crypt crc rawread encname resource match timefn rdwrfn consio options errhnd rarvm secpassword rijndael getbits sha1 sha256 blake2s hash extinfo extract volume list find unpack headers threadpool rs16 cmddata ui filestr recvol rs scantree qopen hc_decompress_rar -NATIVE_OBJS += $(foreach OBJ,$(OBJS_UNRAR),obj/$(OBJ).NATIVE.o) -LINUX_OBJS += $(foreach OBJ,$(OBJS_UNRAR),obj/$(OBJ).LINUX.o) -WIN_OBJS += $(foreach OBJ,$(OBJS_UNRAR),obj/$(OBJ).WIN.o) +NATIVE_OBJS += $(foreach OBJ,$(OBJS_UNRAR),obj/$(OBJ).UNRAR.NATIVE.o) +LINUX_OBJS += $(foreach OBJ,$(OBJS_UNRAR),obj/$(OBJ).UNRAR.LINUX.o) +WIN_OBJS += $(foreach OBJ,$(OBJS_UNRAR),obj/$(OBJ).UNRAR.WIN.o) endif endif @@ -562,25 +562,25 @@ obj/%.NATIVE.o: src/%.c $(CC) -c $(CCFLAGS) $(CFLAGS_NATIVE) $< -o $@ -fpic ifeq ($(USE_SYSTEM_LZMA),0) -obj/%.NATIVE.o: $(DEPS_LZMA_PATH)/%.c +obj/%.LZMA.NATIVE.o: $(DEPS_LZMA_PATH)/%.c $(CC) -c $(CCFLAGS) $(CFLAGS_NATIVE) $< -o $@ -fpic endif ifeq ($(USE_SYSTEM_ZLIB),0) -obj/%.NATIVE.o: $(DEPS_ZLIB_PATH)/%.c +obj/%.ZLIB.NATIVE.o: $(DEPS_ZLIB_PATH)/%.c $(CC) -c $(CCFLAGS) $(CFLAGS_NATIVE) $(CFLAGS_ZLIB) $< -o $@ -fpic endif ifeq ($(USE_SYSTEM_XXHASH),0) ifeq ($(ENABLE_BRAIN),1) -obj/%.NATIVE.o: $(DEPS_XXHASH_PATH)/%.c +obj/%.XXHASH.NATIVE.o: $(DEPS_XXHASH_PATH)/%.c $(CC) -c $(CCFLAGS) $(CFLAGS_NATIVE) $< -o $@ -fpic endif endif ifeq ($(ENABLE_UNRAR),1) ifeq ($(USE_SYSTEM_UNRAR),0) -obj/%.NATIVE.o: $(DEPS_UNRAR_PATH)/%.cpp +obj/%.UNRAR.NATIVE.o: $(DEPS_UNRAR_PATH)/%.cpp $(CXX) -c $(CXXFLAGS) $(CFLAGS_NATIVE) $(CFLAGS_UNRAR) $< -o $@ -fpic endif endif @@ -735,37 +735,37 @@ obj/%.WIN.o: src/%.c $(CC_WIN) $(CCFLAGS) $(CFLAGS_CROSS_WIN) -c -o $@ $< ifeq ($(USE_SYSTEM_LZMA),0) -obj/%.LINUX.o: $(DEPS_LZMA_PATH)/%.c +obj/%.LZMA.LINUX.o: $(DEPS_LZMA_PATH)/%.c $(CC_LINUX) $(CCFLAGS) $(CFLAGS_CROSS_LINUX) -c -o $@ $< -obj/%.WIN.o: $(DEPS_LZMA_PATH)/%.c +obj/%.LZMA.WIN.o: $(DEPS_LZMA_PATH)/%.c $(CC_WIN) $(CCFLAGS) $(CFLAGS_CROSS_WIN) -c -o $@ $< endif ifeq ($(USE_SYSTEM_ZLIB),0) -obj/%.LINUX.o: $(DEPS_ZLIB_PATH)/%.c +obj/%.ZLIB.LINUX.o: $(DEPS_ZLIB_PATH)/%.c $(CC_LINUX) $(CCFLAGS) $(CFLAGS_CROSS_LINUX) $(CFLAGS_ZLIB) -c -o $@ $< -obj/%.WIN.o: $(DEPS_ZLIB_PATH)/%.c +obj/%.ZLIB.WIN.o: $(DEPS_ZLIB_PATH)/%.c $(CC_WIN) $(CCFLAGS) $(CFLAGS_CROSS_WIN) $(CFLAGS_ZLIB) -c -o $@ $< endif ifeq ($(USE_SYSTEM_XXHASH),0) ifeq ($(ENABLE_BRAIN),1) -obj/%.LINUX.o: $(DEPS_XXHASH_PATH)/%.c +obj/%.XXHASH.LINUX.o: $(DEPS_XXHASH_PATH)/%.c $(CC_LINUX) $(CCFLAGS) $(CFLAGS_CROSS_LINUX) -c -o $@ $< -obj/%.WIN.o: $(DEPS_XXHASH_PATH)/%.c +obj/%.XXHASH.WIN.o: $(DEPS_XXHASH_PATH)/%.c $(CC_WIN) $(CCFLAGS) $(CFLAGS_CROSS_WIN) -c -o $@ $< endif endif ifeq ($(ENABLE_UNRAR),1) ifeq ($(USE_SYSTEM_UNRAR),0) -obj/%.LINUX.o: $(DEPS_UNRAR_PATH)/%.cpp +obj/%.UNRAR.LINUX.o: $(DEPS_UNRAR_PATH)/%.cpp $(CXX_LINUX) $(CXXFLAGS) $(CFLAGS_CROSS_LINUX) $(CFLAGS_UNRAR) -c -o $@ $< -obj/%.WIN.o: $(DEPS_UNRAR_PATH)/%.cpp +obj/%.UNRAR.WIN.o: $(DEPS_UNRAR_PATH)/%.cpp $(CXX_WIN) $(CXXFLAGS) $(CFLAGS_CROSS_WIN) $(CFLAGS_UNRAR) -c -o $@ $< endif endif From 3cacbe0e1f89301f3ae025969a0378af66f8a1f9 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Wed, 25 Aug 2021 16:56:55 +0300 Subject: [PATCH 02/11] Add support for xz compressed files --- include/types.h | 3 + src/Makefile | 2 +- src/filehandling.c | 438 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 349 insertions(+), 94 deletions(-) diff --git a/include/types.h b/include/types.h index 92c4b2e85..b204c64ae 100644 --- a/include/types.h +++ b/include/types.h @@ -1067,6 +1067,8 @@ typedef struct link_speed // file handling +typedef struct xzfile xzfile_t; + typedef struct hc_fp { int fd; @@ -1074,6 +1076,7 @@ typedef struct hc_fp FILE *pfp; // plain fp gzFile gfp; // gzip fp unzFile ufp; // zip fp + xzfile_t *xfp; // xz fp int bom_size; diff --git a/src/Makefile b/src/Makefile index 96503483c..4875f2c9d 100644 --- a/src/Makefile +++ b/src/Makefile @@ -370,7 +370,7 @@ LINUX_OBJS := $(foreach OBJ,$(OBJS_ALL),obj/$(OBJ).LINUX.o) WIN_OBJS := $(foreach OBJ,$(OBJS_ALL),obj/$(OBJ).WIN.o) ifeq ($(USE_SYSTEM_LZMA),0) -OBJS_LZMA := Alloc Lzma2Dec LzmaDec +OBJS_LZMA := 7zCrc 7zCrcOpt 7zFile 7zStream Alloc Bra Bra86 BraIA64 CpuArch Delta Lzma2Dec LzmaDec MtDec Sha256 Sha256Opt Threads Xz XzCrc64 XzCrc64Opt XzDec NATIVE_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).LZMA.NATIVE.o) LINUX_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).LZMA.LINUX.o) diff --git a/src/filehandling.c b/src/filehandling.c index 48c2831b2..fb2ac66b8 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -9,6 +9,38 @@ #include "shared.h" #include "filehandling.h" +#include +#include <7ZCrc.h> +#include <7zFile.h> +#include + +/* Maybe _LZMA_NO_SYSTEM_SIZE_T defined? */ +#if defined (__clang__) || defined (__GNUC__) +#include +_Static_assert(sizeof(size_t) == sizeof(SizeT), "Check why sizeof(size_t) != sizeof(SizeT)"); +#endif + +#ifndef XZFILE_BUFFER_SIZE +#define XZFILE_BUFFER_SIZE 1024 * 1024 +#endif + +static bool xz_initialized = false; + +static const ISzAlloc xz_alloc = { hc_lzma_alloc, hc_lzma_free }; + +struct xzfile +{ + CAlignOffsetAlloc alloc; + Byte *inBuf; + bool inEof; + SizeT inLen; + SizeT inPos; + Int64 inProcessed; + CFileSeqInStream inStream; + Int64 outProcessed; + CXzUnpacker state; +}; + #if defined (__CYGWIN__) // workaround for zlib with cygwin build int _wopen (const char *path, int oflag, ...) @@ -30,6 +62,7 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) fp->pfp = NULL; fp->gfp = NULL; fp->ufp = NULL; + fp->xfp = NULL; fp->bom_size = 0; fp->path = NULL; fp->mode = NULL; @@ -73,6 +106,7 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) bool is_gzip = false; bool is_zip = false; + bool is_xz = false; int fd_tmp = open (path, O_RDONLY); @@ -84,10 +118,11 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) { if (check[0] == 0x1f && check[1] == 0x8b && check[2] == 0x08) is_gzip = true; if (check[0] == 0x50 && check[1] == 0x4b && check[2] == 0x03 && check[3] == 0x04) is_zip = true; + if (memcmp(check, XZ_SIG, XZ_SIG_SIZE) == 0) is_xz = true; // compressed files with BOM will be undetected! - if (is_gzip == false && is_zip == false) + if (is_gzip == false && is_zip == false && is_xz == false) { fp->bom_size = hc_string_bom_size (check); } @@ -107,32 +142,101 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) if (fp->fd == -1) return false; - if (is_zip == false) + if (is_gzip) { - if (is_gzip) - { - if ((fp->gfp = gzdopen (fp->fd, mode)) == NULL) return false; - } - else - { - if ((fp->pfp = fdopen (fp->fd, mode)) == NULL) return false; - - if (fp->bom_size) - { - // atm just skip bom - - const int nread = fread (check, sizeof (char), fp->bom_size, fp->pfp); - - if (nread != fp->bom_size) return false; - } - } + if ((fp->gfp = gzdopen (fp->fd, mode)) == NULL) return false; } - else + else if (is_zip) { if ((fp->ufp = unzOpen64 (path)) == NULL) return false; if (unzOpenCurrentFile (fp->ufp) != UNZ_OK) return false; } + else if (is_xz) + { + if (xz_initialized == false) + { + CrcGenerateTable (); + xz_initialized = true; + } + + xzfile_t *xfp = (xzfile_t *) hccalloc (1, sizeof(*xfp)); + if (xfp == NULL) return false; + + /* prepare cache line aligned memory allocator */ + AlignOffsetAlloc_CreateVTable (&xfp->alloc); + xfp->alloc.numAlignBits = 7; + xfp->alloc.baseAlloc = &xz_alloc; + ISzAllocPtr alloc = &xfp->alloc.vt; + xfp->inBuf = (Byte *) ISzAlloc_Alloc (alloc, XZFILE_BUFFER_SIZE); + if (xfp->inBuf == NULL) + { + hcfree (xfp); + close (fp->fd); + return false; + } + + /* open file */ + CFileSeqInStream *inStream = &xfp->inStream; + FileSeqInStream_CreateVTable (inStream); + CSzFile *file = &inStream->file; + File_Construct (file); + WRes wres = InFile_Open (file, path); + if (wres != SZ_OK) + { + ISzAlloc_Free (alloc, xfp->inBuf); + hcfree (xfp); + close (fp->fd); + return false; + } + + /* fill buffer */ + SizeT inLen = XZFILE_BUFFER_SIZE; + SRes res = ISeqInStream_Read (&inStream->vt, xfp->inBuf, &inLen); + if (res != SZ_OK || inLen == 0) + { + File_Close (file); + ISzAlloc_Free (alloc, xfp->inBuf); + hcfree (xfp); + close (fp->fd); + return false; + } + + xfp->inLen = inLen; + + /* read headers */ + SizeT outLen = 0; + ECoderStatus status; + CXzUnpacker *state = &xfp->state; + XzUnpacker_Construct (state, alloc); + res = XzUnpacker_Code (state, NULL, &outLen, xfp->inBuf, &inLen, false, CODER_FINISH_ANY, &status); + if (res != SZ_OK) + { + XzUnpacker_Free (state); + File_Close (file); + ISzAlloc_Free (alloc, xfp->inBuf); + hcfree (xfp); + close (fp->fd); + return false; + } + + xfp->inPos = inLen; + xfp->inProcessed = inLen; + fp->xfp = xfp; + } + else + { + if ((fp->pfp = fdopen (fp->fd, mode)) == NULL) return false; + + if (fp->bom_size) + { + // atm just skip bom + + const int nread = fread (check, sizeof (char), fp->bom_size, fp->pfp); + + if (nread != fp->bom_size) return false; + } + } fp->path = path; fp->mode = mode; @@ -149,6 +253,7 @@ bool hc_fopen_raw (HCFILE *fp, const char *path, const char *mode) fp->pfp = NULL; fp->gfp = NULL; fp->ufp = NULL; + fp->xfp = NULL; fp->bom_size = 0; fp->path = NULL; fp->mode = NULL; @@ -213,17 +318,9 @@ size_t hc_fread (void *ptr, size_t size, size_t nmemb, HCFILE *fp) if (fp == NULL) return n; - if (fp->gfp) - { - n = gzfread (ptr, size, nmemb, fp->gfp); - } - else if (fp->ufp) - { - unsigned s = size * nmemb; + if (ptr == NULL || size == 0 || nmemb == 0) return 0; - n = unzReadCurrentFile (fp->ufp, ptr, s); - } - else if (fp->pfp) + if (fp->pfp) { #if defined (_WIN) @@ -267,6 +364,50 @@ size_t hc_fread (void *ptr, size_t size, size_t nmemb, HCFILE *fp) n = fread (ptr, size, nmemb, fp->pfp); #endif } + else if (fp->gfp) + { + n = gzfread (ptr, size, nmemb, fp->gfp); + } + else if (fp->ufp) + { + unsigned s = size * nmemb; + + n = unzReadCurrentFile (fp->ufp, ptr, s); + } + else if (fp->xfp) + { + Byte *outBuf = (Byte *) ptr; + SizeT outLen = (SizeT) size * nmemb; + SizeT outPos = 0; + SRes res = SZ_OK; + xzfile_t *xfp = fp->xfp; + + do + { + /* fill buffer if needed */ + if (xfp->inLen == xfp->inPos && !xfp->inEof) + { + xfp->inPos = 0; + xfp->inLen = XZFILE_BUFFER_SIZE; + res = ISeqInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); + if (res != SZ_OK || xfp->inLen == 0) xfp->inEof = true; + } + + /* decode */ + ECoderStatus status; + SizeT inLeft = xfp->inLen - xfp->inPos; + SizeT outLeft = outLen - outPos; + res = XzUnpacker_Code (&xfp->state, outBuf + outPos, &outLeft, xfp->inBuf + xfp->inPos, &inLeft, inLeft == 0, CODER_FINISH_ANY, &status); + xfp->inPos += inLeft; + xfp->inProcessed += inLeft; + if (res != SZ_OK) return -1; + if (inLeft == 0 && outLeft == 0) break; + outPos += outLeft; + xfp->outProcessed += outLeft; + } while (outPos < outLen); + + n = outPos; + } return n; } @@ -277,14 +418,7 @@ size_t hc_fwrite (const void *ptr, size_t size, size_t nmemb, HCFILE *fp) if (fp == NULL) return n; - if (fp->gfp) - { - n = gzfwrite (ptr, size, nmemb, fp->gfp); - } - else if (fp->ufp) - { - } - else if (fp->pfp) + if (fp->pfp) { #if defined (_WIN) @@ -328,6 +462,10 @@ size_t hc_fwrite (const void *ptr, size_t size, size_t nmemb, HCFILE *fp) n = fwrite (ptr, size, nmemb, fp->pfp); #endif } + else if (fp->gfp) + { + n = gzfwrite (ptr, size, nmemb, fp->gfp); + } return n; } @@ -338,7 +476,11 @@ int hc_fseek (HCFILE *fp, off_t offset, int whence) if (fp == NULL) return r; - if (fp->gfp) + if (fp->pfp) + { + r = fseeko (fp->pfp, offset, whence); + } + else if (fp->gfp) { r = gzseek (fp->gfp, offset, whence); } @@ -363,9 +505,9 @@ int hc_fseek (HCFILE *fp, off_t offset, int whence) // r = unzSetOffset (fp->ufp, offset); */ } - else if (fp->pfp) + else if (fp->xfp) { - r = fseeko (fp->pfp, offset, whence); + /* TODO */ } return r; @@ -375,7 +517,11 @@ void hc_rewind (HCFILE *fp) { if (fp == NULL) return; - if (fp->gfp) + if (fp->pfp) + { + rewind (fp->pfp); + } + else if (fp->gfp) { gzrewind (fp->gfp); } @@ -383,9 +529,37 @@ void hc_rewind (HCFILE *fp) { unzGoToFirstFile (fp->ufp); } - else if (fp->pfp) + else if (fp->xfp) { - rewind (fp->pfp); + xzfile_t *xfp = fp->xfp; + + /* cleanup */ + xfp->inEof = false; + xfp->inLen = 0; + xfp->inPos = 0; + xfp->inProcessed = 0; + xfp->outProcessed = 0; + + /* reset */ + Int64 begin = 0; + CFileSeqInStream *inStream = &xfp->inStream; + File_Seek (&inStream->file, &begin, SZ_SEEK_SET); + CXzUnpacker *state = &xfp->state; + XzUnpacker_Init (&xfp->state); + + /* fill buffer */ + SizeT inLen = XZFILE_BUFFER_SIZE; + SRes res = ISeqInStream_Read (&inStream->vt, xfp->inBuf, &inLen); + if (res != SZ_OK || inLen == 0) return; + + xfp->inLen = inLen; + + /* read headers */ + SizeT outLen = 0; + ECoderStatus status; + XzUnpacker_Code (state, NULL, &outLen, xfp->inBuf, &inLen, false, CODER_FINISH_ANY, &status); + xfp->inPos = inLen; + xfp->inProcessed = inLen; } } @@ -393,6 +567,7 @@ int hc_fstat (HCFILE *fp, struct stat *buf) { if (fp == NULL || buf == NULL || fp->fd == -1) return -1; + /* TODO: For compressed files hc_ftell() reports uncompressed bytes, but hc_fstat() reports compressed bytes */ return fstat (fp->fd, buf); } @@ -402,7 +577,11 @@ off_t hc_ftell (HCFILE *fp) if (fp == NULL) return -1; - if (fp->gfp) + if (fp->pfp) + { + n = ftello (fp->pfp); + } + else if (fp->gfp) { n = (off_t) gztell (fp->gfp); } @@ -410,9 +589,11 @@ off_t hc_ftell (HCFILE *fp) { n = unztell (fp->ufp); } - else if (fp->pfp) + else if (fp->xfp) { - n = ftello (fp->pfp); + /* uncompressed bytes */ + const xzfile_t *xfp = fp->xfp; + n = (off_t) xfp->outProcessed; } return n; @@ -424,28 +605,29 @@ int hc_fputc (int c, HCFILE *fp) if (fp == NULL) return r; - if (fp->gfp) - { - r = gzputc (fp->gfp, c); - } - else if (fp->ufp) - { - } - else if (fp->pfp) + if (fp->pfp) { r = fputc (c, fp->pfp); } + else if (fp->gfp) + { + r = gzputc (fp->gfp, c); + } return r; } int hc_fgetc (HCFILE *fp) { - int r = -1; + int r = EOF; if (fp == NULL) return r; - if (fp->gfp) + if (fp->pfp) + { + r = fgetc (fp->pfp); + } + else if (fp->gfp) { r = gzgetc (fp->gfp); } @@ -455,9 +637,32 @@ int hc_fgetc (HCFILE *fp) if (unzReadCurrentFile (fp->ufp, &c, 1) == 1) r = (int) c; } - else if (fp->pfp) + else if (fp->xfp) { - r = fgetc (fp->pfp); + Byte out; + SRes res = SZ_OK; + xzfile_t *xfp = fp->xfp; + + /* fill buffer if needed */ + if (xfp->inLen == xfp->inPos && !xfp->inEof) + { + xfp->inPos = 0; + xfp->inLen = XZFILE_BUFFER_SIZE; + res = ISeqInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); + if (res != SZ_OK || xfp->inLen == 0) xfp->inEof = true; + } + + /* decode single byte */ + ECoderStatus status; + SizeT inLeft = xfp->inLen - xfp->inPos; + SizeT outLeft = 1; + res = XzUnpacker_Code (&xfp->state, &out, &outLeft, xfp->inBuf + xfp->inPos, &inLeft, inLeft == 0, CODER_FINISH_ANY, &status); + if (inLeft == 0 && outLeft == 0) return r; + xfp->inPos += inLeft; + xfp->inProcessed += inLeft; + if (res != SZ_OK) return r; + xfp->outProcessed++; + r = (int) out; } return r; @@ -467,9 +672,13 @@ char *hc_fgets (char *buf, int len, HCFILE *fp) { char *r = NULL; - if (fp == NULL) return r; + if (fp == NULL || len <= 0) return r; - if (fp->gfp) + if (fp->pfp) + { + r = fgets (buf, len, fp->pfp); + } + else if (fp->gfp) { r = gzgets (fp->gfp, buf, len); } @@ -477,9 +686,46 @@ char *hc_fgets (char *buf, int len, HCFILE *fp) { if (unzReadCurrentFile (fp->ufp, buf, len) > 0) r = buf; } - else if (fp->pfp) + else if (fp->xfp) { - r = fgets (buf, len, fp->pfp); + Byte *outBuf = (Byte *) buf; + SizeT outLen = (SizeT) len - 1; + SRes res = SZ_OK; + xzfile_t *xfp = fp->xfp; + + while (outLen > 0) + { + /* fill buffer if needed */ + if (xfp->inLen == xfp->inPos && !xfp->inEof) + { + xfp->inPos = 0; + xfp->inLen = XZFILE_BUFFER_SIZE; + res = ISeqInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); + if (res != SZ_OK || xfp->inLen == 0) xfp->inEof = true; + } + + /* decode single byte */ + ECoderStatus status; + SizeT inLeft = xfp->inLen - xfp->inPos; + SizeT outLeft = 1; + res = XzUnpacker_Code (&xfp->state, outBuf, &outLeft, xfp->inBuf + xfp->inPos, &inLeft, inLeft == 0, CODER_FINISH_ANY, &status); + if (inLeft == 0 && outLeft == 0) break; + xfp->inPos += inLeft; + xfp->inProcessed += inLeft; + if (res != SZ_OK) break; + xfp->outProcessed++; + if (*outBuf++ == '\n') + { + /* success */ + r = buf; + break; + } + + outLen--; + } + + /* always NULL terminate */ + *outBuf = 0; } return r; @@ -491,17 +737,14 @@ int hc_vfprintf (HCFILE *fp, const char *format, va_list ap) if (fp == NULL) return r; - if (fp->gfp) - { - r = gzvprintf (fp->gfp, format, ap); - } - else if (fp->ufp) - { - } - else if (fp->pfp) + if (fp->pfp) { r = vfprintf (fp->pfp, format, ap); } + else if (fp->gfp) + { + r = gzvprintf (fp->gfp, format, ap); + } return r; } @@ -516,17 +759,14 @@ int hc_fprintf (HCFILE *fp, const char *format, ...) va_start (ap, format); - if (fp->gfp) - { - r = gzvprintf (fp->gfp, format, ap); - } - else if (fp->ufp) - { - } - else if (fp->pfp) + if (fp->pfp) { r = vfprintf (fp->pfp, format, ap); } + else if (fp->gfp) + { + r = gzvprintf (fp->gfp, format, ap); + } va_end (ap); @@ -557,7 +797,11 @@ int hc_feof (HCFILE *fp) if (fp == NULL) return r; - if (fp->gfp) + if (fp->pfp) + { + r = feof (fp->pfp); + } + else if (fp->gfp) { r = gzeof (fp->gfp); } @@ -565,9 +809,10 @@ int hc_feof (HCFILE *fp) { r = unzeof (fp->ufp); } - else if (fp->pfp) + else if (fp->xfp) { - r = feof (fp->pfp); + const xzfile_t *xfp = fp->xfp; + r = (xfp->inEof && xfp->inPos == xfp->inLen); } return r; @@ -577,17 +822,14 @@ void hc_fflush (HCFILE *fp) { if (fp == NULL) return; - if (fp->gfp) - { - gzflush (fp->gfp, Z_SYNC_FLUSH); - } - else if (fp->ufp) - { - } - else if (fp->pfp) + if (fp->pfp) { fflush (fp->pfp); } + else if (fp->gfp) + { + gzflush (fp->gfp, Z_SYNC_FLUSH); + } } void hc_fsync (HCFILE *fp) @@ -610,7 +852,11 @@ void hc_fclose (HCFILE *fp) { if (fp == NULL) return; - if (fp->gfp) + if (fp->pfp) + { + fclose (fp->pfp); + } + else if (fp->gfp) { gzclose (fp->gfp); } @@ -622,15 +868,21 @@ void hc_fclose (HCFILE *fp) close (fp->fd); } - else if (fp->pfp) + else if (fp->xfp) { - fclose (fp->pfp); + xzfile_t *xfp = fp->xfp; + XzUnpacker_Free (&xfp->state); + File_Close (&xfp->inStream.file); + ISzAlloc_Free (&xfp->alloc.vt, xfp->inBuf); + hcfree (xfp); + close (fp->fd); } fp->fd = -1; fp->pfp = NULL; fp->gfp = NULL; fp->ufp = NULL; + fp->xfp = NULL; fp->path = NULL; fp->mode = NULL; From 639068548ee0a6a6e579e12395b79114764b2a78 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Wed, 25 Aug 2021 18:23:49 +0300 Subject: [PATCH 03/11] Define _7ZIP_ST to build single-threaded LZMA SDK --- src/Makefile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Makefile b/src/Makefile index 4875f2c9d..221744699 100644 --- a/src/Makefile +++ b/src/Makefile @@ -192,6 +192,10 @@ CFLAGS += -Wno-enum-conversion CFLAGS += -Wno-typedef-redefinition endif +ifeq ($(USE_SYSTEM_LZMA),0) +CFLAGS_LZMA += -D_7ZIP_ST +endif + ## because ZLIB ifeq ($(USE_SYSTEM_ZLIB),0) CFLAGS_ZLIB += -Wno-implicit-fallthrough @@ -370,7 +374,7 @@ LINUX_OBJS := $(foreach OBJ,$(OBJS_ALL),obj/$(OBJ).LINUX.o) WIN_OBJS := $(foreach OBJ,$(OBJS_ALL),obj/$(OBJ).WIN.o) ifeq ($(USE_SYSTEM_LZMA),0) -OBJS_LZMA := 7zCrc 7zCrcOpt 7zFile 7zStream Alloc Bra Bra86 BraIA64 CpuArch Delta Lzma2Dec LzmaDec MtDec Sha256 Sha256Opt Threads Xz XzCrc64 XzCrc64Opt XzDec +OBJS_LZMA := 7zCrc 7zCrcOpt 7zFile 7zStream Alloc Bra Bra86 BraIA64 CpuArch Delta LzmaDec Lzma2Dec Sha256 Sha256Opt Xz XzCrc64 XzCrc64Opt XzDec NATIVE_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).LZMA.NATIVE.o) LINUX_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).LZMA.LINUX.o) @@ -563,7 +567,7 @@ obj/%.NATIVE.o: src/%.c ifeq ($(USE_SYSTEM_LZMA),0) obj/%.LZMA.NATIVE.o: $(DEPS_LZMA_PATH)/%.c - $(CC) -c $(CCFLAGS) $(CFLAGS_NATIVE) $< -o $@ -fpic + $(CC) -c $(CCFLAGS) $(CFLAGS_NATIVE) $(CFLAGS_LZMA) $< -o $@ -fpic endif ifeq ($(USE_SYSTEM_ZLIB),0) @@ -736,10 +740,10 @@ obj/%.WIN.o: src/%.c ifeq ($(USE_SYSTEM_LZMA),0) obj/%.LZMA.LINUX.o: $(DEPS_LZMA_PATH)/%.c - $(CC_LINUX) $(CCFLAGS) $(CFLAGS_CROSS_LINUX) -c -o $@ $< + $(CC_LINUX) $(CCFLAGS) $(CFLAGS_CROSS_LINUX) $(CFLAGS_LZMA) -c -o $@ $< obj/%.LZMA.WIN.o: $(DEPS_LZMA_PATH)/%.c - $(CC_WIN) $(CCFLAGS) $(CFLAGS_CROSS_WIN) -c -o $@ $< + $(CC_WIN) $(CCFLAGS) $(CFLAGS_CROSS_WIN) $(CFLAGS_LZMA) -c -o $@ $< endif ifeq ($(USE_SYSTEM_ZLIB),0) From 47bb7620eef5216a476075d7a96987646e462e64 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 26 Aug 2021 13:01:25 +0300 Subject: [PATCH 04/11] Use fastest SHA-256 implementation supported by CPU --- src/filehandling.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/filehandling.c b/src/filehandling.c index fb2ac66b8..2d6e6f316 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -157,6 +157,7 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) if (xz_initialized == false) { CrcGenerateTable (); + Sha256Prepare (); xz_initialized = true; } @@ -399,7 +400,7 @@ size_t hc_fread (void *ptr, size_t size, size_t nmemb, HCFILE *fp) SizeT outLeft = outLen - outPos; res = XzUnpacker_Code (&xfp->state, outBuf + outPos, &outLeft, xfp->inBuf + xfp->inPos, &inLeft, inLeft == 0, CODER_FINISH_ANY, &status); xfp->inPos += inLeft; - xfp->inProcessed += inLeft; + xfp->inProcessed += inLeft; if (res != SZ_OK) return -1; if (inLeft == 0 && outLeft == 0) break; outPos += outLeft; @@ -659,7 +660,7 @@ int hc_fgetc (HCFILE *fp) res = XzUnpacker_Code (&xfp->state, &out, &outLeft, xfp->inBuf + xfp->inPos, &inLeft, inLeft == 0, CODER_FINISH_ANY, &status); if (inLeft == 0 && outLeft == 0) return r; xfp->inPos += inLeft; - xfp->inProcessed += inLeft; + xfp->inProcessed += inLeft; if (res != SZ_OK) return r; xfp->outProcessed++; r = (int) out; @@ -672,7 +673,7 @@ char *hc_fgets (char *buf, int len, HCFILE *fp) { char *r = NULL; - if (fp == NULL || len <= 0) return r; + if (fp == NULL || buf == NULL || len <= 0) return r; if (fp->pfp) { @@ -711,7 +712,7 @@ char *hc_fgets (char *buf, int len, HCFILE *fp) res = XzUnpacker_Code (&xfp->state, outBuf, &outLeft, xfp->inBuf + xfp->inPos, &inLeft, inLeft == 0, CODER_FINISH_ANY, &status); if (inLeft == 0 && outLeft == 0) break; xfp->inPos += inLeft; - xfp->inProcessed += inLeft; + xfp->inProcessed += inLeft; if (res != SZ_OK) break; xfp->outProcessed++; if (*outBuf++ == '\n') From 985252e283db6b138636a31c80d9f7d14cbe6683 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 26 Aug 2021 13:46:37 +0300 Subject: [PATCH 05/11] Silence GCC misleading indentation warning for LZMA SDK --- src/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Makefile b/src/Makefile index 221744699..154407343 100644 --- a/src/Makefile +++ b/src/Makefile @@ -194,6 +194,7 @@ endif ifeq ($(USE_SYSTEM_LZMA),0) CFLAGS_LZMA += -D_7ZIP_ST +CFLAGS_LZMA += -Wno-misleading-indentation endif ## because ZLIB From 4dcdbebb2fed76bde3ab4a24a79fbbe3981b92e7 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Thu, 26 Aug 2021 16:41:17 +0300 Subject: [PATCH 06/11] Scan the xz file to determinate the number of blocks and streams, and its uncompressed size --- src/Makefile | 2 +- src/filehandling.c | 90 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/Makefile b/src/Makefile index 154407343..e57748daf 100644 --- a/src/Makefile +++ b/src/Makefile @@ -375,7 +375,7 @@ LINUX_OBJS := $(foreach OBJ,$(OBJS_ALL),obj/$(OBJ).LINUX.o) WIN_OBJS := $(foreach OBJ,$(OBJS_ALL),obj/$(OBJ).WIN.o) ifeq ($(USE_SYSTEM_LZMA),0) -OBJS_LZMA := 7zCrc 7zCrcOpt 7zFile 7zStream Alloc Bra Bra86 BraIA64 CpuArch Delta LzmaDec Lzma2Dec Sha256 Sha256Opt Xz XzCrc64 XzCrc64Opt XzDec +OBJS_LZMA := 7zCrc 7zCrcOpt 7zFile 7zStream Alloc Bra Bra86 BraIA64 CpuArch Delta LzmaDec Lzma2Dec Sha256 Sha256Opt Xz XzCrc64 XzCrc64Opt XzDec XzIn NATIVE_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).LZMA.NATIVE.o) LINUX_OBJS += $(foreach OBJ,$(OBJS_LZMA),obj/$(OBJ).LZMA.LINUX.o) diff --git a/src/filehandling.c b/src/filehandling.c index 2d6e6f316..15e3cd13a 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -31,14 +31,17 @@ static const ISzAlloc xz_alloc = { hc_lzma_alloc, hc_lzma_free }; struct xzfile { CAlignOffsetAlloc alloc; + UInt64 inBlocks; Byte *inBuf; bool inEof; SizeT inLen; SizeT inPos; Int64 inProcessed; - CFileSeqInStream inStream; + CFileInStream inStream; Int64 outProcessed; + UInt64 outSize; CXzUnpacker state; + CXzs streams; }; #if defined (__CYGWIN__) @@ -177,9 +180,9 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) return false; } - /* open file */ - CFileSeqInStream *inStream = &xfp->inStream; - FileSeqInStream_CreateVTable (inStream); + /* open the file */ + CFileInStream *inStream = &xfp->inStream; + FileInStream_CreateVTable (inStream); CSzFile *file = &inStream->file; File_Construct (file); WRes wres = InFile_Open (file, path); @@ -191,11 +194,39 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) return false; } - /* fill buffer */ + /* scan the file */ + CLookToRead2 lookStream; + LookToRead2_CreateVTable (&lookStream, false); + lookStream.buf = xfp->inBuf; + lookStream.bufSize = XZFILE_BUFFER_SIZE; + lookStream.realStream = &inStream->vt; + LookToRead2_Init (&lookStream); + Xzs_Construct (&xfp->streams); + Int64 offset = 0; + SRes res = Xzs_ReadBackward (&xfp->streams, &lookStream.vt, &offset, NULL, alloc); + if (res != SZ_OK || offset != 0) + { + Xzs_Free (&xfp->streams, alloc); + File_Close (file); + ISzAlloc_Free (alloc, xfp->inBuf); + hcfree (xfp); + close (fp->fd); + return false; + } + + xfp->inBlocks = Xzs_GetNumBlocks (&xfp->streams); + xfp->outSize = Xzs_GetUnpackSize (&xfp->streams); + + /* seek to start of the file and fill the buffer */ SizeT inLen = XZFILE_BUFFER_SIZE; - SRes res = ISeqInStream_Read (&inStream->vt, xfp->inBuf, &inLen); + res = ISeekInStream_Seek (&inStream->vt, &offset, SZ_SEEK_SET); + if (res == SZ_OK) + { + res = ISeekInStream_Read (&inStream->vt, xfp->inBuf, &inLen); + } if (res != SZ_OK || inLen == 0) { + Xzs_Free (&xfp->streams, alloc); File_Close (file); ISzAlloc_Free (alloc, xfp->inBuf); hcfree (xfp); @@ -214,6 +245,7 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) if (res != SZ_OK) { XzUnpacker_Free (state); + Xzs_Free (&xfp->streams, alloc); File_Close (file); ISzAlloc_Free (alloc, xfp->inBuf); hcfree (xfp); @@ -390,7 +422,7 @@ size_t hc_fread (void *ptr, size_t size, size_t nmemb, HCFILE *fp) { xfp->inPos = 0; xfp->inLen = XZFILE_BUFFER_SIZE; - res = ISeqInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); + res = ISeekInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); if (res != SZ_OK || xfp->inLen == 0) xfp->inEof = true; } @@ -542,15 +574,16 @@ void hc_rewind (HCFILE *fp) xfp->outProcessed = 0; /* reset */ - Int64 begin = 0; - CFileSeqInStream *inStream = &xfp->inStream; - File_Seek (&inStream->file, &begin, SZ_SEEK_SET); + Int64 offset = 0; + CFileInStream *inStream = &xfp->inStream; + SRes res = ISeekInStream_Seek (&inStream->vt, &offset, SZ_SEEK_SET); + if (res != SZ_OK) return; CXzUnpacker *state = &xfp->state; XzUnpacker_Init (&xfp->state); - /* fill buffer */ + /* fill the buffer */ SizeT inLen = XZFILE_BUFFER_SIZE; - SRes res = ISeqInStream_Read (&inStream->vt, xfp->inBuf, &inLen); + res = ISeekInStream_Read (&inStream->vt, xfp->inBuf, &inLen); if (res != SZ_OK || inLen == 0) return; xfp->inLen = inLen; @@ -566,10 +599,29 @@ void hc_rewind (HCFILE *fp) int hc_fstat (HCFILE *fp, struct stat *buf) { - if (fp == NULL || buf == NULL || fp->fd == -1) return -1; + int r = -1; - /* TODO: For compressed files hc_ftell() reports uncompressed bytes, but hc_fstat() reports compressed bytes */ - return fstat (fp->fd, buf); + if (fp == NULL || buf == NULL || fp->fd == -1) return r; + + r = fstat (fp->fd, buf); + if (r != 0) return r; + + if (fp->gfp) + { + /* TODO: For compressed files hc_ftell() reports uncompressed bytes, but hc_fstat() reports compressed bytes */ + } + else if (fp->ufp) + { + /* TODO: For compressed files hc_ftell() reports uncompressed bytes, but hc_fstat() reports compressed bytes */ + } + else if (fp->xfp) + { + /* uncompressed bytes */ + const xzfile_t *xfp = fp->xfp; + buf->st_size = (off_t) xfp->outSize; + } + + return r; } off_t hc_ftell (HCFILE *fp) @@ -649,7 +701,7 @@ int hc_fgetc (HCFILE *fp) { xfp->inPos = 0; xfp->inLen = XZFILE_BUFFER_SIZE; - res = ISeqInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); + res = ISeekInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); if (res != SZ_OK || xfp->inLen == 0) xfp->inEof = true; } @@ -701,7 +753,7 @@ char *hc_fgets (char *buf, int len, HCFILE *fp) { xfp->inPos = 0; xfp->inLen = XZFILE_BUFFER_SIZE; - res = ISeqInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); + res = ISeekInStream_Read (&xfp->inStream.vt, xfp->inBuf, &xfp->inLen); if (res != SZ_OK || xfp->inLen == 0) xfp->inEof = true; } @@ -872,9 +924,11 @@ void hc_fclose (HCFILE *fp) else if (fp->xfp) { xzfile_t *xfp = fp->xfp; + ISzAllocPtr alloc = &xfp->alloc.vt; XzUnpacker_Free (&xfp->state); + Xzs_Free (&xfp->streams, alloc); File_Close (&xfp->inStream.file); - ISzAlloc_Free (&xfp->alloc.vt, xfp->inBuf); + ISzAlloc_Free (alloc, xfp->inBuf); hcfree (xfp); close (fp->fd); } From f502d80459c7e41af8dee333a05a109865e73075 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Fri, 27 Aug 2021 11:09:37 +0300 Subject: [PATCH 07/11] Fix include file case --- src/filehandling.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/filehandling.c b/src/filehandling.c index 15e3cd13a..a36740fcc 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -10,7 +10,7 @@ #include "filehandling.h" #include -#include <7ZCrc.h> +#include <7zCrc.h> #include <7zFile.h> #include From b2870dadfd32a59a639d99b6519b0154f3d33ba1 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Sat, 28 Aug 2021 20:05:59 +0300 Subject: [PATCH 08/11] Fix include file case --- src/filehandling.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/filehandling.c b/src/filehandling.c index a36740fcc..c60361621 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -12,7 +12,7 @@ #include #include <7zCrc.h> #include <7zFile.h> -#include +#include /* Maybe _LZMA_NO_SYSTEM_SIZE_T defined? */ #if defined (__clang__) || defined (__GNUC__) From 8324d3cd07f8a76ab70cc2eb461691914acb8cd4 Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Sun, 29 Aug 2021 01:31:56 +0300 Subject: [PATCH 09/11] Initialize both 32bit CrcGenerateTable() and 64bit Crc64GenerateTable() --- src/filehandling.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/filehandling.c b/src/filehandling.c index c60361621..9866ddd61 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -13,6 +13,7 @@ #include <7zCrc.h> #include <7zFile.h> #include +#include /* Maybe _LZMA_NO_SYSTEM_SIZE_T defined? */ #if defined (__clang__) || defined (__GNUC__) @@ -157,9 +158,11 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) } else if (is_xz) { + /* thread safe on little endian */ if (xz_initialized == false) { CrcGenerateTable (); + Crc64GenerateTable (); Sha256Prepare (); xz_initialized = true; } From 106fe88e4e94282873978d98e5f3039da60890ce Mon Sep 17 00:00:00 2001 From: Jukka Ojanen Date: Mon, 30 Aug 2021 16:28:54 +0300 Subject: [PATCH 10/11] Report the uncompressed size only if it's known --- src/filehandling.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/filehandling.c b/src/filehandling.c index 9866ddd61..eefdffacf 100644 --- a/src/filehandling.c +++ b/src/filehandling.c @@ -18,7 +18,7 @@ /* Maybe _LZMA_NO_SYSTEM_SIZE_T defined? */ #if defined (__clang__) || defined (__GNUC__) #include -_Static_assert(sizeof(size_t) == sizeof(SizeT), "Check why sizeof(size_t) != sizeof(SizeT)"); +_Static_assert(sizeof (size_t) == sizeof (SizeT), "Check why sizeof(size_t) != sizeof(SizeT)"); #endif #ifndef XZFILE_BUFFER_SIZE @@ -167,7 +167,7 @@ bool hc_fopen (HCFILE *fp, const char *path, const char *mode) xz_initialized = true; } - xzfile_t *xfp = (xzfile_t *) hccalloc (1, sizeof(*xfp)); + xzfile_t *xfp = (xzfile_t *) hccalloc (1, sizeof (*xfp)); if (xfp == NULL) return false; /* prepare cache line aligned memory allocator */ @@ -619,9 +619,12 @@ int hc_fstat (HCFILE *fp, struct stat *buf) } else if (fp->xfp) { - /* uncompressed bytes */ + /* check that the uncompressed size is known */ const xzfile_t *xfp = fp->xfp; - buf->st_size = (off_t) xfp->outSize; + if (xfp->outSize != (UInt64)((Int64)-1)) + { + buf->st_size = (off_t) xfp->outSize; + } } return r; From b85d753d1525e884271c7fa539eb70e325429d2c Mon Sep 17 00:00:00 2001 From: Chick3nman Date: Wed, 1 Sep 2021 18:09:40 -0500 Subject: [PATCH 11/11] Add 3070 Ti to hctune --- hashcat.hctune | 1 + 1 file changed, 1 insertion(+) diff --git a/hashcat.hctune b/hashcat.hctune index 5b991c35b..cc5873c2b 100644 --- a/hashcat.hctune +++ b/hashcat.hctune @@ -271,6 +271,7 @@ GeForce_RTX_2080_Ti ALIAS_nv_sm50_or_higher GeForce_RTX_3060 ALIAS_nv_sm50_or_higher GeForce_RTX_3060_Ti ALIAS_nv_sm50_or_higher GeForce_RTX_3070 ALIAS_nv_sm50_or_higher +GeForce_RTX_3070_Ti ALIAS_nv_sm50_or_higher GeForce_RTX_3080 ALIAS_nv_sm50_or_higher GeForce_RTX_3090 ALIAS_nv_sm50_or_higher