Marek Marczykowski-Górecki
6 years ago
parent
42fa533e79
commit
9e354a6dfb
@ -0,0 +1,83 @@
|
||||
From 32a07d4156a281084c90a4b78affc8b0b32a26fc Mon Sep 17 00:00:00 2001
|
||||
From: intrigeri <intrigeri@boum.org>
|
||||
Date: Mon, 21 Nov 2016 11:41:28 +0000
|
||||
Subject: [PATCH] If SOURCE_DATE_EPOCH is set, also clamp content timestamps
|
||||
with that value.
|
||||
|
||||
Based on a patch by Alexander Couzens <lynxis@fe...> posted on
|
||||
https://sourceforge.net/p/squashfs/mailman/message/34673610/
|
||||
---
|
||||
squashfs-tools/mksquashfs.c | 15 ++++++++++++---
|
||||
1 file changed, 12 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
|
||||
index b49e956..9f020bf 100644
|
||||
--- a/squashfs-tools/mksquashfs.c
|
||||
+++ b/squashfs-tools/mksquashfs.c
|
||||
@@ -137,6 +137,9 @@ unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0;
|
||||
/* inode lookup table */
|
||||
squashfs_inode *inode_lookup_table = NULL;
|
||||
|
||||
+/* clamp all timestamps to SOURCE_DATE_EPOCH */
|
||||
+time_t content_clamp_time = -1;
|
||||
+
|
||||
/* override filesystem creation time */
|
||||
time_t mkfs_fixed_time = -1;
|
||||
|
||||
@@ -2246,6 +2249,8 @@ restat:
|
||||
pathname_reader(dir_ent), strerror(errno));
|
||||
goto read_err;
|
||||
}
|
||||
+ if(content_clamp_time != -1 && buf2.st_mtime >= content_clamp_time)
|
||||
+ buf2.st_mtime = content_clamp_time;
|
||||
|
||||
if(read_size != buf2.st_size) {
|
||||
close(file);
|
||||
@@ -3101,7 +3106,7 @@ void dir_scan(squashfs_inode *inode, char *pathname,
|
||||
buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR;
|
||||
buf.st_uid = getuid();
|
||||
buf.st_gid = getgid();
|
||||
- buf.st_mtime = time(NULL);
|
||||
+ buf.st_mtime = content_clamp_time != -1 ? content_clamp_time : time(NULL);
|
||||
buf.st_dev = 0;
|
||||
buf.st_ino = 0;
|
||||
dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0);
|
||||
@@ -3110,6 +3115,8 @@ void dir_scan(squashfs_inode *inode, char *pathname,
|
||||
/* source directory has disappeared? */
|
||||
BAD_ERROR("Cannot stat source directory %s because %s\n",
|
||||
pathname, strerror(errno));
|
||||
+ if(content_clamp_time != -1 && buf.st_mtime >= content_clamp_time)
|
||||
+ buf.st_mtime = content_clamp_time;
|
||||
dir_ent->inode = lookup_inode(&buf);
|
||||
}
|
||||
|
||||
@@ -3365,6 +3372,8 @@ struct dir_info *dir_scan1(char *filename, char *subpath,
|
||||
free_dir_entry(dir_ent);
|
||||
continue;
|
||||
}
|
||||
+ if(content_clamp_time != -1 && buf.st_mtime >= content_clamp_time)
|
||||
+ buf.st_mtime = content_clamp_time;
|
||||
|
||||
if((buf.st_mode & S_IFMT) != S_IFREG &&
|
||||
(buf.st_mode & S_IFMT) != S_IFDIR &&
|
||||
@@ -3544,7 +3553,7 @@ void dir_scan2(struct dir_info *dir, struct pseudo *pseudo)
|
||||
buf.st_gid = pseudo_ent->dev->gid;
|
||||
buf.st_rdev = makedev(pseudo_ent->dev->major,
|
||||
pseudo_ent->dev->minor);
|
||||
- buf.st_mtime = time(NULL);
|
||||
+ buf.st_mtime = content_clamp_time != -1 ? content_clamp_time : time(NULL);
|
||||
buf.st_ino = pseudo_ino ++;
|
||||
|
||||
if(pseudo_ent->dev->type == 'd') {
|
||||
@@ -5674,7 +5683,7 @@ printOptions:
|
||||
"%lu but was found to be: %llu \n", ULONG_MAX, epoch);
|
||||
EXIT_MKSQUASHFS();
|
||||
}
|
||||
- mkfs_fixed_time = (time_t)epoch;
|
||||
+ mkfs_fixed_time = content_clamp_time = (time_t)epoch;
|
||||
}
|
||||
|
||||
/*
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@ -0,0 +1,90 @@
|
||||
From 0ab12a8585373be2de5129e14d979c62e7a90d82 Mon Sep 17 00:00:00 2001
|
||||
From: Chris Lamb <lamby@debian.org>
|
||||
Date: Mon, 21 Nov 2016 09:33:05 +0100
|
||||
Subject: [PATCH] If SOURCE_DATE_EPOCH is set, override timestamps with that
|
||||
value.
|
||||
|
||||
See https://reproducible-builds.org/specs/source-date-epoch/ for more
|
||||
information about this environment variable.
|
||||
|
||||
Based on a patch by Alexander Couzens <lynxis@fe...> posted on
|
||||
https://sourceforge.net/p/squashfs/mailman/message/34673610/
|
||||
|
||||
Signed-off-by: Chris Lamb <lamby@debian.org>
|
||||
---
|
||||
squashfs-tools/mksquashfs.c | 38 ++++++++++++++++++++++++++++++++++++-
|
||||
1 file changed, 37 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
|
||||
index c2098bd..b49e956 100644
|
||||
--- a/squashfs-tools/mksquashfs.c
|
||||
+++ b/squashfs-tools/mksquashfs.c
|
||||
@@ -137,6 +137,9 @@ unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0;
|
||||
/* inode lookup table */
|
||||
squashfs_inode *inode_lookup_table = NULL;
|
||||
|
||||
+/* override filesystem creation time */
|
||||
+time_t mkfs_fixed_time = -1;
|
||||
+
|
||||
/* in memory directory data */
|
||||
#define I_COUNT_SIZE 128
|
||||
#define DIR_ENTRIES 32
|
||||
@@ -5104,6 +5107,9 @@ int main(int argc, char *argv[])
|
||||
int total_mem = get_default_phys_mem();
|
||||
int progress = TRUE;
|
||||
int force_progress = FALSE;
|
||||
+ char *source_date_epoch, *endptr;
|
||||
+ unsigned long long epoch;
|
||||
+
|
||||
struct file_buffer **fragment = NULL;
|
||||
|
||||
if(argc > 1 && strcmp(argv[1], "-version") == 0) {
|
||||
@@ -5641,6 +5647,36 @@ printOptions:
|
||||
}
|
||||
}
|
||||
|
||||
+ /* if SOURCE_DATE_EPOCH is set, use that timestamp for the mkfs time */
|
||||
+ source_date_epoch = getenv("SOURCE_DATE_EPOCH");
|
||||
+ if(source_date_epoch) {
|
||||
+ errno = 0;
|
||||
+ epoch = strtoull(source_date_epoch, &endptr, 10);
|
||||
+ if((errno == ERANGE && (epoch == ULLONG_MAX || epoch == 0))
|
||||
+ || (errno != 0 && epoch == 0)) {
|
||||
+ ERROR("Environment variable $SOURCE_DATE_EPOCH: "
|
||||
+ "strtoull: %s\n", strerror(errno));
|
||||
+ EXIT_MKSQUASHFS();
|
||||
+ }
|
||||
+ if(endptr == source_date_epoch) {
|
||||
+ ERROR("Environment variable $SOURCE_DATE_EPOCH: "
|
||||
+ "No digits were found: %s\n", endptr);
|
||||
+ EXIT_MKSQUASHFS();
|
||||
+ }
|
||||
+ if(*endptr != '\0') {
|
||||
+ ERROR("Environment variable $SOURCE_DATE_EPOCH: "
|
||||
+ "Trailing garbage: %s\n", endptr);
|
||||
+ EXIT_MKSQUASHFS();
|
||||
+ }
|
||||
+ if(epoch > ULONG_MAX) {
|
||||
+ ERROR("Environment variable $SOURCE_DATE_EPOCH: "
|
||||
+ "value must be smaller than or equal to "
|
||||
+ "%lu but was found to be: %llu \n", ULONG_MAX, epoch);
|
||||
+ EXIT_MKSQUASHFS();
|
||||
+ }
|
||||
+ mkfs_fixed_time = (time_t)epoch;
|
||||
+ }
|
||||
+
|
||||
/*
|
||||
* Some compressors may need the options to be checked for validity
|
||||
* once all the options have been processed
|
||||
@@ -5975,7 +6011,7 @@ printOptions:
|
||||
sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments,
|
||||
always_use_fragments, duplicate_checking, exportable,
|
||||
no_xattrs, comp_opts);
|
||||
- sBlk.mkfs_time = time(NULL);
|
||||
+ sBlk.mkfs_time = mkfs_fixed_time != -1 ? mkfs_fixed_time : time(NULL);
|
||||
|
||||
disable_info();
|
||||
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@ -0,0 +1,220 @@
|
||||
From afc0c76a170bd17cbd29bbec6ae6d2227e398570 Mon Sep 17 00:00:00 2001
|
||||
From: Alexander Couzens <lynxis@fe80.eu>
|
||||
Date: Fri, 13 Jan 2017 22:00:37 +0100
|
||||
Subject: [PATCH] remove frag_deflator_thread
|
||||
|
||||
frag_deflator_thread compress fragments.
|
||||
Replace the deflator_thread with a function and
|
||||
use the function instead of the to_frag queue.
|
||||
---
|
||||
squashfs-tools/info.c | 5 ---
|
||||
squashfs-tools/mksquashfs.c | 76 +++++++++++++------------------------
|
||||
squashfs-tools/mksquashfs.h | 2 +-
|
||||
squashfs-tools/restore.c | 15 +-------
|
||||
4 files changed, 30 insertions(+), 68 deletions(-)
|
||||
|
||||
diff --git a/squashfs-tools/info.c b/squashfs-tools/info.c
|
||||
index 7968c77..028d578 100644
|
||||
--- a/squashfs-tools/info.c
|
||||
+++ b/squashfs-tools/info.c
|
||||
@@ -96,11 +96,6 @@ void dump_state()
|
||||
printf("compressed block queue (deflate thread(s) -> main thread)\n");
|
||||
dump_seq_queue(to_main, 0);
|
||||
|
||||
- printf("uncompressed packed fragment queue (main thread -> fragment"
|
||||
- " deflate thread(s))\n");
|
||||
- dump_queue(to_frag);
|
||||
-
|
||||
-
|
||||
printf("locked frag queue (compressed frags waiting while multi-block"
|
||||
" file is written)\n");
|
||||
dump_queue(locked_fragment);
|
||||
diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
|
||||
index cf48e40..cacf14c 100644
|
||||
--- a/squashfs-tools/mksquashfs.c
|
||||
+++ b/squashfs-tools/mksquashfs.c
|
||||
@@ -270,10 +270,10 @@ unsigned int sid_count = 0, suid_count = 0, sguid_count = 0;
|
||||
struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
|
||||
struct cache *bwriter_buffer, *fwriter_buffer;
|
||||
struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
|
||||
- *to_frag, *locked_fragment, *to_process_frag;
|
||||
+ *locked_fragment, *to_process_frag;
|
||||
struct seq_queue *to_main;
|
||||
pthread_t reader_thread, writer_thread, main_thread;
|
||||
-pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
|
||||
+pthread_t *deflator_thread, *frag_thread;
|
||||
pthread_t *restore_thread = NULL;
|
||||
pthread_mutex_t fragment_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
pthread_mutex_t pos_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
@@ -323,7 +323,7 @@ struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
|
||||
void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
|
||||
unsigned short get_checksum_mem(char *buff, int bytes);
|
||||
void check_usable_phys_mem(int total_mem);
|
||||
-
|
||||
+void frag_deflator(struct file_buffer *file_buffer);
|
||||
|
||||
void prep_exit()
|
||||
{
|
||||
@@ -1540,7 +1540,7 @@ void write_fragment(struct file_buffer *fragment)
|
||||
pthread_mutex_lock(&fragment_mutex);
|
||||
fragment_table[fragment->block].unused = 0;
|
||||
fragments_outstanding ++;
|
||||
- queue_put(to_frag, fragment);
|
||||
+ frag_deflator(fragment);
|
||||
pthread_cleanup_pop(1);
|
||||
}
|
||||
|
||||
@@ -2412,51 +2412,34 @@ void *deflator(void *arg)
|
||||
}
|
||||
|
||||
|
||||
-void *frag_deflator(void *arg)
|
||||
+void frag_deflator(struct file_buffer *file_buffer)
|
||||
{
|
||||
- void *stream = NULL;
|
||||
- int res;
|
||||
-
|
||||
- res = compressor_init(comp, &stream, block_size, 1);
|
||||
- if(res)
|
||||
- BAD_ERROR("frag_deflator:: compressor_init failed\n");
|
||||
|
||||
- pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
|
||||
-
|
||||
- while(1) {
|
||||
- int c_byte, compressed_size;
|
||||
- struct file_buffer *file_buffer = queue_get(to_frag);
|
||||
- struct file_buffer *write_buffer =
|
||||
+ int c_byte, compressed_size;
|
||||
+ struct file_buffer *write_buffer =
|
||||
cache_get(fwriter_buffer, file_buffer->block);
|
||||
|
||||
- c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
|
||||
- file_buffer->size, block_size, noF, 1);
|
||||
- compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
|
||||
- write_buffer->size = compressed_size;
|
||||
- pthread_mutex_lock(&fragment_mutex);
|
||||
- if(fragments_locked == FALSE) {
|
||||
- fragment_table[file_buffer->block].size = c_byte;
|
||||
- fragment_table[file_buffer->block].start_block = bytes;
|
||||
- write_buffer->block = bytes;
|
||||
- bytes += compressed_size;
|
||||
- fragments_outstanding --;
|
||||
- pthread_mutex_unlock(&fragment_mutex);
|
||||
- queue_put(to_writer, write_buffer);
|
||||
- TRACE("Writing fragment %lld, uncompressed size %d, "
|
||||
- "compressed size %d\n", file_buffer->block,
|
||||
- file_buffer->size, compressed_size);
|
||||
- } else {
|
||||
- add_pending_fragment(write_buffer, c_byte,
|
||||
- file_buffer->block);
|
||||
- pthread_mutex_unlock(&fragment_mutex);
|
||||
- }
|
||||
- cache_block_put(file_buffer);
|
||||
+ c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
|
||||
+ file_buffer->size, block_size, noF, 1);
|
||||
+ compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
|
||||
+ write_buffer->size = compressed_size;
|
||||
+ if(fragments_locked == FALSE) {
|
||||
+ fragment_table[file_buffer->block].size = c_byte;
|
||||
+ fragment_table[file_buffer->block].start_block = bytes;
|
||||
+ write_buffer->block = bytes;
|
||||
+ bytes += compressed_size;
|
||||
+ fragments_outstanding --;
|
||||
+ queue_put(to_writer, write_buffer);
|
||||
+ TRACE("Writing fragment %lld, uncompressed size %d, "
|
||||
+ "compressed size %d\n", file_buffer->block,
|
||||
+ file_buffer->size, compressed_size);
|
||||
+ } else {
|
||||
+ add_pending_fragment(write_buffer, c_byte,
|
||||
+ file_buffer->block);
|
||||
}
|
||||
-
|
||||
- pthread_cleanup_pop(0);
|
||||
+ cache_block_put(file_buffer);
|
||||
}
|
||||
|
||||
-
|
||||
struct file_buffer *get_file_buffer()
|
||||
{
|
||||
struct file_buffer *file_buffer = seq_queue_get(to_main);
|
||||
@@ -4257,19 +4240,17 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
|
||||
multiply_overflow(processors * 3, sizeof(pthread_t)))
|
||||
BAD_ERROR("Processors too large\n");
|
||||
|
||||
- deflator_thread = malloc(processors * 3 * sizeof(pthread_t));
|
||||
+ deflator_thread = malloc(processors * 2 * sizeof(pthread_t));
|
||||
if(deflator_thread == NULL)
|
||||
MEM_ERROR();
|
||||
|
||||
- frag_deflator_thread = &deflator_thread[processors];
|
||||
- frag_thread = &frag_deflator_thread[processors];
|
||||
+ frag_thread = &deflator_thread[processors];
|
||||
|
||||
to_reader = queue_init(1);
|
||||
to_deflate = queue_init(reader_size);
|
||||
to_process_frag = queue_init(reader_size);
|
||||
to_writer = queue_init(bwriter_size + fwriter_size);
|
||||
from_writer = queue_init(1);
|
||||
- to_frag = queue_init(fragment_size);
|
||||
locked_fragment = queue_init(fragment_size);
|
||||
to_main = seq_queue_init();
|
||||
reader_buffer = cache_init(block_size, reader_size, 0, 0);
|
||||
@@ -4285,9 +4266,6 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
|
||||
for(i = 0; i < processors; i++) {
|
||||
if(pthread_create(&deflator_thread[i], NULL, deflator, NULL))
|
||||
BAD_ERROR("Failed to create thread\n");
|
||||
- if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
|
||||
- NULL) != 0)
|
||||
- BAD_ERROR("Failed to create thread\n");
|
||||
if(pthread_create(&frag_thread[i], NULL, frag_thrd,
|
||||
(void *) destination_file) != 0)
|
||||
BAD_ERROR("Failed to create thread\n");
|
||||
diff --git a/squashfs-tools/mksquashfs.h b/squashfs-tools/mksquashfs.h
|
||||
index 55708a3..dc5bde4 100644
|
||||
--- a/squashfs-tools/mksquashfs.h
|
||||
+++ b/squashfs-tools/mksquashfs.h
|
||||
@@ -135,7 +135,7 @@ struct append_file {
|
||||
extern struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
|
||||
struct cache *bwriter_buffer, *fwriter_buffer;
|
||||
extern struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
|
||||
- *to_frag, *locked_fragment, *to_process_frag;
|
||||
+ *locked_fragment, *to_process_frag;
|
||||
extern struct append_file **file_mapping;
|
||||
extern struct seq_queue *to_main;
|
||||
extern pthread_mutex_t fragment_mutex, dup_mutex;
|
||||
diff --git a/squashfs-tools/restore.c b/squashfs-tools/restore.c
|
||||
index 5e336b3..a7aaf2e 100644
|
||||
--- a/squashfs-tools/restore.c
|
||||
+++ b/squashfs-tools/restore.c
|
||||
@@ -47,8 +47,8 @@
|
||||
#define TRUE 1
|
||||
|
||||
extern pthread_t reader_thread, writer_thread, main_thread;
|
||||
-extern pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
|
||||
-extern struct queue *to_deflate, *to_writer, *to_frag, *to_process_frag;
|
||||
+extern pthread_t *deflator_thread, *frag_thread;
|
||||
+extern struct queue *to_deflate, *to_writer, *to_process_frag;
|
||||
extern struct seq_queue *to_main;
|
||||
extern void restorefs();
|
||||
extern int processors;
|
||||
@@ -120,17 +120,6 @@ void *restore_thrd(void *arg)
|
||||
pthread_cancel(main_thread);
|
||||
pthread_join(main_thread, NULL);
|
||||
|
||||
- /* then flush the main thread to fragment deflator thread(s)
|
||||
- * queue. The fragment deflator thread(s) will idle
|
||||
- */
|
||||
- queue_flush(to_frag);
|
||||
-
|
||||
- /* now kill the fragment deflator thread(s) */
|
||||
- for(i = 0; i < processors; i++)
|
||||
- pthread_cancel(frag_deflator_thread[i]);
|
||||
- for(i = 0; i < processors; i++)
|
||||
- pthread_join(frag_deflator_thread[i], NULL);
|
||||
-
|
||||
/*
|
||||
* then flush the main thread/fragment deflator thread(s)
|
||||
* to writer thread queue. The writer thread will idle
|
||||
--
|
||||
2.17.1
|
||||
|
||||
Loading…
Reference in new issue