Marek Marczykowski-Górecki
6 years ago
parent
45c201932b
commit
94ccc1a19a
@ -0,0 +1,45 @@
|
||||
From 1b0ff7f98bdce87deb7bc61d6c227be21fa43a94 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
|
||||
<marmarek@invisiblethingslab.com>
|
||||
Date: Thu, 4 Oct 2018 23:36:15 +0200
|
||||
Subject: [PATCH 1/6] Use $SOURCE_DATE_EPOCH (if set) in discinfo file
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
Organization: Invisible Things Lab
|
||||
Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
|
||||
This helps the output image to be reproducible.
|
||||
|
||||
Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
---
|
||||
pungi/compose_metadata/discinfo.py | 6 +++++-
|
||||
1 file changed, 5 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/pungi/compose_metadata/discinfo.py b/pungi/compose_metadata/discinfo.py
|
||||
index df61ca0..758feef 100644
|
||||
--- a/pungi/compose_metadata/discinfo.py
|
||||
+++ b/pungi/compose_metadata/discinfo.py
|
||||
@@ -32,6 +32,7 @@ __all__ = (
|
||||
)
|
||||
|
||||
|
||||
+import os
|
||||
import time
|
||||
|
||||
|
||||
@@ -43,7 +44,10 @@ def write_discinfo(file_path, description, arch, disc_numbers=None, timestamp=No
|
||||
if not isinstance(disc_numbers, list):
|
||||
raise TypeError("Invalid type: disc_numbers type is %s; expected: <list>" % type(disc_numbers))
|
||||
if not timestamp:
|
||||
- timestamp = "%f" % time.time()
|
||||
+ if 'SOURCE_DATE_EPOCH' in os.environ:
|
||||
+ timestamp = os.environ['SOURCE_DATE_EPOCH']
|
||||
+ else:
|
||||
+ timestamp = "%f" % time.time()
|
||||
with open(file_path, "w") as f:
|
||||
f.write("%s\n" % timestamp)
|
||||
f.write("%s\n" % description)
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@ -0,0 +1,49 @@
|
||||
From d33b8f9995070d68472c25779dfa543d6d7535db Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
|
||||
<marmarek@invisiblethingslab.com>
|
||||
Date: Thu, 4 Oct 2018 23:37:35 +0200
|
||||
Subject: [PATCH 2/6] Use xorriso instead of genisoimage
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
Organization: Invisible Things Lab
|
||||
Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
|
||||
xorriso make the image reproducible (given the same input files),
|
||||
including support for SOURCE_DATE_EPOCH in various metadata.
|
||||
|
||||
Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
---
|
||||
pungi.spec | 2 +-
|
||||
pungi/gather.py | 2 +-
|
||||
2 files changed, 2 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/pungi.spec b/pungi.spec
|
||||
index 6a23a63..d7bdc66 100644
|
||||
--- a/pungi.spec
|
||||
+++ b/pungi.spec
|
||||
@@ -35,7 +35,7 @@ Requires: jigdo
|
||||
Requires: cvs
|
||||
Requires: yum-utils
|
||||
Requires: isomd5sum
|
||||
-Requires: genisoimage
|
||||
+Requires: xorriso
|
||||
Requires: gettext
|
||||
Requires: syslinux
|
||||
Requires: git
|
||||
diff --git a/pungi/gather.py b/pungi/gather.py
|
||||
index 20cc33d..15dfcee 100644
|
||||
--- a/pungi/gather.py
|
||||
+++ b/pungi/gather.py
|
||||
@@ -1709,7 +1709,7 @@ class Pungi(PungiBase):
|
||||
clean=True) # This is risky...
|
||||
|
||||
# setup the base command
|
||||
- mkisofs = ['/usr/bin/mkisofs']
|
||||
+ mkisofs = ['/usr/bin/xorriso', '-as', 'mkisofs']
|
||||
mkisofs.extend(['-v', '-U', '-J', '-R', '-T', '-m', 'repoview', '-m', 'boot.iso']) # common mkisofs flags
|
||||
|
||||
x86bootargs = ['-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat',
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@ -0,0 +1,34 @@
|
||||
From 61ed5d6ea5b1beedb59b3962d0df99f0b3c69402 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
|
||||
<marmarek@invisiblethingslab.com>
|
||||
Date: Thu, 4 Oct 2018 23:38:57 +0200
|
||||
Subject: [PATCH 3/6] Use constant MBR ID for isohybrid
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
Organization: Invisible Things Lab
|
||||
Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
|
||||
If not set explicitly, isohybrid choose it randomly, which harm
|
||||
reproducibility.
|
||||
|
||||
Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
---
|
||||
pungi/gather.py | 1 +
|
||||
1 file changed, 1 insertion(+)
|
||||
|
||||
diff --git a/pungi/gather.py b/pungi/gather.py
|
||||
index 15dfcee..6f52bc6 100644
|
||||
--- a/pungi/gather.py
|
||||
+++ b/pungi/gather.py
|
||||
@@ -1731,6 +1731,7 @@ class Pungi(PungiBase):
|
||||
ppcbootargs.append('-hfs-bless') # must be last
|
||||
|
||||
isohybrid = ['/usr/bin/isohybrid']
|
||||
+ isohybrid.extend(['--id', '42'])
|
||||
|
||||
# Check the size of the tree
|
||||
# This size checking method may be bunk, accepting patches...
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@ -0,0 +1,67 @@
|
||||
From 57e49f366a34e3d8fdb020d7f19bc2fec8547ec9 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
|
||||
<marmarek@invisiblethingslab.com>
|
||||
Date: Thu, 4 Oct 2018 23:42:19 +0200
|
||||
Subject: [PATCH 4/6] Make sure .treeinfo file is sorted
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
Organization: Invisible Things Lab
|
||||
Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
|
||||
OrderedDict used by default by ConfigParser isn't enough because order
|
||||
of entries being added may not be deterministic (depends on directory
|
||||
list order). To solve this problem, use SortedDict as a base.
|
||||
|
||||
Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
---
|
||||
pungi.spec | 2 ++
|
||||
pungi/gather.py | 5 +++++
|
||||
2 files changed, 7 insertions(+)
|
||||
|
||||
diff --git a/pungi.spec b/pungi.spec
|
||||
index d7bdc66..dcb1986 100644
|
||||
--- a/pungi.spec
|
||||
+++ b/pungi.spec
|
||||
@@ -17,6 +17,7 @@ BuildRequires: python-jsonschema
|
||||
BuildRequires: python-enum34
|
||||
BuildRequires: python2-dnf
|
||||
BuildRequires: python2-multilib
|
||||
+BuildRequires: python2-dict-sorted
|
||||
|
||||
Requires: createrepo >= 0.4.11
|
||||
Requires: yum => 3.4.3-28
|
||||
@@ -44,6 +45,7 @@ Requires: libguestfs-tools-c
|
||||
Requires: python-enum34
|
||||
Requires: python2-dnf
|
||||
Requires: python2-multilib
|
||||
+Requires: python2-dict-sorted
|
||||
|
||||
BuildArch: noarch
|
||||
|
||||
diff --git a/pungi/gather.py b/pungi/gather.py
|
||||
index 6f52bc6..1035036 100644
|
||||
--- a/pungi/gather.py
|
||||
+++ b/pungi/gather.py
|
||||
@@ -26,6 +26,7 @@ import urlgrabber.progress
|
||||
import subprocess
|
||||
import createrepo
|
||||
import ConfigParser
|
||||
+from sdict import AlphaSortedDict
|
||||
from fnmatch import fnmatch
|
||||
|
||||
import arch as arch_module
|
||||
@@ -95,6 +96,10 @@ def is_package(po):
|
||||
class MyConfigParser(ConfigParser.ConfigParser):
|
||||
"""A subclass of ConfigParser which does not lowercase options"""
|
||||
|
||||
+ def __init__(self, *args, **kwargs):
|
||||
+ kwargs['dict_type'] = AlphaSortedDict
|
||||
+ ConfigParser.ConfigParser.__init__(self, *args, **kwargs)
|
||||
+
|
||||
def optionxform(self, optionstr):
|
||||
return optionstr
|
||||
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@ -0,0 +1,60 @@
|
||||
From 8eaee0ada8caa8b509b96867b06b876ef606d64f Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
|
||||
<marmarek@invisiblethingslab.com>
|
||||
Date: Thu, 4 Oct 2018 23:44:06 +0200
|
||||
Subject: [PATCH 5/6] Set repodata mtime to SOURCE_DATE_EPOCH
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
Organization: Invisible Things Lab
|
||||
Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
|
||||
repodata/repomd.xml include timestamps of all the other repodata files.
|
||||
Even when those files are created reproducibly, they have current
|
||||
modification time. In general case this is a good thing (ease checking
|
||||
if repodata cache is up to date). But in case of composing installation
|
||||
image, it breaks reproducibility.
|
||||
Avoid this by reseting mtime of repodata/* to $SOURCE_DATE_EPOCH, just
|
||||
before creating repomd.xml.
|
||||
|
||||
Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
---
|
||||
pungi/gather.py | 12 ++++++++++++
|
||||
1 file changed, 12 insertions(+)
|
||||
|
||||
diff --git a/pungi/gather.py b/pungi/gather.py
|
||||
index 1035036..dbe38f7 100644
|
||||
--- a/pungi/gather.py
|
||||
+++ b/pungi/gather.py
|
||||
@@ -25,6 +25,7 @@ import logging
|
||||
import urlgrabber.progress
|
||||
import subprocess
|
||||
import createrepo
|
||||
+import glob
|
||||
import ConfigParser
|
||||
from sdict import AlphaSortedDict
|
||||
from fnmatch import fnmatch
|
||||
@@ -1409,9 +1410,20 @@ class Pungi(PungiBase):
|
||||
conf.baseurl = baseurl
|
||||
if compress_type:
|
||||
conf.compress_type = compress_type
|
||||
+ if 'SOURCE_DATE_EPOCH' in os.environ:
|
||||
+ conf.revision = os.environ['SOURCE_DATE_EPOCH']
|
||||
repomatic = createrepo.MetaDataGenerator(conf)
|
||||
self.logger.info('Making repodata')
|
||||
repomatic.doPkgMetadata()
|
||||
+
|
||||
+ # set mtime to $SOURCE_DATE_EPOCH, do that just before creating
|
||||
+ # repomd.xml, because it includes timestamps of referenced files
|
||||
+ if 'SOURCE_DATE_EPOCH' in os.environ:
|
||||
+ s_d_e = int(os.environ['SOURCE_DATE_EPOCH'])
|
||||
+ for repo_file in glob.glob(
|
||||
+ os.path.join(conf.outputdir, conf.tempdir, "*")):
|
||||
+ os.utime(repo_file, (s_d_e, s_d_e))
|
||||
+
|
||||
repomatic.doRepoMetadata()
|
||||
repomatic.doFinalMove()
|
||||
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@ -0,0 +1,53 @@
|
||||
From aaae0547c9bfefac7aa0d431cc4065eb369a7605 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
|
||||
<marmarek@invisiblethingslab.com>
|
||||
Date: Fri, 5 Oct 2018 15:26:36 +0200
|
||||
Subject: [PATCH 6/6] Monkey patch createrepo to clamp repodata mtime
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
Organization: Invisible Things Lab
|
||||
Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
|
||||
Unfortunately some files are created during repomatic.doRepoMetadata(),
|
||||
so clamping mtime before the call isn't enough. To limit changes to just
|
||||
one component, monkey patch createrepo function.
|
||||
|
||||
Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
|
||||
---
|
||||
pungi/gather.py | 19 +++++++++++++++++++
|
||||
1 file changed, 19 insertions(+)
|
||||
|
||||
diff --git a/pungi/gather.py b/pungi/gather.py
|
||||
index dbe38f7..9963dc6 100644
|
||||
--- a/pungi/gather.py
|
||||
+++ b/pungi/gather.py
|
||||
@@ -1424,6 +1424,25 @@ class Pungi(PungiBase):
|
||||
os.path.join(conf.outputdir, conf.tempdir, "*")):
|
||||
os.utime(repo_file, (s_d_e, s_d_e))
|
||||
|
||||
+ # unfortunately the above is not enough, because some files are created
|
||||
+ # during doRepoMetadata(). This include:
|
||||
+ # - compressed sqlite versions of the metadata - needed by repoview
|
||||
+ # - group file (compressed and not) - this is needed, so the timestamp
|
||||
+ # problem needs to be solved anyway
|
||||
+ orig_createRepoDataObject = repomatic._createRepoDataObject
|
||||
+ def wrapped_createRepoDataObject(*args, **kwargs):
|
||||
+ repodata = orig_createRepoDataObject(*args, **kwargs)
|
||||
+ if int(repodata.timestamp) > s_d_e:
|
||||
+ repodata.timestamp = str(s_d_e)
|
||||
+ return repodata
|
||||
+ repomatic._createRepoDataObject = wrapped_createRepoDataObject
|
||||
+
|
||||
+ orig_compressFile = createrepo.utils.compressFile
|
||||
+ def wrapped_compressFile(source, dest, compress_type):
|
||||
+ orig_compressFile(source, dest, compress_type)
|
||||
+ os.utime(dest, (s_d_e, s_d_e))
|
||||
+ createrepo.utils.compressFile = wrapped_compressFile
|
||||
+
|
||||
repomatic.doRepoMetadata()
|
||||
repomatic.doFinalMove()
|
||||
|
||||
--
|
||||
2.17.1
|
||||
|
||||
Loading…
Reference in new issue