From 94ccc1a19a1c19a2a59feffa39e77f4711fc699e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 4 Oct 2018 23:51:29 +0200 Subject: [PATCH] pungi: add patches for making the output reproducible --- ...E_DATE_EPOCH-if-set-in-discinfo-file.patch | 45 +++++++++++++ ...2-Use-xorriso-instead-of-genisoimage.patch | 49 ++++++++++++++ ...03-Use-constant-MBR-ID-for-isohybrid.patch | 34 ++++++++++ ...4-Make-sure-.treeinfo-file-is-sorted.patch | 67 +++++++++++++++++++ ...-repodata-mtime-to-SOURCE_DATE_EPOCH.patch | 60 +++++++++++++++++ ...h-createrepo-to-clamp-repodata-mtime.patch | 53 +++++++++++++++ pungi/pungi.spec | 17 ++++- 7 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 pungi/0001-Use-SOURCE_DATE_EPOCH-if-set-in-discinfo-file.patch create mode 100644 pungi/0002-Use-xorriso-instead-of-genisoimage.patch create mode 100644 pungi/0003-Use-constant-MBR-ID-for-isohybrid.patch create mode 100644 pungi/0004-Make-sure-.treeinfo-file-is-sorted.patch create mode 100644 pungi/0005-Set-repodata-mtime-to-SOURCE_DATE_EPOCH.patch create mode 100644 pungi/0006-Monkey-patch-createrepo-to-clamp-repodata-mtime.patch diff --git a/pungi/0001-Use-SOURCE_DATE_EPOCH-if-set-in-discinfo-file.patch b/pungi/0001-Use-SOURCE_DATE_EPOCH-if-set-in-discinfo-file.patch new file mode 100644 index 0000000..a1198f5 --- /dev/null +++ b/pungi/0001-Use-SOURCE_DATE_EPOCH-if-set-in-discinfo-file.patch @@ -0,0 +1,45 @@ +From 1b0ff7f98bdce87deb7bc61d6c227be21fa43a94 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= + +Date: Thu, 4 Oct 2018 23:36:15 +0200 +Subject: [PATCH 1/6] Use $SOURCE_DATE_EPOCH (if set) in discinfo file +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Organization: Invisible Things Lab +Cc: Marek Marczykowski-Górecki + +This helps the output image to be reproducible. + +Signed-off-by: Marek Marczykowski-Górecki +--- + pungi/compose_metadata/discinfo.py | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/pungi/compose_metadata/discinfo.py b/pungi/compose_metadata/discinfo.py +index df61ca0..758feef 100644 +--- a/pungi/compose_metadata/discinfo.py ++++ b/pungi/compose_metadata/discinfo.py +@@ -32,6 +32,7 @@ __all__ = ( + ) + + ++import os + import time + + +@@ -43,7 +44,10 @@ def write_discinfo(file_path, description, arch, disc_numbers=None, timestamp=No + if not isinstance(disc_numbers, list): + raise TypeError("Invalid type: disc_numbers type is %s; expected: " % type(disc_numbers)) + if not timestamp: +- timestamp = "%f" % time.time() ++ if 'SOURCE_DATE_EPOCH' in os.environ: ++ timestamp = os.environ['SOURCE_DATE_EPOCH'] ++ else: ++ timestamp = "%f" % time.time() + with open(file_path, "w") as f: + f.write("%s\n" % timestamp) + f.write("%s\n" % description) +-- +2.17.1 + diff --git a/pungi/0002-Use-xorriso-instead-of-genisoimage.patch b/pungi/0002-Use-xorriso-instead-of-genisoimage.patch new file mode 100644 index 0000000..bf553aa --- /dev/null +++ b/pungi/0002-Use-xorriso-instead-of-genisoimage.patch @@ -0,0 +1,49 @@ +From d33b8f9995070d68472c25779dfa543d6d7535db Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= + +Date: Thu, 4 Oct 2018 23:37:35 +0200 +Subject: [PATCH 2/6] Use xorriso instead of genisoimage +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Organization: Invisible Things Lab +Cc: Marek Marczykowski-Górecki + +xorriso make the image reproducible (given the same input files), +including support for SOURCE_DATE_EPOCH in various metadata. + +Signed-off-by: Marek Marczykowski-Górecki +--- + pungi.spec | 2 +- + pungi/gather.py | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/pungi.spec b/pungi.spec +index 6a23a63..d7bdc66 100644 +--- a/pungi.spec ++++ b/pungi.spec +@@ -35,7 +35,7 @@ Requires: jigdo + Requires: cvs + Requires: yum-utils + Requires: isomd5sum +-Requires: genisoimage ++Requires: xorriso + Requires: gettext + Requires: syslinux + Requires: git +diff --git a/pungi/gather.py b/pungi/gather.py +index 20cc33d..15dfcee 100644 +--- a/pungi/gather.py ++++ b/pungi/gather.py +@@ -1709,7 +1709,7 @@ class Pungi(PungiBase): + clean=True) # This is risky... + + # setup the base command +- mkisofs = ['/usr/bin/mkisofs'] ++ mkisofs = ['/usr/bin/xorriso', '-as', 'mkisofs'] + mkisofs.extend(['-v', '-U', '-J', '-R', '-T', '-m', 'repoview', '-m', 'boot.iso']) # common mkisofs flags + + x86bootargs = ['-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', +-- +2.17.1 + diff --git a/pungi/0003-Use-constant-MBR-ID-for-isohybrid.patch b/pungi/0003-Use-constant-MBR-ID-for-isohybrid.patch new file mode 100644 index 0000000..2f5910a --- /dev/null +++ b/pungi/0003-Use-constant-MBR-ID-for-isohybrid.patch @@ -0,0 +1,34 @@ +From 61ed5d6ea5b1beedb59b3962d0df99f0b3c69402 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= + +Date: Thu, 4 Oct 2018 23:38:57 +0200 +Subject: [PATCH 3/6] Use constant MBR ID for isohybrid +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Organization: Invisible Things Lab +Cc: Marek Marczykowski-Górecki + +If not set explicitly, isohybrid choose it randomly, which harm +reproducibility. + +Signed-off-by: Marek Marczykowski-Górecki +--- + pungi/gather.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/pungi/gather.py b/pungi/gather.py +index 15dfcee..6f52bc6 100644 +--- a/pungi/gather.py ++++ b/pungi/gather.py +@@ -1731,6 +1731,7 @@ class Pungi(PungiBase): + ppcbootargs.append('-hfs-bless') # must be last + + isohybrid = ['/usr/bin/isohybrid'] ++ isohybrid.extend(['--id', '42']) + + # Check the size of the tree + # This size checking method may be bunk, accepting patches... +-- +2.17.1 + diff --git a/pungi/0004-Make-sure-.treeinfo-file-is-sorted.patch b/pungi/0004-Make-sure-.treeinfo-file-is-sorted.patch new file mode 100644 index 0000000..dbd62c3 --- /dev/null +++ b/pungi/0004-Make-sure-.treeinfo-file-is-sorted.patch @@ -0,0 +1,67 @@ +From 57e49f366a34e3d8fdb020d7f19bc2fec8547ec9 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= + +Date: Thu, 4 Oct 2018 23:42:19 +0200 +Subject: [PATCH 4/6] Make sure .treeinfo file is sorted +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Organization: Invisible Things Lab +Cc: Marek Marczykowski-Górecki + +OrderedDict used by default by ConfigParser isn't enough because order +of entries being added may not be deterministic (depends on directory +list order). To solve this problem, use SortedDict as a base. + +Signed-off-by: Marek Marczykowski-Górecki +--- + pungi.spec | 2 ++ + pungi/gather.py | 5 +++++ + 2 files changed, 7 insertions(+) + +diff --git a/pungi.spec b/pungi.spec +index d7bdc66..dcb1986 100644 +--- a/pungi.spec ++++ b/pungi.spec +@@ -17,6 +17,7 @@ BuildRequires: python-jsonschema + BuildRequires: python-enum34 + BuildRequires: python2-dnf + BuildRequires: python2-multilib ++BuildRequires: python2-dict-sorted + + Requires: createrepo >= 0.4.11 + Requires: yum => 3.4.3-28 +@@ -44,6 +45,7 @@ Requires: libguestfs-tools-c + Requires: python-enum34 + Requires: python2-dnf + Requires: python2-multilib ++Requires: python2-dict-sorted + + BuildArch: noarch + +diff --git a/pungi/gather.py b/pungi/gather.py +index 6f52bc6..1035036 100644 +--- a/pungi/gather.py ++++ b/pungi/gather.py +@@ -26,6 +26,7 @@ import urlgrabber.progress + import subprocess + import createrepo + import ConfigParser ++from sdict import AlphaSortedDict + from fnmatch import fnmatch + + import arch as arch_module +@@ -95,6 +96,10 @@ def is_package(po): + class MyConfigParser(ConfigParser.ConfigParser): + """A subclass of ConfigParser which does not lowercase options""" + ++ def __init__(self, *args, **kwargs): ++ kwargs['dict_type'] = AlphaSortedDict ++ ConfigParser.ConfigParser.__init__(self, *args, **kwargs) ++ + def optionxform(self, optionstr): + return optionstr + +-- +2.17.1 + diff --git a/pungi/0005-Set-repodata-mtime-to-SOURCE_DATE_EPOCH.patch b/pungi/0005-Set-repodata-mtime-to-SOURCE_DATE_EPOCH.patch new file mode 100644 index 0000000..b1eb1e9 --- /dev/null +++ b/pungi/0005-Set-repodata-mtime-to-SOURCE_DATE_EPOCH.patch @@ -0,0 +1,60 @@ +From 8eaee0ada8caa8b509b96867b06b876ef606d64f Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= + +Date: Thu, 4 Oct 2018 23:44:06 +0200 +Subject: [PATCH 5/6] Set repodata mtime to SOURCE_DATE_EPOCH +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Organization: Invisible Things Lab +Cc: Marek Marczykowski-Górecki + +repodata/repomd.xml include timestamps of all the other repodata files. +Even when those files are created reproducibly, they have current +modification time. In general case this is a good thing (ease checking +if repodata cache is up to date). But in case of composing installation +image, it breaks reproducibility. +Avoid this by reseting mtime of repodata/* to $SOURCE_DATE_EPOCH, just +before creating repomd.xml. + +Signed-off-by: Marek Marczykowski-Górecki +--- + pungi/gather.py | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + +diff --git a/pungi/gather.py b/pungi/gather.py +index 1035036..dbe38f7 100644 +--- a/pungi/gather.py ++++ b/pungi/gather.py +@@ -25,6 +25,7 @@ import logging + import urlgrabber.progress + import subprocess + import createrepo ++import glob + import ConfigParser + from sdict import AlphaSortedDict + from fnmatch import fnmatch +@@ -1409,9 +1410,20 @@ class Pungi(PungiBase): + conf.baseurl = baseurl + if compress_type: + conf.compress_type = compress_type ++ if 'SOURCE_DATE_EPOCH' in os.environ: ++ conf.revision = os.environ['SOURCE_DATE_EPOCH'] + repomatic = createrepo.MetaDataGenerator(conf) + self.logger.info('Making repodata') + repomatic.doPkgMetadata() ++ ++ # set mtime to $SOURCE_DATE_EPOCH, do that just before creating ++ # repomd.xml, because it includes timestamps of referenced files ++ if 'SOURCE_DATE_EPOCH' in os.environ: ++ s_d_e = int(os.environ['SOURCE_DATE_EPOCH']) ++ for repo_file in glob.glob( ++ os.path.join(conf.outputdir, conf.tempdir, "*")): ++ os.utime(repo_file, (s_d_e, s_d_e)) ++ + repomatic.doRepoMetadata() + repomatic.doFinalMove() + +-- +2.17.1 + diff --git a/pungi/0006-Monkey-patch-createrepo-to-clamp-repodata-mtime.patch b/pungi/0006-Monkey-patch-createrepo-to-clamp-repodata-mtime.patch new file mode 100644 index 0000000..ae3bce7 --- /dev/null +++ b/pungi/0006-Monkey-patch-createrepo-to-clamp-repodata-mtime.patch @@ -0,0 +1,53 @@ +From aaae0547c9bfefac7aa0d431cc4065eb369a7605 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= + +Date: Fri, 5 Oct 2018 15:26:36 +0200 +Subject: [PATCH 6/6] Monkey patch createrepo to clamp repodata mtime +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Organization: Invisible Things Lab +Cc: Marek Marczykowski-Górecki + +Unfortunately some files are created during repomatic.doRepoMetadata(), +so clamping mtime before the call isn't enough. To limit changes to just +one component, monkey patch createrepo function. + +Signed-off-by: Marek Marczykowski-Górecki +--- + pungi/gather.py | 19 +++++++++++++++++++ + 1 file changed, 19 insertions(+) + +diff --git a/pungi/gather.py b/pungi/gather.py +index dbe38f7..9963dc6 100644 +--- a/pungi/gather.py ++++ b/pungi/gather.py +@@ -1424,6 +1424,25 @@ class Pungi(PungiBase): + os.path.join(conf.outputdir, conf.tempdir, "*")): + os.utime(repo_file, (s_d_e, s_d_e)) + ++ # unfortunately the above is not enough, because some files are created ++ # during doRepoMetadata(). This include: ++ # - compressed sqlite versions of the metadata - needed by repoview ++ # - group file (compressed and not) - this is needed, so the timestamp ++ # problem needs to be solved anyway ++ orig_createRepoDataObject = repomatic._createRepoDataObject ++ def wrapped_createRepoDataObject(*args, **kwargs): ++ repodata = orig_createRepoDataObject(*args, **kwargs) ++ if int(repodata.timestamp) > s_d_e: ++ repodata.timestamp = str(s_d_e) ++ return repodata ++ repomatic._createRepoDataObject = wrapped_createRepoDataObject ++ ++ orig_compressFile = createrepo.utils.compressFile ++ def wrapped_compressFile(source, dest, compress_type): ++ orig_compressFile(source, dest, compress_type) ++ os.utime(dest, (s_d_e, s_d_e)) ++ createrepo.utils.compressFile = wrapped_compressFile ++ + repomatic.doRepoMetadata() + repomatic.doFinalMove() + +-- +2.17.1 + diff --git a/pungi/pungi.spec b/pungi/pungi.spec index 373e79a..31d3d6c 100644 --- a/pungi/pungi.spec +++ b/pungi/pungi.spec @@ -14,6 +14,13 @@ Patch3: disable-efi.patch Patch4: Hacky-way-to-pass-gpgkey-to-lorax.patch #Patch5: fix-recursive-partition-table-on-iso-image.patch #Patch6: disable-upgrade.patch +Patch7: 0001-Use-SOURCE_DATE_EPOCH-if-set-in-discinfo-file.patch +Patch8: 0002-Use-xorriso-instead-of-genisoimage.patch +Patch9: 0003-Use-constant-MBR-ID-for-isohybrid.patch +Patch10: 0004-Make-sure-.treeinfo-file-is-sorted.patch +Patch11: 0005-Set-repodata-mtime-to-SOURCE_DATE_EPOCH.patch +Patch12: 0006-Monkey-patch-createrepo-to-clamp-repodata-mtime.patch + BuildRequires: python-nose, python-mock BuildRequires: python-devel, python-setuptools, python2-productmd >= 1.3 BuildRequires: python-lockfile, kobo, kobo-rpmlib, python-kickstart, createrepo_c @@ -24,6 +31,7 @@ BuildRequires: python-jsonschema BuildRequires: python-enum34 BuildRequires: python2-dnf BuildRequires: python2-multilib +BuildRequires: python2-dict-sorted #deps for doc building BuildRequires: python-sphinx, texlive-latex-bin-bin, texlive-collection-fontsrecommended @@ -52,7 +60,7 @@ Requires: koji >= 1.10.1-13 Requires: cvs Requires: yum-utils Requires: isomd5sum -Requires: genisoimage +Requires: xorriso Requires: gettext # this is x86 only #Requires: syslinux @@ -61,6 +69,7 @@ Requires: python-jsonschema Requires: python-enum34 Requires: python2-dnf Requires: python2-multilib +Requires: python2-dict-sorted BuildArch: noarch @@ -86,6 +95,12 @@ notification to Fedora Message Bus. %patch4 -p1 #%%patch5 -p1 #%%patch6 -p1 +%patch7 -p1 +%patch8 -p1 +%patch9 -p1 +%patch10 -p1 +%patch11 -p1 +%patch12 -p1 %build %{__python} setup.py build