pungi: add patches for making the output reproducible

pungi/0001-Use-SOURCE_DATE_EPOCH-if-set-in-discinfo-file.patch

@@ -0,0 +1,45 @@
+From 1b0ff7f98bdce87deb7bc61d6c227be21fa43a94 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
+ <marmarek@invisiblethingslab.com>
+Date: Thu, 4 Oct 2018 23:36:15 +0200
+Subject: [PATCH 1/6] Use $SOURCE_DATE_EPOCH (if set) in discinfo file
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+Organization: Invisible Things Lab
+Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+
+This helps the output image to be reproducible.
+
+Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+---
+ pungi/compose_metadata/discinfo.py | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/pungi/compose_metadata/discinfo.py b/pungi/compose_metadata/discinfo.py
+index df61ca0..758feef 100644
+--- a/pungi/compose_metadata/discinfo.py
++++ b/pungi/compose_metadata/discinfo.py
+@@ -32,6 +32,7 @@ __all__ = (
+ )
+ 
+ 
++import os
+ import time
+ 
+ 
+@@ -43,7 +44,10 @@ def write_discinfo(file_path, description, arch, disc_numbers=None, timestamp=No
+     if not isinstance(disc_numbers, list):
+         raise TypeError("Invalid type: disc_numbers type is %s; expected: <list>" % type(disc_numbers))
+     if not timestamp:
+-        timestamp = "%f" % time.time()
++        if 'SOURCE_DATE_EPOCH' in os.environ:
++            timestamp = os.environ['SOURCE_DATE_EPOCH']
++        else:
++            timestamp = "%f" % time.time()
+     with open(file_path, "w") as f:
+         f.write("%s\n" % timestamp)
+         f.write("%s\n" % description)
+-- 
+2.17.1
+

pungi/0002-Use-xorriso-instead-of-genisoimage.patch

@@ -0,0 +1,49 @@
+From d33b8f9995070d68472c25779dfa543d6d7535db Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
+ <marmarek@invisiblethingslab.com>
+Date: Thu, 4 Oct 2018 23:37:35 +0200
+Subject: [PATCH 2/6] Use xorriso instead of genisoimage
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+Organization: Invisible Things Lab
+Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+
+xorriso make the image reproducible (given the same input files),
+including support for SOURCE_DATE_EPOCH in various metadata.
+
+Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+---
+ pungi.spec      | 2 +-
+ pungi/gather.py | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/pungi.spec b/pungi.spec
+index 6a23a63..d7bdc66 100644
+--- a/pungi.spec
++++ b/pungi.spec
+@@ -35,7 +35,7 @@ Requires:       jigdo
+ Requires:       cvs
+ Requires:       yum-utils
+ Requires:       isomd5sum
+-Requires:       genisoimage
++Requires:       xorriso
+ Requires:       gettext
+ Requires:       syslinux
+ Requires:       git
+diff --git a/pungi/gather.py b/pungi/gather.py
+index 20cc33d..15dfcee 100644
+--- a/pungi/gather.py
++++ b/pungi/gather.py
+@@ -1709,7 +1709,7 @@ class Pungi(PungiBase):
+                            clean=True) # This is risky...
+ 
+         # setup the base command
+-        mkisofs = ['/usr/bin/mkisofs']
++        mkisofs = ['/usr/bin/xorriso', '-as', 'mkisofs']
+         mkisofs.extend(['-v', '-U', '-J', '-R', '-T', '-m', 'repoview', '-m', 'boot.iso']) # common mkisofs flags
+ 
+         x86bootargs = ['-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', 
+-- 
+2.17.1
+

pungi/0003-Use-constant-MBR-ID-for-isohybrid.patch

@@ -0,0 +1,34 @@
+From 61ed5d6ea5b1beedb59b3962d0df99f0b3c69402 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
+ <marmarek@invisiblethingslab.com>
+Date: Thu, 4 Oct 2018 23:38:57 +0200
+Subject: [PATCH 3/6] Use constant MBR ID for isohybrid
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+Organization: Invisible Things Lab
+Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+
+If not set explicitly, isohybrid choose it randomly, which harm
+reproducibility.
+
+Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+---
+ pungi/gather.py | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/pungi/gather.py b/pungi/gather.py
+index 15dfcee..6f52bc6 100644
+--- a/pungi/gather.py
++++ b/pungi/gather.py
+@@ -1731,6 +1731,7 @@ class Pungi(PungiBase):
+         ppcbootargs.append('-hfs-bless') # must be last
+ 
+         isohybrid = ['/usr/bin/isohybrid']
++        isohybrid.extend(['--id', '42'])
+ 
+         # Check the size of the tree
+         # This size checking method may be bunk, accepting patches...
+-- 
+2.17.1
+

pungi/0004-Make-sure-.treeinfo-file-is-sorted.patch

@@ -0,0 +1,67 @@
+From 57e49f366a34e3d8fdb020d7f19bc2fec8547ec9 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
+ <marmarek@invisiblethingslab.com>
+Date: Thu, 4 Oct 2018 23:42:19 +0200
+Subject: [PATCH 4/6] Make sure .treeinfo file is sorted
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+Organization: Invisible Things Lab
+Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+
+OrderedDict used by default by ConfigParser isn't enough because order
+of entries being added may not be deterministic (depends on directory
+list order). To solve this problem, use SortedDict as a base.
+
+Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+---
+ pungi.spec      | 2 ++
+ pungi/gather.py | 5 +++++
+ 2 files changed, 7 insertions(+)
+
+diff --git a/pungi.spec b/pungi.spec
+index d7bdc66..dcb1986 100644
+--- a/pungi.spec
++++ b/pungi.spec
+@@ -17,6 +17,7 @@ BuildRequires:  python-jsonschema
+ BuildRequires:  python-enum34
+ BuildRequires:  python2-dnf
+ BuildRequires:  python2-multilib
++BuildRequires:  python2-dict-sorted
+ 
+ Requires:       createrepo >= 0.4.11
+ Requires:       yum => 3.4.3-28
+@@ -44,6 +45,7 @@ Requires:       libguestfs-tools-c
+ Requires:       python-enum34
+ Requires:       python2-dnf
+ Requires:       python2-multilib
++Requires:       python2-dict-sorted
+ 
+ BuildArch:      noarch
+ 
+diff --git a/pungi/gather.py b/pungi/gather.py
+index 6f52bc6..1035036 100644
+--- a/pungi/gather.py
++++ b/pungi/gather.py
+@@ -26,6 +26,7 @@ import urlgrabber.progress
+ import subprocess
+ import createrepo
+ import ConfigParser
++from sdict import AlphaSortedDict
+ from fnmatch import fnmatch
+ 
+ import arch as arch_module
+@@ -95,6 +96,10 @@ def is_package(po):
+ class MyConfigParser(ConfigParser.ConfigParser):
+     """A subclass of ConfigParser which does not lowercase options"""
+ 
++    def __init__(self, *args, **kwargs):
++        kwargs['dict_type'] = AlphaSortedDict
++        ConfigParser.ConfigParser.__init__(self, *args, **kwargs)
++
+     def optionxform(self, optionstr):
+         return optionstr
+ 
+-- 
+2.17.1
+

pungi/0005-Set-repodata-mtime-to-SOURCE_DATE_EPOCH.patch

@@ -0,0 +1,60 @@
+From 8eaee0ada8caa8b509b96867b06b876ef606d64f Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
+ <marmarek@invisiblethingslab.com>
+Date: Thu, 4 Oct 2018 23:44:06 +0200
+Subject: [PATCH 5/6] Set repodata mtime to SOURCE_DATE_EPOCH
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+Organization: Invisible Things Lab
+Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+
+repodata/repomd.xml include timestamps of all the other repodata files.
+Even when those files are created reproducibly, they have current
+modification time. In general case this is a good thing (ease checking
+if repodata cache is up to date). But in case of composing installation
+image, it breaks reproducibility.
+Avoid this by reseting mtime of repodata/* to $SOURCE_DATE_EPOCH, just
+before creating repomd.xml.
+
+Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+---
+ pungi/gather.py | 12 ++++++++++++
+ 1 file changed, 12 insertions(+)
+
+diff --git a/pungi/gather.py b/pungi/gather.py
+index 1035036..dbe38f7 100644
+--- a/pungi/gather.py
++++ b/pungi/gather.py
+@@ -25,6 +25,7 @@ import logging
+ import urlgrabber.progress
+ import subprocess
+ import createrepo
++import glob
+ import ConfigParser
+ from sdict import AlphaSortedDict
+ from fnmatch import fnmatch
+@@ -1409,9 +1410,20 @@ class Pungi(PungiBase):
+             conf.baseurl = baseurl
+         if compress_type:
+             conf.compress_type = compress_type
++        if 'SOURCE_DATE_EPOCH' in os.environ:
++            conf.revision = os.environ['SOURCE_DATE_EPOCH']
+         repomatic = createrepo.MetaDataGenerator(conf)
+         self.logger.info('Making repodata')
+         repomatic.doPkgMetadata()
++
++        # set mtime to $SOURCE_DATE_EPOCH, do that just before creating
++        # repomd.xml, because it includes timestamps of referenced files
++        if 'SOURCE_DATE_EPOCH' in os.environ:
++            s_d_e = int(os.environ['SOURCE_DATE_EPOCH'])
++            for repo_file in glob.glob(
++                    os.path.join(conf.outputdir, conf.tempdir, "*")):
++                os.utime(repo_file, (s_d_e, s_d_e))
++
+         repomatic.doRepoMetadata()
+         repomatic.doFinalMove()
+         
+-- 
+2.17.1
+

pungi/0006-Monkey-patch-createrepo-to-clamp-repodata-mtime.patch

@@ -0,0 +1,53 @@
+From aaae0547c9bfefac7aa0d431cc4065eb369a7605 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
+ <marmarek@invisiblethingslab.com>
+Date: Fri, 5 Oct 2018 15:26:36 +0200
+Subject: [PATCH 6/6] Monkey patch createrepo to clamp repodata mtime
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+Organization: Invisible Things Lab
+Cc: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+
+Unfortunately some files are created during repomatic.doRepoMetadata(),
+so clamping mtime before the call isn't enough. To limit changes to just
+one component, monkey patch createrepo function.
+
+Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+---
+ pungi/gather.py | 19 +++++++++++++++++++
+ 1 file changed, 19 insertions(+)
+
+diff --git a/pungi/gather.py b/pungi/gather.py
+index dbe38f7..9963dc6 100644
+--- a/pungi/gather.py
++++ b/pungi/gather.py
+@@ -1424,6 +1424,25 @@ class Pungi(PungiBase):
+                     os.path.join(conf.outputdir, conf.tempdir, "*")):
+                 os.utime(repo_file, (s_d_e, s_d_e))
+ 
++            # unfortunately the above is not enough, because some files are created
++            # during doRepoMetadata(). This include:
++            #  - compressed sqlite versions of the metadata - needed by repoview
++            #  - group file (compressed and not) - this is needed, so the timestamp
++            #    problem needs to be solved anyway
++            orig_createRepoDataObject = repomatic._createRepoDataObject
++            def wrapped_createRepoDataObject(*args, **kwargs):
++                repodata = orig_createRepoDataObject(*args, **kwargs)
++                if int(repodata.timestamp) > s_d_e:
++                    repodata.timestamp = str(s_d_e)
++                return repodata
++            repomatic._createRepoDataObject = wrapped_createRepoDataObject
++
++            orig_compressFile = createrepo.utils.compressFile
++            def wrapped_compressFile(source, dest, compress_type):
++                orig_compressFile(source, dest, compress_type)
++                os.utime(dest, (s_d_e, s_d_e))
++            createrepo.utils.compressFile = wrapped_compressFile
++
+         repomatic.doRepoMetadata()
+         repomatic.doFinalMove()
+         
+-- 
+2.17.1
+

pungi/pungi.spec

@@ -14,6 +14,13 @@ Patch3:         disable-efi.patch
 Patch4:         Hacky-way-to-pass-gpgkey-to-lorax.patch
 #Patch5:         fix-recursive-partition-table-on-iso-image.patch
 #Patch6:         disable-upgrade.patch
+Patch7:         0001-Use-SOURCE_DATE_EPOCH-if-set-in-discinfo-file.patch
+Patch8:         0002-Use-xorriso-instead-of-genisoimage.patch
+Patch9:         0003-Use-constant-MBR-ID-for-isohybrid.patch
+Patch10:       	0004-Make-sure-.treeinfo-file-is-sorted.patch
+Patch11:       	0005-Set-repodata-mtime-to-SOURCE_DATE_EPOCH.patch
+Patch12:        0006-Monkey-patch-createrepo-to-clamp-repodata-mtime.patch
+
 BuildRequires:  python-nose, python-mock
 BuildRequires:  python-devel, python-setuptools, python2-productmd >= 1.3
 BuildRequires:  python-lockfile, kobo, kobo-rpmlib, python-kickstart, createrepo_c
@@ -24,6 +31,7 @@ BuildRequires:  python-jsonschema
 BuildRequires:  python-enum34
 BuildRequires:  python2-dnf
 BuildRequires:  python2-multilib
+BuildRequires:  python2-dict-sorted
 
 #deps for doc building
 BuildRequires:  python-sphinx, texlive-latex-bin-bin, texlive-collection-fontsrecommended
@@ -52,7 +60,7 @@ Requires:       koji >= 1.10.1-13
 Requires:       cvs
 Requires:       yum-utils
 Requires:       isomd5sum
-Requires:       genisoimage
+Requires:       xorriso
 Requires:       gettext
 # this is x86 only 
 #Requires:       syslinux
@@ -61,6 +69,7 @@ Requires:       python-jsonschema
 Requires:       python-enum34
 Requires:       python2-dnf
 Requires:       python2-multilib
+Requires:       python2-dict-sorted
 
 BuildArch:      noarch
 
@@ -86,6 +95,12 @@ notification to Fedora Message Bus.
 %patch4 -p1
 #%%patch5 -p1
 #%%patch6 -p1
+%patch7 -p1
+%patch8 -p1
+%patch9 -p1
+%patch10 -p1
+%patch11 -p1
+%patch12 -p1
 
 %build
 %{__python} setup.py build