mirror of
https://github.com/projectatomic/rpmdistro-gitoverlay.git
synced 2026-02-06 18:45:47 +01:00
56b1e02f01
https://pagure.io/fedora-atomic-host-continuous/issue/16 Fedora started using richdeps in the `BuildRequires`, which finally brought down our hacky tower to do Fedora RPM builds from a CentOS 7 host. Complicating all of this is that mock in Fedora is python3. So we'd have to maintain compatibility with both 2/3 and that sucks. Let's do a hard 3 port for now.
275 lines
12 KiB
Python
275 lines
12 KiB
Python
# Copyright (C) 2015 Colin Walters <walters@verbum.org>
|
|
#
|
|
# This library is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU Lesser General Public
|
|
# License as published by the Free Software Foundation; either
|
|
# version 2 of the License, or (at your option) any later version.
|
|
#
|
|
# This library is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# Lesser General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Lesser General Public
|
|
# License along with this library; if not, write to the
|
|
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
# Boston, MA 02111-1307, USA.
|
|
|
|
import os
|
|
import json
|
|
import argparse
|
|
import subprocess
|
|
import errno
|
|
import shutil
|
|
import tempfile
|
|
|
|
from .utils import log, fatal, ensuredir, rmrf, ensure_clean_dir, run_sync
|
|
from .basetask_resolve import BaseTaskResolve
|
|
from . import specfile
|
|
from .git import GitRemote
|
|
|
|
def require_key(conf, key):
|
|
try:
|
|
return conf[key]
|
|
except KeyError:
|
|
fatal("Missing config key {0}".format(key))
|
|
|
|
class TaskResolve(BaseTaskResolve):
|
|
def __init__(self):
|
|
BaseTaskResolve.__init__(self)
|
|
self._srpm_mock_initialized = None
|
|
|
|
def _json_dumper(self, obj):
|
|
if isinstance(obj, GitRemote):
|
|
return obj.url
|
|
else:
|
|
return obj
|
|
|
|
def _tar_czf_with_prefix(self, dirpath, prefix, output):
|
|
dn = os.path.dirname(dirpath)
|
|
bn = os.path.basename(dirpath)
|
|
run_sync(['tar', '--exclude-vcs', '-czf', output, '--transform', 's,^' + bn + ',' + prefix + ',', bn],
|
|
cwd=dn)
|
|
|
|
def _strip_all_prefixes(self, s, prefixes):
|
|
for prefix in prefixes:
|
|
if s.startswith(prefix):
|
|
s = s[len(prefix):]
|
|
return s
|
|
|
|
def _rpm_verrel(self, component, upstream_tag, upstream_rev, distgit_desc):
|
|
override_version = component.get('override-version')
|
|
gitdesc = upstream_rev or ''
|
|
if distgit_desc is not None:
|
|
if gitdesc != '':
|
|
gitdesc += '.'
|
|
gitdesc += distgit_desc.replace('-', '.')
|
|
if override_version is not None:
|
|
return [override_version, gitdesc]
|
|
rpm_version = upstream_tag or '0'
|
|
# Some common patterns out there.
|
|
known_tag_prefixes = ['v']
|
|
for prefix in [component['pkgname'],component['pkgname'].upper()]:
|
|
for suffix in ['-', '_']:
|
|
known_tag_prefixes.append(prefix + suffix)
|
|
rpm_version = self._strip_all_prefixes(rpm_version, known_tag_prefixes)
|
|
rpm_version = rpm_version.replace('-', '.')
|
|
return [rpm_version, gitdesc]
|
|
|
|
def _generate_srcsnap_impl(self, component, upstream_tag, upstream_rev, upstream_co,
|
|
distgit_desc, distgit_co, target):
|
|
distgit = component.get('distgit')
|
|
if distgit is not None:
|
|
patches_action = distgit.get('patches', None)
|
|
else:
|
|
patches_action = None
|
|
|
|
upstream_desc = upstream_rev
|
|
if upstream_tag is not None:
|
|
upstream_desc = upstream_tag + '-' + upstream_desc
|
|
|
|
[rpm_version, rpm_release] = self._rpm_verrel(component, upstream_tag, upstream_rev, distgit_desc)
|
|
|
|
spec_fn = specfile.spec_fn(spec_dir=distgit_co)
|
|
spec = specfile.Spec(distgit_co + '/' + spec_fn)
|
|
|
|
if upstream_desc is not None:
|
|
tar_dirname = '{0}-{1}'.format(component['name'], upstream_desc)
|
|
tarname = tar_dirname + '.tar.gz'
|
|
tmp_tarpath = distgit_co + '/' + tarname
|
|
self._tar_czf_with_prefix(upstream_co, tar_dirname, tmp_tarpath)
|
|
rmrf(upstream_co)
|
|
has_zero = spec.get_tag('Source0', allow_empty=True) is not None
|
|
source_tag = 'Source'
|
|
if has_zero:
|
|
source_tag += '0'
|
|
spec.set_tag(source_tag, tarname)
|
|
# This is a currently ad-hoc convention
|
|
spec.set_global('commit', upstream_rev)
|
|
spec.set_tag('Version', rpm_version)
|
|
spec.set_setup_dirname(tar_dirname)
|
|
spec.set_tag('Release', rpm_release + '%{?dist}')
|
|
|
|
# Anything useful there you should find in upstream dist-git or equivalent.
|
|
spec.delete_changelog()
|
|
# Forcibly override
|
|
# spec.set_tag('Epoch', '99')
|
|
if patches_action in (None, 'keep'):
|
|
pass
|
|
elif patches_action == 'drop':
|
|
spec.wipe_patches()
|
|
else:
|
|
fatal("Component '{0}': Unknown patches action '{1}'".format(component['name'],
|
|
patches_action))
|
|
spec.save()
|
|
spec._txt = '# NOTE: AUTO-GENERATED by rpmdistro-gitoverlay; DO NOT EDIT\n' + spec._txt
|
|
|
|
sources_path = distgit_co + '/sources'
|
|
if os.path.exists(sources_path):
|
|
# Exec as an external binary because pyrpkg is python 2 only, and
|
|
# mock is Python 3 only. Sigh.
|
|
subprocess.check_call([PKGLIBDIR + '/rpkg-prep-sources', # noqa pylint: disable=undefined-variable
|
|
'--distgit-name='+distgit['name'],
|
|
'--distgit-url='+distgit['src'].url,
|
|
'--distgit-co='+distgit_co,
|
|
'--lookaside-mirror='+self.lookaside_mirror])
|
|
|
|
shutil.move(distgit_co, self.tmp_snapshotdir + '/' + target)
|
|
|
|
def _generate_srcsnap(self, component):
|
|
upstream_src = component.get('src')
|
|
if upstream_src is not None:
|
|
upstream_rev = component['revision']
|
|
[upstream_tag, upstream_rev] = self.mirror.describe(upstream_src, upstream_rev)
|
|
upstream_desc = upstream_rev
|
|
if upstream_tag is not None:
|
|
upstream_desc = upstream_tag + '-' + upstream_desc
|
|
else:
|
|
upstream_rev = upstream_tag = upstream_desc = None
|
|
|
|
distgit = component.get('distgit')
|
|
if distgit is not None:
|
|
distgit_src = distgit['src']
|
|
distgit_rev = distgit['revision']
|
|
[distgit_tag, distgit_rev] = self.mirror.describe(distgit_src, distgit_rev)
|
|
distgit_desc = distgit_rev
|
|
if distgit_tag is not None:
|
|
distgit_desc = distgit_tag + '-' + distgit_desc
|
|
else:
|
|
distgit_desc = None
|
|
|
|
assert (upstream_desc or distgit_desc) is not None
|
|
|
|
[rpm_version, rpm_release] = self._rpm_verrel(component, upstream_tag, upstream_rev, distgit_desc)
|
|
|
|
srcsnap_name = "{0}-{1}-{2}.srcsnap".format(component['pkgname'], rpm_version, rpm_release)
|
|
tmpdir = tempfile.mkdtemp('', 'rdgo-srpms', self.tmpdir)
|
|
try:
|
|
if upstream_src is not None:
|
|
upstream_co = tmpdir + '/' + component['name']
|
|
self.mirror.checkout(upstream_src, upstream_rev, upstream_co)
|
|
else:
|
|
upstream_co = None
|
|
|
|
if distgit is not None:
|
|
distgit_topdir = tmpdir + '/' + 'distgit'
|
|
ensure_clean_dir(distgit_topdir)
|
|
# Create a directory whose name matches the module
|
|
# name, which helps fedpkg/rhpkg.
|
|
distgit_co = distgit_topdir + '/' + distgit['name']
|
|
self.mirror.checkout(distgit_src, distgit_rev, distgit_co)
|
|
else:
|
|
specfn = self._find_spec(upstream_co)
|
|
if specfn is None:
|
|
fatal("Failed to find .spec (or .spec.in) file")
|
|
if specfn.endswith('.in'):
|
|
dest_specfn = tmpdir + '/' + os.path.basename(specfn[:-3])
|
|
else:
|
|
dest_specfn = tmpdir
|
|
shutil.copy2(specfn, dest_specfn)
|
|
distgit_co = tmpdir
|
|
|
|
self._generate_srcsnap_impl(component, upstream_tag, upstream_rev, upstream_co,
|
|
distgit_desc, distgit_co,
|
|
srcsnap_name)
|
|
finally:
|
|
if 'PRESERVE_TEMP' not in os.environ:
|
|
rmrf(tmpdir)
|
|
return srcsnap_name
|
|
|
|
def run(self, argv):
|
|
parser = argparse.ArgumentParser(description="Create snapshot.json")
|
|
parser.add_argument('--tempdir', action='store', default=None,
|
|
help='Path to directory for temporary working files')
|
|
parser.add_argument('--fetch-all', action='store_true', help='Fetch all git repositories')
|
|
parser.add_argument('-f', '--fetch', action='append', default=[],
|
|
help='Fetch the specified git repository')
|
|
parser.add_argument('--override-giturl', action='store',
|
|
help='If the provided git URL if it is in the overlay, prepare to override it, otherwise exit 77')
|
|
parser.add_argument('--override-gitbranch', action='store',
|
|
help='Use with --override-giturl to specify a branch')
|
|
parser.add_argument('--override-gitrepo-from', action='store',
|
|
help='Pull from this local git repository')
|
|
parser.add_argument('--override-gitrepo-from-rev', action='store',
|
|
help='Use with --override-gitrepo-from to specify an expected revision')
|
|
parser.add_argument('--touch-if-changed', action='store', default=None,
|
|
help='Create or update timestamp on target path if a change occurred')
|
|
parser.add_argument('-b', '--build', action='store_true',
|
|
help='If fetch changes, automatically do a build')
|
|
|
|
opts = parser.parse_args(argv)
|
|
|
|
srcdir = self.workdir + '/src'
|
|
if not os.path.isdir(srcdir):
|
|
fatal("Missing src/ directory; run 'rpmdistro-gitoverlay init'?")
|
|
if os.path.islink(srcdir):
|
|
fatal("src/ directory is a symbolic link; is this a thin clone?")
|
|
|
|
self._load_overlay()
|
|
|
|
self.tmpdir = opts.tempdir
|
|
self.old_snapshotdir = self.workdir + '/old-snapshot'
|
|
self.snapshotdir = self.workdir + '/snapshot'
|
|
self.tmp_snapshotdir = self.snapshotdir + '.tmp'
|
|
ensure_clean_dir(self.tmp_snapshotdir)
|
|
|
|
ensuredir(self.lookaside_mirror)
|
|
|
|
expanded = self._expand_overlay(fetchall=opts.fetch_all, fetch=opts.fetch,
|
|
override_giturl=opts.override_giturl,
|
|
override_gitbranch=opts.override_gitbranch,
|
|
override_gitrepo_from=opts.override_gitrepo_from,
|
|
override_gitrepo_from_rev=opts.override_gitrepo_from_rev)
|
|
|
|
for component in expanded['components']:
|
|
srcsnap = self._generate_srcsnap(component)
|
|
component['srcsnap'] = os.path.basename(srcsnap)
|
|
|
|
snapshot_path = self.snapshotdir + '/snapshot.json'
|
|
snapshot_tmppath = self.tmp_snapshotdir + '/snapshot.json'
|
|
with open(snapshot_tmppath, 'w') as f:
|
|
json.dump(expanded, f, indent=4, sort_keys=True, default=self._json_dumper)
|
|
|
|
rmrf(self.old_snapshotdir)
|
|
|
|
changed = True
|
|
if (os.path.exists(snapshot_path) and subprocess.call(['cmp', '-s', snapshot_path, snapshot_tmppath]) == 0):
|
|
changed = False
|
|
if changed:
|
|
try:
|
|
os.rename(self.snapshotdir, self.old_snapshotdir)
|
|
except OSError as e:
|
|
if e.errno != errno.ENOENT:
|
|
raise
|
|
os.rename(self.tmp_snapshotdir, self.snapshotdir)
|
|
log("Wrote: " + self.snapshotdir)
|
|
if opts.touch_if_changed:
|
|
with open(opts.touch_if_changed, 'a'):
|
|
log("Updated timestamp of {}".format(opts.touch_if_changed))
|
|
os.utime(opts.touch_if_changed, None)
|
|
if opts.build:
|
|
os.execlp('rpmdistro-gitoverlay', 'rpmdistro-gitoverlay', 'build')
|
|
else:
|
|
rmrf(self.tmp_snapshotdir)
|
|
log("No changes.")
|