From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 127137 invoked by alias); 24 Nov 2016 22:26:58 -0000 Mailing-List: contact libc-alpha-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Archive: List-Post: List-Help: , Sender: libc-alpha-owner@sourceware.org Received: (qmail 127123 invoked by uid 89); 24 Nov 2016 22:26:58 -0000 Authentication-Results: sourceware.org; auth=none X-Virus-Found: No X-Spam-SWARE-Status: No, score=0.5 required=5.0 tests=AWL,BAYES_50,RCVD_IN_DNSWL_NONE,SPF_PASS,URIBL_RED autolearn=ham version=3.3.2 spammy=trans, mpfr, glibcgit, glibc.git X-HELO: relay1.mentorg.com Date: Thu, 24 Nov 2016 22:26:00 -0000 From: Joseph Myers To: Subject: Make build-many-glibcs.py track component versions requested and used [committed] Message-ID: User-Agent: Alpine 2.20 (DEB 67 2015-01-07) MIME-Version: 1.0 Content-Type: text/plain; charset="US-ASCII" X-ClientProxiedBy: svr-ies-mbx-01.mgc.mentorg.com (139.181.222.1) To svr-ies-mbx-01.mgc.mentorg.com (139.181.222.1) X-SW-Source: 2016-11/txt/msg00907.txt.bz2 This patch makes build-many-glibcs.py record the component versions checked out, and whether those versions were explicitly requested or defaults, in a file versions.json in the source directory. The preferred version of a component is the first of: one explicitly specified on the command line; one explicitly specified on the command line in a previous run of build-many-glibcs.py; a default version for that component. Thus, once you've run build-many-glibcs.py checkout once with the updated script (making sure to specify the right versions of any components previously checked out with a non-default version), in future you can just run it without version specifiers and it will know when a default-version component has changed its default version and so should be checked out again. Because you might have local changes and not want a default-version component checkout replaced, you need to pass the --replace-sources option to allow the script to delete and replace a component source directory automatically; otherwise, it will give an error if a version has changed. The script does not try to change branches of git or SVN checkouts without checking out from scratch; if the version number requested has changed and --replace-sources is used, the relevant source directory will be removed completely and a new one checked out from scratch. Apart from allowing automatic updates of components with default versions, this also facilitates bots reporting on the versions used in a given build. versions.json contains not just information on the version number and whether that was requested explicitly, but also git or SVN revision information intended to be used in email reports from bots. Committed. 2016-11-24 Joseph Myers * scripts/build-many-glibcs.py: Import json module. (Context.__init__): Take replace_sources argument. Load versions.json. (Context.load_versions_json): New function. (Context.store_json): Likewise. (Context.store_versions_json): Likewise. (Context.set_component_version): Likewise. (Context.checkout): Update versions.json. Check for and handle changes of version. Prefer previously explicitly specified version to default version. (Context.checkout_vcs): Return a revision identifier. (Context.git_checkout): Likewise. (Context.gcc_checkout): Likewise. (get_parser): Add --replace-sources option. (main): Pass replace_sources argument to Context call. diff --git a/scripts/build-many-glibcs.py b/scripts/build-many-glibcs.py index 1447e22..72f22a8 100755 --- a/scripts/build-many-glibcs.py +++ b/scripts/build-many-glibcs.py @@ -32,6 +32,7 @@ configurations for which compilers or glibc are to be built. """ import argparse +import json import os import re import shutil @@ -44,12 +45,14 @@ import urllib.request class Context(object): """The global state associated with builds in a given directory.""" - def __init__(self, topdir, parallelism, keep, action): + def __init__(self, topdir, parallelism, keep, replace_sources, action): """Initialize the context.""" self.topdir = topdir self.parallelism = parallelism self.keep = keep + self.replace_sources = replace_sources self.srcdir = os.path.join(topdir, 'src') + self.versions_json = os.path.join(self.srcdir, 'versions.json') self.installdir = os.path.join(topdir, 'install') self.host_libraries_installdir = os.path.join(self.installdir, 'host-libraries') @@ -65,6 +68,7 @@ class Context(object): self.glibc_configs = {} self.makefile_pieces = ['.PHONY: all\n'] self.add_all_configs() + self.load_versions_json() def get_build_triplet(self): """Determine the build triplet with config.guess.""" @@ -559,6 +563,32 @@ class Context(object): for c in configs: self.glibc_configs[c].build() + def load_versions_json(self): + """Load information about source directory versions.""" + if not os.access(self.versions_json, os.F_OK): + self.versions = {} + return + with open(self.versions_json, 'r') as f: + self.versions = json.load(f) + + def store_json(self, data, filename): + """Store information in a JSON file.""" + filename_tmp = filename + '.tmp' + with open(filename_tmp, 'w') as f: + json.dump(data, f, indent=2, sort_keys=True) + os.rename(filename_tmp, filename) + + def store_versions_json(self): + """Store information about source directory versions.""" + self.store_json(self.versions, self.versions_json) + + def set_component_version(self, component, version, explicit, revision): + """Set the version information for a component.""" + self.versions[component] = {'version': version, + 'explicit': explicit, + 'revision': revision} + self.store_versions_json() + def checkout(self, versions): """Check out the desired component versions.""" default_versions = {'binutils': 'vcs-2.27', @@ -569,6 +599,7 @@ class Context(object): 'mpc': '1.0.3', 'mpfr': '3.1.5'} use_versions = {} + explicit_versions = {} for v in versions: found_v = False for k in default_versions.keys(): @@ -579,6 +610,7 @@ class Context(object): print('error: multiple versions for %s' % k) exit(1) use_versions[k] = vx + explicit_versions[k] = True found_v = True break if not found_v: @@ -586,19 +618,36 @@ class Context(object): exit(1) for k in default_versions.keys(): if k not in use_versions: - use_versions[k] = default_versions[k] + if k in self.versions and self.versions[k]['explicit']: + use_versions[k] = self.versions[k]['version'] + explicit_versions[k] = True + else: + use_versions[k] = default_versions[k] + explicit_versions[k] = False os.makedirs(self.srcdir, exist_ok=True) for k in sorted(default_versions.keys()): update = os.access(self.component_srcdir(k), os.F_OK) v = use_versions[k] + if (update and + k in self.versions and + v != self.versions[k]['version']): + if not self.replace_sources: + print('error: version of %s has changed from %s to %s, ' + 'use --replace-sources to check out again' % + (k, self.versions[k]['version'], v)) + exit(1) + shutil.rmtree(self.component_srcdir(k)) + update = False if v.startswith('vcs-'): - self.checkout_vcs(k, v[4:], update) + revision = self.checkout_vcs(k, v[4:], update) else: self.checkout_tar(k, v, update) + revision = v + self.set_component_version(k, v, explicit_versions[k], revision) def checkout_vcs(self, component, version, update): """Check out the given version of the given component from version - control.""" + control. Return a revision identifier.""" if component == 'binutils': git_url = 'git://sourceware.org/git/binutils-gdb.git' if version == 'mainline': @@ -606,7 +655,7 @@ class Context(object): else: trans = str.maketrans({'.': '_'}) git_branch = 'binutils-%s-branch' % version.translate(trans) - self.git_checkout(component, git_url, git_branch, update) + return self.git_checkout(component, git_url, git_branch, update) elif component == 'gcc': if version == 'mainline': branch = 'trunk' @@ -614,21 +663,22 @@ class Context(object): trans = str.maketrans({'.': '_'}) branch = 'branches/gcc-%s-branch' % version.translate(trans) svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch - self.gcc_checkout(svn_url, update) + return self.gcc_checkout(svn_url, update) elif component == 'glibc': git_url = 'git://sourceware.org/git/glibc.git' if version == 'mainline': git_branch = 'master' else: git_branch = 'release/%s/master' % version - self.git_checkout(component, git_url, git_branch, update) + r = self.git_checkout(component, git_url, git_branch, update) self.fix_glibc_timestamps() + return r else: print('error: component %s coming from VCS' % component) exit(1) def git_checkout(self, component, git_url, git_branch, update): - """Check out a component from git.""" + """Check out a component from git. Return a commit identifier.""" if update: subprocess.run(['git', 'remote', 'prune', 'origin'], cwd=self.component_srcdir(component), check=True) @@ -637,6 +687,11 @@ class Context(object): else: subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url, self.component_srcdir(component)], check=True) + r = subprocess.run(['git', 'rev-parse', 'HEAD'], + cwd=self.component_srcdir(component), + stdout=subprocess.PIPE, + check=True, universal_newlines=True).stdout + return r.rstrip() def fix_glibc_timestamps(self): """Fix timestamps in a glibc checkout.""" @@ -652,12 +707,16 @@ class Context(object): subprocess.run(['touch', to_touch], check=True) def gcc_checkout(self, svn_url, update): - """Check out GCC from SVN.""" + """Check out GCC from SVN. Return the revision number.""" if not update: subprocess.run(['svn', 'co', '-q', svn_url, self.component_srcdir('gcc')], check=True) subprocess.run(['contrib/gcc_update', '--silent'], cwd=self.component_srcdir('gcc'), check=True) + r = subprocess.run(['svnversion', self.component_srcdir('gcc')], + stdout=subprocess.PIPE, + check=True, universal_newlines=True).stdout + return r.rstrip() def checkout_tar(self, component, version, update): """Check out the given version of the given component from a @@ -1125,6 +1184,9 @@ def get_parser(): help='Whether to keep all build directories, ' 'none or only those from failed builds', default='none', choices=('none', 'all', 'failed')) + parser.add_argument('--replace-sources', action='store_true', + help='Remove and replace source directories ' + 'with the wrong version of a component') parser.add_argument('topdir', help='Toplevel working directory') parser.add_argument('action', @@ -1142,7 +1204,8 @@ def main(argv): parser = get_parser() opts = parser.parse_args(argv) topdir = os.path.abspath(opts.topdir) - ctx = Context(topdir, opts.parallelism, opts.keep, opts.action) + ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources, + opts.action) ctx.run_builds(opts.action, opts.configs) -- Joseph S. Myers joseph@codesourcery.com