mirror of
https://github.com/flatcar/scripts.git
synced 2025-08-08 21:46:58 +02:00
63c51c4d3c
Preflight prebuilts reference the last full build, so they should always be complete as long as the last full build is still there. Also add host prebuilts to the preflight prebuilts. BUG=chromium-os:5311 TEST=Run unit tests for cbuildbot and prebuilt.py. Test runs of cbuildbot.py with --dryrun. Review URL: http://codereview.chromium.org/5344002 Change-Id: Id95f94c02cc2f6cbd70a029d4f8b94617b7cc071
308 lines
9.3 KiB
Python
308 lines
9.3 KiB
Python
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
#
|
|
# Adapted from portage/getbinpkg.py -- Portage binary-package helper functions
|
|
# Copyright 2003-2004 Gentoo Foundation
|
|
# Distributed under the terms of the GNU General Public License v2
|
|
|
|
import operator
|
|
import os
|
|
import tempfile
|
|
import time
|
|
import urllib2
|
|
import urlparse
|
|
|
|
class PackageIndex(object):
|
|
"""A parser for the Portage Packages index file.
|
|
|
|
The Portage Packages index file serves to keep track of what packages are
|
|
included in a tree. It contains the following sections:
|
|
1) The header. The header tracks general key/value pairs that don't apply
|
|
to any specific package. E.g., it tracks the base URL of the packages
|
|
file, and the number of packages included in the file. The header is
|
|
terminated by a blank line.
|
|
2) The body. The body is a list of packages. Each package contains a list
|
|
of key/value pairs. Packages are either terminated by a blank line or
|
|
by the end of the file. Every package has a CPV entry, which serves as
|
|
a unique identifier for the package.
|
|
"""
|
|
|
|
def __init__(self):
|
|
"""Constructor."""
|
|
|
|
# The header tracks general key/value pairs that don't apply to any
|
|
# specific package. E.g., it tracks the base URL of the packages.
|
|
self.header = {}
|
|
|
|
# A list of packages (stored as a list of dictionaries).
|
|
self.packages = []
|
|
|
|
# Whether or not the PackageIndex has been modified since the last time it
|
|
# was written.
|
|
self.modified = False
|
|
|
|
def _PopulateDuplicateDB(self, db):
|
|
"""Populate db with SHA1 -> URL mapping for packages.
|
|
|
|
Args:
|
|
db: Dictionary to populate with SHA1 -> URL mapping for packages.
|
|
"""
|
|
|
|
uri = self.header['URI']
|
|
for pkg in self.packages:
|
|
cpv, sha1 = pkg['CPV'], pkg['SHA1']
|
|
path = pkg.get('PATH', cpv + '.tbz2')
|
|
db[sha1] = urlparse.urljoin(uri, path)
|
|
|
|
def _ReadPkgIndex(self, pkgfile):
|
|
"""Read a list of key/value pairs from the Packages file into a dictionary.
|
|
|
|
Both header entries and package entries are lists of key/value pairs, so
|
|
they can both be read by this function. Entries can be terminated by empty
|
|
lines or by the end of the file.
|
|
|
|
This function will read lines from the specified file until it encounters
|
|
the a blank line or the end of the file.
|
|
|
|
Keys and values in the Packages file are separated by a colon and a space.
|
|
Keys may contain capital letters, numbers, and underscores, but may not
|
|
contain colons. Values may contain any character except a newline. In
|
|
particular, it is normal for values to contain colons.
|
|
|
|
Lines that have content, and do not contain a valid key/value pair, are
|
|
ignored. This is for compatibility with the Portage package parser, and
|
|
to allow for future extensions to the Packages file format.
|
|
|
|
All entries must contain at least one key/value pair. If the end of the
|
|
fils is reached, an empty dictionary is returned.
|
|
|
|
Args:
|
|
pkgfile: A python file object.
|
|
|
|
Returns the dictionary of key-value pairs that was read from the file.
|
|
"""
|
|
d = {}
|
|
for line in pkgfile:
|
|
line = line.rstrip('\n')
|
|
if not line:
|
|
assert d, 'Packages entry must contain at least one key/value pair'
|
|
break
|
|
line = line.split(': ', 1)
|
|
if len(line) == 2:
|
|
k, v = line
|
|
d[k] = v
|
|
return d
|
|
|
|
def _WritePkgIndex(self, pkgfile, entry):
|
|
"""Write header entry or package entry to packages file.
|
|
|
|
The keys and values will be separated by a colon and a space. The entry
|
|
will be terminated by a blank line.
|
|
|
|
Args:
|
|
pkgfile: A python file object.
|
|
entry: A dictionary of the key/value pairs to write.
|
|
"""
|
|
lines = ['%s: %s' % (k, v) for k, v in sorted(entry.items()) if v]
|
|
pkgfile.write('%s\n\n' % '\n'.join(lines))
|
|
|
|
def _ReadHeader(self, pkgfile):
|
|
"""Read header of packages file.
|
|
|
|
Args:
|
|
pkgfile: A python file object.
|
|
"""
|
|
assert not self.header, 'Should only read header once.'
|
|
self.header = self._ReadPkgIndex(pkgfile)
|
|
|
|
def _ReadBody(self, pkgfile):
|
|
"""Read body of packages file.
|
|
|
|
Before calling this function, you must first read the header (using
|
|
_ReadHeader).
|
|
|
|
Args:
|
|
pkgfile: A python file object.
|
|
"""
|
|
assert self.header, 'Should read header first.'
|
|
assert not self.packages, 'Should only read body once.'
|
|
|
|
# Read all of the sections in the body by looping until we reach the end
|
|
# of the file.
|
|
while True:
|
|
d = self._ReadPkgIndex(pkgfile)
|
|
if not d:
|
|
break
|
|
if 'CPV' in d:
|
|
self.packages.append(d)
|
|
|
|
def Read(self, pkgfile):
|
|
"""Read the entire packages file.
|
|
|
|
Args:
|
|
pkgfile: A python file object.
|
|
"""
|
|
self._ReadHeader(pkgfile)
|
|
self._ReadBody(pkgfile)
|
|
|
|
def RemoveFilteredPackages(self, filter_fn):
|
|
"""Remove packages which match filter_fn.
|
|
|
|
Args:
|
|
filter_fn: A function which operates on packages. If it returns True,
|
|
the package should be removed.
|
|
"""
|
|
|
|
filtered = [p for p in self.packages if not filter_fn(p)]
|
|
if filtered != self.packages:
|
|
self.modified = True
|
|
self.packages = filtered
|
|
|
|
def ResolveDuplicateUploads(self, pkgindexes):
|
|
"""Point packages at files that have already been uploaded.
|
|
|
|
For each package in our index, check if there is an existing package that
|
|
has already been uploaded to the same base URI. If so, point that package
|
|
at the existing file, so that we don't have to upload the file.
|
|
|
|
Args:
|
|
pkgindexes: A list of PackageIndex objects containing info about packages
|
|
that have already been uploaded.
|
|
|
|
Returns:
|
|
A list of the packages that still need to be uploaded.
|
|
"""
|
|
db = {}
|
|
for pkgindex in pkgindexes:
|
|
pkgindex._PopulateDuplicateDB(db)
|
|
|
|
uploads = []
|
|
base_uri = self.header['URI']
|
|
for pkg in self.packages:
|
|
sha1 = pkg['SHA1']
|
|
uri = db.get(sha1)
|
|
if uri and uri.startswith(base_uri):
|
|
pkg['PATH'] = uri[len(base_uri):].lstrip('/')
|
|
else:
|
|
uploads.append(pkg)
|
|
return uploads
|
|
|
|
def SetUploadLocation(self, base_uri, path_prefix):
|
|
"""Set upload location to base_uri + path_prefix.
|
|
|
|
Args:
|
|
base_uri: Base URI for all packages in the file. We set
|
|
self.header['URI'] to this value, so all packages must live under
|
|
this directory.
|
|
path_prefix: Path prefix to use for all current packages in the file.
|
|
This will be added to the beginning of the path for every package.
|
|
"""
|
|
self.header['URI'] = base_uri
|
|
for pkg in self.packages:
|
|
pkg['PATH'] = urlparse.urljoin(path_prefix, pkg['CPV'] + '.tbz2')
|
|
|
|
def Write(self, pkgfile):
|
|
"""Write a packages file to disk.
|
|
|
|
If 'modified' flag is set, the TIMESTAMP and PACKAGES fields in the header
|
|
will be updated before writing to disk.
|
|
|
|
Args:
|
|
pkgfile: A python file object.
|
|
"""
|
|
if self.modified:
|
|
self.header['TIMESTAMP'] = str(long(time.time()))
|
|
self.header['PACKAGES'] = str(len(self.packages))
|
|
self.modified = False
|
|
self._WritePkgIndex(pkgfile, self.header)
|
|
for metadata in sorted(self.packages, key=operator.itemgetter('CPV')):
|
|
self._WritePkgIndex(pkgfile, metadata)
|
|
|
|
def WriteToNamedTemporaryFile(self):
|
|
"""Write pkgindex to a temporary file.
|
|
|
|
Args:
|
|
pkgindex: The PackageIndex object.
|
|
|
|
Returns:
|
|
A temporary file containing the packages from pkgindex.
|
|
"""
|
|
f = tempfile.NamedTemporaryFile()
|
|
self.Write(f)
|
|
f.flush()
|
|
f.seek(0)
|
|
return f
|
|
|
|
|
|
def _RetryUrlOpen(url, tries=3):
|
|
"""Open the specified url, retrying if we run into temporary errors.
|
|
|
|
We retry for both network errors and 5xx Server Errors. We do not retry
|
|
for HTTP errors with a non-5xx code.
|
|
|
|
Args:
|
|
url: The specified url.
|
|
tries: The number of times to try.
|
|
|
|
Returns:
|
|
The result of urllib2.urlopen(url).
|
|
"""
|
|
for i in range(tries):
|
|
try:
|
|
return urllib2.urlopen(url)
|
|
except urllib2.HTTPError as e:
|
|
if i + 1 >= tries or e.code < 500:
|
|
raise
|
|
else:
|
|
print 'Cannot GET %s: %s' % (url, str(e))
|
|
except urllib2.URLError as e:
|
|
if i + 1 >= tries:
|
|
raise
|
|
else:
|
|
print 'Cannot GET %s: %s' % (url, str(e))
|
|
print 'Sleeping for 10 seconds before retrying...'
|
|
time.sleep(10)
|
|
|
|
|
|
def GrabRemotePackageIndex(binhost_url):
|
|
"""Grab the latest binary package database from the specified URL.
|
|
|
|
Args:
|
|
binhost_url: Base URL of remote packages (PORTAGE_BINHOST).
|
|
|
|
Returns:
|
|
A PackageIndex object, if the Packages file can be retrieved. If the
|
|
server returns status code 404, None is returned.
|
|
"""
|
|
|
|
url = urlparse.urljoin(binhost_url, 'Packages')
|
|
try:
|
|
f = _RetryUrlOpen(url)
|
|
except urllib2.HTTPError as e:
|
|
if e.code == 404:
|
|
return None
|
|
raise
|
|
|
|
pkgindex = PackageIndex()
|
|
pkgindex.Read(f)
|
|
pkgindex.header.setdefault('URI', binhost_url)
|
|
f.close()
|
|
return pkgindex
|
|
|
|
|
|
def GrabLocalPackageIndex(package_path):
|
|
"""Read a local packages file from disk into a PackageIndex() object.
|
|
|
|
Args:
|
|
package_path: Directory containing Packages file.
|
|
|
|
Returns:
|
|
A PackageIndex object.
|
|
"""
|
|
packages_file = file(os.path.join(package_path, 'Packages'))
|
|
pkgindex = PackageIndex()
|
|
pkgindex.Read(packages_file)
|
|
packages_file.close()
|
|
return pkgindex
|