From f5948251b6b833ede7a7b2980cbbcf26f63f7427 Mon Sep 17 00:00:00 2001 From: Scott Zawalski Date: Mon, 4 Oct 2010 15:15:44 -0700 Subject: [PATCH] Change prebuilt filters to use the private ebuilds to generate a definitive list. Add unittests to test an empty filter list, loading filters, and filtering specific packages based off of mock filters. --- prebuilt.py | 43 +++++++++++++++++------- prebuilt_unittest.py | 80 +++++++++++++++++++++++++++++++++----------- 2 files changed, 91 insertions(+), 32 deletions(-) diff --git a/prebuilt.py b/prebuilt.py index 7a59b39f88..d3d41fd5da 100755 --- a/prebuilt.py +++ b/prebuilt.py @@ -7,6 +7,7 @@ import datetime import multiprocessing import optparse import os +import re import sys from chromite.lib import cros_build_lib @@ -42,6 +43,14 @@ _BOTO_CONFIG = '/home/chrome-bot/external-boto' _GS_BOARD_PATH = 'board/%(board)s/%(version)s/' # We only support amd64 right now _GS_HOST_PATH = 'host/%s' % _HOST_TARGET +# Private overlays to look at for builds to filter +# relative to build path +_PRIVATE_OVERLAY_DIR = 'src/private-overlays' + + +class FiltersEmpty(Exception): + """Raised when filters are used but none are found.""" + pass def UpdateLocalFile(filename, key, value): """Update the key in file with the value passed. @@ -110,18 +119,28 @@ def GetVersion(): return datetime.datetime.now().strftime('%d.%m.%y.%H%M%S') -def LoadFilterFile(filter_file): +def LoadPrivateFilters(build_path): """Load a file with keywords on a per line basis. Args: filter_file: file to load into _FILTER_PACKAGES """ - filter_fh = open(filter_file) - try: - _FILTER_PACKAGES.update([filter.strip() for filter in filter_fh]) - finally: - filter_fh.close() - return _FILTER_PACKAGES + # TODO(scottz): eventually use manifest.xml to find the proper + # private overlay path. + filter_path = os.path.join(build_path, _PRIVATE_OVERLAY_DIR) + files = cros_build_lib.ListFiles(filter_path) + filters = [] + for file in files: + if file.endswith('.ebuild'): + basename = os.path.basename(file) + match = re.match('(.*?)-\d.*.ebuild', basename) + if match: + filters.append(match.group(1)) + + if not filters: + raise FiltersEmpty('No filters were returned') + + _FILTER_PACKAGES.update(filters) def ShouldFilterPackage(file_path): @@ -277,9 +296,9 @@ def main(): parser.add_option('-u', '--upload', dest='upload', default=None, help='Upload to GS bucket') - parser.add_option('-f', '--filter', dest='filter_file', - default=None, - help='File to use for filtering GS bucket uploads') + parser.add_option('-f', '--filters', dest='filters', action='store_true', + default=False, + help='Turn on filtering of private ebuild packages') options, args = parser.parse_args() # Setup boto environment for gsutil to use @@ -290,8 +309,8 @@ def main(): if not options.upload: usage(parser, 'Error: you need to provide a gsutil upload bucket -u') - if options.filter_file: - LoadFilterFile(options.filter_file) + if options.filters: + LoadPrivateFilters(options.build_path) git_file = None if options.git_sync: diff --git a/prebuilt_unittest.py b/prebuilt_unittest.py index f5cd97f7ed..16817320f9 100755 --- a/prebuilt_unittest.py +++ b/prebuilt_unittest.py @@ -6,6 +6,7 @@ import mox import os import prebuilt +import shutil import tempfile import unittest from chromite.lib import cros_build_lib @@ -61,35 +62,74 @@ class TestUpdateFile(unittest.TestCase): class TestPrebuiltFilters(unittest.TestCase): def setUp(self): - self.FAUX_FILTERS = set(['oob', 'bibby', 'bob']) - temp_fd, self.filter_filename = tempfile.mkstemp() - os.write(temp_fd, '\n'.join(self.FAUX_FILTERS)) - os.close(temp_fd) + self.tmp_dir = tempfile.mkdtemp() + self.private_dir = os.path.join(self.tmp_dir, + prebuilt._PRIVATE_OVERLAY_DIR) + self.private_structure_base = 'chromeos-overlay/chromeos-base' + self.private_pkgs = ['test-package/salt-flavor-0.1.r3.ebuild', + 'easy/alpha_beta-0.1.41.r3.ebuild', + 'dev/j-t-r-0.1.r3.ebuild',] + self.expected_filters = set(['salt-flavor', 'alpha_beta', 'j-t-r']) + def tearDown(self): - os.remove(self.filter_filename) + if self.tmp_dir: + shutil.rmtree(self.tmp_dir) - def testLoadFilterFile(self): - """ - Call filter packages with a list of packages that should be filtered - and ensure they are. - """ - loaded_filters = prebuilt.LoadFilterFile(self.filter_filename) - self.assertEqual(self.FAUX_FILTERS, loaded_filters) + def _CreateNestedDir(self, tmp_dir, dir_structure): + for entry in dir_structure: + full_path = os.path.join(os.path.join(tmp_dir, entry)) + # ensure dirs are created + try: + os.makedirs(os.path.dirname(full_path)) + if full_path.endswith('/'): + # we only want to create directories + return + except OSError, err: + if err.errno == errno.EEXIST: + # we don't care if the dir already exists + pass + else: + raise + # create dummy files + tmp = open(full_path, 'w') + tmp.close() + def _LoadPrivateMockFilters(self): + """Load mock filters as defined in the setUp function.""" + dir_structure = [os.path.join(self.private_structure_base, entry) + for entry in self.private_pkgs] + + self._CreateNestedDir(self.private_dir, dir_structure) + prebuilt.LoadPrivateFilters(self.tmp_dir) + def testFilterPattern(self): """Check that particular packages are filtered properly.""" - prebuilt.LoadFilterFile(self.filter_filename) - file_list = ['/usr/local/package/oob', - '/usr/local/package/other/path/valid', - '/var/tmp/bibby.file', - '/tmp/b/o/b'] - expected_list = ['/usr/local/package/other/path/valid', - '/tmp/b/o/b'] - filtered_list = [file for file in file_list if not + self._LoadPrivateMockFilters() + packages = ['/some/dir/area/j-t-r-0.1.r3.tbz', + '/var/pkgs/new/alpha_beta-0.2.3.4.tbz', + '/usr/local/cache/good-0.1.3.tbz', + '/usr-blah/b_d/salt-flavor-0.0.3.tbz'] + expected_list = ['/usr/local/cache/good-0.1.3.tbz'] + filtered_list = [file for file in packages if not prebuilt.ShouldFilterPackage(file)] self.assertEqual(expected_list, filtered_list) + def testLoadPrivateFilters(self): + self._LoadPrivateMockFilters() + prebuilt.LoadPrivateFilters(self.tmp_dir) + self.assertEqual(self.expected_filters, prebuilt._FILTER_PACKAGES) + + def testEmptyFiltersErrors(self): + """Ensure LoadPrivateFilters errors if an empty list is generated.""" + os.makedirs(os.path.join(self.tmp_dir, prebuilt._PRIVATE_OVERLAY_DIR)) + try: + prebuilt.LoadPrivateFilters(self.tmp_dir) + except prebuilt.FiltersEmpty: + return + + self.fail('Exception was not raised for empty list') + class TestPrebuilt(unittest.TestCase): fake_path = '/b/cbuild/build/chroot/build/x86-dogfood/'