Change prebuilt filters to use the private ebuilds to generate a

definitive list.
Add unittests to test an empty filter list, loading filters, and
filtering specific packages based off of mock filters.

prebuilt.py

@@ -7,6 +7,7 @@ import datetime
 import multiprocessing
 import optparse
 import os
+import re
 import sys
 
 from chromite.lib import cros_build_lib
@@ -42,6 +43,14 @@ _BOTO_CONFIG = '/home/chrome-bot/external-boto'
 _GS_BOARD_PATH = 'board/%(board)s/%(version)s/'
 # We only support amd64 right now
 _GS_HOST_PATH = 'host/%s' % _HOST_TARGET
+# Private overlays to look at for builds to filter
+# relative to build path
+_PRIVATE_OVERLAY_DIR = 'src/private-overlays'
+
+
+class FiltersEmpty(Exception):
+  """Raised when filters are used but none are found."""
+  pass
 
 def UpdateLocalFile(filename, key, value):
   """Update the key in file with the value passed.
@@ -110,18 +119,28 @@ def GetVersion():
   return datetime.datetime.now().strftime('%d.%m.%y.%H%M%S')
 
 
-def LoadFilterFile(filter_file):
+def LoadPrivateFilters(build_path):
   """Load a file with keywords on a per line basis.
 
   Args:
     filter_file: file to load into _FILTER_PACKAGES
   """
-  filter_fh = open(filter_file)
+  # TODO(scottz): eventually use manifest.xml to find the proper
-  try:
+  # private overlay path.
-    _FILTER_PACKAGES.update([filter.strip() for filter in filter_fh])
+  filter_path = os.path.join(build_path, _PRIVATE_OVERLAY_DIR)
-  finally:
+  files = cros_build_lib.ListFiles(filter_path)
-    filter_fh.close()
+  filters = []
-  return _FILTER_PACKAGES
+  for file in files:
+    if file.endswith('.ebuild'):
+      basename = os.path.basename(file)
+      match = re.match('(.*?)-\d.*.ebuild', basename)
+      if match:
+        filters.append(match.group(1))
+
+  if not filters:
+    raise FiltersEmpty('No filters were returned')
+
+  _FILTER_PACKAGES.update(filters)
 
 
 def ShouldFilterPackage(file_path):
@@ -277,9 +296,9 @@ def main():
   parser.add_option('-u', '--upload', dest='upload',
                     default=None,
                     help='Upload to GS bucket')
-  parser.add_option('-f', '--filter', dest='filter_file',
+  parser.add_option('-f', '--filters', dest='filters', action='store_true',
-                    default=None,
+                    default=False,
-                    help='File to use for filtering GS bucket uploads')
+                    help='Turn on filtering of private ebuild packages')
 
   options, args = parser.parse_args()
   # Setup boto environment for gsutil to use
@@ -290,8 +309,8 @@ def main():
   if not options.upload:
     usage(parser, 'Error: you need to provide a gsutil upload bucket -u')
 
-  if options.filter_file:
+  if options.filters:
-    LoadFilterFile(options.filter_file)
+    LoadPrivateFilters(options.build_path)
 
   git_file = None
   if options.git_sync:

prebuilt_unittest.py

@@ -6,6 +6,7 @@
 import mox
 import os
 import prebuilt
+import shutil
 import tempfile
 import unittest
 from chromite.lib import cros_build_lib
@@ -61,35 +62,74 @@ class TestUpdateFile(unittest.TestCase):
 class TestPrebuiltFilters(unittest.TestCase):
 
   def setUp(self):
-    self.FAUX_FILTERS = set(['oob', 'bibby', 'bob'])
+    self.tmp_dir = tempfile.mkdtemp()
-    temp_fd, self.filter_filename = tempfile.mkstemp()
+    self.private_dir = os.path.join(self.tmp_dir,
-    os.write(temp_fd, '\n'.join(self.FAUX_FILTERS))
+                                    prebuilt._PRIVATE_OVERLAY_DIR)
-    os.close(temp_fd)
+    self.private_structure_base = 'chromeos-overlay/chromeos-base'
+    self.private_pkgs = ['test-package/salt-flavor-0.1.r3.ebuild',
+                         'easy/alpha_beta-0.1.41.r3.ebuild',
+                         'dev/j-t-r-0.1.r3.ebuild',]
+    self.expected_filters = set(['salt-flavor', 'alpha_beta', 'j-t-r'])
+ 
 
   def tearDown(self):
-    os.remove(self.filter_filename)
+    if self.tmp_dir:
+      shutil.rmtree(self.tmp_dir)
 
-  def testLoadFilterFile(self):
+  def _CreateNestedDir(self, tmp_dir, dir_structure):
-    """
+    for entry in dir_structure:
-    Call filter packages with a list of packages that should be filtered
+      full_path = os.path.join(os.path.join(tmp_dir, entry))
-    and ensure they are.
+      # ensure dirs are created
-    """
+      try:
-    loaded_filters = prebuilt.LoadFilterFile(self.filter_filename)
+        os.makedirs(os.path.dirname(full_path))
-    self.assertEqual(self.FAUX_FILTERS, loaded_filters)
+        if full_path.endswith('/'):
+          # we only want to create directories
+          return
+      except OSError, err:
+        if err.errno == errno.EEXIST:
+          # we don't care if the dir already exists
+          pass
+        else:
+          raise
+      # create dummy files
+      tmp = open(full_path, 'w')
+      tmp.close()
+
+  def _LoadPrivateMockFilters(self):
+    """Load mock filters as defined in the setUp function."""
+    dir_structure = [os.path.join(self.private_structure_base, entry)
+                     for entry in self.private_pkgs]
+
+    self._CreateNestedDir(self.private_dir, dir_structure)
+    prebuilt.LoadPrivateFilters(self.tmp_dir)
  
   def testFilterPattern(self):
     """Check that particular packages are filtered properly."""
-    prebuilt.LoadFilterFile(self.filter_filename)
+    self._LoadPrivateMockFilters()
-    file_list = ['/usr/local/package/oob',
+    packages = ['/some/dir/area/j-t-r-0.1.r3.tbz',
-                 '/usr/local/package/other/path/valid',
+                '/var/pkgs/new/alpha_beta-0.2.3.4.tbz',
-                 '/var/tmp/bibby.file',
+                '/usr/local/cache/good-0.1.3.tbz',
-                 '/tmp/b/o/b']
+                '/usr-blah/b_d/salt-flavor-0.0.3.tbz']
-    expected_list = ['/usr/local/package/other/path/valid',
+    expected_list = ['/usr/local/cache/good-0.1.3.tbz']
-                     '/tmp/b/o/b']
+    filtered_list = [file for file in packages if not
-    filtered_list = [file for file in file_list if not
                      prebuilt.ShouldFilterPackage(file)]
     self.assertEqual(expected_list, filtered_list)
 
+  def testLoadPrivateFilters(self):
+    self._LoadPrivateMockFilters()
+    prebuilt.LoadPrivateFilters(self.tmp_dir)
+    self.assertEqual(self.expected_filters, prebuilt._FILTER_PACKAGES)
+
+  def testEmptyFiltersErrors(self):
+    """Ensure LoadPrivateFilters errors if an empty list is generated."""
+    os.makedirs(os.path.join(self.tmp_dir, prebuilt._PRIVATE_OVERLAY_DIR))
+    try:
+      prebuilt.LoadPrivateFilters(self.tmp_dir)
+    except prebuilt.FiltersEmpty:
+      return
+
+    self.fail('Exception was not raised for empty list')
+
 
 class TestPrebuilt(unittest.TestCase):
   fake_path = '/b/cbuild/build/chroot/build/x86-dogfood/'