From 6fecd3101f3ff4f8f51b84588b6be6cd934c161b Mon Sep 17 00:00:00 2001 From: James Forcier Date: Thu, 28 Jun 2018 11:29:00 -0700 Subject: [PATCH 1/2] metadata/repoman: add repoman configuration There are a few repoman checks that create a lot of false positives, and some that we want to consider warnings. --- .../metadata/repoman/qa_data.yaml | 65 ++++++++++++++++ .../metadata/repoman/repository.yaml | 78 +++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 sdk_container/src/third_party/coreos-overlay/metadata/repoman/qa_data.yaml create mode 100644 sdk_container/src/third_party/coreos-overlay/metadata/repoman/repository.yaml diff --git a/sdk_container/src/third_party/coreos-overlay/metadata/repoman/qa_data.yaml b/sdk_container/src/third_party/coreos-overlay/metadata/repoman/qa_data.yaml new file mode 100644 index 0000000000..17c0e0b477 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/metadata/repoman/qa_data.yaml @@ -0,0 +1,65 @@ +--- + +version: 1 +repoman_version: 2.3.3 + +missingvars: + - LICENSE + +qacats: + - EAPI.definition + - EAPI.deprecated + - EAPI.incompatible + - EAPI.unsupported + - LICENSE.invalid + - LICENSE.missing + - LICENSE.syntax + - dependency.syntax + - digest.assumed + - digest.missing + - ebuild.invalidname + - ebuild.majorsyn + - ebuild.syntax + - file.name + - inherit.missing + - manifest.bad + - repo.eapi.banned + - variable.invalidchar + +qawarnings: + - DESCRIPTION.missing + - DESCRIPTION.toolong + - HOMEPAGE.virtual + - IUSE.invalid + - IUSE.missing + - KEYWORDS.stupid + - LICENSE.deprecated + - LICENSE.virtual + - digest.unused + - ebuild.absdosym + - ebuild.invalidname + - ebuild.minorsyn + - ebuild.patches + - file.UTF8 + - file.executable + - file.size + - inherit.unused + - inherit.deprecated + - portage.internal + - repo.eapi.deprecated + - upstream.workaround + - uri.https + - variable.readonly + +valid_restrict: + - binchecks + - bindist + - fetch + - installsources + - mirror + - preserve-libs + - primaryuri + - splitdebug + - strip + - test + - userpriv diff --git a/sdk_container/src/third_party/coreos-overlay/metadata/repoman/repository.yaml b/sdk_container/src/third_party/coreos-overlay/metadata/repoman/repository.yaml new file mode 100644 index 0000000000..d38f71c6db --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/metadata/repoman/repository.yaml @@ -0,0 +1,78 @@ +--- + +version: 1 +repoman_version: 2.3.3 + +# The few modules disabled here generate unreasonable false positives for Container Linux: +# * The `keywords` module generates KEYWORDS.dropped and KEYWORDS.invalid errors. These are more or +# less irrelevant for Container Linux, as they tend to be for platforms with non-Linux kernels, +# which aren't a concern for us. These could likely be marked as warnings in qa_data.yaml, but +# they aren't really even worth fixing on their own, so there's no need to run them at all. +# * The `pkgmetadata` module checks metadata.xml. Most of this isn't relevant to Container Linux, +# users will not be interacting with the overlay themselves, and so any metadata specified in +# metadata.xml will only ever be seen by users of the SDK. As with `keywords`, proper metadata +# might be nice to have, but considering metadata issues as warnings or errors here adds more +# noise than signal. +# * The `profile` module, among other issues, generates a huge number (>10000) of dependency.bad +# errors. This is due to unmodified (or minimally modified) ebuilds from Gentoo upstream that +# depend on ebuilds not present in coreos-overlay or portage-stable. These dependencies are hidden +# behind USE flags or otherwise unused in Container Linux, so the missing ebuilds aren't a +# problem, but the amount of noise generated by this module makes it unusable. +# +# TODO(csssuf): The `profile` module also contains the LICENSE.invalid check which _is_ valuable +# for Container Linux, but repoman currently provides no way to disable checks individually; they +# can only be marked as warnings, or their containing modules disabled entirely. Unfortunately the +# `profile` module contains so much noise that leaving it enabled isn't an option, so we lose the +# ability to use the LICENSE.invalid check too. If/when repoman has a fix for this, use it. +scan_modules: + description + eapi + ebuild_metadata + fetches + files + -keywords + live + manifests + multicheck + -pkgmetadata + -profile + restrict + ruby + +linechecks_modules: + assignment + eapi3assignment + implicitdepend + hasq + useq + preservelib + bindnow + inherit + dosym + definition + srcprepare + eapi3deprecated + pkgpretend + eapi4incompatible + eapi4gonevars + paralleldisabled + autodefault + -gentooheader + nooffset + nesteddie + patches + emakeparallel + srccompileeconf + srcunpackpatches + portageinternal + portageinternalvariableassignment + quote + quoteda + httpsuri + builtwith + uselesscds + uselessdodoc + whitespace + blankline + addpredict + noasneeded From 1533663dfc90427d0193271a35e4741744969caf Mon Sep 17 00:00:00 2001 From: James Forcier Date: Thu, 28 Jun 2018 11:32:12 -0700 Subject: [PATCH 2/2] travis: add basic travis configuration for repoman --- .../third_party/coreos-overlay/.travis.yml | 21 +++++++++++++++++ .../coreos-overlay/.travis/coreos.conf | 8 +++++++ .../coreos-overlay/.travis/setup_repoman.sh | 23 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 sdk_container/src/third_party/coreos-overlay/.travis.yml create mode 100644 sdk_container/src/third_party/coreos-overlay/.travis/coreos.conf create mode 100755 sdk_container/src/third_party/coreos-overlay/.travis/setup_repoman.sh diff --git a/sdk_container/src/third_party/coreos-overlay/.travis.yml b/sdk_container/src/third_party/coreos-overlay/.travis.yml new file mode 100644 index 0000000000..5afcfddad5 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/.travis.yml @@ -0,0 +1,21 @@ +# Derived from https://github.com/mrueg/repoman-travis/ + +sudo: required +language: python +python: + - pypy + +# For some reason portage is defaulting to lbzip2/lbunzip2 here and then complaining when they don't +# exist, so use plain old b{,un}zip2. +env: + - PORTAGE_VER="2.3.40" PORTAGE_BZIP2_COMMAND="bzip2" PORTAGE_BUNZIP2_COMMAND="bunzip2" + +before_install: + - sudo apt-get -qq update + - pip install lxml pyyaml + +before_script: ./.travis/setup_repoman.sh + +script: + - cd /usr/coreos-overlay + - /tmp/portage/portage-${PORTAGE_VER}/repoman/bin/repoman --experimental-repository-modules=y -dx full diff --git a/sdk_container/src/third_party/coreos-overlay/.travis/coreos.conf b/sdk_container/src/third_party/coreos-overlay/.travis/coreos.conf new file mode 100644 index 0000000000..f95be5e178 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/.travis/coreos.conf @@ -0,0 +1,8 @@ +[DEFAULT] +main-repo = portage-stable + +[coreos] +location = /usr/coreos-overlay + +[portage-stable] +location = /usr/portage diff --git a/sdk_container/src/third_party/coreos-overlay/.travis/setup_repoman.sh b/sdk_container/src/third_party/coreos-overlay/.travis/setup_repoman.sh new file mode 100755 index 0000000000..bfad0f2cf6 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/.travis/setup_repoman.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euo pipefail + +# Set up directory permissions/portage user and group. +sudo chmod a+rwX /etc/passwd /etc/group /etc /usr +echo "portage:x:250:250:portage:/var/tmp/portage:/bin/false" >> /etc/passwd +echo "portage::250:portage,travis" >> /etc/group + +# Shuffle portage repositories around into the right places. +mkdir -p /etc/portage/repos.conf /usr/coreos-overlay +mv * /usr/coreos-overlay/ +mv .git /usr/coreos-overlay/ +git clone https://github.com/coreos/portage-stable /usr/portage/ +cp .travis/coreos.conf /etc/portage/repos.conf/ +ln -s /usr/coreos-overlay/profiles/coreos/amd64/sdk /etc/portage/make.profile +mkdir -p /usr/portage/metadata/{dtd,xml-schema} +wget -O /usr/portage/metadata/dtd/metadata.dtd https://www.gentoo.org/dtd/metadata.dtd +wget -O /usr/portage/metadata/xml-schema/metadata.xsd https://www.gentoo.org/xml-schema/metadata.xsd + +# Download portage. +mkdir /tmp/portage && cd /tmp/portage +wget -qO - "https://gitweb.gentoo.org/proj/portage.git/snapshot/portage-${PORTAGE_VER}.tar.gz" | tar xz