diff --git a/.bazelrc b/.bazelrc index 0d3b799900..2b9ed65152 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,20 +1,16 @@ # By default, we build and run ONOS using the Bazel-provided remote JDK. The # specific JDK version is defined in tools/build/jdk/BUILD. # -# If you prefer to use your locally installed JDK, replace the --javabase and -# --host_javabase arguments with the following lines. ABSOLUTE_JAVABASE should -# point to the "JAVA_HOME" of your JDK. +# If you prefer to use your locally installed JDK use local_java_repositoy +# in the WORKSPACE file. # -# build --javabase=@bazel_tools//tools/jdk:absolute_javabase -# build --host_javabase=@bazel_tools//tools/jdk:absolute_javabase -# build --define=ABSOLUTE_JAVABASE= -# build --define=RUN_WITH_ABSOLUTE_JAVABASE=true +# https://bazel.build/docs/bazel-and-java -build --javabase=@org_onosproject_onos//tools/build/jdk:default_jdk -build --host_javabase=@org_onosproject_onos//tools/build/jdk:default_jdk - -build --java_toolchain=@org_onosproject_onos//tools/build/jdk:default_toolchain -build --host_java_toolchain=@org_onosproject_onos//tools/build/jdk:default_toolchain +# Builds using remotejdk_11, executes using remotejdk_11 or local_jdk +build --java_language_version=11 +build --java_runtime_version=remotejdk_11 +build --tool_java_language_version=11 +build --tool_java_runtime_version=remotejdk_11 build --host_force_python=PY2 @@ -22,7 +18,7 @@ build --nouse_ijars build --experimental_strict_action_env # For external/io_grpc_grpc_java/core/src/main/java/io/grpc/internal/RetriableStream.java -build --javacopt="-Xep:GuardedBy:WARN -Xep:ComparableType:WARN" +build --javacopt="-Xep:GuardedBy:WARN -Xep:ComparableType:WARN -Xep:ReturnValueIgnored:OFF" # intellij-bazel plugin needs this build --incompatible_new_actions_api=false diff --git a/.bazelversion b/.bazelversion index 47b6be3faf..91e1ea48a9 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -3.7.2 \ No newline at end of file +6.0.0-pre.20220421.3 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f4fb78443e..129e7cfa0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,19 @@ +# Copyright 2015-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# With this dockerfile you can build a ONOS Docker container + ARG JOBS=2 ARG PROFILE=default ARG TAG=11.0.8-11.41.23 @@ -21,7 +37,7 @@ RUN apt-get update && apt-get install -y ${BUILD_DEPS} # Install Bazelisk, which will download the version of bazel specified in # .bazelversion -RUN curl -L -o bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.5.0/bazelisk-linux-amd64 +RUN curl -L -o bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 RUN chmod +x bazelisk && mv bazelisk /usr/bin # Build-stage environment variables @@ -36,16 +52,15 @@ WORKDIR ${ONOS_ROOT} # Build ONOS using the JDK pre-installed in the base image, instead of the # Bazel-provided remote one. By doing wo we make sure to build with the most # updated JDK, including bug and security fixes, independently of the Bazel -# version. +# version. NOTE that WORKSPACE-docker file defines dockerjdk ARG JOBS ARG JAVA_PATH ARG PROFILE -RUN bazelisk build onos \ +RUN cat WORKSPACE-docker >> WORKSPACE && bazelisk build onos \ --jobs ${JOBS} \ --verbose_failures \ - --javabase=@bazel_tools//tools/jdk:absolute_javabase \ - --host_javabase=@bazel_tools//tools/jdk:absolute_javabase \ - --define=ABSOLUTE_JAVABASE=${JAVA_PATH} \ + --java_runtime_version=dockerjdk_11 \ + --tool_java_runtime_version=dockerjdk_11 \ --define profile=${PROFILE} # We extract the tar in the build environment to avoid having to put the tar in diff --git a/WORKSPACE-docker b/WORKSPACE-docker new file mode 100644 index 0000000000..1b2b0d7047 --- /dev/null +++ b/WORKSPACE-docker @@ -0,0 +1,16 @@ + +# Appended by the Dockerfiles to help bazel finding the local path of +# the Azul jvm. When we build docker images we use the jvm available +# in the base image to compile ONOS. In order to do that we have to +# provide the absolute path by using the local_java_repository. + +# local_java_repository for docker based builds +load("@bazel_tools//tools/jdk:local_java_repository.bzl", "local_java_repository") + +ABSOLUTE_JAVABASE = "/usr/lib/jvm/zulu11-ca-amd64" + +local_java_repository( + name = "dockerjdk", + version = "11", + java_home = ABSOLUTE_JAVABASE, +) diff --git a/tools/build/bazel/grpc_workspace.bzl b/tools/build/bazel/grpc_workspace.bzl index f2f3dfc8be..1a680da6ce 100644 --- a/tools/build/bazel/grpc_workspace.bzl +++ b/tools/build/bazel/grpc_workspace.bzl @@ -1,7 +1,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") GRPC_JAVA_VERSION = "1.22.1" -GRPC_SHA = "e6915ea60ab8a6e17de86bd94a8db320b0115cc214db19bef8f2ba4af2dab430" +GRPC_SHA = "073395219d611c21a460d21dfe5cb77550030d6396b400f796e568b5175c2ec0" GAPIS_COMMIT = "37cc0e5acae50ee91f00827a7010c3b07dfa5311" GAPIS_SHA = "17d023f48ea290f25edaf25a967973b5a42ce6d71b1570862f302d95aa8b9f77" @@ -10,9 +10,9 @@ def generate_grpc(): # grpc-java fork that fixes the OSGi split brain problem. http_archive( name = "io_grpc_grpc_java", - urls = ["https://github.com/opennetworkinglab/grpc-java/archive/v%s-patched.zip" % GRPC_JAVA_VERSION], + urls = ["https://github.com/opennetworkinglab/grpc-java/archive/v%s-fix-host_javabase.zip" % GRPC_JAVA_VERSION], sha256 = GRPC_SHA, - strip_prefix = "grpc-java-%s-patched" % GRPC_JAVA_VERSION, + strip_prefix = "grpc-java-%s-fix-host_javabase" % GRPC_JAVA_VERSION, ) # Google APIs protos (status.proto, etc.) diff --git a/tools/build/bazel/osgi_java_library.bzl b/tools/build/bazel/osgi_java_library.bzl index 491444f441..0236db4348 100644 --- a/tools/build/bazel/osgi_java_library.bzl +++ b/tools/build/bazel/osgi_java_library.bzl @@ -124,6 +124,8 @@ def _bnd_impl(ctx): bundle_classpath, karaf_commands, fragment_host, + # enable/disable osgi-wrap logging + "false", ] ctx.actions.run( @@ -507,7 +509,7 @@ def osgi_jar_with_tests( native.java_library( name = name + "-native", srcs = native_srcs, - resource_jars = resource_jars + [name + "_cfgdef_jar"], + resources = resource_jars + [name + "_cfgdef_jar"], deps = deps, visibility = visibility, javacopts = javacopts, @@ -516,13 +518,14 @@ def osgi_jar_with_tests( native.java_library( name = name + "-native", srcs = native_srcs, - resource_jars = [name + "_cfgdef_jar"], - resources = native_resources, + resources = native_resources + [name + "_cfgdef_jar"], deps = deps, visibility = visibility, javacopts = javacopts, ) + # NOTE that the additional resource_jars are modified by + # osgi-wrap because java_library does not decompress them. karaf_command_packages_string = ",".join(karaf_command_packages) _bnd( name = name, diff --git a/tools/build/bazel/topo_BUILD b/tools/build/bazel/topo_BUILD index 9a42836a57..a3418b2c61 100644 --- a/tools/build/bazel/topo_BUILD +++ b/tools/build/bazel/topo_BUILD @@ -16,6 +16,6 @@ genrule( name = "topo_device_proto_sed", srcs = [":device/device.proto"], outs = ["new/device/device.proto"], - cmd = "sed -e 's:import \"gogoproto.*;::g;s: ..gogoproto\..*:;:g;s:import \"%s:import \":g;s:^syntax = \"proto3\";:&\\\n option java_package = \"org.onosproject.uonos\";:g' $(location :device/device.proto) >> \"$@\"" + cmd = "sed -e 's:import \"gogoproto.*;::g;s: ..gogoproto\\..*:;:g;s:import \"%s:import \":g;s:^syntax = \"proto3\";:&\\\n option java_package = \"org.onosproject.uonos\";:g' $(location :device/device.proto) >> \"$@\"" % IMPORT_PREFIX, ) diff --git a/tools/build/jdk/BUILD b/tools/build/jdk/BUILD index 00550315f4..c99f398289 100644 --- a/tools/build/jdk/BUILD +++ b/tools/build/jdk/BUILD @@ -1,24 +1,3 @@ -# This is where we define the JDK used to build ONOS, as well as the language -# source and target values passed to javac. The :default_toolchain and -# :default_jdk are expected to be passed as arguments when invoking bazel build -# (see onos/.bazelrc) - -load("//tools/build/bazel:variables.bzl", "DEFAULT_JAVA_VERSION") -load("@bazel_tools//tools/jdk:default_java_toolchain.bzl", "default_java_toolchain") - -default_java_toolchain( - name = "default_toolchain", - source_version = DEFAULT_JAVA_VERSION, - target_version = DEFAULT_JAVA_VERSION, - visibility = ["//visibility:public"], -) - -alias( - name = "default_jdk", - actual = "@bazel_tools//tools/jdk:remote_jdk11", - visibility = ["//visibility:public"], -) - # We use the following rule to package the same JDK used for building and make # it available for external scripts as their JAVA_HOME, such as for `bazel run # onos-local`. diff --git a/tools/dev/Dockerfile-yourkit b/tools/dev/Dockerfile-yourkit index d54460efee..176c3b3d37 100644 --- a/tools/dev/Dockerfile-yourkit +++ b/tools/dev/Dockerfile-yourkit @@ -41,7 +41,7 @@ RUN apt-get update && apt-get install -y ${BUILD_DEPS} # Install Bazelisk, which will download the version of bazel specified in # .bazelversion -RUN curl -L -o bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.5.0/bazelisk-linux-amd64 +RUN curl -L -o bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 RUN chmod +x bazelisk && mv bazelisk /usr/bin # Build-stage environment variables @@ -56,16 +56,15 @@ WORKDIR ${ONOS_ROOT} # Build ONOS using the JDK pre-installed in the base image, instead of the # Bazel-provided remote one. By doing wo we make sure to build with the most # updated JDK, including bug and security fixes, independently of the Bazel -# version. +# version. NOTE that WORKSPACE-docker file defines dockerjdk ARG JOBS ARG JAVA_PATH ARG PROFILE -RUN bazelisk build onos \ +RUN cat WORKSPACE-docker >> WORKSPACE && bazelisk build onos \ --jobs ${JOBS} \ --verbose_failures \ - --javabase=@bazel_tools//tools/jdk:absolute_javabase \ - --host_javabase=@bazel_tools//tools/jdk:absolute_javabase \ - --define=ABSOLUTE_JAVABASE=${JAVA_PATH} \ + --java_runtime_version=dockerjdk_11 \ + --tool_java_runtime_version=dockerjdk_11 \ --define profile=${PROFILE} # We extract the tar in the build environment to avoid having to put the tar in diff --git a/utils/osgiwrap/src/main/java/org/onlab/osgiwrap/OSGiWrapper.java b/utils/osgiwrap/src/main/java/org/onlab/osgiwrap/OSGiWrapper.java index 2bdc2b360f..aaba624c8f 100644 --- a/utils/osgiwrap/src/main/java/org/onlab/osgiwrap/OSGiWrapper.java +++ b/utils/osgiwrap/src/main/java/org/onlab/osgiwrap/OSGiWrapper.java @@ -26,13 +26,12 @@ import aQute.bnd.osgi.Resource; import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.ByteStreams; -import org.apache.felix.scrplugin.bnd.SCRDescriptorBndPlugin; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; @@ -48,6 +47,7 @@ import java.util.Objects; import java.util.Set; import java.util.jar.JarEntry; import java.util.jar.JarInputStream; +import java.util.jar.JarOutputStream; import java.util.jar.Manifest; import static com.google.common.io.Files.createParentDirs; @@ -55,9 +55,14 @@ import static com.google.common.io.Files.write; import static java.nio.file.Files.walkFileTree; /** - * BND-based wrapper to convert Buck JARs to OSGi-compatible JARs. + * BND-based wrapper to convert Buck/Bazel JARs to OSGi-compatible JARs. */ public class OSGiWrapper { + // Resources that need to be modified in the original jar + private static final String _CFGDEF_JAR = "_cfgdef.jar"; + private static final String MODEL_SRCJAR = "model.srcjar"; + private static final String SCHEMA_JAR = "schema.jar"; + private static final String NONE = "NONE"; private String inputJar; @@ -87,10 +92,11 @@ public class OSGiWrapper { private String karafCommands; private String fragmentHost; + private boolean debug; // FIXME should consider using Commons CLI, etc. public static void main(String[] args) { - if (args.length < 17) { + if (args.length < 18) { System.err.println("Not enough args"); System.exit(1); } @@ -112,7 +118,8 @@ public class OSGiWrapper { String bundleClasspath = args[14]; String karafCommands = args[15]; String fragmentHost = args[16]; - String desc = Joiner.on(' ').join(Arrays.copyOfRange(args, 17, args.length)); + String debug = args[17]; + String desc = Joiner.on(' ').join(Arrays.copyOfRange(args, 18, args.length)); OSGiWrapper wrapper = new OSGiWrapper(jar, output, cp, name, group, @@ -126,7 +133,8 @@ public class OSGiWrapper { destdir, bundleClasspath, karafCommands, - fragmentHost); + fragmentHost, + debug); wrapper.log(wrapper + "\n"); if (!wrapper.execute()) { System.err.printf("Error generating %s\n", name); @@ -152,7 +160,8 @@ public class OSGiWrapper { String destdir, String bundleClasspath, String karafCommands, - String fragmentHost) { + String fragmentHost, + String debug) { this.inputJar = inputJar; this.classpath = Lists.newArrayList(classpath.split(":")); if (!this.classpath.contains(inputJar)) { @@ -186,6 +195,7 @@ public class OSGiWrapper { this.karafCommands = karafCommands; this.fragmentHost = fragmentHost; + this.debug = Boolean.parseBoolean(debug); } private void setProperties(Analyzer analyzer) { @@ -228,8 +238,8 @@ public class OSGiWrapper { public boolean execute() { Analyzer analyzer = new Builder(); try { - // Extract the input jar contents into the specified output directory - expandJar(inputJar, new File(destdir)); + // Sanitize the input jar content + inputJar = modifyJar(inputJar); Jar jar = new Jar(new File(inputJar)); // where our data is analyzer.setJar(jar); // give bnd the contents @@ -246,15 +256,6 @@ public class OSGiWrapper { // Analyze the target JAR first analyzer.analyze(); - //// Scan the JAR for Felix SCR annotations and generate XML files - //Map properties = Maps.newHashMap(); - //// destdir hack - //properties.put("destdir", destdir); - //SCRDescriptorBndPlugin scrDescriptorBndPlugin = new SCRDescriptorBndPlugin(); - //scrDescriptorBndPlugin.setProperties(properties); - //scrDescriptorBndPlugin.setReporter(analyzer); - //scrDescriptorBndPlugin.analyzeJar(analyzer); - if (includeResources != null) { doIncludeResources(analyzer); } @@ -287,26 +288,66 @@ public class OSGiWrapper { } } - // Expands the specified jar file into the given directory - private void expandJar(String inputJar, File intoDir) throws IOException { - try (JarInputStream jis = new JarInputStream(new FileInputStream(inputJar))) { + // Extract the jar and melds its content with the jar produced by Bazel + private void addJarToJar(JarEntry entryJar, JarInputStream jis, JarOutputStream jos) throws IOException { + File file = new File(new File(destdir), entryJar.getName()); + byte[] data = ByteStreams.toByteArray(jis); + createParentDirs(file); + write(data, file); + // Entry jar input stream which points to the inner jar resources (cfgdef, model,...) + try (JarInputStream innerJis = new JarInputStream(new FileInputStream(file))) { JarEntry entry; - while ((entry = jis.getNextJarEntry()) != null) { + byte[] byteBuff = new byte[1024]; + while ((entry = innerJis.getNextJarEntry()) != null) { if (!entry.isDirectory()) { - byte[] data = ByteStreams.toByteArray(jis); - jis.closeEntry(); if (!entry.getName().contains("..")) { - File file = new File(intoDir, entry.getName()); - createParentDirs(file); - write(data, file); + jos.putNextEntry(entry); + for (int bytesRead; (bytesRead = innerJis.read(byteBuff)) != -1; ) { + jos.write(byteBuff, 0, bytesRead); + } } else { - throw new IOException("Corrupt jar file"); + throw new IOException("Jar " + entryJar + " is corrupted"); } } + innerJis.closeEntry(); } } } + // Modify the specified jar to fix the resource_jars loaded by Bazel. + // Starting from Bazel 5, resource_jars are no longer supported and + // we have to load them as resources. This means that we expand them + // and we set the right path in OSGi-compatible JAR. + private String modifyJar(String inputJar) throws IOException { + // libonos-xxx input and libonos-xxx-new which is the sanitized final jar + try (JarInputStream jis = new JarInputStream(new FileInputStream(inputJar)); + JarOutputStream jos = new JarOutputStream(new FileOutputStream(inputJar + "new"))) { + JarEntry entry; + byte[] byteBuff = new byte[1024]; + while ((entry = jis.getNextJarEntry()) != null) { + if (!entry.isDirectory()) { + if (!entry.getName().contains("..")) { + // We add the content but we don't write them again in the new jar + if (entry.getName().contains(bundleName + OSGiWrapper._CFGDEF_JAR) || + entry.getName().contains(SCHEMA_JAR) || + entry.getName().contains(MODEL_SRCJAR)) { + addJarToJar(entry, jis, jos); + } else { + jos.putNextEntry(entry); + for (int bytesRead; (bytesRead = jis.read(byteBuff)) != -1; ) { + jos.write(byteBuff, 0, bytesRead); + } + } + } else { + throw new IOException("Jar " + inputJar + " is corrupted"); + } + } + jis.closeEntry(); + } + } + return inputJar + "new"; + } + private boolean isWab() { return !Objects.equals(webContext, NONE); } @@ -399,10 +440,10 @@ public class OSGiWrapper { } } - private boolean addFileToJar(Jar jar, String destination, String sourceAbsPath) throws IOException { + private void addFileToJar(Jar jar, String destination, String sourceAbsPath) throws IOException { if (includedResources.contains(sourceAbsPath)) { log("Skipping already included resource: %s\n", sourceAbsPath); - return false; + return; } File file = new File(sourceAbsPath); if (!file.isFile()) { @@ -412,16 +453,17 @@ public class OSGiWrapper { Resource resource = new FileResource(file); if (jar.getResource(destination) != null) { warn("Skipping duplicate resource: %s\n", destination); - return false; + return; } jar.putResource(destination, resource); includedResources.add(sourceAbsPath); log("Adding resource: %s\n", destination); - return true; } private void log(String format, Object... objects) { - //System.err.printf(format, objects); + if (debug) { + System.out.printf(format, objects); + } } private void warn(String format, Object... objects) {