Support for the protobuf standard lib in gRPC buck tools

Required for building P4Runtime stuff.

Also:
- support in grpc_jar for building external protobuf files
- minor code refactoring/cleanups

Change-Id: I50c09f967cc9257366eb028d4ea1502767d8d4a0

bucklets/grpc.bucklet

@@ -2,17 +2,15 @@ include_defs('//onos.defs')
 include_defs('//bucklets/onos.bucklet')
 
 
-PROTOC_VERSION = '3.2.0'
+DEFAULT_PROTOC_VERSION = '3.2.0'
-GRPC_PLUGIN_VERSION = '1.3.0'
+DEFAULT_GRPC_PLUGIN_VERSION = '1.3.0'
 
-PROTOC_EXECUTABLE_BASE_URL = "https://repo1.maven.org/maven2/com/google/protobuf/protoc"
+PROTOC_RELEASE_BASE_URL = "https://github.com/google/protobuf/releases/download"
 GRPC_PLUGIN_BASE_URL = "https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java"
 
 PROTOC_SHA1S = {
-    "protoc-3.3.0-osx-x86_64.exe":"3070e439f9557bb72fb04df631f29d7556c9029c",
+    "protoc-3.2.0-linux-x86_64.zip":"f418d246d183a534d9bd749e614f639a55f6829b",
-    "protoc-3.3.0-linux-x86_64.exe":"e6a95fc7477c602cc402ed976d3edbd82c841879",
+    "protoc-3.2.0-osx-x86_64.zip":"81f4fac3200ba2cb12a98df0a9ee4d1c584e9210"
-    "protoc-3.2.0-linux-x86_64.exe":"086893ffdc1023e503ccd0ee522ca1e6046b12a7",
-    "protoc-3.2.0-osx-x86_64.exe":"87f532ef51bb314d2c5d2ba7842b39cbbdb60323"
 }
 
 GRPC_JAVA_SHA1S = {
@@ -29,43 +27,47 @@ def get_system_arch():
         os = "osx"
     return "%s-%s" % ( os, arch)
 
-def fetch_protoc_binary(
+def fetch_protoc(
-        protoc_version = PROTOC_VERSION
+        version
     ):
-    file_name = "protoc-%s-%s.exe" % (protoc_version, get_system_arch())
+    file_name = "protoc-%s-%s.zip" % (version, get_system_arch())
     if file_name not in PROTOC_SHA1S:
-        raise Exception('Cannot download %s, architecture not supported' % file_name)
+        raise Exception('Cannot download %s, architecture or version not supported' % file_name)
     remote_file(
-        name = 'fetch-protoc-binary-' + protoc_version,
+        name = 'protoc-release-' + version,
-        out = 'protoc-binary',
+        url = PROTOC_RELEASE_BASE_URL + '/v' + version + '/' + file_name,
-        url = PROTOC_EXECUTABLE_BASE_URL + '/' + protoc_version + '/' + file_name,
         sha1 = PROTOC_SHA1S[file_name],
     )
     genrule(
-        name = 'prepare-protoc-executable-' + protoc_version,
+        name = 'protoc-exe-' + version,
-        srcs = [ ':fetch-protoc-binary-' + protoc_version ],
+        bash = 'jar xf $(location :protoc-release-' + version + ') bin/protoc && ' +
-        bash = 'cp $(location :fetch-protoc-binary-' + protoc_version +') $OUT && chmod +x $OUT',
+                'mv bin/protoc $OUT && ' +
+                'chmod +x $OUT',
+        out = 'protoc.exe',
         executable = True,
         visibility = [ "PUBLIC" ],
-        out = 'protoc.exe',
+    )
+    genrule(
+        name = 'protoc-lib-' + version,
+        bash = 'jar xf $(location :protoc-release-' + version + ') include && mv include $OUT',
+        out = 'include',
+        visibility = [ "PUBLIC" ],
     )
 
-def fetch_grpc_plugin_binary(
+def fetch_grpc_plugin(
-        grpc_plugin_version = GRPC_PLUGIN_VERSION
+        version
-    ):
+):
-    file_name = "protoc-gen-grpc-java-%s-%s.exe" % (grpc_plugin_version, get_system_arch())
+    file_name = "protoc-gen-grpc-java-%s-%s.exe" % (version, get_system_arch())
     if file_name not in GRPC_JAVA_SHA1S:
-        raise Exception('Cannot download %s, architecture not supported' % file_name)
+        raise Exception('Cannot download %s, architecture or version not supported' % file_name)
     remote_file(
-        name = 'fetch-grpc-plugin-binary-' + grpc_plugin_version,
+        name = 'grpc-plugin-binary-' + version,
-        out = 'grpc-plugin-binary',
+        url = GRPC_PLUGIN_BASE_URL + '/' + version + '/' + file_name,
-        url = GRPC_PLUGIN_BASE_URL + '/' + grpc_plugin_version + '/' + file_name,
         sha1 = GRPC_JAVA_SHA1S[file_name],
     )
     genrule(
-        name = 'prepare-grpc-plugin-executable-' + grpc_plugin_version,
+        name = 'grpc-plugin-exe-' + version,
-        srcs = [ ':fetch-grpc-plugin-binary-' + grpc_plugin_version ],
+        bash = 'cp $(location :grpc-plugin-binary-' + version + ') $OUT && chmod +x $OUT',
-        bash = 'cp $(location :fetch-grpc-plugin-binary-' + grpc_plugin_version + ') $OUT && chmod +x $OUT',
         executable = True,
         visibility = [ "PUBLIC" ],
         out = 'grpc-plugin.exe',
@@ -84,9 +86,12 @@ def grpc_jar(
     # listed in the first listed proto_path which contains the specified
     # file
     proto_paths = [],
+    srcs = [],
+    src_string = '', # Useful to build proto files external to the ONOS sources, using BUCK's environment variables.
     proto_match_patterns = [ "**/proto/**/*.proto" ],
-    protoc_version = PROTOC_VERSION,
+    protoc_version = DEFAULT_PROTOC_VERSION,
-    plugin_version = GRPC_PLUGIN_VERSION,
+    plugin_version = DEFAULT_GRPC_PLUGIN_VERSION,
+    include_std_lib = False,
     **kwargs
     ):
 
@@ -94,7 +99,12 @@ def grpc_jar(
     if name is None:
         name = _get_name()
 
-    #Create the string for the proto_path arguments (order matters, similar to classpath)
+    # Create the string for the proto_path arguments (order matters, similar to classpath)
+
+    if include_std_lib:
+        # Add protoc standard lib to the includes
+        proto_paths = ['$(location //incubator/protobuf-dependencies:protoc-lib-' + protoc_version + ')'] + proto_paths
+
     if len(proto_paths) != 0:
         proto_paths_string = "-I=" + reduce(lambda a,b:  a +" -I=" + b, proto_paths)
     else:
@@ -107,9 +117,9 @@ def grpc_jar(
         out = 'grpc.src.zip',
         cmd = '$(location //buck-tools:grpc) $OUT '
               + '\"' + proto_paths_string + '\" '
-              + '$(location //incubator/protobuf-dependencies:prepare-protoc-executable-'+ protoc_version + ') '
+              + '$(location //incubator/protobuf-dependencies:protoc-exe-'+ protoc_version + ') '
-              + '$(location //incubator/grpc-dependencies:prepare-grpc-plugin-executable-' + plugin_version + ') '
+              + '$(location //incubator/grpc-dependencies:grpc-plugin-exe-' + plugin_version + ') '
-              + '$SRCS',
+              + '$SRCS' + src_string,
     )
 
     osgi_jar(

incubator/grpc-dependencies/BUCK

@@ -1,3 +1,3 @@
 include_defs('//bucklets/grpc.bucklet')
 
-fetch_grpc_plugin_binary()
+fetch_grpc_plugin(version = '1.3.0')

incubator/protobuf-dependencies/BUCK

@@ -1,3 +1,3 @@
 include_defs('//bucklets/grpc.bucklet')
 
-fetch_protoc_binary()
+fetch_protoc(version = '3.2.0')