Add generated Go code for P4Runtime

The generated files are placed under the go/ directory, and the 2 Go packages which can be imported by other projects are "github.com/p4lang/p4runtime/go/p4/v1" and "github.com/p4lang/p4runtime/go/p4/config/v1". As part of CI, we now check that the generated Go files are up-to-date with respect to the checked-in Protobuf files. We also check that the go.mod file is "tidy". When updating the Protobuf files, a contributor can re-generated the Go files with `./build/update_go.sh`. Fixes #260
p4lang · Apr 30, 2020 · 4f6973d · 4f6973d
1 parent 169e9f8
commit 4f6973d
Show file tree

Hide file tree

Showing 18 changed files with 9,769 additions and 15 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -6,12 +6,13 @@ env:
   - secure: "kgFEYWugXGaFM/cEh3xUImFcbaq3nFcHT4P9G1XmiJ5JjpT7+5NkvCLydYKEBPEQuIVH+DRxrK1LqySWHC2qhBHE/AyB/0hpQ6JsYOL6LKPzPYUwSJhFKFHxxP1OtBrS5f51DUQ5wCPI6gF6sQP2l24QIVjIfgh3FSkpXX7RzUFF2eC2MS3Tg/NljsD5kGqCGdzaguaFculUuPpRqNixqZZXiM2BjDHHwGWT2rH7NcP4u6x2ycaTcUfN/XnnfDTrIkunicYte/x+l0QOMNXRZ3L/gs1Wgia4UT28P6Bld3MPVUa3sJe+Nz3eqDBZ/I8nyIm7tEvM3oNjLjT5i4JjDWQYffRkc2upVDim4d1boCbx+6zGBrKjP/IJ0AoqG0IBEKaYqfJkHEB8o2mKgp+ib8am69M3Ap3QAPjHGCmlix13+2IHsevoWw/dFnjP8ipb8YTOX/R9xV9cKLiTp4mrcGbDrPUJoadQ2RghNRcqBroS463pPZTsi/9UxcfHGMSWdDv7dqJJw/Y/fPT9mqBBpvX7+Mw2NKNB4AsC+ZCMWVP2kw36LuTww6mjpNy6NgfoctcNqB9rfMnyRKWDAfK4EaFfR5tx6P8xlupNqSnzqTqQzqCDrsTrFyXIXSLGqulz1ZC+RucM0jkCMcjpS+EsFxLUexAgZLl59/bHPEakIvo="
 
 install:
-  - docker build -t p4runtime-ci -f CI/Dockerfile .
+  - docker build -t p4runtime-ci -f build/Dockerfile .
 
 # We check that the P4Runtime Protobuf files are correct (by compiling them with
 # protoc), then we generate the HTML for the spec.
 script:
-  - docker run -t p4runtime-ci /p4runtime/CI/compile_protos.sh /tmp/ || travis_terminate 1
+  - docker run -t p4runtime-ci /p4runtime/build/compile_protos.sh /tmp || travis_terminate 1
+  - ./CI/check_go.sh || travis_terminate 1 # TODO: remove previous step
   - docker run -v `pwd`/docs/v1:/usr/src/p4-spec p4lang/p4rt-madoko:latest make || travis_terminate 1
   - ./tools/madokolint.py docs/v1/P4Runtime-Spec.mdk
 

diff --git a/CI/Dockerfile b/CI/Dockerfile
diff --git a/CI/check_go.sh b/CI/check_go.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+THIS_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
+
+pushd $THIS_DIR/.. >/dev/null
+
+rm -rf go/*
+./build/update_go.sh
+
+diff="$(git status --porcelain go go.mod go.sum)"
+
+if [ ! -z "$diff" ]; then
+    echo "The generated Go files are not up-to-date"
+    echo "You can regenerate them with './build/update_go.sh' and commit the changes"
+    exit 1
+fi
+
+popd >/dev/null
+
diff --git a/README.md b/README.md
@@ -51,11 +51,11 @@ devices or switches.
 # Compiling P4Runtime Protobuf files
 
 You can use Docker to run the protoc compiler on the P4Runtime Protobuf files
-and generate the Protobuf & gRPC bindings for C++ and Python:
+and generate the Protobuf & gRPC bindings for C++, Python and Go:
 
 ```
-docker build -t p4runtime -f CI/Dockerfile .
-docker run -v <OUT>:/out/ -t p4runtime /p4runtime/CI/compile_protos.sh /out/
+docker build -t p4runtime -f build/Dockerfile .
+docker run -v <OUT>:/out/ -t p4runtime /p4runtime/build/compile_protos.sh /out/
 ```
 
 This will generate the bindings in the local `<OUT>` directory. **You need to
@@ -125,5 +125,11 @@ processes.
    pull request against the specification that a key committer must review and
    approve.
 
+When updating the Protobuf files in a pull request, you will also need to update
+the generated Go files, which are hosted in this repository under
+[go/](go/). This can be done easily by running `./build/update_go.sh`, providing
+docker is installed and your user is part of the "docker" group (which means
+that the `docker` command can be executed without `sudo`).
+
 
 [P4 Slack Workspace]: https://p4-lang.slack.com/join/shared_invite/enQtODA0NzY4Mjc5MTExLTRlMWVmN2I5ZTY4MTAzMDI3MGQ1OTZjM2ZmM2Q1MWE2YzZjYTQ2ZWMyMGUyYjQ2ZmIxMjFjZDE4ZThiN2ZkZWI
diff --git a/build/Dockerfile b/build/Dockerfile
@@ -0,0 +1,28 @@
+FROM p4lang/third-party:latest
+LABEL maintainer="P4 API Working Group <p4-dev@lists.p4.org>"
+LABEL description="Dockerfile used for CI testing of p4lang/p4runtime"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends software-properties-common git curl
+
+ARG GO_VERSION=1.14.2
+
+RUN curl -o go.tar.gz https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz && \
+    tar -C /usr/local -xzf go.tar.gz && \
+    rm go.tar.gz
+
+ENV PATH="${PATH}:/usr/local/go/bin:/root/go/bin"
+
+# latest version of protoc-gen-go does not support GRPC
+# https://github.com/protocolbuffers/protobuf-go/releases/tag/v1.20.0
+ARG PROTOC_GEN_GO_VERSION=v1.3.2
+
+ENV GO111MODULE=on
+
+# required to generate gRPC bindings
+RUN go get google.golang.org/grpc
+RUN go get github.com/golang/protobuf/protoc-gen-go@$PROTOC_GEN_GO_VERSION
+
+COPY . /p4runtime/
+WORKDIR /p4runtime/
+
diff --git a/CI/compile_protos.sh → build/compile_protos.sh b/CI/compile_protos.sh → build/compile_protos.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-# This script is meant for CI testing, to ensure that the P4Runtime Protobuf
-# files are correct and compile with the protoc compiler (CPP, gRPC and Python).
+# This ensures that the P4Runtime Protobuf files are correct and compile with
+# the protoc compiler (CPP, gRPC, Python and Go).
 
 if [ $# -ne 1 ]; then
     echo "Usage: compile_protos.sh <BUILD_DIR>"
@@ -52,6 +52,7 @@ PROTOFLAGS="-I$GOOGLE_PROTO_DIR -I$PROTO_DIR"
 mkdir -p $BUILD_DIR/cpp_out
 mkdir -p $BUILD_DIR/grpc_out
 mkdir -p $BUILD_DIR/py_out
+mkdir -p $BUILD_DIR/go_out
 
 set -o xtrace
 $PROTOC $PROTOS --cpp_out $BUILD_DIR/cpp_out $PROTOFLAGS
@@ -61,6 +62,13 @@ $PROTOC $PROTOS --grpc_out $BUILD_DIR/grpc_out --plugin=protoc-gen-grpc=$GRPC_CP
 # plugin inserts code into the proto-generated files). But maybe I am just using
 # an old version of the Python plugin.
 $PROTOC $PROTOS --python_out $BUILD_DIR/py_out $PROTOFLAGS --grpc_out $BUILD_DIR/py_out --plugin=protoc-gen-grpc=$GRPC_PY_PLUGIN
+
+# our version of protoc-gen-go requires multiple invocations
+# see https://github.com/golang/protobuf/issues/39
+$PROTOC $PROTO_DIR/p4/config/v1/p4types.proto $PROTO_DIR/p4/config/v1/p4info.proto --go_out=$BUILD_DIR/go_out $PROTOFLAGS
+$PROTOC $PROTO_DIR/p4/v1/p4data.proto --go_out=$BUILD_DIR/go_out $PROTOFLAGS
+$PROTOC $PROTO_DIR/p4/v1/p4runtime.proto --go_out=plugins=grpc:$BUILD_DIR/go_out $PROTOFLAGS
+
 set +o xtrace
 
 rm -rf $tmpdir

diff --git a/build/update_go.sh b/build/update_go.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e
+
+THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+pushd $THIS_DIR/.. >/dev/null
+
+docker build -t p4runtime-ci -f build/Dockerfile .
+
+tmpdir=$(mktemp -d -p /tmp)
+
+docker run --rm \
+       -v "$tmpdir:/tmp/gen" \
+       p4runtime-ci /p4runtime/build/compile_protos.sh /tmp/gen
+
+cp -r $tmpdir/go_out/github.com/p4lang/p4runtime/go/* go/
+
+# Cleanup files owned by root user
+docker run --rm \
+       -v "$tmpdir:/tmp/gen" \
+       p4runtime-ci bash -c "rm -r /tmp/gen/*"
+
+docker run --rm -u "$(id -u):$(id -g)" \
+       -e "GOCACHE=/tmp/gocache" \
+       -e "GOPATH=/tmp/gopath" \
+       -v "$(pwd):/p4runtime" \
+       -w /p4runtime \
+       golang:1.14 bash -c "go mod tidy"
+
+rm -rf $tmpdir
+
+popd >/dev/null
diff --git a/docs/README.md b/docs/README.md
@@ -169,6 +169,7 @@ this [section of the Madoko
 spec](http://madoko.org/reference.html#sec-smart-quotes-symbols-and-direct-links).
 
 ## Document Figures
+
 Each image in the specification has a corresponding `.odg` file under
 `assets/`. These are LibreOffice drawing files. The files are rendered into
 `.svg` and `.png` images (for HTML and PDF output, resepectively) at build time,

diff --git a/go.mod b/go.mod
@@ -0,0 +1,9 @@
+module github.com/p4lang/p4runtime
+
+go 1.14
+
+require (
+	github.com/golang/protobuf v1.4.0
+	google.golang.org/genproto v0.0.0-20200413115906-b5235f65be36
+	google.golang.org/grpc v1.28.1
+)
diff --git a/go.sum b/go.sum
@@ -0,0 +1,71 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200413115906-b5235f65be36 h1:j7CmVRD4Kec0+f8VuBAc2Ak2MFfXm5Q2/RxuJLL+76E=
+google.golang.org/genproto v0.0.0-20200413115906-b5235f65be36/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k=
+google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=