refactor(proto): host module proto files within modules (#18758)

.github/workflows/proto-registry.yml

@@ -1,20 +1,23 @@
-name: Buf-Push
+# name: Buf Push
 # Protobuf runs buf (https://buf.build/) push updated proto files to https://buf.build/cosmos/cosmos-sdk
 # This workflow is only run when a .proto file has been changed
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - "proto/**"
+# on:
+#   push:
+#     tags:
+#       - "v[0-9]+.[0-9]+.[0-9]+" # Push events to matching v*, i.e. v1.0, v20.15.10
+#       - "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" # Push events to matching v*-rc*, i.e. v1.0-rc1, v20.15.10-rc2
+#       - "v[0-9]+.[0-9]+.[0-9]+-beta[0-9]+" # Push events to matching v*-beta*, i.e. v1.0-beta1, v20.15.10-beta2
+#       - "v[0-9]+.[0-9]+.[0-9]+-alpha[0-9]+" # Push events to matching v*-alpha*, i.e. v1.0-alpha1, v20.15.10-alpha2
+#     paths:
+#       - "proto/**"
 
-jobs:
-  push:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: bufbuild/[email protected]
-      - uses: bufbuild/buf-push-action@v1
-        with:
-          input: "proto"
-          buf_token: ${{ secrets.BUF_TOKEN }}
+# jobs:
+#   root:
+#     runs-on: ubuntu-latest
+#     name: "Push to buf.build/cosmos/cosmos-sdk"
+#     steps:
+#       - uses: actions/checkout@v4
+#       - uses: bufbuild/buf-setup-action@v1.29.0
+#       - run: buf push proto --tag ${{ github.ref_type == 'tag' && github.ref_name || github.sha }} # https://github.com/bufbuild/buf-push-action/issues/20
+
+## TODO at each module tag to their own buf repository

Makefile

@@ -425,7 +425,6 @@ proto-all: proto-format proto-lint proto-gen
 
 #? proto-gen: Generate Protobuf files
 proto-gen:
-	@echo "Generating Protobuf files"
 	@$(protoImage) sh ./scripts/protocgen.sh
 
 #? proto-swagger-gen: Generate Protobuf Swagger

scripts/protocgen.sh

@@ -9,32 +9,52 @@ find ./ -name "*.proto" -exec clang-format -i {} \;
 
 set -e
 
-echo "Generating gogo proto code"
-cd proto
-proto_dirs=$(find ./cosmos ./amino -path -prune -o -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq | grep -v '^./cosmos/store/')
+home=$PWD
+
+echo "Generating proto code"
+proto_dirs=$(find ./ -name 'buf.yaml' -print0 | xargs -0 -n1 dirname | sort | uniq)
 for dir in $proto_dirs; do
-  for file in $(find "${dir}" -maxdepth 1 -name '*.proto'); do
-    # this regex checks if a proto file has its go_package set to cosmossdk.io/api/...
-    # gogo proto files SHOULD ONLY be generated if this is false
-    # we don't want gogo proto to run for proto files which are natively built for google.golang.org/protobuf
-    if grep -q "option go_package" "$file" && grep -H -o -c 'option go_package.*cosmossdk.io/api' "$file" | grep -q ':0$'; then
-      buf generate --template buf.gen.gogo.yaml $file
+  echo "Generating proto code for $dir"
+
+  cd $dir
+  # check if buf.gen.pulsar.yaml exists in the proto directory
+  if [ -f "buf.gen.pulsar.yaml" ]; then
+    buf generate --template buf.gen.pulsar.yaml
+    # move generated files to the right places
+    if [ -d "../cosmos" -a "$dir" != "./proto" ]; then
+      cp -r ../cosmos $home/api
+      rm -rf ../cosmos
+    fi
+  fi
+
+  # check if buf.gen.gogo.yaml exists in the proto directory
+  if [ -f "buf.gen.gogo.yaml" ]; then
+      for file in $(find . -maxdepth 5 -name '*.proto' | grep -v '^./cosmos/store/'); do
+        # this regex checks if a proto file has its go_package set to cosmossdk.io/api/...
+        # gogo proto files SHOULD ONLY be generated if this is false
+        # we don't want gogo proto to run for proto files which are natively built for google.golang.org/protobuf
+        if grep -q "option go_package" "$file" && grep -H -o -c 'option go_package.*cosmossdk.io/api' "$file" | grep -q ':0$'; then
+          buf generate --template buf.gen.gogo.yaml $file
+        fi
+    done
+
+    # move generated files to the right places
+    if [ -d "../cosmossdk.io" -a "$dir" != "./proto" ]; then
+      cp -r ../cosmossdk.io/* $home
+      rm -rf ../cosmossdk.io
     fi
-  done
-done
 
-cd ..
+    if [ -d "../github.com" -a "$dir" != "./proto" ]; then
+      cp -r ../github.com/cosmos/cosmos-sdk/* $home
+      rm -rf ../github.com
+    fi
+  fi
 
-# generate tests proto code
-(cd testutil/testdata; buf generate)
-(cd baseapp/testutil; buf generate)
-(cd tests/integration/tx/internal; make codegen)
+  cd $home
+done
 
-# move proto files to the right places
+# move generated files to the right places
 cp -r github.com/cosmos/cosmos-sdk/* ./
-cp -r cosmossdk.io/** ./
-rm -rf github.com cosmossdk.io
-
-go mod tidy
+rm -rf github.com
 
-./scripts/protocgen-pulsar.sh
+go mod tidy

x/accounts/proto/buf.gen.gogo.yaml

@@ -0,0 +1,8 @@
+version: v1
+plugins:
+  - name: gocosmos
+    out: ..
+    opt: plugins=grpc,Mgoogle/protobuf/any.proto=github.com/cosmos/cosmos-sdk/codec/types
+  - name: grpc-gateway
+    out: ..
+    opt: logtostderr=true,allow_colon_final_segments=true

x/accounts/proto/buf.gen.pulsar.yaml

@@ -0,0 +1,18 @@
+version: v1
+managed:
+  enabled: true
+  go_package_prefix:
+    default: cosmossdk.io/api
+    except:
+      - buf.build/googleapis/googleapis
+      - buf.build/cosmos/gogo-proto
+      - buf.build/cosmos/cosmos-proto
+    override:
+      buf.build/cosmos/cosmos-sdk: cosmossdk.io/api
+plugins:
+  - name: go-pulsar
+    out: ..
+    opt: paths=source_relative
+  - name: go-grpc
+    out: ..
+    opt: paths=source_relative,Mcosmos/app/v1alpha1/module.proto=cosmossdk.io/api/cosmos/app/v1alpha1

x/accounts/proto/buf.lock

@@ -0,0 +1,28 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: cosmos
+    repository: cosmos-proto
+    commit: 1935555c206d4afb9e94615dfd0fad31
+    digest: shake256:c74d91a3ac7ae07d579e90eee33abf9b29664047ac8816500cf22c081fec0d72d62c89ce0bebafc1f6fec7aa5315be72606717740ca95007248425102c365377
+  - remote: buf.build
+    owner: cosmos
+    repository: cosmos-sdk
+    commit: cf13c7d232dd405180c2af616fa8a075
+    digest: shake256:769a38e306a98339b549bc96991c97fae8bd3ceb1a7646c7bfe9a74e406ab068372970fbc5abda1891e2f3c36527cf2d3a25f631739d36900787226e564bb612
+  - remote: buf.build
+    owner: cosmos
+    repository: gogo-proto
+    commit: 5e5b9fdd01804356895f8f79a6f1ddc1
+    digest: shake256:0b85da49e2e5f9ebc4806eae058e2f56096ff3b1c59d1fb7c190413dd15f45dd456f0b69ced9059341c80795d2b6c943de15b120a9e0308b499e43e4b5fc2952
+  - remote: buf.build
+    owner: googleapis
+    repository: googleapis
+    commit: 28151c0d0a1641bf938a7672c500e01d
+    digest: shake256:49215edf8ef57f7863004539deff8834cfb2195113f0b890dd1f67815d9353e28e668019165b9d872395871eeafcbab3ccfdb2b5f11734d3cca95be9e8d139de
+  - remote: buf.build
+    owner: protocolbuffers
+    repository: wellknowntypes
+    commit: 657250e6a39648cbb169d079a60bd9ba
+    digest: shake256:00de25001b8dd2e29d85fc4bcc3ede7aed886d76d67f5e0f7a9b320b90f871d3eb73507d50818d823a0512f3f8db77a11c043685528403e31ff3fef18323a9fb

x/accounts/proto/buf.yaml

@@ -0,0 +1,18 @@
+version: v1
+name: buf.build/mods/accounts
+deps:
+  - buf.build/cosmos/cosmos-sdk # pin the Cosmos SDK version
+  - buf.build/cosmos/cosmos-proto
+  - buf.build/cosmos/gogo-proto
+  - buf.build/googleapis/googleapis
+lint:
+  use:
+    - DEFAULT
+    - COMMENTS
+    - FILE_LOWER_SNAKE_CASE
+  except:
+    - UNARY_RPC
+    - COMMENT_FIELD
+    - SERVICE_SUFFIX
+    - PACKAGE_VERSION_SUFFIX
+    - RPC_REQUEST_STANDARD_NAME

x/auth/proto/buf.gen.gogo.yaml

@@ -0,0 +1,8 @@
+version: v1
+plugins:
+  - name: gocosmos
+    out: ..
+    opt: plugins=grpc,Mgoogle/protobuf/any.proto=github.com/cosmos/cosmos-sdk/codec/types
+  - name: grpc-gateway
+    out: ..
+    opt: logtostderr=true,allow_colon_final_segments=true

x/auth/proto/buf.gen.pulsar.yaml

@@ -0,0 +1,18 @@
+version: v1
+managed:
+  enabled: true
+  go_package_prefix:
+    default: cosmossdk.io/api
+    except:
+      - buf.build/googleapis/googleapis
+      - buf.build/cosmos/gogo-proto
+      - buf.build/cosmos/cosmos-proto
+    override:
+      buf.build/cosmos/cosmos-sdk: cosmossdk.io/api
+plugins:
+  - name: go-pulsar
+    out: ..
+    opt: paths=source_relative
+  - name: go-grpc
+    out: ..
+    opt: paths=source_relative

x/auth/proto/buf.lock

@@ -0,0 +1,28 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: cosmos
+    repository: cosmos-proto
+    commit: 1935555c206d4afb9e94615dfd0fad31
+    digest: shake256:c74d91a3ac7ae07d579e90eee33abf9b29664047ac8816500cf22c081fec0d72d62c89ce0bebafc1f6fec7aa5315be72606717740ca95007248425102c365377
+  - remote: buf.build
+    owner: cosmos
+    repository: cosmos-sdk
+    commit: cf13c7d232dd405180c2af616fa8a075
+    digest: shake256:769a38e306a98339b549bc96991c97fae8bd3ceb1a7646c7bfe9a74e406ab068372970fbc5abda1891e2f3c36527cf2d3a25f631739d36900787226e564bb612
+  - remote: buf.build
+    owner: cosmos
+    repository: gogo-proto
+    commit: 5e5b9fdd01804356895f8f79a6f1ddc1
+    digest: shake256:0b85da49e2e5f9ebc4806eae058e2f56096ff3b1c59d1fb7c190413dd15f45dd456f0b69ced9059341c80795d2b6c943de15b120a9e0308b499e43e4b5fc2952
+  - remote: buf.build
+    owner: googleapis
+    repository: googleapis
+    commit: 28151c0d0a1641bf938a7672c500e01d
+    digest: shake256:49215edf8ef57f7863004539deff8834cfb2195113f0b890dd1f67815d9353e28e668019165b9d872395871eeafcbab3ccfdb2b5f11734d3cca95be9e8d139de
+  - remote: buf.build
+    owner: protocolbuffers
+    repository: wellknowntypes
+    commit: 657250e6a39648cbb169d079a60bd9ba
+    digest: shake256:00de25001b8dd2e29d85fc4bcc3ede7aed886d76d67f5e0f7a9b320b90f871d3eb73507d50818d823a0512f3f8db77a11c043685528403e31ff3fef18323a9fb

x/auth/proto/buf.yaml

@@ -0,0 +1,18 @@
+version: v1
+name: buf.build/mods/auth
+deps:
+  - buf.build/cosmos/cosmos-sdk # pin the Cosmos SDK version
+  - buf.build/cosmos/cosmos-proto
+  - buf.build/cosmos/gogo-proto
+  - buf.build/googleapis/googleapis
+lint:
+  use:
+    - DEFAULT
+    - COMMENTS
+    - FILE_LOWER_SNAKE_CASE
+  except:
+    - UNARY_RPC
+    - COMMENT_FIELD
+    - SERVICE_SUFFIX
+    - PACKAGE_VERSION_SUFFIX
+    - RPC_REQUEST_STANDARD_NAME

x/authz/proto/buf.gen.gogo.yaml

@@ -0,0 +1,8 @@
+version: v1
+plugins:
+  - name: gocosmos
+    out: ..
+    opt: plugins=grpc,Mgoogle/protobuf/any.proto=github.com/cosmos/cosmos-sdk/codec/types
+  - name: grpc-gateway
+    out: ..
+    opt: logtostderr=true,allow_colon_final_segments=true

x/authz/proto/buf.gen.pulsar.yaml

@@ -0,0 +1,18 @@
+version: v1
+managed:
+  enabled: true
+  go_package_prefix:
+    default: cosmossdk.io/api
+    except:
+      - buf.build/googleapis/googleapis
+      - buf.build/cosmos/gogo-proto
+      - buf.build/cosmos/cosmos-proto
+    override:
+      buf.build/cosmos/cosmos-sdk: cosmossdk.io/api
+plugins:
+  - name: go-pulsar
+    out: ..
+    opt: paths=source_relative
+  - name: go-grpc
+    out: ..
+    opt: paths=source_relative

x/authz/proto/buf.lock

@@ -0,0 +1,28 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: cosmos
+    repository: cosmos-proto
+    commit: 1935555c206d4afb9e94615dfd0fad31
+    digest: shake256:c74d91a3ac7ae07d579e90eee33abf9b29664047ac8816500cf22c081fec0d72d62c89ce0bebafc1f6fec7aa5315be72606717740ca95007248425102c365377
+  - remote: buf.build
+    owner: cosmos
+    repository: cosmos-sdk
+    commit: cf13c7d232dd405180c2af616fa8a075
+    digest: shake256:769a38e306a98339b549bc96991c97fae8bd3ceb1a7646c7bfe9a74e406ab068372970fbc5abda1891e2f3c36527cf2d3a25f631739d36900787226e564bb612
+  - remote: buf.build
+    owner: cosmos
+    repository: gogo-proto
+    commit: 5e5b9fdd01804356895f8f79a6f1ddc1
+    digest: shake256:0b85da49e2e5f9ebc4806eae058e2f56096ff3b1c59d1fb7c190413dd15f45dd456f0b69ced9059341c80795d2b6c943de15b120a9e0308b499e43e4b5fc2952
+  - remote: buf.build
+    owner: googleapis
+    repository: googleapis
+    commit: 28151c0d0a1641bf938a7672c500e01d
+    digest: shake256:49215edf8ef57f7863004539deff8834cfb2195113f0b890dd1f67815d9353e28e668019165b9d872395871eeafcbab3ccfdb2b5f11734d3cca95be9e8d139de
+  - remote: buf.build
+    owner: protocolbuffers
+    repository: wellknowntypes
+    commit: 657250e6a39648cbb169d079a60bd9ba
+    digest: shake256:00de25001b8dd2e29d85fc4bcc3ede7aed886d76d67f5e0f7a9b320b90f871d3eb73507d50818d823a0512f3f8db77a11c043685528403e31ff3fef18323a9fb

x/authz/proto/buf.yaml

@@ -0,0 +1,18 @@
+version: v1
+name: buf.build/mods/authz
+deps:
+  - buf.build/cosmos/cosmos-sdk # pin the Cosmos SDK version
+  - buf.build/cosmos/cosmos-proto
+  - buf.build/cosmos/gogo-proto
+  - buf.build/googleapis/googleapis
+lint:
+  use:
+    - DEFAULT
+    - COMMENTS
+    - FILE_LOWER_SNAKE_CASE
+  except:
+    - UNARY_RPC
+    - COMMENT_FIELD
+    - SERVICE_SUFFIX
+    - PACKAGE_VERSION_SUFFIX
+    - RPC_REQUEST_STANDARD_NAME

x/bank/proto/buf.gen.gogo.yaml

@@ -0,0 +1,8 @@
+version: v1
+plugins:
+  - name: gocosmos
+    out: ..
+    opt: plugins=grpc,Mgoogle/protobuf/any.proto=github.com/cosmos/cosmos-sdk/codec/types
+  - name: grpc-gateway
+    out: ..
+    opt: logtostderr=true,allow_colon_final_segments=true