add ch annotated dummy package as a sibling manifest

docs/release-notes/change-log.md

@@ -11,6 +11,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## Unreleased
 
+### CLI
+
+- A manifest can now import `sf/substreams/sink/sql/schema/v1/schema.proto` without
+  vendoring a copy of it. The file is a system protobuf, but `protoparse` needs the
+  source on disk to honour its extensions, so an import previously failed with
+  `no such file`. It is now served from an embedded copy, the same way
+  `sf/substreams/options.proto` already was.
+
 ### Docs
 
 - Document `Feed.Delete` on the Remote Feed Hosted Store guide: remote-feed clients can
@@ -141,6 +149,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Tests
 
+- The `tests_e2e/dummy` directory gains a `substreams.clickhouse.yaml` sibling manifest
+  packing `e2e_clickhouse`, whose `map_events_clickhouse` module emits
+  `test.clickhouse.Events`. That message carries the `(schema.table)` ClickHouse
+  annotations, so the package sinks with `substreams sink clickhouse` without further
+  setup. Kept out of `substreams.yaml` and given its own message so the annotations do
+  not change the module hashes of the existing e2e modules.
+
 - The `tests_e2e/dummy` package gains three modules for exercising Hosted Stores against a
   staging environment. `map_hosted_store_feed`, packed into `e2e-v0.3.0.spkg`, emits
   `SinkEntries` and can be given to a Substreams Feed Hosted Store as its output module; it

manifest/protobuf.go

@@ -149,6 +149,9 @@ func loadLocalProtobufs(pkg *pbsubstreams.Package, manif *Manifest) ([]*desc.Fil
 			if strings.HasSuffix(filename, sfproto.OptionsPath) {
 				return io.NopCloser(bytes.NewReader(sfproto.OptionsSource)), nil
 			}
+			if strings.HasSuffix(filename, sfproto.SQLSchemaPath) {
+				return io.NopCloser(bytes.NewReader(sfproto.SQLSchemaSource)), nil
+			}
 			return os.Open(filename)
 		},
 		LookupImportProto: func(file string) (*descriptorpb.FileDescriptorProto, error) {
@@ -310,6 +313,9 @@ func loadProtobufFromDirectory(pkg *pbsubstreams.Package, protoPath string) ([]*
 			if strings.HasSuffix(filename, sfproto.OptionsPath) {
 				return io.NopCloser(bytes.NewReader(sfproto.OptionsSource)), nil
 			}
+			if strings.HasSuffix(filename, sfproto.SQLSchemaPath) {
+				return io.NopCloser(bytes.NewReader(sfproto.SQLSchemaSource)), nil
+			}
 			return os.Open(filename)
 		},
 		LookupImportProto: func(file string) (*descriptorpb.FileDescriptorProto, error) {

manifest/sink_test.go

@@ -1,8 +1,11 @@
 package manifest
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
+	pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1"
 	"github.com/stretchr/testify/require"
 )
 
@@ -22,3 +25,28 @@ func TestSystemProtobufs_ResolveSQLSinkServiceTypes(t *testing.T) {
 		require.NotNil(t, msgDesc)
 	}
 }
+
+// A manifest that imports the SQL sink schema must parse without the file being
+// on disk. It is a system protobuf, but protoparse needs the source to honour
+// its extensions, so it is served from an embedded copy.
+func TestSQLSchemaProto_ResolvesWithoutLocalCopy(t *testing.T) {
+	dir := t.TempDir()
+	protoDir := filepath.Join(dir, "proto")
+	require.NoError(t, os.MkdirAll(protoDir, 0o755))
+
+	contract := `syntax = "proto3";
+package test.annotated;
+import "sf/substreams/sink/sql/schema/v1/schema.proto";
+message Row {
+  option (schema.table) = {
+    name: "Row"
+    clickhouse_table_options: { order_by_fields: [{name: "id"}] }
+  };
+  string id = 1 [(schema.field) = { primary_key: true }];
+}`
+	require.NoError(t, os.WriteFile(filepath.Join(protoDir, "contract.proto"), []byte(contract), 0o644))
+
+	pkg := &pbsubstreams.Package{}
+	_, err := loadProtobufFromDirectory(pkg, protoDir)
+	require.NoError(t, err)
+}

proto/embed.go

@@ -8,3 +8,10 @@ var OptionsPath = "sf/substreams/options.proto"
 
 //go:embed sf/substreams/options.proto
 var OptionsSource []byte
+
+// SQL sink schema annotations. Like options.proto, protoparse needs the source
+// on disk to honour the extensions, and a manifest that imports it has no copy.
+var SQLSchemaPath = "sf/substreams/sink/sql/schema/v1/schema.proto"
+
+//go:embed sf/substreams/sink/sql/schema/v1/schema.proto
+var SQLSchemaSource []byte

tests_e2e/dummy/proto/contract.proto

@@ -2,11 +2,23 @@ syntax = "proto3";
 
 package test.output;
 
+import "sf/substreams/sink/sql/schema/v1/schema.proto";
+
 message Events {
   repeated Event event = 1;
 }
+// Annotated for the ClickHouse sink, which has no default ordering and refuses
+// to create a table without one. Postgres ignores these options.
 message Event {
-  string evt_tx_hash = 1;
+  option (schema.table) = {
+    name: "Event"
+    clickhouse_table_options: {
+      order_by_fields: [{name: "evt_tx_hash"}, {name: "evt_block_number"}]
+    }
+  };
+
+  // ClickHouse requires the primary key to be a prefix of the sorting key.
+  string evt_tx_hash = 1 [(schema.field) = { primary_key: true }];
   string evt_from = 2;
   string evt_to = 3;
   uint64 evt_block_number = 4;

tests_e2e/dummy/src/pb/.last_generated_hash

@@ -1 +1 @@
-d937e490350d7325bd1c9e9c6bb7d40b008237a117a5d0bc241f69d4f92656ed
+cb8b989881857369c71f2726366159d38f11bab49fe54f60ca4bcb1a64581377

tests_e2e/dummy/src/pb/test.output.rs

@@ -6,9 +6,12 @@ pub struct Events {
     #[prost(message, repeated, tag="1")]
     pub event: ::prost::alloc::vec::Vec<Event>,
 }
+/// Annotated for the ClickHouse sink, which has no default ordering and refuses
+/// to create a table without one. Postgres ignores these options.
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct Event {
+    /// ClickHouse requires the primary key to be a prefix of the sorting key.
     #[prost(string, tag="1")]
     pub evt_tx_hash: ::prost::alloc::string::String,
     #[prost(string, tag="2")]

tests_e2e/dummy/substreams.remote-feed.yaml

@@ -25,7 +25,11 @@ protobuf:
   importPaths:
     - ./proto
   excludePaths:
-    - sf/substreams
+    # Narrower than "sf/substreams": the sink/sql schema stays, since
+    # contract.proto imports it for the ClickHouse annotations.
+    - sf/substreams/rpc
+    - sf/substreams/v1
+    - sf/substreams/index
     - sf/firehose
     - google
 

tests_e2e/dummy/substreams.substreams-feed.yaml

@@ -24,7 +24,11 @@ protobuf:
   importPaths:
     - ./proto
   excludePaths:
-    - sf/substreams
+    # Narrower than "sf/substreams": the sink/sql schema stays, since
+    # contract.proto imports it for the ClickHouse annotations.
+    - sf/substreams/rpc
+    - sf/substreams/v1
+    - sf/substreams/index
     - sf/firehose
     - google
 

tests_e2e/dummy/substreams.yaml

@@ -1,7 +1,7 @@
 specVersion: v0.1.0
 package:
   name: e2e
-  version: v0.3.0
+  version: v0.4.0
 
 protobuf:
   descriptorSets:
@@ -16,7 +16,11 @@ protobuf:
   importPaths:
     - ./proto
   excludePaths:
-    - sf/substreams
+    # Narrower than "sf/substreams": the sink/sql schema stays, since
+    # contract.proto imports it for the ClickHouse annotations.
+    - sf/substreams/rpc
+    - sf/substreams/v1
+    - sf/substreams/index
     - sf/firehose
     - google