diff --git a/docs/release-notes/change-log.md b/docs/release-notes/change-log.md index 6f5abd395..e7a94e215 100644 --- a/docs/release-notes/change-log.md +++ b/docs/release-notes/change-log.md @@ -11,6 +11,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased +### CLI + +- A manifest can now import `sf/substreams/sink/sql/schema/v1/schema.proto` without + vendoring a copy of it. The file is a system protobuf, but `protoparse` needs the + source on disk to honour its extensions, so an import previously failed with + `no such file`. It is now served from an embedded copy, the same way + `sf/substreams/options.proto` already was. + ### Docs - Document `Feed.Delete` on the Remote Feed Hosted Store guide: remote-feed clients can @@ -141,6 +149,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Tests +- The `tests_e2e/dummy` directory gains a `substreams.clickhouse.yaml` sibling manifest + packing `e2e_clickhouse`, whose `map_events_clickhouse` module emits + `test.clickhouse.Events`. That message carries the `(schema.table)` ClickHouse + annotations, so the package sinks with `substreams sink clickhouse` without further + setup. Kept out of `substreams.yaml` and given its own message so the annotations do + not change the module hashes of the existing e2e modules. + - The `tests_e2e/dummy` package gains three modules for exercising Hosted Stores against a staging environment. `map_hosted_store_feed`, packed into `e2e-v0.3.0.spkg`, emits `SinkEntries` and can be given to a Substreams Feed Hosted Store as its output module; it diff --git a/manifest/protobuf.go b/manifest/protobuf.go index c62790890..7a8ce67f6 100644 --- a/manifest/protobuf.go +++ b/manifest/protobuf.go @@ -149,6 +149,9 @@ func loadLocalProtobufs(pkg *pbsubstreams.Package, manif *Manifest) ([]*desc.Fil if strings.HasSuffix(filename, sfproto.OptionsPath) { return io.NopCloser(bytes.NewReader(sfproto.OptionsSource)), nil } + if strings.HasSuffix(filename, sfproto.SQLSchemaPath) { + return io.NopCloser(bytes.NewReader(sfproto.SQLSchemaSource)), nil + } return os.Open(filename) }, LookupImportProto: func(file string) (*descriptorpb.FileDescriptorProto, error) { @@ -310,6 +313,9 @@ func loadProtobufFromDirectory(pkg *pbsubstreams.Package, protoPath string) ([]* if strings.HasSuffix(filename, sfproto.OptionsPath) { return io.NopCloser(bytes.NewReader(sfproto.OptionsSource)), nil } + if strings.HasSuffix(filename, sfproto.SQLSchemaPath) { + return io.NopCloser(bytes.NewReader(sfproto.SQLSchemaSource)), nil + } return os.Open(filename) }, LookupImportProto: func(file string) (*descriptorpb.FileDescriptorProto, error) { diff --git a/manifest/sink_test.go b/manifest/sink_test.go index 5dfe58422..b3b42476b 100644 --- a/manifest/sink_test.go +++ b/manifest/sink_test.go @@ -1,8 +1,11 @@ package manifest import ( + "os" + "path/filepath" "testing" + pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1" "github.com/stretchr/testify/require" ) @@ -22,3 +25,28 @@ func TestSystemProtobufs_ResolveSQLSinkServiceTypes(t *testing.T) { require.NotNil(t, msgDesc) } } + +// A manifest that imports the SQL sink schema must parse without the file being +// on disk. It is a system protobuf, but protoparse needs the source to honour +// its extensions, so it is served from an embedded copy. +func TestSQLSchemaProto_ResolvesWithoutLocalCopy(t *testing.T) { + dir := t.TempDir() + protoDir := filepath.Join(dir, "proto") + require.NoError(t, os.MkdirAll(protoDir, 0o755)) + + contract := `syntax = "proto3"; +package test.annotated; +import "sf/substreams/sink/sql/schema/v1/schema.proto"; +message Row { + option (schema.table) = { + name: "Row" + clickhouse_table_options: { order_by_fields: [{name: "id"}] } + }; + string id = 1 [(schema.field) = { primary_key: true }]; +}` + require.NoError(t, os.WriteFile(filepath.Join(protoDir, "contract.proto"), []byte(contract), 0o644)) + + pkg := &pbsubstreams.Package{} + _, err := loadProtobufFromDirectory(pkg, protoDir) + require.NoError(t, err) +} diff --git a/proto/embed.go b/proto/embed.go index 4a726fa7b..fd56ce660 100644 --- a/proto/embed.go +++ b/proto/embed.go @@ -8,3 +8,10 @@ var OptionsPath = "sf/substreams/options.proto" //go:embed sf/substreams/options.proto var OptionsSource []byte + +// SQL sink schema annotations. Like options.proto, protoparse needs the source +// on disk to honour the extensions, and a manifest that imports it has no copy. +var SQLSchemaPath = "sf/substreams/sink/sql/schema/v1/schema.proto" + +//go:embed sf/substreams/sink/sql/schema/v1/schema.proto +var SQLSchemaSource []byte diff --git a/tests_e2e/dummy/e2e-v0.3.0.spkg b/tests_e2e/dummy/e2e-v0.4.0.spkg similarity index 55% rename from tests_e2e/dummy/e2e-v0.3.0.spkg rename to tests_e2e/dummy/e2e-v0.4.0.spkg index 6b5826252..586243afa 100644 Binary files a/tests_e2e/dummy/e2e-v0.3.0.spkg and b/tests_e2e/dummy/e2e-v0.4.0.spkg differ diff --git a/tests_e2e/dummy/proto/contract.proto b/tests_e2e/dummy/proto/contract.proto index dbf3c7129..69494785f 100644 --- a/tests_e2e/dummy/proto/contract.proto +++ b/tests_e2e/dummy/proto/contract.proto @@ -2,11 +2,23 @@ syntax = "proto3"; package test.output; +import "sf/substreams/sink/sql/schema/v1/schema.proto"; + message Events { repeated Event event = 1; } +// Annotated for the ClickHouse sink, which has no default ordering and refuses +// to create a table without one. Postgres ignores these options. message Event { - string evt_tx_hash = 1; + option (schema.table) = { + name: "Event" + clickhouse_table_options: { + order_by_fields: [{name: "evt_tx_hash"}, {name: "evt_block_number"}] + } + }; + + // ClickHouse requires the primary key to be a prefix of the sorting key. + string evt_tx_hash = 1 [(schema.field) = { primary_key: true }]; string evt_from = 2; string evt_to = 3; uint64 evt_block_number = 4; diff --git a/tests_e2e/dummy/src/pb/.last_generated_hash b/tests_e2e/dummy/src/pb/.last_generated_hash index d2f1213ef..875c16c96 100644 --- a/tests_e2e/dummy/src/pb/.last_generated_hash +++ b/tests_e2e/dummy/src/pb/.last_generated_hash @@ -1 +1 @@ -d937e490350d7325bd1c9e9c6bb7d40b008237a117a5d0bc241f69d4f92656ed \ No newline at end of file +cb8b989881857369c71f2726366159d38f11bab49fe54f60ca4bcb1a64581377 \ No newline at end of file diff --git a/tests_e2e/dummy/src/pb/test.output.rs b/tests_e2e/dummy/src/pb/test.output.rs index 0c40a3840..951643dee 100644 --- a/tests_e2e/dummy/src/pb/test.output.rs +++ b/tests_e2e/dummy/src/pb/test.output.rs @@ -6,9 +6,12 @@ pub struct Events { #[prost(message, repeated, tag="1")] pub event: ::prost::alloc::vec::Vec, } +/// Annotated for the ClickHouse sink, which has no default ordering and refuses +/// to create a table without one. Postgres ignores these options. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Event { + /// ClickHouse requires the primary key to be a prefix of the sorting key. #[prost(string, tag="1")] pub evt_tx_hash: ::prost::alloc::string::String, #[prost(string, tag="2")] diff --git a/tests_e2e/dummy/substreams.remote-feed.yaml b/tests_e2e/dummy/substreams.remote-feed.yaml index be6023cba..471840e0a 100644 --- a/tests_e2e/dummy/substreams.remote-feed.yaml +++ b/tests_e2e/dummy/substreams.remote-feed.yaml @@ -25,7 +25,11 @@ protobuf: importPaths: - ./proto excludePaths: - - sf/substreams + # Narrower than "sf/substreams": the sink/sql schema stays, since + # contract.proto imports it for the ClickHouse annotations. + - sf/substreams/rpc + - sf/substreams/v1 + - sf/substreams/index - sf/firehose - google diff --git a/tests_e2e/dummy/substreams.substreams-feed.yaml b/tests_e2e/dummy/substreams.substreams-feed.yaml index 1f8f6cb81..c89d13bcf 100644 --- a/tests_e2e/dummy/substreams.substreams-feed.yaml +++ b/tests_e2e/dummy/substreams.substreams-feed.yaml @@ -24,7 +24,11 @@ protobuf: importPaths: - ./proto excludePaths: - - sf/substreams + # Narrower than "sf/substreams": the sink/sql schema stays, since + # contract.proto imports it for the ClickHouse annotations. + - sf/substreams/rpc + - sf/substreams/v1 + - sf/substreams/index - sf/firehose - google diff --git a/tests_e2e/dummy/substreams.yaml b/tests_e2e/dummy/substreams.yaml index f45aff80e..7f0ceb159 100644 --- a/tests_e2e/dummy/substreams.yaml +++ b/tests_e2e/dummy/substreams.yaml @@ -1,7 +1,7 @@ specVersion: v0.1.0 package: name: e2e - version: v0.3.0 + version: v0.4.0 protobuf: descriptorSets: @@ -16,7 +16,11 @@ protobuf: importPaths: - ./proto excludePaths: - - sf/substreams + # Narrower than "sf/substreams": the sink/sql schema stays, since + # contract.proto imports it for the ClickHouse annotations. + - sf/substreams/rpc + - sf/substreams/v1 + - sf/substreams/index - sf/firehose - google