From 571c1982437fc3a9a0bebca293418c22ba64e270 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Wed, 19 Nov 2025 13:45:44 -0800 Subject: [PATCH 001/100] Fix issue 2245 - Creating more than 41 vlabels causes crash in drop_graph (#2248) Fixed issue 2245 - Creating more than 41 vlabels causes drop_grapth to fail with "label (relation) cache corrupted" and crashing out on the following command. This was due to corruption of the label_relation_cache during the HASH_DELETE process. As the issue was with a cache flush routine, it was necessary to fix them all. Here is the list of the flush functions that were fixed - static void flush_graph_name_cache(void) static void flush_graph_namespace_cache(void) static void flush_label_name_graph_cache(void) static void flush_label_graph_oid_cache(void) static void flush_label_relation_cache(void) static void flush_label_seq_name_graph_cache(void) Added regression tests. modified: regress/expected/catalog.out modified: regress/sql/catalog.sql modified: src/backend/utils/cache/ag_cache.c --- regress/expected/catalog.out | 141 ++++++++++++++++++++++- regress/sql/catalog.sql | 42 ++++++- src/backend/utils/cache/ag_cache.c | 174 ++++++++++------------------- 3 files changed, 241 insertions(+), 116 deletions(-) diff --git a/regress/expected/catalog.out b/regress/expected/catalog.out index d06a0ce67..a15fa4698 100644 --- a/regress/expected/catalog.out +++ b/regress/expected/catalog.out @@ -457,7 +457,146 @@ NOTICE: graph does not exist (1 row) DROP FUNCTION raise_notice(TEXT); --- dropping the graph +-- +-- Fix issue 2245 - Creating more than 41 vlabels causes drop_graph to fail with +-- label (relation) cache corrupted +-- +-- this result will change if another graph was created prior to this point. +SELECT count(*) FROM ag_label; + count +------- + 2 +(1 row) + +SELECT * FROM create_graph('issue_2245'); +NOTICE: graph "issue_2245" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('issue_2245', $$ + CREATE (a1:Part1 {part_num: '123'}), (a2:Part2 {part_num: '345'}), (a3:Part3 {part_num: '456'}), + (a4:Part4 {part_num: '789'}), (a5:Part5 {part_num: '123'}), (a6:Part6 {part_num: '345'}), + (a7:Part7 {part_num: '456'}), (a8:Part8 {part_num: '789'}), (a9:Part9 {part_num: '123'}), + (a10:Part10 {part_num: '345'}), (a11:Part11 {part_num: '456'}), (a12:Part12 {part_num: '789'}), + (a13:Part13 {part_num: '123'}), (a14:Part14 {part_num: '345'}), (a15:Part15 {part_num: '456'}), + (a16:Part16 {part_num: '789'}), (a17:Part17 {part_num: '123'}), (a18:Part18 {part_num: '345'}), + (a19:Part19 {part_num: '456'}), (a20:Part20 {part_num: '789'}), (a21:Part21 {part_num: '123'}), + (a22:Part22 {part_num: '345'}), (a23:Part23 {part_num: '456'}), (a24:Part24 {part_num: '789'}), + (a25:Part25 {part_num: '123'}), (a26:Part26 {part_num: '345'}), (a27:Part27 {part_num: '456'}), + (a28:Part28 {part_num: '789'}), (a29:Part29 {part_num: '789'}), (a30:Part30 {part_num: '123'}), + (a31:Part31 {part_num: '345'}), (a32:Part32 {part_num: '456'}), (a33:Part33 {part_num: '789'}), + (a34:Part34 {part_num: '123'}), (a35:Part35 {part_num: '345'}), (a36:Part36 {part_num: '456'}), + (a37:Part37 {part_num: '789'}), (a38:Part38 {part_num: '123'}), (a39:Part39 {part_num: '345'}), + (a40:Part40 {part_num: '456'}), (a41:Part41 {part_num: '789'}), (a42:Part42 {part_num: '345'}), + (a43:Part43 {part_num: '456'}), (a44:Part44 {part_num: '789'}), (a45:Part45 {part_num: '456'}), + (a46:Part46 {part_num: '789'}), (a47:Part47 {part_num: '456'}), (a48:Part48 {part_num: '789'}), + (a49:Part49 {part_num: '789'}), (a50:Part50 {part_num: '456'}), (a51:Part51 {part_num: '789'}) + $$) AS (result agtype); + result +-------- +(0 rows) + +SELECT count(*) FROM ag_label; + count +------- + 55 +(1 row) + +SELECT drop_graph('issue_2245', true); +NOTICE: drop cascades to 53 other objects +DETAIL: drop cascades to table issue_2245._ag_label_vertex +drop cascades to table issue_2245._ag_label_edge +drop cascades to table issue_2245."Part1" +drop cascades to table issue_2245."Part2" +drop cascades to table issue_2245."Part3" +drop cascades to table issue_2245."Part4" +drop cascades to table issue_2245."Part5" +drop cascades to table issue_2245."Part6" +drop cascades to table issue_2245."Part7" +drop cascades to table issue_2245."Part8" +drop cascades to table issue_2245."Part9" +drop cascades to table issue_2245."Part10" +drop cascades to table issue_2245."Part11" +drop cascades to table issue_2245."Part12" +drop cascades to table issue_2245."Part13" +drop cascades to table issue_2245."Part14" +drop cascades to table issue_2245."Part15" +drop cascades to table issue_2245."Part16" +drop cascades to table issue_2245."Part17" +drop cascades to table issue_2245."Part18" +drop cascades to table issue_2245."Part19" +drop cascades to table issue_2245."Part20" +drop cascades to table issue_2245."Part21" +drop cascades to table issue_2245."Part22" +drop cascades to table issue_2245."Part23" +drop cascades to table issue_2245."Part24" +drop cascades to table issue_2245."Part25" +drop cascades to table issue_2245."Part26" +drop cascades to table issue_2245."Part27" +drop cascades to table issue_2245."Part28" +drop cascades to table issue_2245."Part29" +drop cascades to table issue_2245."Part30" +drop cascades to table issue_2245."Part31" +drop cascades to table issue_2245."Part32" +drop cascades to table issue_2245."Part33" +drop cascades to table issue_2245."Part34" +drop cascades to table issue_2245."Part35" +drop cascades to table issue_2245."Part36" +drop cascades to table issue_2245."Part37" +drop cascades to table issue_2245."Part38" +drop cascades to table issue_2245."Part39" +drop cascades to table issue_2245."Part40" +drop cascades to table issue_2245."Part41" +drop cascades to table issue_2245."Part42" +drop cascades to table issue_2245."Part43" +drop cascades to table issue_2245."Part44" +drop cascades to table issue_2245."Part45" +drop cascades to table issue_2245."Part46" +drop cascades to table issue_2245."Part47" +drop cascades to table issue_2245."Part48" +drop cascades to table issue_2245."Part49" +drop cascades to table issue_2245."Part50" +drop cascades to table issue_2245."Part51" +NOTICE: graph "issue_2245" has been dropped + drop_graph +------------ + +(1 row) + +-- this result should be the same as the one before the create_graph +SELECT count(*) FROM ag_label; + count +------- + 2 +(1 row) + +-- create the graph again +SELECT * FROM create_graph('issue_2245'); +NOTICE: graph "issue_2245" has been created + create_graph +-------------- + +(1 row) + +SELECT count(*) FROM ag_label; + count +------- + 4 +(1 row) + +-- dropping the graphs +SELECT drop_graph('issue_2245', true); +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table issue_2245._ag_label_vertex +drop cascades to table issue_2245._ag_label_edge +NOTICE: graph "issue_2245" has been dropped + drop_graph +------------ + +(1 row) + SELECT drop_graph('graph', true); NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table graph._ag_label_vertex diff --git a/regress/sql/catalog.sql b/regress/sql/catalog.sql index 85fc4e8ab..bb72c3495 100644 --- a/regress/sql/catalog.sql +++ b/regress/sql/catalog.sql @@ -193,5 +193,45 @@ SELECT raise_notice('graph1'); DROP FUNCTION raise_notice(TEXT); --- dropping the graph +-- +-- Fix issue 2245 - Creating more than 41 vlabels causes drop_graph to fail with +-- label (relation) cache corrupted +-- + +-- this result will change if another graph was created prior to this point. +SELECT count(*) FROM ag_label; + +SELECT * FROM create_graph('issue_2245'); +SELECT * FROM cypher('issue_2245', $$ + CREATE (a1:Part1 {part_num: '123'}), (a2:Part2 {part_num: '345'}), (a3:Part3 {part_num: '456'}), + (a4:Part4 {part_num: '789'}), (a5:Part5 {part_num: '123'}), (a6:Part6 {part_num: '345'}), + (a7:Part7 {part_num: '456'}), (a8:Part8 {part_num: '789'}), (a9:Part9 {part_num: '123'}), + (a10:Part10 {part_num: '345'}), (a11:Part11 {part_num: '456'}), (a12:Part12 {part_num: '789'}), + (a13:Part13 {part_num: '123'}), (a14:Part14 {part_num: '345'}), (a15:Part15 {part_num: '456'}), + (a16:Part16 {part_num: '789'}), (a17:Part17 {part_num: '123'}), (a18:Part18 {part_num: '345'}), + (a19:Part19 {part_num: '456'}), (a20:Part20 {part_num: '789'}), (a21:Part21 {part_num: '123'}), + (a22:Part22 {part_num: '345'}), (a23:Part23 {part_num: '456'}), (a24:Part24 {part_num: '789'}), + (a25:Part25 {part_num: '123'}), (a26:Part26 {part_num: '345'}), (a27:Part27 {part_num: '456'}), + (a28:Part28 {part_num: '789'}), (a29:Part29 {part_num: '789'}), (a30:Part30 {part_num: '123'}), + (a31:Part31 {part_num: '345'}), (a32:Part32 {part_num: '456'}), (a33:Part33 {part_num: '789'}), + (a34:Part34 {part_num: '123'}), (a35:Part35 {part_num: '345'}), (a36:Part36 {part_num: '456'}), + (a37:Part37 {part_num: '789'}), (a38:Part38 {part_num: '123'}), (a39:Part39 {part_num: '345'}), + (a40:Part40 {part_num: '456'}), (a41:Part41 {part_num: '789'}), (a42:Part42 {part_num: '345'}), + (a43:Part43 {part_num: '456'}), (a44:Part44 {part_num: '789'}), (a45:Part45 {part_num: '456'}), + (a46:Part46 {part_num: '789'}), (a47:Part47 {part_num: '456'}), (a48:Part48 {part_num: '789'}), + (a49:Part49 {part_num: '789'}), (a50:Part50 {part_num: '456'}), (a51:Part51 {part_num: '789'}) + $$) AS (result agtype); + +SELECT count(*) FROM ag_label; +SELECT drop_graph('issue_2245', true); + +-- this result should be the same as the one before the create_graph +SELECT count(*) FROM ag_label; + +-- create the graph again +SELECT * FROM create_graph('issue_2245'); +SELECT count(*) FROM ag_label; + +-- dropping the graphs +SELECT drop_graph('issue_2245', true); SELECT drop_graph('graph', true); diff --git a/src/backend/utils/cache/ag_cache.c b/src/backend/utils/cache/ag_cache.c index e3c4d0794..493ffcfa9 100644 --- a/src/backend/utils/cache/ag_cache.c +++ b/src/backend/utils/cache/ag_cache.c @@ -286,52 +286,34 @@ static void invalidate_graph_caches(Datum arg, int cache_id, uint32 hash_value) static void flush_graph_name_cache(void) { - HASH_SEQ_STATUS hash_seq; - - hash_seq_init(&hash_seq, graph_name_cache_hash); - for (;;) + /* + * If the graph_name_cache exists, destroy it. This will avoid any + * potential corruption issues. + */ + if (graph_name_cache_hash) { - graph_name_cache_entry *entry; - void *removed; - - entry = hash_seq_search(&hash_seq); - if (!entry) - { - break; - } - removed = hash_search(graph_name_cache_hash, &entry->name, HASH_REMOVE, - NULL); - if (!removed) - { - ereport(ERROR, (errmsg_internal("graph (name) cache corrupted"))); - } + hash_destroy(graph_name_cache_hash); + graph_name_cache_hash = NULL; } + + /* recreate the graph_name_cache */ + create_graph_name_cache(); } static void flush_graph_namespace_cache(void) { - HASH_SEQ_STATUS hash_seq; - - hash_seq_init(&hash_seq, graph_namespace_cache_hash); - for (;;) + /* + * If the graph_namespace_cache exists, destroy it. This will avoid any + * potential corruption issues. + */ + if (graph_namespace_cache_hash) { - graph_namespace_cache_entry *entry; - void *removed; - - entry = hash_seq_search(&hash_seq); - if (!entry) - { - break; - } - - removed = hash_search(graph_namespace_cache_hash, &entry->namespace, - HASH_REMOVE, NULL); - if (!removed) - { - ereport(ERROR, - (errmsg_internal("graph (namespace) cache corrupted"))); - } + hash_destroy(graph_namespace_cache_hash); + graph_namespace_cache_hash = NULL; } + + /* recreate the graph_namespace_cache */ + create_graph_namespace_cache(); } graph_cache_data *search_graph_name_cache(const char *name) @@ -664,27 +646,18 @@ static void invalidate_label_name_graph_cache(Oid relid) static void flush_label_name_graph_cache(void) { - HASH_SEQ_STATUS hash_seq; - - hash_seq_init(&hash_seq, label_name_graph_cache_hash); - for (;;) + /* + * If the label_name_graph_cache exists, destroy it. This will avoid any + * potential corruption issues. + */ + if (label_name_graph_cache_hash) { - label_name_graph_cache_entry *entry; - void *removed; - - entry = hash_seq_search(&hash_seq); - if (!entry) - { - break; - } - removed = hash_search(label_name_graph_cache_hash, &entry->key, - HASH_REMOVE, NULL); - if (!removed) - { - ereport(ERROR, - (errmsg_internal("label (name, graph) cache corrupted"))); - } + hash_destroy(label_name_graph_cache_hash); + label_name_graph_cache_hash = NULL; } + + /* recreate the label_name_graph_cache */ + create_label_name_graph_cache(); } static void invalidate_label_graph_oid_cache(Oid relid) @@ -722,27 +695,18 @@ static void invalidate_label_graph_oid_cache(Oid relid) static void flush_label_graph_oid_cache(void) { - HASH_SEQ_STATUS hash_seq; - - hash_seq_init(&hash_seq, label_graph_oid_cache_hash); - for (;;) + /* + * If the label_graph_oid_cache exists, destroy it. This will avoid any + * potential corruption issues. + */ + if (label_graph_oid_cache_hash) { - label_graph_oid_cache_entry *entry; - void *removed; - - entry = hash_seq_search(&hash_seq); - if (!entry) - { - break; - } - removed = hash_search(label_graph_oid_cache_hash, &entry->key, - HASH_REMOVE, NULL); - if (!removed) - { - ereport(ERROR, - (errmsg_internal("label (graph, id) cache corrupted"))); - } + hash_destroy(label_graph_oid_cache_hash); + label_graph_oid_cache_hash = NULL; } + + /* recreate the label_graph_oid_cache */ + create_label_graph_oid_cache(); } static void invalidate_label_relation_cache(Oid relid) @@ -765,27 +729,18 @@ static void invalidate_label_relation_cache(Oid relid) static void flush_label_relation_cache(void) { - HASH_SEQ_STATUS hash_seq; - - hash_seq_init(&hash_seq, label_relation_cache_hash); - for (;;) + /* + * If the label_relation_cache exists, destroy it. This will avoid any + * potential corruption issues. + */ + if (label_relation_cache_hash) { - label_relation_cache_entry *entry; - void *removed; - - entry = hash_seq_search(&hash_seq); - if (!entry) - { - break; - } - removed = hash_search(label_relation_cache_hash, &entry->relation, - HASH_REMOVE, NULL); - if (!removed) - { - ereport(ERROR, - (errmsg_internal("label (relation) cache corrupted"))); - } + hash_destroy(label_relation_cache_hash); + label_relation_cache_hash = NULL; } + + /* recreate the label_relation_cache */ + create_label_relation_cache(); } static void invalidate_label_seq_name_graph_cache(Oid relid) @@ -823,27 +778,18 @@ static void invalidate_label_seq_name_graph_cache(Oid relid) static void flush_label_seq_name_graph_cache(void) { - HASH_SEQ_STATUS hash_seq; - - hash_seq_init(&hash_seq, label_seq_name_graph_cache_hash); - for (;;) + /* + * If the label_seq_name_graph_cache exists, destroy it. This will + * avoid any potential corruption issues by deleting entries. + */ + if (label_seq_name_graph_cache_hash) { - label_seq_name_graph_cache_entry *entry; - void *removed; - - entry = hash_seq_search(&hash_seq); - if (!entry) - { - break; - } - removed = hash_search(label_seq_name_graph_cache_hash, &entry->key, - HASH_REMOVE, NULL); - if (!removed) - { - ereport(ERROR, - (errmsg_internal("label (seq_name, graph) cache corrupted"))); - } + hash_destroy(label_seq_name_graph_cache_hash); + label_seq_name_graph_cache_hash = NULL; } + + /* recreate the label_seq_name_graph_cache */ + create_label_seq_name_graph_cache(); } label_cache_data *search_label_name_graph_cache(const char *name, Oid graph) From 5aed9ecc5b492a47dc0e449421cf344de58b5edd Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Wed, 3 Dec 2025 22:17:09 +0500 Subject: [PATCH 002/100] Add index on id columns (#2117) - Whenever a label will be created, indices on id columns will be created by default. In case of vertex, a unique index on id column will be created, which will also serve as a unique constraint. In case of edge, a non-unique index on start_id and end_id columns will be created. - This change is expected to improve the performance of queries that involve joins. From some performance tests, it was observed that the performance of queries improved alot. - Loader was updated to insert tuples in indices as well. This has caused to slow the loader down a bit, but it was necessary. - A bug related to command ids in cypher_delete executor was also fixed. --- regress/expected/age_load.out | 14 -- regress/expected/cypher_match.out | 72 ++++---- regress/expected/cypher_merge.out | 2 +- regress/expected/cypher_subquery.out | 2 +- regress/expected/cypher_vle.out | 24 +-- regress/expected/expr.out | 32 ++-- regress/expected/graph_generation.out | 20 +-- regress/expected/index.out | 97 +++++++++-- regress/expected/map_projection.out | 2 +- regress/sql/age_load.sql | 6 - regress/sql/index.sql | 34 ++-- src/backend/commands/label_commands.c | 78 ++++++++- src/backend/executor/cypher_delete.c | 4 + src/backend/utils/load/ag_load_edges.c | 75 +-------- src/backend/utils/load/ag_load_labels.c | 215 +----------------------- src/backend/utils/load/age_load.c | 193 +++++++++++++++++---- src/include/utils/load/ag_load_labels.h | 5 - src/include/utils/load/age_load.h | 16 +- 18 files changed, 424 insertions(+), 467 deletions(-) diff --git a/regress/expected/age_load.out b/regress/expected/age_load.out index b638e636b..5f2bdab78 100644 --- a/regress/expected/age_load.out +++ b/regress/expected/age_load.out @@ -43,13 +43,6 @@ SELECT load_labels_from_file('agload_test_graph', 'Country', (1 row) --- A temporary table should have been created with 54 ids; 1 from CREATE and 53 from file -SELECT COUNT(*)=54 FROM "_agload_test_graph_ag_vertex_ids"; - ?column? ----------- - t -(1 row) - -- Sequence should be equal to max entry id i.e. 248 SELECT currval('agload_test_graph."Country_id_seq"')=248; ?column? @@ -74,13 +67,6 @@ NOTICE: VLabel "City" has been created (1 row) --- Temporary table should have 54+72485 rows now -SELECT COUNT(*)=54+72485 FROM "_agload_test_graph_ag_vertex_ids"; - ?column? ----------- - t -(1 row) - -- Sequence should be equal to max entry id i.e. 146941 SELECT currval('agload_test_graph."City_id_seq"')=146941; ?column? diff --git a/regress/expected/cypher_match.out b/regress/expected/cypher_match.out index e83ba3b93..72ca1cd71 100644 --- a/regress/expected/cypher_match.out +++ b/regress/expected/cypher_match.out @@ -79,8 +79,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 1125899906842625, "label": "v1", "properties": {"id": "initial"}}::vertex, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex]::path [{"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1125899906842625, "label": "v1", "properties": {"id": "initial"}}::vertex]::path + [{"id": 1125899906842625, "label": "v1", "properties": {"id": "initial"}}::vertex, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex]::path (2 rows) SELECT * FROM cypher('cypher_match', $$ @@ -88,8 +88,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a ---------------------------------------------------------------------------------- - {"id": 1125899906842625, "label": "v1", "properties": {"id": "initial"}}::vertex {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex + {"id": 1125899906842625, "label": "v1", "properties": {"id": "initial"}}::vertex (2 rows) SELECT * FROM cypher('cypher_match', $$ @@ -97,8 +97,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a ---------------------------------------------------------------------------------- - {"id": 1125899906842625, "label": "v1", "properties": {"id": "initial"}}::vertex {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex + {"id": 1125899906842625, "label": "v1", "properties": {"id": "initial"}}::vertex (2 rows) SELECT * FROM cypher('cypher_match', $$ @@ -115,8 +115,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a --------------------------------------------------------------------------------------------------------------------------- - {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge (2 rows) SELECT * FROM cypher('cypher_match', $$ @@ -132,10 +132,10 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a --------------------------------------------------------------------------------------------------------------------------- - {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge - {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge + {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge (4 rows) SELECT * FROM cypher('cypher_match', $$ @@ -143,10 +143,10 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a --------------------------------------------------------------------------------------------------------------------------- - {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge - {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge + {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge (4 rows) SELECT * FROM cypher('cypher_match', $$ @@ -154,10 +154,10 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a --------------------------------------------------------------------------------------------------------------------------- - {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge - {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge + {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge (4 rows) SELECT * FROM cypher('cypher_match', $$ @@ -165,8 +165,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a --------------------------------------------------------------------------------------------------------------------------- - {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge (2 rows) SELECT * FROM cypher('cypher_match', $$ @@ -174,8 +174,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a ---------------------------------------------------------------------------------- - {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex {"id": 1125899906842625, "label": "v1", "properties": {"id": "initial"}}::vertex + {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex (2 rows) -- Right Path Test @@ -250,8 +250,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a --------------------------------------------------------------------------------------------------------------------------- - {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge (2 rows) --Left Path Test @@ -308,8 +308,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (a agtype); a --------------------------------------------------------------------------------------------------------------------------- - {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge (2 rows) --Divergent Path Tests @@ -412,8 +412,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (i agtype); i --------------------------------------------------------------------------------------------------------------------------- - {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge {"id": 2533274790395905, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685251, "properties": {}}::edge + {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge (2 rows) SELECT * FROM cypher('cypher_match', $$ @@ -712,8 +712,8 @@ $$) AS (r0 agtype); {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge {"id": 1970324836974593, "label": "e2", "end_id": 1688849860263939, "start_id": 1688849860263938, "properties": {}}::edge - {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge {"id": 2533274790395905, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685251, "properties": {}}::edge + {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge (6 rows) SELECT * FROM cypher('cypher_match', $$ @@ -775,8 +775,8 @@ $$) AS (r1 agtype); {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge {"id": 1970324836974593, "label": "e2", "end_id": 1688849860263939, "start_id": 1688849860263938, "properties": {}}::edge {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge - {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge {"id": 1970324836974593, "label": "e2", "end_id": 1688849860263939, "start_id": 1688849860263938, "properties": {}}::edge + {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge {"id": 1970324836974593, "label": "e2", "end_id": 1688849860263939, "start_id": 1688849860263938, "properties": {}}::edge (12 rows) @@ -1055,8 +1055,8 @@ SELECT * FROM cypher('cypher_match', {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex | {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge | {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex | {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge | {"id": 1688849860263937, "label": "v2", "properties": {"id": "initial"}}::vertex {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex | {"id": 1970324836974593, "label": "e2", "end_id": 1688849860263939, "start_id": 1688849860263938, "properties": {}}::edge | {"id": 1688849860263939, "label": "v2", "properties": {"id": "end"}}::vertex - {"id": 2251799813685249, "label": "v3", "properties": {"id": "initial"}}::vertex | {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex {"id": 2251799813685251, "label": "v3", "properties": {"id": "end"}}::vertex | {"id": 2533274790395905, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685251, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex + {"id": 2251799813685249, "label": "v3", "properties": {"id": "initial"}}::vertex | {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex (6 rows) SELECT * FROM cypher('cypher_match', @@ -1068,8 +1068,8 @@ AS (u agtype, e agtype, v agtype); {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex | {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge | {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex | {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge | {"id": 1688849860263937, "label": "v2", "properties": {"id": "initial"}}::vertex {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex | {"id": 1970324836974593, "label": "e2", "end_id": 1688849860263939, "start_id": 1688849860263938, "properties": {}}::edge | {"id": 1688849860263939, "label": "v2", "properties": {"id": "end"}}::vertex - {"id": 2251799813685249, "label": "v3", "properties": {"id": "initial"}}::vertex | {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex {"id": 2251799813685251, "label": "v3", "properties": {"id": "end"}}::vertex | {"id": 2533274790395905, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685251, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex + {"id": 2251799813685249, "label": "v3", "properties": {"id": "initial"}}::vertex | {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex (6 rows) -- Property Constraint in EXISTS @@ -1123,8 +1123,8 @@ AS (u agtype, e agtype, v agtype); {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex | {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge | {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex | {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge | {"id": 1688849860263937, "label": "v2", "properties": {"id": "initial"}}::vertex {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex | {"id": 1970324836974593, "label": "e2", "end_id": 1688849860263939, "start_id": 1688849860263938, "properties": {}}::edge | {"id": 1688849860263939, "label": "v2", "properties": {"id": "end"}}::vertex - {"id": 2251799813685249, "label": "v3", "properties": {"id": "initial"}}::vertex | {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex {"id": 2251799813685251, "label": "v3", "properties": {"id": "end"}}::vertex | {"id": 2533274790395905, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685251, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex + {"id": 2251799813685249, "label": "v3", "properties": {"id": "initial"}}::vertex | {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex {"id": 2814749767106561, "label": "loop", "properties": {"id": "initial"}}::vertex | {"id": 3096224743817217, "label": "self", "end_id": 2814749767106561, "start_id": 2814749767106561, "properties": {}}::edge | {"id": 2814749767106561, "label": "loop", "properties": {"id": "initial"}}::vertex (7 rows) @@ -1156,8 +1156,8 @@ AS (u agtype, e agtype, v agtype); {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex | {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge | {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex | {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge | {"id": 1688849860263937, "label": "v2", "properties": {"id": "initial"}}::vertex {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex | {"id": 1970324836974593, "label": "e2", "end_id": 1688849860263939, "start_id": 1688849860263938, "properties": {}}::edge | {"id": 1688849860263939, "label": "v2", "properties": {"id": "end"}}::vertex - {"id": 2251799813685249, "label": "v3", "properties": {"id": "initial"}}::vertex | {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex {"id": 2251799813685251, "label": "v3", "properties": {"id": "end"}}::vertex | {"id": 2533274790395905, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685251, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex + {"id": 2251799813685249, "label": "v3", "properties": {"id": "initial"}}::vertex | {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge | {"id": 2251799813685250, "label": "v3", "properties": {"id": "middle"}}::vertex {"id": 2814749767106561, "label": "loop", "properties": {"id": "initial"}}::vertex | {"id": 3096224743817217, "label": "self", "end_id": 2814749767106561, "start_id": 2814749767106561, "properties": {}}::edge | {"id": 2814749767106561, "label": "loop", "properties": {"id": "initial"}}::vertex (7 rows) @@ -2164,8 +2164,8 @@ SELECT * FROM cypher('cypher_match', $$ MATCH p=(u)-[]-()-[]-(u) RETURN p $$)as p ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 281474976710667, "label": "", "properties": {"name": "Dave"}}::vertex, {"id": 4785074604081155, "label": "knows", "end_id": 281474976710667, "start_id": 281474976710668, "properties": {}}::edge, {"id": 281474976710668, "label": "", "properties": {"name": "John"}}::vertex, {"id": 4785074604081156, "label": "knows", "end_id": 281474976710668, "start_id": 281474976710667, "properties": {}}::edge, {"id": 281474976710667, "label": "", "properties": {"name": "Dave"}}::vertex]::path - [{"id": 281474976710668, "label": "", "properties": {"name": "John"}}::vertex, {"id": 4785074604081155, "label": "knows", "end_id": 281474976710667, "start_id": 281474976710668, "properties": {}}::edge, {"id": 281474976710667, "label": "", "properties": {"name": "Dave"}}::vertex, {"id": 4785074604081156, "label": "knows", "end_id": 281474976710668, "start_id": 281474976710667, "properties": {}}::edge, {"id": 281474976710668, "label": "", "properties": {"name": "John"}}::vertex]::path [{"id": 281474976710667, "label": "", "properties": {"name": "Dave"}}::vertex, {"id": 4785074604081156, "label": "knows", "end_id": 281474976710668, "start_id": 281474976710667, "properties": {}}::edge, {"id": 281474976710668, "label": "", "properties": {"name": "John"}}::vertex, {"id": 4785074604081155, "label": "knows", "end_id": 281474976710667, "start_id": 281474976710668, "properties": {}}::edge, {"id": 281474976710667, "label": "", "properties": {"name": "Dave"}}::vertex]::path + [{"id": 281474976710668, "label": "", "properties": {"name": "John"}}::vertex, {"id": 4785074604081155, "label": "knows", "end_id": 281474976710667, "start_id": 281474976710668, "properties": {}}::edge, {"id": 281474976710667, "label": "", "properties": {"name": "Dave"}}::vertex, {"id": 4785074604081156, "label": "knows", "end_id": 281474976710668, "start_id": 281474976710667, "properties": {}}::edge, {"id": 281474976710668, "label": "", "properties": {"name": "John"}}::vertex]::path [{"id": 281474976710668, "label": "", "properties": {"name": "John"}}::vertex, {"id": 4785074604081156, "label": "knows", "end_id": 281474976710668, "start_id": 281474976710667, "properties": {}}::edge, {"id": 281474976710667, "label": "", "properties": {"name": "Dave"}}::vertex, {"id": 4785074604081155, "label": "knows", "end_id": 281474976710667, "start_id": 281474976710668, "properties": {}}::edge, {"id": 281474976710668, "label": "", "properties": {"name": "John"}}::vertex]::path (4 rows) @@ -2407,15 +2407,15 @@ SELECT * FROM cypher('cypher_match', $$ MATCH (a {name:a.name}) MATCH (a {age:a. SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relationship}]->(b) RETURN p $$) as (a agtype); a ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 281474976710659, "label": "", "properties": {"age": 3, "name": "orphan"}}::vertex, {"id": 4785074604081154, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710659, "properties": {"years": 4, "relationship": "enemies"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path [{"id": 281474976710661, "label": "", "properties": {"age": 4, "name": "T"}}::vertex, {"id": 4785074604081153, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710661, "properties": {"years": 3, "relationship": "friends"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path + [{"id": 281474976710659, "label": "", "properties": {"age": 3, "name": "orphan"}}::vertex, {"id": 4785074604081154, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710659, "properties": {"years": 4, "relationship": "enemies"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path (2 rows) SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relationship, years: u.years}]->(b) RETURN p $$) as (a agtype); a ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 281474976710659, "label": "", "properties": {"age": 3, "name": "orphan"}}::vertex, {"id": 4785074604081154, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710659, "properties": {"years": 4, "relationship": "enemies"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path [{"id": 281474976710661, "label": "", "properties": {"age": 4, "name": "T"}}::vertex, {"id": 4785074604081153, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710661, "properties": {"years": 3, "relationship": "friends"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path + [{"id": 281474976710659, "label": "", "properties": {"age": 3, "name": "orphan"}}::vertex, {"id": 4785074604081154, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710659, "properties": {"years": 4, "relationship": "enemies"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path (2 rows) SELECT * FROM cypher('cypher_match', $$ MATCH p=(a {name:a.name})-[u {relationship: u.relationship}]->(b {age:b.age}) RETURN p $$) as (a agtype); @@ -3514,19 +3514,17 @@ SELECT count(*) FROM cypher('test_enable_containment', $$ MATCH p=(x:Customer)-[ (1 row) SELECT * FROM cypher('test_enable_containment', $$ EXPLAIN (costs off) MATCH (x:Customer)-[:bought ={store: 'Amazon', addr:{city: 'Vancouver', street: 30}}]->(y:Product) RETURN 0 $$) as (a agtype); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Hash Join - Hash Cond: (y.id = _age_default_alias_0.end_id) - -> Seq Scan on "Product" y - -> Hash - -> Hash Join - Hash Cond: (x.id = _age_default_alias_0.start_id) - -> Seq Scan on "Customer" x - -> Hash - -> Seq Scan on bought _age_default_alias_0 - Filter: ((agtype_access_operator(VARIADIC ARRAY[properties, '"store"'::agtype]) = '"Amazon"'::agtype) AND (agtype_access_operator(VARIADIC ARRAY[properties, '"addr"'::agtype]) = '{"city": "Vancouver", "street": 30}'::agtype)) -(10 rows) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + -> Nested Loop + -> Seq Scan on bought _age_default_alias_0 + Filter: ((agtype_access_operator(VARIADIC ARRAY[properties, '"store"'::agtype]) = '"Amazon"'::agtype) AND (agtype_access_operator(VARIADIC ARRAY[properties, '"addr"'::agtype]) = '{"city": "Vancouver", "street": 30}'::agtype)) + -> Index Only Scan using "Customer_pkey" on "Customer" x + Index Cond: (id = _age_default_alias_0.start_id) + -> Index Only Scan using "Product_pkey" on "Product" y + Index Cond: (id = _age_default_alias_0.end_id) +(8 rows) SELECT * FROM cypher('test_enable_containment', $$ EXPLAIN (costs off) MATCH (x:Customer ={school: { name: 'XYZ College',program: { major: 'Psyc', degree: 'BSc'} },phone: [ 123456789, 987654321, 456987123 ]}) RETURN 0 $$) as (a agtype); QUERY PLAN diff --git a/regress/expected/cypher_merge.out b/regress/expected/cypher_merge.out index 238a4c472..56a23f513 100644 --- a/regress/expected/cypher_merge.out +++ b/regress/expected/cypher_merge.out @@ -655,8 +655,8 @@ $$) AS (name agtype, bornIn agtype, city agtype); name | bornin | city -------------------+--------------+----------------------------------------------------------------------------------------- "Rob Reiner" | "New York" | {"id": 1970324836974593, "label": "City", "properties": {"name": "New York"}}::vertex - "Martin Sheen" | "Ohio" | {"id": 1970324836974595, "label": "City", "properties": {"name": "Ohio"}}::vertex "Michael Douglas" | "New Jersey" | {"id": 1970324836974594, "label": "City", "properties": {"name": "New Jersey"}}::vertex + "Martin Sheen" | "Ohio" | {"id": 1970324836974595, "label": "City", "properties": {"name": "Ohio"}}::vertex (3 rows) --validate diff --git a/regress/expected/cypher_subquery.out b/regress/expected/cypher_subquery.out index ff5672bca..934549f82 100644 --- a/regress/expected/cypher_subquery.out +++ b/regress/expected/cypher_subquery.out @@ -668,8 +668,8 @@ SELECT * FROM cypher('subquery', $$ MATCH (a:person)-[]->(b) RETURN a $$) AS (result agtype); result ------------------------------------------------------------------------------------------------- - {"id": 844424930131975, "label": "person", "properties": {"age": 6, "name": "Calvin"}}::vertex {"id": 844424930131977, "label": "person", "properties": {"age": 8, "name": "Charlie"}}::vertex + {"id": 844424930131975, "label": "person", "properties": {"age": 6, "name": "Calvin"}}::vertex {"id": 844424930131977, "label": "person", "properties": {"age": 8, "name": "Charlie"}}::vertex (3 rows) diff --git a/regress/expected/cypher_vle.out b/regress/expected/cypher_vle.out index 9cbb3420c..57f930d98 100644 --- a/regress/expected/cypher_vle.out +++ b/regress/expected/cypher_vle.out @@ -508,37 +508,37 @@ SELECT * FROM cypher('cypher_vle', $$MATCH p=(u)-[e*0..0]->(v) RETURN id(u), p, SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[*0..0]->()-[]->() RETURN p $$) AS (p agtype); p ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1970324836974593, "label": "self_loop", "end_id": 1407374883553281, "start_id": 1407374883553281, "properties": {"name": "self loop", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path - [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path (13 rows) SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[]->()-[*0..0]->() RETURN p $$) AS (p agtype); p ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1970324836974593, "label": "self_loop", "end_id": 1407374883553281, "start_id": 1407374883553281, "properties": {"name": "self loop", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path (13 rows) -- diff --git a/regress/expected/expr.out b/regress/expected/expr.out index 513ea142f..d0546d8a3 100644 --- a/regress/expected/expr.out +++ b/regress/expected/expr.out @@ -2644,10 +2644,10 @@ SELECT * FROM cypher('expr', $$ MATCH (v) RETURN v $$) AS (expression agtype); SELECT * FROM cypher('expr', $$ MATCH ()-[e]-() RETURN e $$) AS (expression agtype); expression --------------------------------------------------------------------------------------------------------------------------- - {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge - {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge + {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge (4 rows) -- id() @@ -2656,10 +2656,10 @@ SELECT * FROM cypher('expr', $$ $$) AS (id agtype); id ------------------ - 1407374883553281 - 1407374883553281 1407374883553282 1407374883553282 + 1407374883553281 + 1407374883553281 (4 rows) SELECT * FROM cypher('expr', $$ @@ -2698,10 +2698,10 @@ SELECT * FROM cypher('expr', $$ $$) AS (start_id agtype); start_id ------------------ - 1125899906842626 - 1125899906842626 1125899906842625 1125899906842625 + 1125899906842626 + 1125899906842626 (4 rows) -- should return null @@ -2731,10 +2731,10 @@ SELECT * FROM cypher('expr', $$ $$) AS (end_id agtype); end_id ------------------ - 1125899906842627 - 1125899906842627 1125899906842626 1125899906842626 + 1125899906842627 + 1125899906842627 (4 rows) -- should return null @@ -2764,10 +2764,10 @@ SELECT * FROM cypher('expr', $$ $$) AS (id agtype, start_id agtype, startNode agtype); id | start_id | startnode ------------------+------------------+---------------------------------------------------------------------------------- - 1407374883553281 | 1125899906842626 | {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex - 1407374883553281 | 1125899906842626 | {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex 1407374883553282 | 1125899906842625 | {"id": 1125899906842625, "label": "v1", "properties": {"id": "initial"}}::vertex 1407374883553282 | 1125899906842625 | {"id": 1125899906842625, "label": "v1", "properties": {"id": "initial"}}::vertex + 1407374883553281 | 1125899906842626 | {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex + 1407374883553281 | 1125899906842626 | {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex (4 rows) -- should return null @@ -2797,10 +2797,10 @@ SELECT * FROM cypher('expr', $$ $$) AS (id agtype, end_id agtype, endNode agtype); id | end_id | endnode ------------------+------------------+--------------------------------------------------------------------------------- - 1407374883553281 | 1125899906842627 | {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex - 1407374883553281 | 1125899906842627 | {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex 1407374883553282 | 1125899906842626 | {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex 1407374883553282 | 1125899906842626 | {"id": 1125899906842626, "label": "v1", "properties": {"id": "middle"}}::vertex + 1407374883553281 | 1125899906842627 | {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex + 1407374883553281 | 1125899906842627 | {"id": 1125899906842627, "label": "v1", "properties": {"id": "end"}}::vertex (4 rows) -- should return null @@ -7496,10 +7496,10 @@ SELECT * FROM cypher('opt_forms', $$MATCH (u) RETURN *$$) AS (result agtype); SELECT * FROM cypher('opt_forms', $$MATCH (u)--(v) RETURN u.i, v.i$$) AS (u agtype, v agtype); u | v ---+--- - 2 | 3 - 3 | 2 1 | 2 2 | 1 + 2 | 3 + 3 | 2 (4 rows) SELECT * FROM cypher('opt_forms', $$MATCH (u)-->(v) RETURN u.i, v.i$$) AS (u agtype, v agtype); @@ -7686,12 +7686,12 @@ SELECT * FROM cypher('keys', $$MATCH (v) RETURN keys(v)$$) AS (vertex_keys agtyp SELECT * FROM cypher('keys', $$MATCH ()-[e]-() RETURN keys(e)$$) AS (edge_keys agtype); edge_keys ----------- - [] - [] ["song"] ["song"] + [] ["song"] ["song"] + [] (6 rows) SELECT * FROM cypher('keys', $$RETURN keys({a:1,b:'two',c:[1,2,3]})$$) AS (keys agtype); diff --git a/regress/expected/graph_generation.out b/regress/expected/graph_generation.out index 235052a08..ca511eafa 100644 --- a/regress/expected/graph_generation.out +++ b/regress/expected/graph_generation.out @@ -43,15 +43,15 @@ SELECT * FROM cypher('gp1', $$MATCH (a)-[e]->(b) RETURN e$$) as (n agtype); n ---------------------------------------------------------------------------------------------------------------------------- {"id": 1125899906842625, "label": "edges", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {}}::edge - {"id": 1125899906842629, "label": "edges", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge {"id": 1125899906842626, "label": "edges", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge - {"id": 1125899906842630, "label": "edges", "end_id": 844424930131972, "start_id": 844424930131970, "properties": {}}::edge + {"id": 1125899906842629, "label": "edges", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge {"id": 1125899906842627, "label": "edges", "end_id": 844424930131972, "start_id": 844424930131969, "properties": {}}::edge + {"id": 1125899906842630, "label": "edges", "end_id": 844424930131972, "start_id": 844424930131970, "properties": {}}::edge {"id": 1125899906842632, "label": "edges", "end_id": 844424930131972, "start_id": 844424930131971, "properties": {}}::edge + {"id": 1125899906842628, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131969, "properties": {}}::edge {"id": 1125899906842631, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131970, "properties": {}}::edge - {"id": 1125899906842634, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131972, "properties": {}}::edge {"id": 1125899906842633, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131971, "properties": {}}::edge - {"id": 1125899906842628, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131969, "properties": {}}::edge + {"id": 1125899906842634, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131972, "properties": {}}::edge (10 rows) SELECT * FROM create_complete_graph('gp1',5,'edges','vertices'); @@ -140,25 +140,25 @@ SELECT * FROM cypher('gp1', $$MATCH (a)-[e]->(b) RETURN e$$) as (n agtype); n ---------------------------------------------------------------------------------------------------------------------------- {"id": 1125899906842625, "label": "edges", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {}}::edge - {"id": 1125899906842629, "label": "edges", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge {"id": 1125899906842626, "label": "edges", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge + {"id": 1125899906842629, "label": "edges", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge {"id": 1125899906842627, "label": "edges", "end_id": 844424930131972, "start_id": 844424930131969, "properties": {}}::edge - {"id": 1125899906842632, "label": "edges", "end_id": 844424930131972, "start_id": 844424930131971, "properties": {}}::edge {"id": 1125899906842630, "label": "edges", "end_id": 844424930131972, "start_id": 844424930131970, "properties": {}}::edge - {"id": 1125899906842634, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131972, "properties": {}}::edge + {"id": 1125899906842632, "label": "edges", "end_id": 844424930131972, "start_id": 844424930131971, "properties": {}}::edge {"id": 1125899906842628, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131969, "properties": {}}::edge {"id": 1125899906842631, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131970, "properties": {}}::edge {"id": 1125899906842633, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131971, "properties": {}}::edge + {"id": 1125899906842634, "label": "edges", "end_id": 844424930131973, "start_id": 844424930131972, "properties": {}}::edge {"id": 1125899906842635, "label": "edges", "end_id": 844424930131975, "start_id": 844424930131974, "properties": {}}::edge - {"id": 1125899906842639, "label": "edges", "end_id": 844424930131976, "start_id": 844424930131975, "properties": {}}::edge {"id": 1125899906842636, "label": "edges", "end_id": 844424930131976, "start_id": 844424930131974, "properties": {}}::edge + {"id": 1125899906842639, "label": "edges", "end_id": 844424930131976, "start_id": 844424930131975, "properties": {}}::edge {"id": 1125899906842637, "label": "edges", "end_id": 844424930131977, "start_id": 844424930131974, "properties": {}}::edge - {"id": 1125899906842642, "label": "edges", "end_id": 844424930131977, "start_id": 844424930131976, "properties": {}}::edge {"id": 1125899906842640, "label": "edges", "end_id": 844424930131977, "start_id": 844424930131975, "properties": {}}::edge - {"id": 1125899906842644, "label": "edges", "end_id": 844424930131978, "start_id": 844424930131977, "properties": {}}::edge + {"id": 1125899906842642, "label": "edges", "end_id": 844424930131977, "start_id": 844424930131976, "properties": {}}::edge {"id": 1125899906842638, "label": "edges", "end_id": 844424930131978, "start_id": 844424930131974, "properties": {}}::edge {"id": 1125899906842641, "label": "edges", "end_id": 844424930131978, "start_id": 844424930131975, "properties": {}}::edge {"id": 1125899906842643, "label": "edges", "end_id": 844424930131978, "start_id": 844424930131976, "properties": {}}::edge + {"id": 1125899906842644, "label": "edges", "end_id": 844424930131978, "start_id": 844424930131977, "properties": {}}::edge {"id": 1125899906842645, "label": "edges", "end_id": 844424930131978, "start_id": 844424930131969, "properties": {}}::edge (21 rows) diff --git a/regress/expected/index.out b/regress/expected/index.out index f911900ab..3ed7b1c33 100644 --- a/regress/expected/index.out +++ b/regress/expected/index.out @@ -264,18 +264,22 @@ $$) as (n agtype); --- (0 rows) -ALTER TABLE cypher_index."Country" ADD PRIMARY KEY (id); -CREATE UNIQUE INDEX CONCURRENTLY cntry_id_idx ON cypher_index."Country" (id); -ALTER TABLE cypher_index."Country" CLUSTER ON cntry_id_idx; -ALTER TABLE cypher_index."City" ADD PRIMARY KEY (id); -CREATE UNIQUE INDEX city_id_idx ON cypher_index."City" (id); -ALTER TABLE cypher_index."City" CLUSTER ON city_id_idx; -ALTER TABLE cypher_index.has_city -ADD CONSTRAINT has_city_end_fk FOREIGN KEY (end_id) -REFERENCES cypher_index."Country"(id) MATCH FULL; -CREATE INDEX load_has_city_eid_idx ON cypher_index.has_city (end_id); -CREATE INDEX load_has_city_sid_idx ON cypher_index.has_city (start_id); -ALTER TABLE cypher_index."has_city" CLUSTER ON load_has_city_eid_idx; +-- Verify that the incices are created on id columns +SELECT indexname, indexdef FROM pg_indexes WHERE schemaname= 'cypher_index'; + indexname | indexdef +-----------------------------+------------------------------------------------------------------------------------------------ + _ag_label_edge_pkey | CREATE UNIQUE INDEX _ag_label_edge_pkey ON cypher_index._ag_label_edge USING btree (id) + _ag_label_edge_start_id_idx | CREATE INDEX _ag_label_edge_start_id_idx ON cypher_index._ag_label_edge USING btree (start_id) + _ag_label_edge_end_id_idx | CREATE INDEX _ag_label_edge_end_id_idx ON cypher_index._ag_label_edge USING btree (end_id) + _ag_label_vertex_pkey | CREATE UNIQUE INDEX _ag_label_vertex_pkey ON cypher_index._ag_label_vertex USING btree (id) + idx_pkey | CREATE UNIQUE INDEX idx_pkey ON cypher_index.idx USING btree (id) + cypher_index_idx_props_uq | CREATE UNIQUE INDEX cypher_index_idx_props_uq ON cypher_index.idx USING btree (properties) + Country_pkey | CREATE UNIQUE INDEX "Country_pkey" ON cypher_index."Country" USING btree (id) + has_city_start_id_idx | CREATE INDEX has_city_start_id_idx ON cypher_index.has_city USING btree (start_id) + has_city_end_id_idx | CREATE INDEX has_city_end_id_idx ON cypher_index.has_city USING btree (end_id) + City_pkey | CREATE UNIQUE INDEX "City_pkey" ON cypher_index."City" USING btree (id) +(10 rows) + SET enable_mergejoin = ON; SET enable_hashjoin = OFF; SET enable_nestloop = OFF; @@ -288,6 +292,29 @@ $$) as (n agtype); 10 (1 row) +SELECT COUNT(*) FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:Country)<-[e:has_city]-() + RETURN e +$$) as (n agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------------------- + Aggregate + -> Merge Join + Merge Cond: (_age_default_alias_0.id = e.start_id) + -> Merge Append + Sort Key: _age_default_alias_0.id + -> Index Only Scan using _ag_label_vertex_pkey on _ag_label_vertex _age_default_alias_0_1 + -> Index Only Scan using idx_pkey on idx _age_default_alias_0_2 + -> Index Only Scan using "Country_pkey" on "Country" _age_default_alias_0_3 + -> Index Only Scan using "City_pkey" on "City" _age_default_alias_0_4 + -> Sort + Sort Key: e.start_id + -> Merge Join + Merge Cond: (a.id = e.end_id) + -> Index Only Scan using "Country_pkey" on "Country" a + -> Index Scan using has_city_end_id_idx on has_city e +(15 rows) + SET enable_mergejoin = OFF; SET enable_hashjoin = ON; SET enable_nestloop = OFF; @@ -300,17 +327,53 @@ $$) as (n agtype); 10 (1 row) +SELECT COUNT(*) FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:Country)<-[e:has_city]-() + RETURN e +$$) as (n agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------------------- + Aggregate + -> Hash Join + Hash Cond: (_age_default_alias_0.id = e.start_id) + -> Append + -> Index Only Scan using _ag_label_vertex_pkey on _ag_label_vertex _age_default_alias_0_1 + -> Index Only Scan using idx_pkey on idx _age_default_alias_0_2 + -> Index Only Scan using "Country_pkey" on "Country" _age_default_alias_0_3 + -> Index Only Scan using "City_pkey" on "City" _age_default_alias_0_4 + -> Hash + -> Hash Join + Hash Cond: (e.end_id = a.id) + -> Index Scan using has_city_end_id_idx on has_city e + -> Hash + -> Index Only Scan using "Country_pkey" on "Country" a +(14 rows) + SET enable_mergejoin = OFF; SET enable_hashjoin = OFF; SET enable_nestloop = ON; SELECT COUNT(*) FROM cypher('cypher_index', $$ - MATCH (a:Country)<-[e:has_city]-() + EXPLAIN (costs off) MATCH (a:Country)<-[e:has_city]-() RETURN e $$) as (n agtype); - count -------- - 10 -(1 row) + QUERY PLAN +---------------------------------------------------------------------------------------------------------- + Aggregate + -> Nested Loop + -> Nested Loop + -> Index Scan using has_city_start_id_idx on has_city e + -> Index Only Scan using "Country_pkey" on "Country" a + Index Cond: (id = e.end_id) + -> Append + -> Index Only Scan using _ag_label_vertex_pkey on _ag_label_vertex _age_default_alias_0_1 + Index Cond: (id = e.start_id) + -> Index Only Scan using idx_pkey on idx _age_default_alias_0_2 + Index Cond: (id = e.start_id) + -> Index Only Scan using "Country_pkey" on "Country" _age_default_alias_0_3 + Index Cond: (id = e.start_id) + -> Index Only Scan using "City_pkey" on "City" _age_default_alias_0_4 + Index Cond: (id = e.start_id) +(15 rows) SET enable_mergejoin = ON; SET enable_hashjoin = ON; diff --git a/regress/expected/map_projection.out b/regress/expected/map_projection.out index dcb7f0e76..f0c45c557 100644 --- a/regress/expected/map_projection.out +++ b/regress/expected/map_projection.out @@ -152,7 +152,7 @@ $$ $$) as (a agtype); a -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"name": "Christian Bale", "movies": [{"title": "The Prestige"}, {"title": "The Dark Knight"}]}, {"name": "Tom Hanks", "movies": [{"title": "Forrest Gump"}, {"title": "Finch"}, {"title": "The Circle"}]}] + [{"name": "Tom Hanks", "movies": [{"title": "Forrest Gump"}, {"title": "Finch"}, {"title": "The Circle"}]}, {"name": "Christian Bale", "movies": [{"title": "The Prestige"}, {"title": "The Dark Knight"}]}] (1 row) -- drop diff --git a/regress/sql/age_load.sql b/regress/sql/age_load.sql index 425ca5417..180248bf1 100644 --- a/regress/sql/age_load.sql +++ b/regress/sql/age_load.sql @@ -34,9 +34,6 @@ SELECT * FROM cypher('agload_test_graph', $$CREATE (n:Country {__id__:1}) RETURN SELECT load_labels_from_file('agload_test_graph', 'Country', 'age_load/countries.csv', true); --- A temporary table should have been created with 54 ids; 1 from CREATE and 53 from file -SELECT COUNT(*)=54 FROM "_agload_test_graph_ag_vertex_ids"; - -- Sequence should be equal to max entry id i.e. 248 SELECT currval('agload_test_graph."Country_id_seq"')=248; @@ -52,9 +49,6 @@ SELECT load_labels_from_file('agload_test_graph', 'Country', SELECT load_labels_from_file('agload_test_graph', 'City', 'age_load/cities.csv', true); --- Temporary table should have 54+72485 rows now -SELECT COUNT(*)=54+72485 FROM "_agload_test_graph_ag_vertex_ids"; - -- Sequence should be equal to max entry id i.e. 146941 SELECT currval('agload_test_graph."City_id_seq"')=146941; diff --git a/regress/sql/index.sql b/regress/sql/index.sql index aac1dc40e..d9a4331a4 100644 --- a/regress/sql/index.sql +++ b/regress/sql/index.sql @@ -166,26 +166,8 @@ SELECT * FROM cypher('cypher_index', $$ (mx)<-[:has_city]-(:City {city_id: 10, name:"Tijuana", west_coast: false, country_code:"MX"}) $$) as (n agtype); -ALTER TABLE cypher_index."Country" ADD PRIMARY KEY (id); - -CREATE UNIQUE INDEX CONCURRENTLY cntry_id_idx ON cypher_index."Country" (id); -ALTER TABLE cypher_index."Country" CLUSTER ON cntry_id_idx; - -ALTER TABLE cypher_index."City" ADD PRIMARY KEY (id); - -CREATE UNIQUE INDEX city_id_idx ON cypher_index."City" (id); - -ALTER TABLE cypher_index."City" CLUSTER ON city_id_idx; - -ALTER TABLE cypher_index.has_city -ADD CONSTRAINT has_city_end_fk FOREIGN KEY (end_id) -REFERENCES cypher_index."Country"(id) MATCH FULL; - -CREATE INDEX load_has_city_eid_idx ON cypher_index.has_city (end_id); - -CREATE INDEX load_has_city_sid_idx ON cypher_index.has_city (start_id); - -ALTER TABLE cypher_index."has_city" CLUSTER ON load_has_city_eid_idx; +-- Verify that the incices are created on id columns +SELECT indexname, indexdef FROM pg_indexes WHERE schemaname= 'cypher_index'; SET enable_mergejoin = ON; SET enable_hashjoin = OFF; @@ -196,6 +178,11 @@ SELECT COUNT(*) FROM cypher('cypher_index', $$ RETURN e $$) as (n agtype); +SELECT COUNT(*) FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:Country)<-[e:has_city]-() + RETURN e +$$) as (n agtype); + SET enable_mergejoin = OFF; SET enable_hashjoin = ON; SET enable_nestloop = OFF; @@ -205,12 +192,17 @@ SELECT COUNT(*) FROM cypher('cypher_index', $$ RETURN e $$) as (n agtype); +SELECT COUNT(*) FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:Country)<-[e:has_city]-() + RETURN e +$$) as (n agtype); + SET enable_mergejoin = OFF; SET enable_hashjoin = OFF; SET enable_nestloop = ON; SELECT COUNT(*) FROM cypher('cypher_index', $$ - MATCH (a:Country)<-[e:has_city]-() + EXPLAIN (costs off) MATCH (a:Country)<-[e:has_city]-() RETURN e $$) as (n agtype); diff --git a/src/backend/commands/label_commands.c b/src/backend/commands/label_commands.c index f603ec97c..c9fdfad89 100644 --- a/src/backend/commands/label_commands.c +++ b/src/backend/commands/label_commands.c @@ -79,6 +79,10 @@ static void range_var_callback_for_remove_relation(const RangeVar *rel, Oid rel_oid, Oid odl_rel_oid, void *arg); +static void create_index_on_column(char *schema_name, + char *rel_name, + char *colname, + bool unique); PG_FUNCTION_INFO_V1(age_is_valid_label_name); @@ -379,16 +383,24 @@ static void create_table_for_label(char *graph_name, char *label_name, * inheritance system. */ if (list_length(parents) != 0) + { create_stmt->tableElts = NIL; + } else if (label_type == LABEL_TYPE_EDGE) + { create_stmt->tableElts = create_edge_table_elements( graph_name, label_name, schema_name, rel_name, seq_name); + } else if (label_type == LABEL_TYPE_VERTEX) + { create_stmt->tableElts = create_vertex_table_elements( graph_name, label_name, schema_name, rel_name, seq_name); + } else + { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("undefined label type \'%c\'", label_type))); + } create_stmt->inhRelations = parents; create_stmt->partbound = NULL; @@ -409,7 +421,69 @@ static void create_table_for_label(char *graph_name, char *label_name, ProcessUtility(wrapper, "(generated CREATE TABLE command)", false, PROCESS_UTILITY_SUBCOMMAND, NULL, NULL, None_Receiver, NULL); - /* CommandCounterIncrement() is called in ProcessUtility() */ + + /* Create index on id columns */ + if (label_type == LABEL_TYPE_VERTEX) + { + create_index_on_column(schema_name, rel_name, "id", true); + } + else if (label_type == LABEL_TYPE_EDGE) + { + create_index_on_column(schema_name, rel_name, "start_id", false); + create_index_on_column(schema_name, rel_name, "end_id", false); + } +} + +static void create_index_on_column(char *schema_name, + char *rel_name, + char *colname, + bool unique) +{ + IndexStmt *index_stmt; + IndexElem *index_col; + PlannedStmt *index_wrapper; + + index_stmt = makeNode(IndexStmt); + index_col = makeNode(IndexElem); + index_col->name = colname; + index_col->expr = NULL; + index_col->indexcolname = NULL; + index_col->collation = InvalidOid; + index_col->opclass = list_make1(makeString("graphid_ops")); + index_col->opclassopts = NIL; + index_col->ordering = SORTBY_DEFAULT; + index_col->nulls_ordering = SORTBY_NULLS_DEFAULT; + + index_stmt->relation = makeRangeVar(schema_name, rel_name, -1); + index_stmt->accessMethod = "btree"; + index_stmt->tableSpace = NULL; + index_stmt->indexParams = list_make1(index_col); + index_stmt->options = NIL; + index_stmt->whereClause = NULL; + index_stmt->excludeOpNames = NIL; + index_stmt->idxcomment = NULL; + index_stmt->indexOid = InvalidOid; + index_stmt->unique = unique; + index_stmt->nulls_not_distinct = false; + index_stmt->primary = unique; + index_stmt->isconstraint = unique; + index_stmt->deferrable = false; + index_stmt->initdeferred = false; + index_stmt->transformed = false; + index_stmt->concurrent = false; + index_stmt->if_not_exists = false; + index_stmt->reset_default_tblspc = false; + + index_wrapper = makeNode(PlannedStmt); + index_wrapper->commandType = CMD_UTILITY; + index_wrapper->canSetTag = false; + index_wrapper->utilityStmt = (Node *)index_stmt; + index_wrapper->stmt_location = -1; + index_wrapper->stmt_len = 0; + + ProcessUtility(index_wrapper, "(generated CREATE INDEX command)", false, + PROCESS_UTILITY_SUBCOMMAND, NULL, NULL, None_Receiver, + NULL); } /* @@ -468,7 +542,7 @@ static List *create_vertex_table_elements(char *graph_name, char *label_name, /* "id" graphid PRIMARY KEY DEFAULT "ag_catalog"."_graphid"(...) */ id = makeColumnDef(AG_VERTEX_COLNAME_ID, GRAPHIDOID, -1, InvalidOid); - id->constraints = list_make2(build_pk_constraint(), + id->constraints = list_make2(build_not_null_constraint(), build_id_default(graph_name, label_name, schema_name, seq_name)); diff --git a/src/backend/executor/cypher_delete.c b/src/backend/executor/cypher_delete.c index 6bb869833..5f9aa561d 100644 --- a/src/backend/executor/cypher_delete.c +++ b/src/backend/executor/cypher_delete.c @@ -340,6 +340,10 @@ static void delete_entity(EState *estate, ResultRelInfo *resultRelInfo, } /* increment the command counter */ CommandCounterIncrement(); + + /* Update command id in estate */ + estate->es_snapshot->curcid = GetCurrentCommandId(false); + estate->es_output_cid = GetCurrentCommandId(false); } else if (lock_result != TM_Invisible && lock_result != TM_SelfModified) { diff --git a/src/backend/utils/load/ag_load_edges.c b/src/backend/utils/load/ag_load_edges.c index 30dc4761d..67049431c 100644 --- a/src/backend/utils/load/ag_load_edges.c +++ b/src/backend/utils/load/ag_load_edges.c @@ -22,11 +22,6 @@ #include "utils/load/ag_load_edges.h" #include "utils/load/csv.h" -void init_edge_batch_insert(batch_insert_state **batch_state, - char *label_name, Oid graph_oid); -void finish_edge_batch_insert(batch_insert_state **batch_state, - char *label_name, Oid graph_oid); - void edge_field_cb(void *field, size_t field_len, void *data) { @@ -131,7 +126,7 @@ void edge_row_cb(int delim __attribute__((unused)), void *data) if (batch_state->num_tuples >= batch_state->max_tuples) { /* Insert the batch when it is full (i.e. BATCH_SIZE) */ - insert_batch(batch_state, cr->label_name, cr->graph_oid); + insert_batch(batch_state); batch_state->num_tuples = 0; } } @@ -223,7 +218,7 @@ int create_edges_from_csv_file(char *file_path, cr.load_as_agtype = load_as_agtype; /* Initialize the batch insert state */ - init_edge_batch_insert(&cr.batch_state, label_name, graph_oid); + init_batch_insert(&cr.batch_state, label_name, graph_oid); while ((bytes_read=fread(buf, 1, 1024, fp)) > 0) { @@ -238,7 +233,7 @@ int create_edges_from_csv_file(char *file_path, csv_fini(&p, edge_field_cb, edge_row_cb, &cr); /* Finish any remaining batch inserts */ - finish_edge_batch_insert(&cr.batch_state, label_name, graph_oid); + finish_batch_insert(&cr.batch_state); if (ferror(fp)) { @@ -250,66 +245,4 @@ int create_edges_from_csv_file(char *file_path, free(cr.fields); csv_free(&p); return EXIT_SUCCESS; -} - -/* - * Initialize the batch insert state for edges. - */ -void init_edge_batch_insert(batch_insert_state **batch_state, - char *label_name, Oid graph_oid) -{ - Relation relation; - int i; - - // Open a temporary relation to get the tuple descriptor - relation = table_open(get_label_relation(label_name, graph_oid), AccessShareLock); - - // Initialize the batch insert state - *batch_state = (batch_insert_state *) palloc0(sizeof(batch_insert_state)); - (*batch_state)->max_tuples = BATCH_SIZE; - (*batch_state)->slots = palloc(sizeof(TupleTableSlot *) * BATCH_SIZE); - (*batch_state)->num_tuples = 0; - - // Create slots - for (i = 0; i < BATCH_SIZE; i++) - { - (*batch_state)->slots[i] = MakeSingleTupleTableSlot( - RelationGetDescr(relation), - &TTSOpsHeapTuple); - } - - table_close(relation, AccessShareLock); -} - -/* - * Finish the batch insert for edges. Insert the - * remaining tuples in the batch state and clean up. - */ -void finish_edge_batch_insert(batch_insert_state **batch_state, - char *label_name, Oid graph_oid) -{ - int i; - Relation relation; - - if ((*batch_state)->num_tuples > 0) - { - insert_batch(*batch_state, label_name, graph_oid); - (*batch_state)->num_tuples = 0; - } - - // Open a temporary relation to ensure resources are properly cleaned up - relation = table_open(get_label_relation(label_name, graph_oid), AccessShareLock); - - // Free slots - for (i = 0; i < BATCH_SIZE; i++) - { - ExecDropSingleTupleTableSlot((*batch_state)->slots[i]); - } - - // Clean up batch state - pfree_if_not_null((*batch_state)->slots); - pfree_if_not_null(*batch_state); - *batch_state = NULL; - - table_close(relation, AccessShareLock); -} +} \ No newline at end of file diff --git a/src/backend/utils/load/ag_load_labels.c b/src/backend/utils/load/ag_load_labels.c index 2ab223346..4a04f3cd8 100644 --- a/src/backend/utils/load/ag_load_labels.c +++ b/src/backend/utils/load/ag_load_labels.c @@ -24,18 +24,6 @@ #include "utils/load/ag_load_labels.h" #include "utils/load/csv.h" -static void setup_temp_table_for_vertex_ids(char *graph_name); -static void insert_batch_in_temp_table(batch_insert_state *batch_state, - Oid graph_oid, Oid relid); -static void init_vertex_batch_insert(batch_insert_state **batch_state, - char *label_name, Oid graph_oid, - Oid temp_table_relid); -static void finish_vertex_batch_insert(batch_insert_state **batch_state, - char *label_name, Oid graph_oid, - Oid temp_table_relid); -static void insert_vertex_batch(batch_insert_state *batch_state, char *label_name, - Oid graph_oid, Oid temp_table_relid); - void vertex_field_cb(void *field, size_t field_len, void *data) { @@ -75,7 +63,6 @@ void vertex_row_cb(int delim __attribute__((unused)), void *data) graphid vertex_id; int64 entry_id; TupleTableSlot *slot; - TupleTableSlot *temp_id_slot; n_fields = cr->cur_field; @@ -114,11 +101,9 @@ void vertex_row_cb(int delim __attribute__((unused)), void *data) /* Get the appropriate slot from the batch state */ slot = batch_state->slots[batch_state->num_tuples]; - temp_id_slot = batch_state->temp_id_slots[batch_state->num_tuples]; /* Clear the slots contents */ ExecClearTuple(slot); - ExecClearTuple(temp_id_slot); /* Fill the values in the slot */ slot->tts_values[0] = GRAPHID_GET_DATUM(vertex_id); @@ -129,20 +114,15 @@ void vertex_row_cb(int delim __attribute__((unused)), void *data) slot->tts_isnull[0] = false; slot->tts_isnull[1] = false; - temp_id_slot->tts_values[0] = GRAPHID_GET_DATUM(vertex_id); - temp_id_slot->tts_isnull[0] = false; - /* Make the slot as containing virtual tuple */ ExecStoreVirtualTuple(slot); - ExecStoreVirtualTuple(temp_id_slot); batch_state->num_tuples++; if (batch_state->num_tuples >= batch_state->max_tuples) { /* Insert the batch when it is full (i.e. BATCH_SIZE) */ - insert_vertex_batch(batch_state, cr->label_name, cr->graph_oid, - cr->temp_table_relid); + insert_batch(batch_state); batch_state->num_tuples = 0; } } @@ -202,7 +182,6 @@ int create_labels_from_csv_file(char *file_path, unsigned char options = 0; csv_vertex_reader cr; char *label_seq_name; - Oid temp_table_relid; if (csv_init(&p, options) != 0) { @@ -210,13 +189,6 @@ int create_labels_from_csv_file(char *file_path, (errmsg("Failed to initialize csv parser\n"))); } - temp_table_relid = RelnameGetRelid(GET_TEMP_VERTEX_ID_TABLE(graph_name)); - if (!OidIsValid(temp_table_relid)) - { - setup_temp_table_for_vertex_ids(graph_name); - temp_table_relid = RelnameGetRelid(GET_TEMP_VERTEX_ID_TABLE(graph_name)); - } - csv_set_space_func(&p, is_space); csv_set_term_func(&p, is_term); @@ -243,7 +215,6 @@ int create_labels_from_csv_file(char *file_path, cr.id_field_exists = id_field_exists; cr.label_seq_relid = get_relname_relid(label_seq_name, graph_oid); cr.load_as_agtype = load_as_agtype; - cr.temp_table_relid = temp_table_relid; if (cr.id_field_exists) { @@ -258,8 +229,7 @@ int create_labels_from_csv_file(char *file_path, } /* Initialize the batch insert state */ - init_vertex_batch_insert(&cr.batch_state, label_name, graph_oid, - cr.temp_table_relid); + init_batch_insert(&cr.batch_state, label_name, graph_oid); while ((bytes_read=fread(buf, 1, 1024, fp)) > 0) { @@ -274,8 +244,7 @@ int create_labels_from_csv_file(char *file_path, csv_fini(&p, vertex_field_cb, vertex_row_cb, &cr); /* Finish any remaining batch inserts */ - finish_vertex_batch_insert(&cr.batch_state, label_name, graph_oid, - cr.temp_table_relid); + finish_batch_insert(&cr.batch_state); if (ferror(fp)) { @@ -288,180 +257,4 @@ int create_labels_from_csv_file(char *file_path, free(cr.fields); csv_free(&p); return EXIT_SUCCESS; -} - -static void insert_vertex_batch(batch_insert_state *batch_state, char *label_name, - Oid graph_oid, Oid temp_table_relid) -{ - insert_batch_in_temp_table(batch_state, graph_oid, temp_table_relid); - insert_batch(batch_state, label_name, graph_oid); -} - -/* - * Create and populate a temporary table with vertex ids that are already - * present in the graph. This table will be used to check if the new vertex - * id generated by loader is a duplicate. - * Unique index is created to enforce uniqueness of the ids. - * - * We dont need this for loading edges since the ids are generated using - * sequence and are unique. - */ -static void setup_temp_table_for_vertex_ids(char *graph_name) -{ - char *create_as_query; - char *index_query; - - create_as_query = psprintf("CREATE TEMP TABLE IF NOT EXISTS %s AS " - "SELECT DISTINCT id FROM \"%s\".%s", - GET_TEMP_VERTEX_ID_TABLE(graph_name), graph_name, - AG_DEFAULT_LABEL_VERTEX); - - index_query = psprintf("CREATE UNIQUE INDEX ON %s (id)", - GET_TEMP_VERTEX_ID_TABLE(graph_name)); - SPI_connect(); - SPI_execute(create_as_query, false, 0); - SPI_execute(index_query, false, 0); - - SPI_finish(); -} - -/* - * Inserts batch of tuples into the temporary table. - * This function also updates the index to check for - * uniqueness of the ids. - */ -static void insert_batch_in_temp_table(batch_insert_state *batch_state, - Oid graph_oid, Oid relid) -{ - int i; - EState *estate; - ResultRelInfo *resultRelInfo; - Relation rel; - List *result; - - rel = table_open(relid, RowExclusiveLock); - - /* Initialize executor state */ - estate = CreateExecutorState(); - - /* Initialize result relation information */ - resultRelInfo = makeNode(ResultRelInfo); - InitResultRelInfo(resultRelInfo, rel, 1, NULL, estate->es_instrument); - estate->es_result_relations = &resultRelInfo; - - /* Open the indices */ - ExecOpenIndices(resultRelInfo, false); - - /* Insert the batch into the temporary table */ - heap_multi_insert(rel, batch_state->temp_id_slots, batch_state->num_tuples, - GetCurrentCommandId(true), 0, NULL); - - for (i = 0; i < batch_state->num_tuples; i++) - { - result = ExecInsertIndexTuples(resultRelInfo, batch_state->temp_id_slots[i], - estate, false, true, NULL, NIL, false); - /* Check if the unique cnstraint is violated */ - if (list_length(result) != 0) - { - Datum id; - bool isnull; - - id = slot_getattr(batch_state->temp_id_slots[i], 1, &isnull); - ereport(ERROR, (errmsg("Cannot insert duplicate vertex id: %ld", - DATUM_GET_GRAPHID(id)), - errhint("Entry id %ld is already used", - get_graphid_entry_id(id)))); - } - } - /* Clean up and close the indices */ - ExecCloseIndices(resultRelInfo); - - FreeExecutorState(estate); - table_close(rel, RowExclusiveLock); - - CommandCounterIncrement(); -} - -/* - * Initialize the batch insert state for vertices. - */ -static void init_vertex_batch_insert(batch_insert_state **batch_state, - char *label_name, Oid graph_oid, - Oid temp_table_relid) -{ - Relation relation; - Oid relid; - - Relation temp_table_relation; - int i; - - /* Open a temporary relation to get the tuple descriptor */ - relid = get_label_relation(label_name, graph_oid); - relation = table_open(relid, AccessShareLock); - - temp_table_relation = table_open(temp_table_relid, AccessShareLock); - - /* Initialize the batch insert state */ - *batch_state = (batch_insert_state *) palloc0(sizeof(batch_insert_state)); - (*batch_state)->max_tuples = BATCH_SIZE; - (*batch_state)->slots = palloc(sizeof(TupleTableSlot *) * BATCH_SIZE); - (*batch_state)->temp_id_slots = palloc(sizeof(TupleTableSlot *) * BATCH_SIZE); - (*batch_state)->num_tuples = 0; - - /* Create slots */ - for (i = 0; i < BATCH_SIZE; i++) - { - (*batch_state)->slots[i] = MakeSingleTupleTableSlot( - RelationGetDescr(relation), - &TTSOpsHeapTuple); - (*batch_state)->temp_id_slots[i] = MakeSingleTupleTableSlot( - RelationGetDescr(temp_table_relation), - &TTSOpsHeapTuple); - } - - table_close(relation, AccessShareLock); - table_close(temp_table_relation, AccessShareLock); -} - -/* - * Finish the batch insert for vertices. Insert the - * remaining tuples in the batch state and clean up. - */ -static void finish_vertex_batch_insert(batch_insert_state **batch_state, - char *label_name, Oid graph_oid, - Oid temp_table_relid) -{ - Relation relation; - Oid relid; - - Relation temp_table_relation; - int i; - - if ((*batch_state)->num_tuples > 0) - { - insert_vertex_batch(*batch_state, label_name, graph_oid, temp_table_relid); - (*batch_state)->num_tuples = 0; - } - - /* Open a temporary relation to ensure resources are properly cleaned up */ - relid = get_label_relation(label_name, graph_oid); - relation = table_open(relid, AccessShareLock); - - temp_table_relation = table_open(temp_table_relid, AccessShareLock); - - /* Free slots */ - for (i = 0; i < BATCH_SIZE; i++) - { - ExecDropSingleTupleTableSlot((*batch_state)->slots[i]); - ExecDropSingleTupleTableSlot((*batch_state)->temp_id_slots[i]); - } - - /* Clean up batch state */ - pfree_if_not_null((*batch_state)->slots); - pfree_if_not_null((*batch_state)->temp_id_slots); - pfree_if_not_null(*batch_state); - *batch_state = NULL; - - table_close(relation, AccessShareLock); - table_close(temp_table_relation, AccessShareLock); -} +} \ No newline at end of file diff --git a/src/backend/utils/load/age_load.c b/src/backend/utils/load/age_load.c index 41233ff18..307ec335a 100644 --- a/src/backend/utils/load/age_load.c +++ b/src/backend/utils/load/age_load.c @@ -18,11 +18,14 @@ */ #include "postgres.h" +#include "catalog/indexing.h" +#include "executor/executor.h" #include "utils/json.h" #include "utils/load/ag_load_edges.h" #include "utils/load/ag_load_labels.h" #include "utils/load/age_load.h" +#include "utils/rel.h" static agtype_value *csv_value_to_agtype_value(char *csv_val); static Oid get_or_create_graph(const Name graph_name); @@ -217,18 +220,35 @@ void insert_edge_simple(Oid graph_oid, char *label_name, graphid edge_id, errmsg("label %s already exists as vertex label", label_name))); } + /* Open the relation */ + label_relation = table_open(get_label_relation(label_name, graph_oid), + RowExclusiveLock); + + /* Form the tuple */ values[0] = GRAPHID_GET_DATUM(edge_id); values[1] = GRAPHID_GET_DATUM(start_id); values[2] = GRAPHID_GET_DATUM(end_id); values[3] = AGTYPE_P_GET_DATUM((edge_properties)); - - label_relation = table_open(get_label_relation(label_name, graph_oid), - RowExclusiveLock); - tuple = heap_form_tuple(RelationGetDescr(label_relation), values, nulls); - heap_insert(label_relation, tuple, - GetCurrentCommandId(true), 0, NULL); + + if (RelationGetForm(label_relation)->relhasindex) + { + /* + * CatalogTupleInsertWithInfo() is originally for PostgreSQL's + * catalog. However, it is used here for convenience. + */ + CatalogIndexState indstate = CatalogOpenIndexes(label_relation); + CatalogTupleInsertWithInfo(label_relation, tuple, indstate); + CatalogCloseIndexes(indstate); + } + else + { + heap_insert(label_relation, tuple, GetCurrentCommandId(true), + 0, NULL); + } + + /* Close the relation */ table_close(label_relation, RowExclusiveLock); CommandCounterIncrement(); } @@ -246,46 +266,75 @@ void insert_vertex_simple(Oid graph_oid, char *label_name, graphid vertex_id, if (get_label_kind(label_name, graph_oid) == LABEL_KIND_EDGE) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("label %s already exists as edge label", label_name))); + errmsg("label %s already exists as edge label", + label_name))); } - values[0] = GRAPHID_GET_DATUM(vertex_id); - values[1] = AGTYPE_P_GET_DATUM((vertex_properties)); - + /* Open the relation */ label_relation = table_open(get_label_relation(label_name, graph_oid), RowExclusiveLock); + + /* Form the tuple */ + values[0] = GRAPHID_GET_DATUM(vertex_id); + values[1] = AGTYPE_P_GET_DATUM((vertex_properties)); tuple = heap_form_tuple(RelationGetDescr(label_relation), values, nulls); - heap_insert(label_relation, tuple, - GetCurrentCommandId(true), 0, NULL); + + if (RelationGetForm(label_relation)->relhasindex) + { + /* + * CatalogTupleInsertWithInfo() is originally for PostgreSQL's + * catalog. However, it is used here for convenience. + */ + CatalogIndexState indstate = CatalogOpenIndexes(label_relation); + CatalogTupleInsertWithInfo(label_relation, tuple, indstate); + CatalogCloseIndexes(indstate); + } + else + { + heap_insert(label_relation, tuple, GetCurrentCommandId(true), + 0, NULL); + } + + /* Close the relation */ table_close(label_relation, RowExclusiveLock); CommandCounterIncrement(); } -void insert_batch(batch_insert_state *batch_state, char *label_name, - Oid graph_oid) +void insert_batch(batch_insert_state *batch_state) { - Relation label_relation; - BulkInsertState bistate; - Oid relid; - - // Get the relation OID - relid = get_label_relation(label_name, graph_oid); - - // Open the relation - label_relation = table_open(relid, RowExclusiveLock); - - // Prepare the BulkInsertState - bistate = GetBulkInsertState(); - - // Perform the bulk insert - heap_multi_insert(label_relation, batch_state->slots, - batch_state->num_tuples, GetCurrentCommandId(true), - 0, bistate); + List *result; + int i; - // Clean up - FreeBulkInsertState(bistate); - table_close(label_relation, RowExclusiveLock); + /* Insert the tuples */ + heap_multi_insert(batch_state->resultRelInfo->ri_RelationDesc, + batch_state->slots, batch_state->num_tuples, + GetCurrentCommandId(true), 0, NULL); + + /* Insert index entries for the tuples */ + if (batch_state->resultRelInfo->ri_NumIndices > 0) + { + for (i = 0; i < batch_state->num_tuples; i++) + { + result = ExecInsertIndexTuples(batch_state->resultRelInfo, + batch_state->slots[i], + batch_state->estate, false, + true, NULL, NIL, false); + + /* Check if the unique constraint is violated */ + if (list_length(result) != 0) + { + Datum id; + bool isnull; + + id = slot_getattr(batch_state->slots[i], 1, &isnull); + ereport(ERROR, (errmsg("Cannot insert duplicate vertex id: %ld", + DATUM_GET_GRAPHID(id)), + errhint("Entry id %ld is already used", + get_graphid_entry_id(id)))); + } + } + } CommandCounterIncrement(); } @@ -475,3 +524,79 @@ static int32 get_or_create_label(Oid graph_oid, char *graph_name, return label_id; } + +/* + * Initialize the batch insert state. + */ +void init_batch_insert(batch_insert_state **batch_state, + char *label_name, Oid graph_oid) +{ + Relation relation; + Oid relid; + EState *estate; + ResultRelInfo *resultRelInfo; + int i; + + /* Open the relation */ + relid = get_label_relation(label_name, graph_oid); + relation = table_open(relid, RowExclusiveLock); + + /* Initialize executor state */ + estate = CreateExecutorState(); + + /* Initialize resultRelInfo */ + resultRelInfo = makeNode(ResultRelInfo); + InitResultRelInfo(resultRelInfo, relation, 1, NULL, estate->es_instrument); + estate->es_result_relations = &resultRelInfo; + + /* Open the indices */ + ExecOpenIndices(resultRelInfo, false); + + /* Initialize the batch insert state */ + *batch_state = (batch_insert_state *) palloc0(sizeof(batch_insert_state)); + (*batch_state)->slots = palloc(sizeof(TupleTableSlot *) * BATCH_SIZE); + (*batch_state)->estate = estate; + (*batch_state)->resultRelInfo = resultRelInfo; + (*batch_state)->max_tuples = BATCH_SIZE; + (*batch_state)->num_tuples = 0; + + /* Create slots */ + for (i = 0; i < BATCH_SIZE; i++) + { + (*batch_state)->slots[i] = MakeSingleTupleTableSlot( + RelationGetDescr(relation), + &TTSOpsHeapTuple); + } +} + +/* + * Finish the batch insert for vertices. Insert the + * tuples remaining in the batch state and clean up. + */ +void finish_batch_insert(batch_insert_state **batch_state) +{ + int i; + + if ((*batch_state)->num_tuples > 0) + { + insert_batch(*batch_state); + (*batch_state)->num_tuples = 0; + } + + /* Free slots */ + for (i = 0; i < BATCH_SIZE; i++) + { + ExecDropSingleTupleTableSlot((*batch_state)->slots[i]); + } + + /* Clean up, close the indices and relation */ + ExecCloseIndices((*batch_state)->resultRelInfo); + table_close((*batch_state)->resultRelInfo->ri_RelationDesc, + RowExclusiveLock); + + /* Clean up batch state */ + FreeExecutorState((*batch_state)->estate); + pfree((*batch_state)->slots); + pfree(*batch_state); + *batch_state = NULL; +} \ No newline at end of file diff --git a/src/include/utils/load/ag_load_labels.h b/src/include/utils/load/ag_load_labels.h index 3a70a5c05..b8ed1572e 100644 --- a/src/include/utils/load/ag_load_labels.h +++ b/src/include/utils/load/ag_load_labels.h @@ -24,10 +24,6 @@ #include "access/heapam.h" #include "utils/load/age_load.h" -#define AGE_VERTIX 1 -#define AGE_EDGE 2 - - struct counts { long unsigned fields; long unsigned allvalues; @@ -51,7 +47,6 @@ typedef struct { char *label_name; int label_id; Oid label_seq_relid; - Oid temp_table_relid; bool id_field_exists; bool load_as_agtype; int curr_seq_num; diff --git a/src/include/utils/load/age_load.h b/src/include/utils/load/age_load.h index b1335581b..72f11493d 100644 --- a/src/include/utils/load/age_load.h +++ b/src/include/utils/load/age_load.h @@ -30,16 +30,13 @@ #ifndef AGE_ENTITY_CREATOR_H #define AGE_ENTITY_CREATOR_H -#define TEMP_VERTEX_ID_TABLE_SUFFIX "_ag_vertex_ids" -#define GET_TEMP_VERTEX_ID_TABLE(graph_name) \ - psprintf("_%s%s", graph_name, TEMP_VERTEX_ID_TABLE_SUFFIX) - #define BATCH_SIZE 1000 -typedef struct +typedef struct batch_insert_state { + EState *estate; + ResultRelInfo *resultRelInfo; TupleTableSlot **slots; - TupleTableSlot **temp_id_slots; int num_tuples; int max_tuples; } batch_insert_state; @@ -57,7 +54,10 @@ void insert_vertex_simple(Oid graph_oid, char *label_name, graphid vertex_id, void insert_edge_simple(Oid graph_oid, char *label_name, graphid edge_id, graphid start_id, graphid end_id, agtype* end_properties); -void insert_batch(batch_insert_state *batch_state, char *label_name, - Oid graph_oid); +void insert_batch(batch_insert_state *batch_state); + +void init_batch_insert(batch_insert_state **batch_state, + char *label_name, Oid graph_oid); +void finish_batch_insert(batch_insert_state **batch_state); #endif /* AGE_ENTITY_CREATOR_H */ From 26f748c42b01b7d6ff5c5ec7c4eb84d97066103a Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Tue, 9 Dec 2025 10:51:01 -0800 Subject: [PATCH 003/100] Fix Issue 2256: segmentation fault when calling coalesce function (#2259) Fixed issue 2256: A segmentation fault occurs when calling the coalesce function in PostgreSQL version 17. This likely predates 17 and includes other similar types of "functions". See issues 1124 (PR 1125) and 1303 (PR 1317) for more details. This issue is due to coalesce() being processed differently from other functions. Additionally, greatest() was found to exhibit the same behavior. They were added to the list of types to ignore during the cypher analyze phase. A few others were added: CaseExpr, XmlExpr, ArrayExpr, & RowExpr. Although, I wasn't able to find cases where these caused crashes. Added regression tests. modified: regress/expected/cypher.out modified: regress/sql/cypher.sql modified: src/backend/parser/cypher_analyze.c --- regress/expected/cypher.out | 16 ++++++++++++++++ regress/sql/cypher.sql | 7 +++++++ src/backend/parser/cypher_analyze.c | 26 +++++++++++++++++++------- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/regress/expected/cypher.out b/regress/expected/cypher.out index 31bafc6cf..53ea9f0c1 100644 --- a/regress/expected/cypher.out +++ b/regress/expected/cypher.out @@ -169,6 +169,22 @@ CREATE TABLE my_edges AS -- create a table of 4 columns, u, e, v, p. should be 5 rows CREATE TABLE my_detailed_paths AS (SELECT * FROM cypher('issue_1767', $$ MATCH p=(u)-[e]->(v) RETURN u,e,v,p $$) as (u agtype, e agtype, v agtype, p agtype)); +-- +-- Issue 2256: A segmentation fault occurs when calling the coalesce function +-- This also occurs with the greatest function too. +-- +SELECT * FROM coalesce(1, 0); + coalesce +---------- + 1 +(1 row) + +SELECT * FROM greatest(1, 0); + greatest +---------- + 1 +(1 row) + -- dump out the tables SELECT * FROM my_vertices; u diff --git a/regress/sql/cypher.sql b/regress/sql/cypher.sql index 7ded61ee6..090c7e704 100644 --- a/regress/sql/cypher.sql +++ b/regress/sql/cypher.sql @@ -94,6 +94,13 @@ CREATE TABLE my_edges AS CREATE TABLE my_detailed_paths AS (SELECT * FROM cypher('issue_1767', $$ MATCH p=(u)-[e]->(v) RETURN u,e,v,p $$) as (u agtype, e agtype, v agtype, p agtype)); +-- +-- Issue 2256: A segmentation fault occurs when calling the coalesce function +-- This also occurs with the greatest function too. +-- +SELECT * FROM coalesce(1, 0); +SELECT * FROM greatest(1, 0); + -- dump out the tables SELECT * FROM my_vertices; SELECT * FROM my_edges; diff --git a/src/backend/parser/cypher_analyze.c b/src/backend/parser/cypher_analyze.c index af3e83c87..a408eea6e 100644 --- a/src/backend/parser/cypher_analyze.c +++ b/src/backend/parser/cypher_analyze.c @@ -171,14 +171,20 @@ static bool convert_cypher_walker(Node *node, ParseState *pstate) * JsonConstructorExpr - wrapper over FuncExpr/Aggref/WindowFunc for * SQL/JSON constructors * - * These are a special case that needs to be ignored. + * Added the following, although only the first 2 caused crashes in tests - + * CoalesceExpr, MinMaxExpr, CaseExpr, XmlExpr, ArrayExpr, RowExpr + * + * These are all special case that needs to be ignored. * */ if (IsA(funcexpr, SQLValueFunction) - || IsA(funcexpr, CoerceViaIO) - || IsA(funcexpr, Var) || IsA(funcexpr, OpExpr) - || IsA(funcexpr, Const) || IsA(funcexpr, BoolExpr) - || IsA(funcexpr, JsonConstructorExpr)) + || IsA(funcexpr, CoerceViaIO) + || IsA(funcexpr, Var) || IsA(funcexpr, OpExpr) + || IsA(funcexpr, Const) || IsA(funcexpr, BoolExpr) + || IsA(funcexpr, JsonConstructorExpr) + || IsA(funcexpr, CoalesceExpr) || IsA(funcexpr, MinMaxExpr) + || IsA(funcexpr, CaseExpr) || IsA(funcexpr, XmlExpr) + || IsA(funcexpr, ArrayExpr) || IsA(funcexpr, RowExpr)) { return false; } @@ -346,14 +352,20 @@ static bool is_func_cypher(FuncExpr *funcexpr) * JsonConstructorExpr - wrapper over FuncExpr/Aggref/WindowFunc for * SQL/JSON constructors * - * These are a special case that needs to be ignored. + * Added the following, although only the first 2 caused crashes in tests - + * CoalesceExpr, MinMaxExpr, CaseExpr, XmlExpr, ArrayExpr, RowExpr + * + * These are all special case that needs to be ignored. * */ if (IsA(funcexpr, SQLValueFunction) || IsA(funcexpr, CoerceViaIO) || IsA(funcexpr, Var) || IsA(funcexpr, OpExpr) || IsA(funcexpr, Const) || IsA(funcexpr, BoolExpr) - || IsA(funcexpr, JsonConstructorExpr)) + || IsA(funcexpr, JsonConstructorExpr) + || IsA(funcexpr, CoalesceExpr) || IsA(funcexpr, MinMaxExpr) + || IsA(funcexpr, CaseExpr) || IsA(funcexpr, XmlExpr) + || IsA(funcexpr, ArrayExpr) || IsA(funcexpr, RowExpr)) { return false; } From fa9973aab32adb0b611b09ee26ae127b92059af2 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Tue, 9 Dec 2025 11:15:26 -0800 Subject: [PATCH 004/100] Adjust 'could not find rte for' ERROR message (#2266) Adjusted the following type of error message. It was mentioned in issue 2263 as being incorrect, which it isn't. However, it did need some clarification added - ERROR: could not find rte for Added a HINT for additional clarity - HINT: variable does not exist within scope of usage For example: CREATE p0=(n0), (n1{k:EXISTS{WITH p0}}) RETURN 1 ERROR: could not find rte for p0 LINE 3: CREATE p0=(n0), (n1{k:EXISTS{WITH p0}}) ^ HINT: variable p0 does not exist within scope of usage Additionally, added pstate->p_expr_kind == EXPR_KIND_INSERT_TARGET to transform_cypher_clause_as_subquery. Updated existing regression tests. Added regression tests from issue. modified: regress/expected/cypher_call.out modified: regress/expected/cypher_subquery.out modified: regress/expected/cypher_union.out modified: regress/expected/cypher_with.out modified: regress/expected/expr.out modified: regress/expected/list_comprehension.out modified: regress/expected/scan.out modified: src/backend/parser/cypher_clause.c modified: src/backend/parser/cypher_expr.c --- regress/expected/cypher_call.out | 2 + regress/expected/cypher_subquery.out | 4 ++ regress/expected/cypher_union.out | 1 + regress/expected/cypher_with.out | 4 ++ regress/expected/expr.out | 66 +++++++++++++++++++++++++ regress/expected/list_comprehension.out | 2 + regress/expected/scan.out | 6 +++ regress/sql/expr.sql | 27 ++++++++++ src/backend/parser/cypher_clause.c | 1 + src/backend/parser/cypher_expr.c | 8 +-- 10 files changed, 118 insertions(+), 3 deletions(-) diff --git a/regress/expected/cypher_call.out b/regress/expected/cypher_call.out index bb1185b96..08f97ba41 100644 --- a/regress/expected/cypher_call.out +++ b/regress/expected/cypher_call.out @@ -125,6 +125,7 @@ SELECT * FROM cypher('cypher_call', $$CALL sqrt(64) YIELD sqrt WHERE a = 8 RETUR ERROR: could not find rte for a LINE 2: ...r('cypher_call', $$CALL sqrt(64) YIELD sqrt WHERE a = 8 RETU... ^ +HINT: variable a does not exist within scope of usage /* MATCH CALL RETURN, should fail */ SELECT * FROM cypher('cypher_call', $$ MATCH (a) CALL sqrt(64) RETURN sqrt $$) as (sqrt agtype); ERROR: Procedure call inside a query does not support naming results implicitly @@ -171,6 +172,7 @@ SELECT * FROM cypher('cypher_call', $$ MATCH (a) CALL sqrt(64) YIELD sqrt WHERE ERROR: could not find rte for b LINE 1: ...all', $$ MATCH (a) CALL sqrt(64) YIELD sqrt WHERE b = 8 RETU... ^ +HINT: variable b does not exist within scope of usage /* CALL MATCH YIELD WHERE UPDATE/RETURN */ SELECT * FROM cypher('cypher_call', $$ CALL sqrt(64) YIELD sqrt WHERE sqrt > 1 CREATE ({n:'c'}) $$) as (a agtype); a diff --git a/regress/expected/cypher_subquery.out b/regress/expected/cypher_subquery.out index 934549f82..456b3a2c9 100644 --- a/regress/expected/cypher_subquery.out +++ b/regress/expected/cypher_subquery.out @@ -135,6 +135,7 @@ SELECT * FROM cypher('subquery', $$ MATCH (a:person) ERROR: could not find rte for c LINE 5: RETURN c ^ +HINT: variable c does not exist within scope of usage --union, no returns SELECT * FROM cypher('subquery', $$ MATCH (a:person) WHERE EXISTS { @@ -341,6 +342,7 @@ SELECT * FROM cypher('subquery', $$ RETURN 1, ERROR: could not find rte for a LINE 4: RETURN a ^ +HINT: variable a does not exist within scope of usage --- COUNT --count pattern subquery in where SELECT * FROM cypher('subquery', $$ MATCH (a:person) @@ -540,6 +542,7 @@ SELECT * FROM cypher('subquery', $$ MATCH (a:person) ERROR: could not find rte for b LINE 2: RETURN a.name, COUNT{MATCH (a) RETURN b} $$) ^ +HINT: variable b does not exist within scope of usage --incorrect nested variable reference SELECT * FROM cypher('subquery', $$ MATCH (a:person) RETURN a.name, COUNT{MATCH (a) @@ -549,6 +552,7 @@ SELECT * FROM cypher('subquery', $$ MATCH (a:person) ERROR: could not find rte for b LINE 4: RETURN b} $$) ^ +HINT: variable b does not exist within scope of usage --count nested with exists SELECT * FROM cypher('subquery', $$ MATCH (a:person) RETURN a.name, diff --git a/regress/expected/cypher_union.out b/regress/expected/cypher_union.out index 063354ddb..14fa56e67 100644 --- a/regress/expected/cypher_union.out +++ b/regress/expected/cypher_union.out @@ -141,6 +141,7 @@ SELECT * FROM cypher('cypher_union', $$MATCH (n) RETURN n UNION ALL MATCH (m) RE ERROR: could not find rte for n LINE 2: ..., $$MATCH (n) RETURN n UNION ALL MATCH (m) RETURN n$$) AS (r... ^ +HINT: variable n does not exist within scope of usage /* *UNION and UNION ALL, type casting */ diff --git a/regress/expected/cypher_with.out b/regress/expected/cypher_with.out index e5f82aa21..99ea320a0 100644 --- a/regress/expected/cypher_with.out +++ b/regress/expected/cypher_with.out @@ -267,6 +267,7 @@ $$) AS (a agtype, b agtype); ERROR: could not find rte for b LINE 4: RETURN m,b ^ +HINT: variable b does not exist within scope of usage SELECT * FROM cypher('cypher_with', $$ MATCH (m)-[]->(b) WITH m AS start_node,b AS end_node @@ -278,6 +279,7 @@ $$) AS (id agtype, node agtype); ERROR: could not find rte for end_node LINE 7: RETURN id(start_node),end_node.name ^ +HINT: variable end_node does not exist within scope of usage -- Clean up SELECT drop_graph('cypher_with', true); NOTICE: drop cascades to 4 other objects @@ -320,6 +322,7 @@ $$) AS (n agtype, d agtype); ERROR: could not find rte for d LINE 8: RETURN c,d ^ +HINT: variable d does not exist within scope of usage -- Issue 396 (should error out) SELECT * FROM cypher('graph',$$ CREATE (v),(u),(w), @@ -338,6 +341,7 @@ $$) as (a agtype,b agtype); ERROR: could not find rte for v LINE 4: RETURN v,path_length ^ +HINT: variable v does not exist within scope of usage -- Clean up SELECT drop_graph('graph', true); NOTICE: drop cascades to 6 other objects diff --git a/regress/expected/expr.out b/regress/expected/expr.out index d0546d8a3..4be3bf90f 100644 --- a/regress/expected/expr.out +++ b/regress/expected/expr.out @@ -3370,6 +3370,7 @@ $$) AS (toBooleanList agtype); ERROR: could not find rte for fail LINE 2: RETURN toBooleanList(fail) ^ +HINT: variable fail does not exist within scope of usage SELECT * FROM cypher('expr', $$ RETURN toBooleanList("fail") $$) AS (toBooleanList agtype); @@ -3513,6 +3514,7 @@ $$) AS (toFloatList agtype); ERROR: could not find rte for failed LINE 2: RETURN toFloatList([failed]) ^ +HINT: variable failed does not exist within scope of usage SELECT * FROM cypher('expr', $$ RETURN toFloatList("failed") $$) AS (toFloatList agtype); @@ -3892,12 +3894,14 @@ $$) AS (toStringList agtype); ERROR: could not find rte for b LINE 2: RETURN toStringList([['a', b]]) ^ +HINT: variable b does not exist within scope of usage SELECT * FROM cypher('expr', $$ RETURN toStringList([test]) $$) AS (toStringList agtype); ERROR: could not find rte for test LINE 2: RETURN toStringList([test]) ^ +HINT: variable test does not exist within scope of usage -- -- reverse(string) -- @@ -7923,6 +7927,7 @@ SELECT * FROM cypher('list', $$ RETURN tail(abc) $$) AS (tail agtype); ERROR: could not find rte for abc LINE 1: SELECT * FROM cypher('list', $$ RETURN tail(abc) $$) AS (tai... ^ +HINT: variable abc does not exist within scope of usage SELECT * FROM cypher('list', $$ RETURN tail() $$) AS (tail agtype); ERROR: function ag_catalog.age_tail() does not exist LINE 1: SELECT * FROM cypher('list', $$ RETURN tail() $$) AS (tail a... @@ -9011,9 +9016,70 @@ SELECT agtype_hash_cmp(agtype_in('[null, null, null, null, null]')); -505290721 (1 row) +-- +-- Issue 2263: AGE returns incorrect error message for EXISTS subquery outer variable reference +-- +-- NOTE: There isn't really anything incorrect about the message. However, +-- it could be more clear. +-- +SELECT * FROM create_graph('issue_2263'); +NOTICE: graph "issue_2263" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('issue_2263', $$ + CREATE a=()-[:T]->(), p=({k:exists{return a}})-[:T]->() + RETURN 1 +$$) AS (one agtype); +ERROR: could not find rte for a +LINE 2: CREATE a=()-[:T]->(), p=({k:exists{return a}})-[:T]->() + ^ +HINT: variable a does not exist within scope of usage +SELECT * FROM cypher('issue_2263', $$ + CREATE p0=(n0), (n1{k:EXISTS{WITH p0}}) + RETURN 1 +$$) AS (one agtype); +ERROR: could not find rte for p0 +LINE 2: CREATE p0=(n0), (n1{k:EXISTS{WITH p0}}) + ^ +HINT: variable p0 does not exist within scope of usage +SELECT * FROM cypher('issue_2263', $$ + CREATE ()-[r4 :T6]->(), ({k2:COUNT{WITH r4.k AS a3 UNWIND [] AS a4 WITH DISTINCT NULL AS a5}}) + RETURN 1 +$$) AS (one agtype); +ERROR: could not find rte for r4 +LINE 2: CREATE ()-[r4 :T6]->(), ({k2:COUNT{WITH r4.k AS a3 UNWIN... + ^ +HINT: variable r4 does not exist within scope of usage +SELECT * FROM cypher('issue_2263', $$ + CREATE (x), ({a1:EXISTS { RETURN COUNT(0) AS a2, keys(x) AS a4 }}) +$$) AS (out agtype); +ERROR: could not find rte for x +LINE 2: ...TE (x), ({a1:EXISTS { RETURN COUNT(0) AS a2, keys(x) AS a4 }... + ^ +HINT: variable x does not exist within scope of usage +SELECT * FROM cypher('issue_2263', $$ + CREATE x = (), ({ a0:COUNT { MATCH () WHERE CASE WHEN true THEN (x IS NULL) END RETURN 0 } }) +$$) AS (out agtype); +ERROR: could not find rte for x +LINE 2: ...({ a0:COUNT { MATCH () WHERE CASE WHEN true THEN (x IS NULL)... + ^ +HINT: variable x does not exist within scope of usage -- -- Cleanup -- +SELECT * FROM drop_graph('issue_2263', true); +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table issue_2263._ag_label_vertex +drop cascades to table issue_2263._ag_label_edge +NOTICE: graph "issue_2263" has been dropped + drop_graph +------------ + +(1 row) + SELECT * FROM drop_graph('issue_1988', true); NOTICE: drop cascades to 4 other objects DETAIL: drop cascades to table issue_1988._ag_label_vertex diff --git a/regress/expected/list_comprehension.out b/regress/expected/list_comprehension.out index 07f777707..5a3756422 100644 --- a/regress/expected/list_comprehension.out +++ b/regress/expected/list_comprehension.out @@ -569,10 +569,12 @@ SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN range(0, 10, 2)],i $$ ERROR: could not find rte for i LINE 1: ..._comprehension', $$ RETURN [i IN range(0, 10, 2)],i $$) AS (... ^ +HINT: variable i does not exist within scope of usage SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN range(0, 10, 2) WHERE i>5 | i^2], i $$) AS (result agtype, i agtype); ERROR: could not find rte for i LINE 1: ...$$ RETURN [i IN range(0, 10, 2) WHERE i>5 | i^2], i $$) AS (... ^ +HINT: variable i does not exist within scope of usage -- Invalid list comprehension SELECT * FROM cypher('list_comprehension', $$ RETURN [1 IN range(0, 10, 2) WHERE 2>5] $$) AS (result agtype); ERROR: Syntax error at or near IN diff --git a/regress/expected/scan.out b/regress/expected/scan.out index d8105a053..46d5676d0 100644 --- a/regress/expected/scan.out +++ b/regress/expected/scan.out @@ -437,36 +437,42 @@ $$) AS t(id text); ERROR: could not find rte for _$09A_z LINE 2: RETURN _$09A_z ^ +HINT: variable _$09A_z does not exist within scope of usage SELECT * FROM cypher('scan', $$ RETURN A $$) AS t(id text); ERROR: could not find rte for A LINE 2: RETURN A ^ +HINT: variable A does not exist within scope of usage SELECT * FROM cypher('scan', $$ RETURN z $$) AS t(id text); ERROR: could not find rte for z LINE 2: RETURN z ^ +HINT: variable z does not exist within scope of usage SELECT * FROM cypher('scan', $$ RETURN `$` $$) AS t(id text); ERROR: could not find rte for $ LINE 2: RETURN `$` ^ +HINT: variable $ does not exist within scope of usage SELECT * FROM cypher('scan', $$ RETURN `0` $$) AS t(id text); ERROR: could not find rte for 0 LINE 2: RETURN `0` ^ +HINT: variable 0 does not exist within scope of usage SELECT * FROM cypher('scan', $$ RETURN ```` $$) AS t(id text); ERROR: could not find rte for ` LINE 2: RETURN ```` ^ +HINT: variable ` does not exist within scope of usage -- zero-length quoted identifier SELECT * FROM cypher('scan', $$ RETURN `` diff --git a/regress/sql/expr.sql b/regress/sql/expr.sql index 83f21856e..466bace41 100644 --- a/regress/sql/expr.sql +++ b/regress/sql/expr.sql @@ -3634,9 +3634,36 @@ SELECT * FROM cypher('issue_1988', $$ SELECT agtype_access_operator(agtype_in('[null, null]')); SELECT agtype_hash_cmp(agtype_in('[null, null, null, null, null]')); +-- +-- Issue 2263: AGE returns incorrect error message for EXISTS subquery outer variable reference +-- +-- NOTE: There isn't really anything incorrect about the message. However, +-- it could be more clear. +-- +SELECT * FROM create_graph('issue_2263'); +SELECT * FROM cypher('issue_2263', $$ + CREATE a=()-[:T]->(), p=({k:exists{return a}})-[:T]->() + RETURN 1 +$$) AS (one agtype); +SELECT * FROM cypher('issue_2263', $$ + CREATE p0=(n0), (n1{k:EXISTS{WITH p0}}) + RETURN 1 +$$) AS (one agtype); +SELECT * FROM cypher('issue_2263', $$ + CREATE ()-[r4 :T6]->(), ({k2:COUNT{WITH r4.k AS a3 UNWIND [] AS a4 WITH DISTINCT NULL AS a5}}) + RETURN 1 +$$) AS (one agtype); +SELECT * FROM cypher('issue_2263', $$ + CREATE (x), ({a1:EXISTS { RETURN COUNT(0) AS a2, keys(x) AS a4 }}) +$$) AS (out agtype); +SELECT * FROM cypher('issue_2263', $$ + CREATE x = (), ({ a0:COUNT { MATCH () WHERE CASE WHEN true THEN (x IS NULL) END RETURN 0 } }) +$$) AS (out agtype); + -- -- Cleanup -- +SELECT * FROM drop_graph('issue_2263', true); SELECT * FROM drop_graph('issue_1988', true); SELECT * FROM drop_graph('issue_1953', true); SELECT * FROM drop_graph('expanded_map', true); diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 468706a87..8383b340e 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -6315,6 +6315,7 @@ transform_cypher_clause_as_subquery(cypher_parsestate *cpstate, pstate->p_expr_kind == EXPR_KIND_OTHER || pstate->p_expr_kind == EXPR_KIND_WHERE || pstate->p_expr_kind == EXPR_KIND_SELECT_TARGET || + pstate->p_expr_kind == EXPR_KIND_INSERT_TARGET || pstate->p_expr_kind == EXPR_KIND_FROM_SUBSELECT); /* diff --git a/src/backend/parser/cypher_expr.c b/src/backend/parser/cypher_expr.c index 993957c83..5f4de86b9 100644 --- a/src/backend/parser/cypher_expr.c +++ b/src/backend/parser/cypher_expr.c @@ -423,9 +423,11 @@ static Node *transform_ColumnRef(cypher_parsestate *cpstate, ColumnRef *cref) else { ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_COLUMN), - errmsg("could not find rte for %s", colname), - parser_errposition(pstate, cref->location))); + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("could not find rte for %s", colname), + errhint("variable %s does not exist within scope of usage", + colname), + parser_errposition(pstate, cref->location))); } if (node == NULL) From 0ea94644f25bdc08c803ff6521bfd412a6690401 Mon Sep 17 00:00:00 2001 From: Aleksey Konovkin Date: Tue, 9 Dec 2025 22:49:05 +0300 Subject: [PATCH 005/100] Fix possible memory and file descriptors leaks (#2258) - Used postgres memory allocation functions instead of standard ones. - Wrapped main loop of csv loader in PG_TRY block for better error handling. --- src/backend/utils/adt/age_global_graph.c | 6 +- src/backend/utils/adt/agtype.c | 39 ++++---- src/backend/utils/load/ag_load_edges.c | 91 ++++++++++-------- src/backend/utils/load/ag_load_labels.c | 115 ++++++++++++----------- src/include/utils/agtype.h | 1 + 5 files changed, 135 insertions(+), 117 deletions(-) diff --git a/src/backend/utils/adt/age_global_graph.c b/src/backend/utils/adt/age_global_graph.c index 6f30060ae..c34e51ee3 100644 --- a/src/backend/utils/adt/age_global_graph.c +++ b/src/backend/utils/adt/age_global_graph.c @@ -1237,12 +1237,10 @@ Datum age_delete_global_graphs(PG_FUNCTION_ARGS) { char *graph_name = NULL; - graph_name = strndup(agtv_temp->val.string.val, - agtv_temp->val.string.len); + graph_name = pnstrdup(agtv_temp->val.string.val, + agtv_temp->val.string.len); success = delete_specific_GRAPH_global_contexts(graph_name); - - free(graph_name); } else { diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index be838cbd0..02fc3221c 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -181,6 +181,17 @@ static agtype_value *agtype_build_map_as_agtype_value(FunctionCallInfo fcinfo); agtype_value *agtype_composite_to_agtype_value_binary(agtype *a); static agtype_value *tostring_helper(Datum arg, Oid type, char *msghdr); + +void *repalloc_check(void *ptr, size_t len) +{ + if (ptr != NULL) + { + return repalloc(ptr, len); + } + + return palloc(len); +} + /* * Due to how pfree can be implemented, it may not check for a passed NULL. This * wrapper does just that, it will only call pfree is the pointer passed is not @@ -5580,7 +5591,7 @@ static char *get_label_name(const char *graph_name, graphid element_graphid) result = NameStr(*DatumGetName(heap_getattr(tuple, Anum_ag_label_name, tupdesc, &column_is_null))); /* duplicate it */ - result = strdup(result); + result = pstrdup(result); /* end the scan and close the relation */ systable_endscan(scan_desc); @@ -5673,8 +5684,8 @@ Datum age_startnode(PG_FUNCTION_ARGS) Assert(AGT_ROOT_IS_SCALAR(agt_arg)); agtv_object = get_ith_agtype_value_from_container(&agt_arg->root, 0); Assert(agtv_object->type == AGTV_STRING); - graph_name = strndup(agtv_object->val.string.val, - agtv_object->val.string.len); + graph_name = pnstrdup(agtv_object->val.string.val, + agtv_object->val.string.len); /* get the edge */ agt_arg = AG_GET_ARG_AGTYPE_P(1); @@ -5708,8 +5719,6 @@ Datum age_startnode(PG_FUNCTION_ARGS) result = get_vertex(graph_name, label_name, start_id); - free(label_name); - return result; } @@ -5738,8 +5747,8 @@ Datum age_endnode(PG_FUNCTION_ARGS) Assert(AGT_ROOT_IS_SCALAR(agt_arg)); agtv_object = get_ith_agtype_value_from_container(&agt_arg->root, 0); Assert(agtv_object->type == AGTV_STRING); - graph_name = strndup(agtv_object->val.string.val, - agtv_object->val.string.len); + graph_name = pnstrdup(agtv_object->val.string.val, + agtv_object->val.string.len); /* get the edge */ agt_arg = AG_GET_ARG_AGTYPE_P(1); @@ -5773,8 +5782,6 @@ Datum age_endnode(PG_FUNCTION_ARGS) result = get_vertex(graph_name, label_name, end_id); - free(label_name); - return result; } @@ -6401,11 +6408,10 @@ Datum age_tofloat(PG_FUNCTION_ARGS) NumericGetDatum(agtv_value->val.numeric))); else if (agtv_value->type == AGTV_STRING) { - string = strndup(agtv_value->val.string.val, - agtv_value->val.string.len); + string = pnstrdup(agtv_value->val.string.val, + agtv_value->val.string.len); result = float8in_internal_null(string, NULL, "double precision", string, &is_valid); - free(string); if (!is_valid) PG_RETURN_NULL(); } @@ -6703,8 +6709,8 @@ Datum age_tointeger(PG_FUNCTION_ARGS) { char *endptr; /* we need a null terminated cstring */ - string = strndup(agtv_value->val.string.val, - agtv_value->val.string.len); + string = pnstrdup(agtv_value->val.string.val, + agtv_value->val.string.len); /* convert it if it is a regular integer string */ result = strtoi64(string, &endptr, 10); @@ -6718,7 +6724,6 @@ Datum age_tointeger(PG_FUNCTION_ARGS) f = float8in_internal_null(string, NULL, "double precision", string, &is_valid); - free(string); /* * If the conversions failed or it's a special float value, * return null. @@ -6731,10 +6736,6 @@ Datum age_tointeger(PG_FUNCTION_ARGS) result = (int64) f; } - else - { - free(string); - } } else { diff --git a/src/backend/utils/load/ag_load_edges.c b/src/backend/utils/load/ag_load_edges.c index 67049431c..931c6e0dc 100644 --- a/src/backend/utils/load/ag_load_edges.c +++ b/src/backend/utils/load/ag_load_edges.c @@ -36,8 +36,8 @@ void edge_field_cb(void *field, size_t field_len, void *data) if (cr->cur_field == cr->alloc) { cr->alloc *= 2; - cr->fields = realloc(cr->fields, sizeof(char *) * cr->alloc); - cr->fields_len = realloc(cr->header, sizeof(size_t *) * cr->alloc); + cr->fields = repalloc_check(cr->fields, sizeof(char *) * cr->alloc); + cr->fields_len = repalloc_check(cr->header, sizeof(size_t *) * cr->alloc); if (cr->fields == NULL) { cr->error = 1; @@ -48,7 +48,7 @@ void edge_field_cb(void *field, size_t field_len, void *data) } cr->fields_len[cr->cur_field] = field_len; cr->curr_row_length += field_len; - cr->fields[cr->cur_field] = strndup((char*)field, field_len); + cr->fields[cr->cur_field] = pnstrdup((char*)field, field_len); cr->cur_field += 1; } @@ -78,13 +78,13 @@ void edge_row_cb(int delim __attribute__((unused)), void *data) { cr->header_num = cr->cur_field; cr->header_row_length = cr->curr_row_length; - cr->header_len = (size_t* )malloc(sizeof(size_t *) * cr->cur_field); - cr->header = malloc((sizeof (char*) * cr->cur_field)); + cr->header_len = (size_t* )palloc(sizeof(size_t *) * cr->cur_field); + cr->header = palloc((sizeof (char*) * cr->cur_field)); for (i = 0; icur_field; i++) { cr->header_len[i] = cr->fields_len[i]; - cr->header[i] = strndup(cr->fields[i], cr->header_len[i]); + cr->header[i] = pnstrdup(cr->fields[i], cr->header_len[i]); } } else @@ -133,7 +133,7 @@ void edge_row_cb(int delim __attribute__((unused)), void *data) for (i = 0; i < n_fields; ++i) { - free(cr->fields[i]); + pfree_if_not_null(cr->fields[i]); } if (cr->error) @@ -192,6 +192,10 @@ int create_edges_from_csv_file(char *file_path, (errmsg("Failed to initialize csv parser\n"))); } + p.malloc_func = palloc; + p.realloc_func = repalloc_check; + p.free_func = pfree_if_not_null; + csv_set_space_func(&p, is_space); csv_set_term_func(&p, is_term); @@ -202,47 +206,52 @@ int create_edges_from_csv_file(char *file_path, (errmsg("Failed to open %s\n", file_path))); } - label_seq_name = get_label_seq_relation_name(label_name); - - memset((void*)&cr, 0, sizeof(csv_edge_reader)); - cr.alloc = 128; - cr.fields = malloc(sizeof(char *) * cr.alloc); - cr.fields_len = malloc(sizeof(size_t *) * cr.alloc); - cr.header_row_length = 0; - cr.curr_row_length = 0; - cr.graph_name = graph_name; - cr.graph_oid = graph_oid; - cr.label_name = label_name; - cr.label_id = label_id; - cr.label_seq_relid = get_relname_relid(label_seq_name, graph_oid); - cr.load_as_agtype = load_as_agtype; - - /* Initialize the batch insert state */ - init_batch_insert(&cr.batch_state, label_name, graph_oid); - - while ((bytes_read=fread(buf, 1, 1024, fp)) > 0) + PG_TRY(); { - if (csv_parse(&p, buf, bytes_read, edge_field_cb, - edge_row_cb, &cr) != bytes_read) + label_seq_name = get_label_seq_relation_name(label_name); + + memset((void*)&cr, 0, sizeof(csv_edge_reader)); + cr.alloc = 128; + cr.fields = palloc(sizeof(char *) * cr.alloc); + cr.fields_len = palloc(sizeof(size_t *) * cr.alloc); + cr.header_row_length = 0; + cr.curr_row_length = 0; + cr.graph_name = graph_name; + cr.graph_oid = graph_oid; + cr.label_name = label_name; + cr.label_id = label_id; + cr.label_seq_relid = get_relname_relid(label_seq_name, graph_oid); + cr.load_as_agtype = load_as_agtype; + + /* Initialize the batch insert state */ + init_batch_insert(&cr.batch_state, label_name, graph_oid); + + while ((bytes_read=fread(buf, 1, 1024, fp)) > 0) { - ereport(ERROR, (errmsg("Error while parsing file: %s\n", - csv_strerror(csv_error(&p))))); + if (csv_parse(&p, buf, bytes_read, edge_field_cb, + edge_row_cb, &cr) != bytes_read) + { + ereport(ERROR, (errmsg("Error while parsing file: %s\n", + csv_strerror(csv_error(&p))))); + } } - } - csv_fini(&p, edge_field_cb, edge_row_cb, &cr); + csv_fini(&p, edge_field_cb, edge_row_cb, &cr); - /* Finish any remaining batch inserts */ - finish_batch_insert(&cr.batch_state); + /* Finish any remaining batch inserts */ + finish_batch_insert(&cr.batch_state); - if (ferror(fp)) + if (ferror(fp)) + { + ereport(ERROR, (errmsg("Error while reading file %s\n", file_path))); + } + } + PG_FINALLY(); { - ereport(ERROR, (errmsg("Error while reading file %s\n", file_path))); + fclose(fp); + csv_free(&p); } + PG_END_TRY(); - fclose(fp); - - free(cr.fields); - csv_free(&p); return EXIT_SUCCESS; -} \ No newline at end of file +} diff --git a/src/backend/utils/load/ag_load_labels.c b/src/backend/utils/load/ag_load_labels.c index 4a04f3cd8..1e86bbda4 100644 --- a/src/backend/utils/load/ag_load_labels.c +++ b/src/backend/utils/load/ag_load_labels.c @@ -39,8 +39,8 @@ void vertex_field_cb(void *field, size_t field_len, void *data) if (cr->cur_field == cr->alloc) { cr->alloc *= 2; - cr->fields = realloc(cr->fields, sizeof(char *) * cr->alloc); - cr->fields_len = realloc(cr->header, sizeof(size_t *) * cr->alloc); + cr->fields = repalloc_check(cr->fields, sizeof(char *) * cr->alloc); + cr->fields_len = repalloc_check(cr->header, sizeof(size_t *) * cr->alloc); if (cr->fields == NULL) { cr->error = 1; @@ -51,7 +51,7 @@ void vertex_field_cb(void *field, size_t field_len, void *data) } cr->fields_len[cr->cur_field] = field_len; cr->curr_row_length += field_len; - cr->fields[cr->cur_field] = strndup((char *) field, field_len); + cr->fields[cr->cur_field] = pnstrdup((char *) field, field_len); cr->cur_field += 1; } @@ -70,13 +70,13 @@ void vertex_row_cb(int delim __attribute__((unused)), void *data) { cr->header_num = cr->cur_field; cr->header_row_length = cr->curr_row_length; - cr->header_len = (size_t* )malloc(sizeof(size_t *) * cr->cur_field); - cr->header = malloc((sizeof (char*) * cr->cur_field)); + cr->header_len = (size_t* )palloc(sizeof(size_t *) * cr->cur_field); + cr->header = palloc((sizeof (char*) * cr->cur_field)); for (i = 0; icur_field; i++) { cr->header_len[i] = cr->fields_len[i]; - cr->header[i] = strndup(cr->fields[i], cr->header_len[i]); + cr->header[i] = pnstrdup(cr->fields[i], cr->header_len[i]); } } else @@ -129,7 +129,7 @@ void vertex_row_cb(int delim __attribute__((unused)), void *data) for (i = 0; i < n_fields; ++i) { - free(cr->fields[i]); + pfree_if_not_null(cr->fields[i]); } if (cr->error) @@ -189,6 +189,10 @@ int create_labels_from_csv_file(char *file_path, (errmsg("Failed to initialize csv parser\n"))); } + p.malloc_func = palloc; + p.realloc_func = repalloc_check; + p.free_func = pfree_if_not_null; + csv_set_space_func(&p, is_space); csv_set_term_func(&p, is_term); @@ -199,62 +203,67 @@ int create_labels_from_csv_file(char *file_path, (errmsg("Failed to open %s\n", file_path))); } - label_seq_name = get_label_seq_relation_name(label_name); - - memset((void*)&cr, 0, sizeof(csv_vertex_reader)); - - cr.alloc = 2048; - cr.fields = malloc(sizeof(char *) * cr.alloc); - cr.fields_len = malloc(sizeof(size_t *) * cr.alloc); - cr.header_row_length = 0; - cr.curr_row_length = 0; - cr.graph_name = graph_name; - cr.graph_oid = graph_oid; - cr.label_name = label_name; - cr.label_id = label_id; - cr.id_field_exists = id_field_exists; - cr.label_seq_relid = get_relname_relid(label_seq_name, graph_oid); - cr.load_as_agtype = load_as_agtype; - - if (cr.id_field_exists) + PG_TRY(); { - /* - * Set the curr_seq_num since we will need it to compare with - * incoming entry_id. - * - * We cant use currval because it will error out if nextval was - * not called before in the session. - */ - cr.curr_seq_num = nextval_internal(cr.label_seq_relid, true); - } + label_seq_name = get_label_seq_relation_name(label_name); + + memset((void*)&cr, 0, sizeof(csv_vertex_reader)); + + cr.alloc = 2048; + cr.fields = palloc(sizeof(char *) * cr.alloc); + cr.fields_len = palloc(sizeof(size_t *) * cr.alloc); + cr.header_row_length = 0; + cr.curr_row_length = 0; + cr.graph_name = graph_name; + cr.graph_oid = graph_oid; + cr.label_name = label_name; + cr.label_id = label_id; + cr.id_field_exists = id_field_exists; + cr.label_seq_relid = get_relname_relid(label_seq_name, graph_oid); + cr.load_as_agtype = load_as_agtype; + + if (cr.id_field_exists) + { + /* + * Set the curr_seq_num since we will need it to compare with + * incoming entry_id. + * + * We cant use currval because it will error out if nextval was + * not called before in the session. + */ + cr.curr_seq_num = nextval_internal(cr.label_seq_relid, true); + } - /* Initialize the batch insert state */ - init_batch_insert(&cr.batch_state, label_name, graph_oid); + /* Initialize the batch insert state */ + init_batch_insert(&cr.batch_state, label_name, graph_oid); - while ((bytes_read=fread(buf, 1, 1024, fp)) > 0) - { - if (csv_parse(&p, buf, bytes_read, vertex_field_cb, - vertex_row_cb, &cr) != bytes_read) + while ((bytes_read=fread(buf, 1, 1024, fp)) > 0) { - ereport(ERROR, (errmsg("Error while parsing file: %s\n", - csv_strerror(csv_error(&p))))); + if (csv_parse(&p, buf, bytes_read, vertex_field_cb, + vertex_row_cb, &cr) != bytes_read) + { + ereport(ERROR, (errmsg("Error while parsing file: %s\n", + csv_strerror(csv_error(&p))))); + } } - } - csv_fini(&p, vertex_field_cb, vertex_row_cb, &cr); + csv_fini(&p, vertex_field_cb, vertex_row_cb, &cr); - /* Finish any remaining batch inserts */ - finish_batch_insert(&cr.batch_state); + /* Finish any remaining batch inserts */ + finish_batch_insert(&cr.batch_state); - if (ferror(fp)) + if (ferror(fp)) + { + ereport(ERROR, (errmsg("Error while reading file %s\n", + file_path))); + } + } + PG_FINALLY(); { - ereport(ERROR, (errmsg("Error while reading file %s\n", - file_path))); + fclose(fp); + csv_free(&p); } + PG_END_TRY(); - fclose(fp); - - free(cr.fields); - csv_free(&p); return EXIT_SUCCESS; } \ No newline at end of file diff --git a/src/include/utils/agtype.h b/src/include/utils/agtype.h index 486775320..ab2ba08cc 100644 --- a/src/include/utils/agtype.h +++ b/src/include/utils/agtype.h @@ -556,6 +556,7 @@ void pfree_agtype_value(agtype_value* value); void pfree_agtype_value_content(agtype_value* value); void pfree_agtype_in_state(agtype_in_state* value); void pfree_if_not_null(void *ptr); +void *repalloc_check(void *ptr, size_t len); agtype_value *agtype_value_from_cstring(char *str, int len); /* Oid accessors for AGTYPE */ Oid get_AGTYPEOID(void); From 898481a8c79381cc9f49ca71a9756a6216b61353 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Wed, 10 Dec 2025 09:08:36 -0800 Subject: [PATCH 006/100] Fix ORDER BY alias resolution with AS in Cypher queries (#2269) NOTE: This PR was partially created with AI tools and reviewed by a human. ORDER BY clauses failed when referencing column aliases from RETURN: MATCH (p:Person) RETURN p.age AS age ORDER BY age DESC ERROR: could not find rte for age Added SQL-99 compliant alias matching to find_target_list_entry() that checks if ORDER BY identifier matches a target list alias before attempting expression transformation. This enables standard SQL behavior for sorting by aliased columns with DESC/DESCENDING/ASC/ASCENDING. Updated regression tests. Added regression tests. modified: regress/expected/cypher_match.out modified: regress/expected/expr.out modified: regress/sql/expr.sql modified: src/backend/parser/cypher_clause.c --- regress/expected/cypher_match.out | 24 ++++---- regress/expected/expr.out | 88 ++++++++++++++++++++++++++++++ regress/sql/expr.sql | 30 ++++++++++ src/backend/parser/cypher_clause.c | 26 +++++++++ 4 files changed, 156 insertions(+), 12 deletions(-) diff --git a/regress/expected/cypher_match.out b/regress/expected/cypher_match.out index 72ca1cd71..a0e284beb 100644 --- a/regress/expected/cypher_match.out +++ b/regress/expected/cypher_match.out @@ -1655,10 +1655,10 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (u agtype, m agtype, l agtype); u | m | l ------------+---------------+------------ - "someone" | "opt_match_e" | "somebody" - "somebody" | "opt_match_e" | "someone" "anybody" | "opt_match_e" | "nobody" "nobody" | "opt_match_e" | "anybody" + "somebody" | "opt_match_e" | "someone" + "someone" | "opt_match_e" | "somebody" (4 rows) SELECT * FROM cypher('cypher_match', $$ @@ -1670,8 +1670,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (n agtype, r agtype, p agtype, m agtype, s agtype, q agtype); n | r | p | m | s | q -----------+---------------+------------+-----------+---------------+------------ - "someone" | "opt_match_e" | "somebody" | "anybody" | "opt_match_e" | "nobody" "anybody" | "opt_match_e" | "nobody" | "someone" | "opt_match_e" | "somebody" + "someone" | "opt_match_e" | "somebody" | "anybody" | "opt_match_e" | "nobody" (2 rows) SELECT * FROM cypher('cypher_match', $$ @@ -1684,18 +1684,18 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (n agtype, r agtype, p agtype, m agtype, s agtype, q agtype); n | r | p | m | s | q ------------+---------------+------------+------------+---------------+------------ - "someone" | "opt_match_e" | "somebody" | "anybody" | "opt_match_e" | "nobody" - "someone" | | | "somebody" | | - "someone" | | | "nobody" | | - "somebody" | | | "someone" | | - "somebody" | | | "anybody" | | - "somebody" | | | "nobody" | | "anybody" | "opt_match_e" | "nobody" | "someone" | "opt_match_e" | "somebody" - "anybody" | | | "somebody" | | "anybody" | | | "nobody" | | - "nobody" | | | "someone" | | - "nobody" | | | "somebody" | | + "anybody" | | | "somebody" | | "nobody" | | | "anybody" | | + "nobody" | | | "somebody" | | + "nobody" | | | "someone" | | + "somebody" | | | "anybody" | | + "somebody" | | | "nobody" | | + "somebody" | | | "someone" | | + "someone" | "opt_match_e" | "somebody" | "anybody" | "opt_match_e" | "nobody" + "someone" | | | "nobody" | | + "someone" | | | "somebody" | | (12 rows) -- Tests to catch match following optional match logic diff --git a/regress/expected/expr.out b/regress/expected/expr.out index 4be3bf90f..926a958d6 100644 --- a/regress/expected/expr.out +++ b/regress/expected/expr.out @@ -6949,6 +6949,94 @@ $$) AS (i agtype); {"key": "value"} (9 rows) +-- +-- Test ORDER BY with AS +-- +SELECT * FROM cypher('order_by', $$ CREATE ({name: 'John', age: 38}) $$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('order_by', $$ CREATE ({name: 'Jill', age: 23}) $$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('order_by', $$ CREATE ({name: 'Ion', age: 34}) $$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('order_by', $$ CREATE ({name: 'Mary', age: 57}) $$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('order_by', $$ CREATE ({name: 'Jerry', age: 34}) $$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('order_by', $$ + MATCH (u) WHERE EXISTS(u.name) RETURN u.name AS name, u.age AS age ORDER BY name +$$) AS (name agtype, age agtype); + name | age +---------+----- + "Ion" | 34 + "Jerry" | 34 + "Jill" | 23 + "John" | 38 + "Mary" | 57 +(5 rows) + +SELECT * FROM cypher('order_by', $$ + MATCH (u) WHERE EXISTS(u.name) RETURN u.name AS name, u.age AS age ORDER BY name ASC +$$) AS (name agtype, age agtype); + name | age +---------+----- + "Ion" | 34 + "Jerry" | 34 + "Jill" | 23 + "John" | 38 + "Mary" | 57 +(5 rows) + +SELECT * FROM cypher('order_by', $$ + MATCH (u) WHERE EXISTS(u.name) RETURN u.name AS name, u.age AS age ORDER BY name DESC +$$) AS (name agtype, age agtype); + name | age +---------+----- + "Mary" | 57 + "John" | 38 + "Jill" | 23 + "Jerry" | 34 + "Ion" | 34 +(5 rows) + +SELECT * FROM cypher('order_by', $$ + MATCH (u) WHERE EXISTS(u.name) RETURN u.name AS name, u.age AS age ORDER BY age ASC, name DESCENDING +$$) AS (name agtype, age agtype); + name | age +---------+----- + "Jill" | 23 + "Jerry" | 34 + "Ion" | 34 + "John" | 38 + "Mary" | 57 +(5 rows) + +SELECT * FROM cypher('order_by', $$ + MATCH (u) WHERE EXISTS(u.name) RETURN u.name AS name, u.age AS age ORDER BY age DESC, name ASCENDING +$$) AS (name agtype, age agtype); + name | age +---------+----- + "Mary" | 57 + "John" | 38 + "Ion" | 34 + "Jerry" | 34 + "Jill" | 23 +(5 rows) + --CASE SELECT create_graph('case_statement'); NOTICE: graph "case_statement" has been created diff --git a/regress/sql/expr.sql b/regress/sql/expr.sql index 466bace41..7bf1f26b2 100644 --- a/regress/sql/expr.sql +++ b/regress/sql/expr.sql @@ -2823,6 +2823,7 @@ SELECT * FROM cypher('order_by', $$CREATE ({i: false})$$) AS (result agtype); SELECT * FROM cypher('order_by', $$CREATE ({i: {key: 'value'}})$$) AS (result agtype); SELECT * FROM cypher('order_by', $$CREATE ({i: [1]})$$) AS (result agtype); + SELECT * FROM cypher('order_by', $$ MATCH (u) RETURN u.i @@ -2835,6 +2836,35 @@ SELECT * FROM cypher('order_by', $$ ORDER BY u.i DESC $$) AS (i agtype); +-- +-- Test ORDER BY with AS +-- +SELECT * FROM cypher('order_by', $$ CREATE ({name: 'John', age: 38}) $$) AS (result agtype); +SELECT * FROM cypher('order_by', $$ CREATE ({name: 'Jill', age: 23}) $$) AS (result agtype); +SELECT * FROM cypher('order_by', $$ CREATE ({name: 'Ion', age: 34}) $$) AS (result agtype); +SELECT * FROM cypher('order_by', $$ CREATE ({name: 'Mary', age: 57}) $$) AS (result agtype); +SELECT * FROM cypher('order_by', $$ CREATE ({name: 'Jerry', age: 34}) $$) AS (result agtype); + +SELECT * FROM cypher('order_by', $$ + MATCH (u) WHERE EXISTS(u.name) RETURN u.name AS name, u.age AS age ORDER BY name +$$) AS (name agtype, age agtype); + +SELECT * FROM cypher('order_by', $$ + MATCH (u) WHERE EXISTS(u.name) RETURN u.name AS name, u.age AS age ORDER BY name ASC +$$) AS (name agtype, age agtype); + +SELECT * FROM cypher('order_by', $$ + MATCH (u) WHERE EXISTS(u.name) RETURN u.name AS name, u.age AS age ORDER BY name DESC +$$) AS (name agtype, age agtype); + +SELECT * FROM cypher('order_by', $$ + MATCH (u) WHERE EXISTS(u.name) RETURN u.name AS name, u.age AS age ORDER BY age ASC, name DESCENDING +$$) AS (name agtype, age agtype); + +SELECT * FROM cypher('order_by', $$ + MATCH (u) WHERE EXISTS(u.name) RETURN u.name AS name, u.age AS age ORDER BY age DESC, name ASCENDING +$$) AS (name agtype, age agtype); + --CASE SELECT create_graph('case_statement'); SELECT * FROM cypher('case_statement', $$CREATE ({id: 1, i: 1, j: null})-[:connected_to {id: 1, k:0}]->({id: 2, i: 'a', j: 'b'})$$) AS (result agtype); diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 8383b340e..a5413bdaa 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -2296,6 +2296,32 @@ static TargetEntry *find_target_list_entry(cypher_parsestate *cpstate, ListCell *lt; TargetEntry *te; + /* + * If the ORDER BY item is a simple identifier, check if it matches + * an alias in the target list. This implements SQL99-compliant + * alias matching for ORDER BY clauses. + */ + if (IsA(node, ColumnRef)) + { + ColumnRef *cref = (ColumnRef *)node; + + if (list_length(cref->fields) == 1) + { + char *name = strVal(linitial(cref->fields)); + + /* Try to match an alias in the target list */ + foreach (lt, *target_list) + { + te = lfirst(lt); + + if (te->resname != NULL && strcmp(te->resname, name) == 0) + { + return te; + } + } + } + } + expr = transform_cypher_expr(cpstate, node, expr_kind); foreach (lt, *target_list) From 91de779aee0ee241bf8022b4822a9745c8bc9fa3 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Thu, 11 Dec 2025 07:32:13 -0800 Subject: [PATCH 007/100] Update grammar file for maintainability (#2270) Consolidated duplicate code, added helper functions, and reviewed the grammar file for issues. NOTE: I used an AI tool to review and cleanup the grammar file. I have reviewed all of the work it did. Improvements: 1. Added KEYWORD_STRDUP macro to eliminate hardcoded string lengths 2. Consolidated EXPLAIN statement handling into make_explain_stmt helper 3. Extracted WITH clause validation into validate_return_item_aliases helper 4. Created make_default_return_node helper for subquery return-less logic Benefits: - Reduced code duplication by ~150 lines - Improved maintainability with helper functions - Eliminated manual string length calculations (error-prone) All 29 existing regression tests pass modified: src/backend/parser/cypher_gram.y --- src/backend/parser/cypher_gram.y | 303 ++++++++++++++----------------- 1 file changed, 140 insertions(+), 163 deletions(-) diff --git a/src/backend/parser/cypher_gram.y b/src/backend/parser/cypher_gram.y index 0bafefe1f..5ba1e6354 100644 --- a/src/backend/parser/cypher_gram.y +++ b/src/backend/parser/cypher_gram.y @@ -41,6 +41,9 @@ #define YYMALLOC palloc #define YYFREE pfree + +/* Helper macro for keyword string duplication */ +#define KEYWORD_STRDUP(kw) pnstrdup((kw), strlen(kw)) %} %locations @@ -276,6 +279,11 @@ static Node *build_list_comprehension_node(Node *var, Node *expr, Node *where, Node *mapping_expr, int location); +/* helper functions */ +static ExplainStmt *make_explain_stmt(List *options); +static void validate_return_item_aliases(List *items, ag_scanner_t scanner); +static cypher_return *make_default_return_node(int location); + %} %% @@ -300,81 +308,59 @@ stmt: * Throw syntax error in this case. */ if (yychar != YYEOF) + { yyerror(&yylloc, scanner, extra, "syntax error"); + } extra->result = $1; extra->extra = NULL; } | EXPLAIN cypher_stmt semicolon_opt { - ExplainStmt *estmt = NULL; - if (yychar != YYEOF) + { yyerror(&yylloc, scanner, extra, "syntax error"); - + } extra->result = $2; - - estmt = makeNode(ExplainStmt); - estmt->query = NULL; - estmt->options = NIL; - extra->extra = (Node *)estmt; + extra->extra = (Node *)make_explain_stmt(NIL); } | EXPLAIN VERBOSE cypher_stmt semicolon_opt { - ExplainStmt *estmt = NULL; - if (yychar != YYEOF) + { yyerror(&yylloc, scanner, extra, "syntax error"); - + } extra->result = $3; - - estmt = makeNode(ExplainStmt); - estmt->query = NULL; - estmt->options = list_make1(makeDefElem("verbose", NULL, @2));; - extra->extra = (Node *)estmt; + extra->extra = (Node *)make_explain_stmt( + list_make1(makeDefElem("verbose", NULL, @2))); } | EXPLAIN ANALYZE cypher_stmt semicolon_opt { - ExplainStmt *estmt = NULL; - if (yychar != YYEOF) + { yyerror(&yylloc, scanner, extra, "syntax error"); - + } extra->result = $3; - - estmt = makeNode(ExplainStmt); - estmt->query = NULL; - estmt->options = list_make1(makeDefElem("analyze", NULL, @2));; - extra->extra = (Node *)estmt; + extra->extra = (Node *)make_explain_stmt( + list_make1(makeDefElem("analyze", NULL, @2))); } | EXPLAIN ANALYZE VERBOSE cypher_stmt semicolon_opt { - ExplainStmt *estmt = NULL; - if (yychar != YYEOF) yyerror(&yylloc, scanner, extra, "syntax error"); - extra->result = $4; - - estmt = makeNode(ExplainStmt); - estmt->query = NULL; - estmt->options = list_make2(makeDefElem("analyze", NULL, @2), - makeDefElem("verbose", NULL, @3));; - extra->extra = (Node *)estmt; + extra->extra = (Node *)make_explain_stmt( + list_make2(makeDefElem("analyze", NULL, @2), + makeDefElem("verbose", NULL, @3))); } | EXPLAIN '(' utility_option_list ')' cypher_stmt semicolon_opt { - ExplainStmt *estmt = NULL; - if (yychar != YYEOF) + { yyerror(&yylloc, scanner, extra, "syntax error"); - + } extra->result = $5; - - estmt = makeNode(ExplainStmt); - estmt->query = NULL; - estmt->options = $3; - extra->extra = (Node *)estmt; + extra->extra = (Node *)make_explain_stmt($3); } ; @@ -652,57 +638,20 @@ single_subquery: single_subquery_no_return: subquery_part_init reading_clause_list { - ColumnRef *cr; - ResTarget *rt; cypher_return *n; /* * since subqueries allow return-less clauses, we add a * return node manually to reflect that syntax */ - cr = makeNode(ColumnRef); - cr->fields = list_make1(makeNode(A_Star)); - cr->location = @1; - - rt = makeNode(ResTarget); - rt->name = NULL; - rt->indirection = NIL; - rt->val = (Node *)cr; - rt->location = @1; - - n = make_ag_node(cypher_return); - n->distinct = false; - n->items = list_make1((Node *)rt); - n->order_by = NULL; - n->skip = NULL; - n->limit = NULL; - + n = make_default_return_node(@1); $$ = list_concat($1, lappend($2, n)); - } - | subquery_pattern + | subquery_pattern { - ColumnRef *cr; - ResTarget *rt; cypher_return *n; - cr = makeNode(ColumnRef); - cr->fields = list_make1(makeNode(A_Star)); - cr->location = @1; - - rt = makeNode(ResTarget); - rt->name = NULL; - rt->indirection = NIL; - rt->val = (Node *)cr; - rt->location = @1; - - n = make_ag_node(cypher_return); - n->distinct = false; - n->items = list_make1((Node *)rt); - n->order_by = NULL; - n->skip = NULL; - n->limit = NULL; - + n = make_default_return_node(@1); $$ = lappend(list_make1($1), n); } ; @@ -958,24 +907,10 @@ limit_opt: with: WITH DISTINCT return_item_list order_by_opt skip_opt limit_opt where_opt { - ListCell *li; cypher_with *n; /* check expressions are aliased */ - foreach(li, $3) - { - ResTarget *item = lfirst(li); - - /* variable does not have to be aliased */ - if (IsA(item->val, ColumnRef) || item->name) - continue; - - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("expression item must be aliased"), - errhint("Items can be aliased by using AS."), - ag_scanner_errposition(item->location, scanner))); - } + validate_return_item_aliases($3, scanner); n = make_ag_node(cypher_with); n->distinct = true; @@ -987,27 +922,12 @@ with: $$ = (Node *)n; } - | WITH return_item_list order_by_opt skip_opt limit_opt - where_opt + | WITH return_item_list order_by_opt skip_opt limit_opt where_opt { - ListCell *li; cypher_with *n; /* check expressions are aliased */ - foreach (li, $2) - { - ResTarget *item = lfirst(li); - - /* variable does not have to be aliased */ - if (IsA(item->val, ColumnRef) || item->name) - continue; - - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("expression item must be aliased"), - errhint("Items can be aliased by using AS."), - ag_scanner_errposition(item->location, scanner))); - } + validate_return_item_aliases($2, scanner); n = make_ag_node(cypher_with); n->distinct = false; @@ -2449,58 +2369,58 @@ qual_op: */ safe_keywords: - ALL { $$ = pnstrdup($1, 3); } - | ANALYZE { $$ = pnstrdup($1, 7); } - | AND { $$ = pnstrdup($1, 3); } - | AS { $$ = pnstrdup($1, 2); } - | ASC { $$ = pnstrdup($1, 3); } - | ASCENDING { $$ = pnstrdup($1, 9); } - | BY { $$ = pnstrdup($1, 2); } - | CALL { $$ = pnstrdup($1, 4); } - | CASE { $$ = pnstrdup($1, 4); } - | COALESCE { $$ = pnstrdup($1, 8); } - | CONTAINS { $$ = pnstrdup($1, 8); } - | COUNT { $$ = pnstrdup($1 ,5); } - | CREATE { $$ = pnstrdup($1, 6); } - | DELETE { $$ = pnstrdup($1, 6); } - | DESC { $$ = pnstrdup($1, 4); } - | DESCENDING { $$ = pnstrdup($1, 10); } - | DETACH { $$ = pnstrdup($1, 6); } - | DISTINCT { $$ = pnstrdup($1, 8); } - | ELSE { $$ = pnstrdup($1, 4); } - | ENDS { $$ = pnstrdup($1, 4); } - | EXISTS { $$ = pnstrdup($1, 6); } - | EXPLAIN { $$ = pnstrdup($1, 7); } - | IN { $$ = pnstrdup($1, 2); } - | IS { $$ = pnstrdup($1, 2); } - | LIMIT { $$ = pnstrdup($1, 6); } - | MATCH { $$ = pnstrdup($1, 6); } - | MERGE { $$ = pnstrdup($1, 6); } - | NOT { $$ = pnstrdup($1, 3); } - | OPERATOR { $$ = pnstrdup($1, 8); } - | OPTIONAL { $$ = pnstrdup($1, 8); } - | OR { $$ = pnstrdup($1, 2); } - | ORDER { $$ = pnstrdup($1, 5); } - | REMOVE { $$ = pnstrdup($1, 6); } - | RETURN { $$ = pnstrdup($1, 6); } - | SET { $$ = pnstrdup($1, 3); } - | SKIP { $$ = pnstrdup($1, 4); } - | STARTS { $$ = pnstrdup($1, 6); } - | THEN { $$ = pnstrdup($1, 4); } - | UNION { $$ = pnstrdup($1, 5); } - | WHEN { $$ = pnstrdup($1, 4); } - | VERBOSE { $$ = pnstrdup($1, 7); } - | WHERE { $$ = pnstrdup($1, 5); } - | WITH { $$ = pnstrdup($1, 4); } - | XOR { $$ = pnstrdup($1, 3); } - | YIELD { $$ = pnstrdup($1, 5); } + ALL { $$ = KEYWORD_STRDUP($1); } + | ANALYZE { $$ = KEYWORD_STRDUP($1); } + | AND { $$ = KEYWORD_STRDUP($1); } + | AS { $$ = KEYWORD_STRDUP($1); } + | ASC { $$ = KEYWORD_STRDUP($1); } + | ASCENDING { $$ = KEYWORD_STRDUP($1); } + | BY { $$ = KEYWORD_STRDUP($1); } + | CALL { $$ = KEYWORD_STRDUP($1); } + | CASE { $$ = KEYWORD_STRDUP($1); } + | COALESCE { $$ = KEYWORD_STRDUP($1); } + | CONTAINS { $$ = KEYWORD_STRDUP($1); } + | COUNT { $$ = KEYWORD_STRDUP($1); } + | CREATE { $$ = KEYWORD_STRDUP($1); } + | DELETE { $$ = KEYWORD_STRDUP($1); } + | DESC { $$ = KEYWORD_STRDUP($1); } + | DESCENDING { $$ = KEYWORD_STRDUP($1); } + | DETACH { $$ = KEYWORD_STRDUP($1); } + | DISTINCT { $$ = KEYWORD_STRDUP($1); } + | ELSE { $$ = KEYWORD_STRDUP($1); } + | ENDS { $$ = KEYWORD_STRDUP($1); } + | EXISTS { $$ = KEYWORD_STRDUP($1); } + | EXPLAIN { $$ = KEYWORD_STRDUP($1); } + | IN { $$ = KEYWORD_STRDUP($1); } + | IS { $$ = KEYWORD_STRDUP($1); } + | LIMIT { $$ = KEYWORD_STRDUP($1); } + | MATCH { $$ = KEYWORD_STRDUP($1); } + | MERGE { $$ = KEYWORD_STRDUP($1); } + | NOT { $$ = KEYWORD_STRDUP($1); } + | OPERATOR { $$ = KEYWORD_STRDUP($1); } + | OPTIONAL { $$ = KEYWORD_STRDUP($1); } + | OR { $$ = KEYWORD_STRDUP($1); } + | ORDER { $$ = KEYWORD_STRDUP($1); } + | REMOVE { $$ = KEYWORD_STRDUP($1); } + | RETURN { $$ = KEYWORD_STRDUP($1); } + | SET { $$ = KEYWORD_STRDUP($1); } + | SKIP { $$ = KEYWORD_STRDUP($1); } + | STARTS { $$ = KEYWORD_STRDUP($1); } + | THEN { $$ = KEYWORD_STRDUP($1); } + | UNION { $$ = KEYWORD_STRDUP($1); } + | WHEN { $$ = KEYWORD_STRDUP($1); } + | VERBOSE { $$ = KEYWORD_STRDUP($1); } + | WHERE { $$ = KEYWORD_STRDUP($1); } + | WITH { $$ = KEYWORD_STRDUP($1); } + | XOR { $$ = KEYWORD_STRDUP($1); } + | YIELD { $$ = KEYWORD_STRDUP($1); } ; conflicted_keywords: - END_P { $$ = pnstrdup($1, 5); } - | FALSE_P { $$ = pnstrdup($1, 7); } - | NULL_P { $$ = pnstrdup($1, 6); } - | TRUE_P { $$ = pnstrdup($1, 6); } + END_P { $$ = KEYWORD_STRDUP($1); } + | FALSE_P { $$ = KEYWORD_STRDUP($1); } + | NULL_P { $$ = KEYWORD_STRDUP($1); } + | TRUE_P { $$ = KEYWORD_STRDUP($1); } ; %% @@ -3384,7 +3304,7 @@ static Node *build_list_comprehension_node(Node *var, Node *expr, /* * Build an ARRAY sublink and attach list_comp as sub-select, - * it will be transformed in to query tree by us and reattached for + * it will be transformed in to query tree by us and reattached for * pg to process. */ sub = makeNode(SubLink); @@ -3396,3 +3316,60 @@ static Node *build_list_comprehension_node(Node *var, Node *expr, return (Node *) node_to_agtype((Node *)sub, "agtype[]", location); } + +/* Helper function to create an ExplainStmt node */ +static ExplainStmt *make_explain_stmt(List *options) +{ + ExplainStmt *estmt = makeNode(ExplainStmt); + estmt->query = NULL; + estmt->options = options; + return estmt; +} + +/* Helper function to validate that return items are properly aliased */ +static void validate_return_item_aliases(List *items, ag_scanner_t scanner) +{ + ListCell *li; + + foreach(li, items) + { + ResTarget *item = lfirst(li); + + /* variable does not have to be aliased */ + if (IsA(item->val, ColumnRef) || item->name) + continue; + + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("expression item must be aliased"), + errhint("Items can be aliased by using AS."), + ag_scanner_errposition(item->location, scanner))); + } +} + +/* Helper function to create a default return node (RETURN *) */ +static cypher_return *make_default_return_node(int location) +{ + ColumnRef *cr; + ResTarget *rt; + cypher_return *n; + + cr = makeNode(ColumnRef); + cr->fields = list_make1(makeNode(A_Star)); + cr->location = location; + + rt = makeNode(ResTarget); + rt->name = NULL; + rt->indirection = NIL; + rt->val = (Node *)cr; + rt->location = location; + + n = make_ag_node(cypher_return); + n->distinct = false; + n->items = list_make1((Node *)rt); + n->order_by = NULL; + n->skip = NULL; + n->limit = NULL; + + return n; +} From 1bb95bf616abe733dd6d5c02aa32c7f419827646 Mon Sep 17 00:00:00 2001 From: jsell-rh Date: Thu, 11 Dec 2025 10:56:37 -0500 Subject: [PATCH 008/100] Convert string to raw string to remove invalid escape sequence warning (#2267) - Changed '\s' to r'\s' --- drivers/python/age/age.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/python/age/age.py b/drivers/python/age/age.py index 817cc6e5a..b1aa82158 100644 --- a/drivers/python/age/age.py +++ b/drivers/python/age/age.py @@ -26,7 +26,7 @@ _EXCEPTION_NoConnection = NoConnection() _EXCEPTION_GraphNotSet = GraphNotSet() -WHITESPACE = re.compile('\s') +WHITESPACE = re.compile(r'\s') class AgeDumper(psycopg.adapt.Dumper): @@ -233,3 +233,4 @@ def cypher(self, cursor:psycopg.cursor, cypherStmt:str, cols:list=None, params:t # def queryCypher(self, cypherStmt:str, columns:list=None , params:tuple=None) -> psycopg.cursor : # return queryCypher(self.connection, self.graphName, cypherStmt, columns, params) + From 838926cc35ae64d3514c656959b749719e904b09 Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Thu, 11 Dec 2025 21:51:12 +0500 Subject: [PATCH 009/100] Migrate python driver configuration to pyproject.toml (#2272) - Add pyproject.toml with package configuration - Simplify setup.py to minimal backward-compatible wrapper. - Updated CI workflow and .gitignore. - Resolves warning about using setup.py directly. --- .github/workflows/python-driver.yaml | 4 +-- .gitignore | 2 ++ drivers/python/README.md | 2 +- drivers/python/pyproject.toml | 48 ++++++++++++++++++++++++++++ drivers/python/setup.py | 28 +++------------- 5 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 drivers/python/pyproject.toml diff --git a/.github/workflows/python-driver.yaml b/.github/workflows/python-driver.yaml index 099b5c871..4dad14638 100644 --- a/.github/workflows/python-driver.yaml +++ b/.github/workflows/python-driver.yaml @@ -24,7 +24,7 @@ jobs: - name: Set up python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - name: Install pre-requisites run: | @@ -33,7 +33,7 @@ jobs: - name: Build run: | - python setup.py install + pip install . - name: Test run: | diff --git a/.gitignore b/.gitignore index a8e809dda..03923b03e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,7 @@ age--*.*.*.sql !age--*--*sql __pycache__ **/__pycache__ +**/.venv +**/apache_age_python.egg-info drivers/python/build diff --git a/drivers/python/README.md b/drivers/python/README.md index 184652275..749b44bfb 100644 --- a/drivers/python/README.md +++ b/drivers/python/README.md @@ -62,7 +62,7 @@ python -m unittest -v test_agtypes.py ### Build from source ``` -python setup.py install +pip install . ``` ### For more information about [Apache AGE](https://age.apache.org/) diff --git a/drivers/python/pyproject.toml b/drivers/python/pyproject.toml new file mode 100644 index 000000000..18112381c --- /dev/null +++ b/drivers/python/pyproject.toml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "apache-age-python" +version = "0.0.7" +description = "Python driver support for Apache AGE" +readme = "README.md" +requires-python = ">=3.9" +license = "Apache-2.0" +keywords = ["Graph Database", "Apache AGE", "PostgreSQL"] +authors = [ + {name = "Ikchan Kwon, Apache AGE", email = "dev-subscribe@age.apache.org"} +] +classifiers = [ + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "psycopg", + "antlr4-python3-runtime==4.11.1", +] + +[project.urls] +Homepage = "https://github.com/apache/age/tree/master/drivers/python" +Download = "https://github.com/apache/age/releases" + +[tool.setuptools] +packages = ["age", "age.gen", "age.networkx"] diff --git a/drivers/python/setup.py b/drivers/python/setup.py index 1da49d9cb..d0eed26be 100644 --- a/drivers/python/setup.py +++ b/drivers/python/setup.py @@ -13,28 +13,10 @@ # specific language governing permissions and limitations # under the License. -from setuptools import setup, find_packages -from age import VERSION +# This setup.py is maintained for backward compatibility. +# All package configuration is in pyproject.toml. For installation, +# use: pip install . -with open("README.md", "r", encoding='utf8') as fh: - long_description = fh.read() +from setuptools import setup -setup( - name = 'apache-age-python', - version = '0.0.7', - description = 'Python driver support for Apache AGE', - long_description=long_description, - long_description_content_type="text/markdown", - author = 'Ikchan Kwon, Apache AGE', - author_email = 'dev-subscribe@age.apache.org', - url = 'https://github.com/apache/age/tree/master/drivers/python', - download_url = 'https://github.com/apache/age/releases' , - license = 'Apache2.0', - install_requires = [ 'psycopg', 'antlr4-python3-runtime==4.11.1'], - packages = ['age', 'age.gen','age.networkx'], - keywords = ['Graph Database', 'Apache AGE', 'PostgreSQL'], - python_requires = '>=3.9', - classifiers = [ - 'Programming Language :: Python :: 3.9' - ] -) +setup() From 48fca83d90f29cf68fd9748db3b2d628f4978749 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Tue, 16 Dec 2025 08:33:28 -0800 Subject: [PATCH 010/100] Restrict age_load commands (#2274) This PR applies restrictions to the following age_load commands - load_labels_from_file() load_edges_from_file() They are now tied to a specific root directory and are required to have a specific file extension to eliminate any attempts to force them to access any other files. Nothing else has changed with the actual command formats or parameters, only that they work out of the /tmp/age directory and only access files with an extension of .csv. Added regression tests and updated the location of the csv files for those regression tests. modified: regress/expected/age_load.out modified: regress/sql/age_load.sql modified: src/backend/utils/load/age_load.c --- regress/expected/age_load.out | 44 +++++++++++++++++- regress/sql/age_load.sql | 38 +++++++++++++++- src/backend/utils/load/age_load.c | 76 ++++++++++++++++++++++++++++--- 3 files changed, 149 insertions(+), 9 deletions(-) diff --git a/regress/expected/age_load.out b/regress/expected/age_load.out index 5f2bdab78..55d1ff1d6 100644 --- a/regress/expected/age_load.out +++ b/regress/expected/age_load.out @@ -16,7 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -\! cp -r regress/age_load/data regress/instance/data/age_load +\! rm -rf /tmp/age/age_load +\! mkdir -p /tmp/age +\! cp -r regress/age_load/data /tmp/age/age_load LOAD 'age'; SET search_path TO ag_catalog; -- Create a country using CREATE clause @@ -401,6 +403,43 @@ SELECT * FROM cypher('agload_conversion', $$ MATCH ()-[e:Edges2]->() RETURN prop {"bool": "false", "string": "nUll", "numeric": "3.14"} (6 rows) +-- +-- Check sandbox +-- +-- check null file name +SELECT load_labels_from_file('agload_conversion', 'Person1', NULL, true, true); +ERROR: file path must not be NULL +SELECT load_edges_from_file('agload_conversion', 'Edges1', NULL, true); +ERROR: file path must not be NULL +-- check no file name +SELECT load_labels_from_file('agload_conversion', 'Person1', '', true, true); +ERROR: file name cannot be zero length +SELECT load_edges_from_file('agload_conversion', 'Edges1', '', true); +ERROR: file name cannot be zero length +-- check for file/path does not exist +SELECT load_labels_from_file('agload_conversion', 'Person1', 'age_load_xxx/conversion_vertices.csv', true, true); +ERROR: File or path does not exist [/tmp/age/age_load_xxx/conversion_vertices.csv] +SELECT load_edges_from_file('agload_conversion', 'Edges1', 'age_load_xxx/conversion_edges.csv', true); +ERROR: File or path does not exist [/tmp/age/age_load_xxx/conversion_edges.csv] +SELECT load_labels_from_file('agload_conversion', 'Person1', 'age_load/conversion_vertices.txt', true, true); +ERROR: File or path does not exist [/tmp/age/age_load/conversion_vertices.txt] +SELECT load_edges_from_file('agload_conversion', 'Edges1', 'age_load/conversion_edges.txt', true); +ERROR: File or path does not exist [/tmp/age/age_load/conversion_edges.txt] +-- check wrong extension +\! touch /tmp/age/age_load/conversion_vertices.txt +\! touch /tmp/age/age_load/conversion_edges.txt +SELECT load_labels_from_file('agload_conversion', 'Person1', 'age_load/conversion_vertices.txt', true, true); +ERROR: You can only load files with extension [.csv]. +SELECT load_edges_from_file('agload_conversion', 'Edges1', 'age_load/conversion_edges.txt', true); +ERROR: You can only load files with extension [.csv]. +-- check outside sandbox directory +SELECT load_labels_from_file('agload_conversion', 'Person1', '../../etc/passwd', true, true); +ERROR: You can only load files located in [/tmp/age/]. +SELECT load_edges_from_file('agload_conversion', 'Edges1', '../../etc/passwd', true); +ERROR: You can only load files located in [/tmp/age/]. +-- +-- Cleanup +-- SELECT drop_graph('agload_conversion', true); NOTICE: drop cascades to 6 other objects DETAIL: drop cascades to table agload_conversion._ag_label_vertex @@ -415,3 +454,6 @@ NOTICE: graph "agload_conversion" has been dropped (1 row) +-- +-- End +-- diff --git a/regress/sql/age_load.sql b/regress/sql/age_load.sql index 180248bf1..cefcfb4ca 100644 --- a/regress/sql/age_load.sql +++ b/regress/sql/age_load.sql @@ -17,7 +17,9 @@ * under the License. */ -\! cp -r regress/age_load/data regress/instance/data/age_load +\! rm -rf /tmp/age/age_load +\! mkdir -p /tmp/age +\! cp -r regress/age_load/data /tmp/age/age_load LOAD 'age'; @@ -160,4 +162,38 @@ SELECT create_elabel('agload_conversion','Edges2'); SELECT load_edges_from_file('agload_conversion', 'Edges2', 'age_load/conversion_edges.csv', false); SELECT * FROM cypher('agload_conversion', $$ MATCH ()-[e:Edges2]->() RETURN properties(e) $$) as (a agtype); +-- +-- Check sandbox +-- +-- check null file name +SELECT load_labels_from_file('agload_conversion', 'Person1', NULL, true, true); +SELECT load_edges_from_file('agload_conversion', 'Edges1', NULL, true); + +-- check no file name +SELECT load_labels_from_file('agload_conversion', 'Person1', '', true, true); +SELECT load_edges_from_file('agload_conversion', 'Edges1', '', true); + +-- check for file/path does not exist +SELECT load_labels_from_file('agload_conversion', 'Person1', 'age_load_xxx/conversion_vertices.csv', true, true); +SELECT load_edges_from_file('agload_conversion', 'Edges1', 'age_load_xxx/conversion_edges.csv', true); +SELECT load_labels_from_file('agload_conversion', 'Person1', 'age_load/conversion_vertices.txt', true, true); +SELECT load_edges_from_file('agload_conversion', 'Edges1', 'age_load/conversion_edges.txt', true); + +-- check wrong extension +\! touch /tmp/age/age_load/conversion_vertices.txt +\! touch /tmp/age/age_load/conversion_edges.txt +SELECT load_labels_from_file('agload_conversion', 'Person1', 'age_load/conversion_vertices.txt', true, true); +SELECT load_edges_from_file('agload_conversion', 'Edges1', 'age_load/conversion_edges.txt', true); + +-- check outside sandbox directory +SELECT load_labels_from_file('agload_conversion', 'Person1', '../../etc/passwd', true, true); +SELECT load_edges_from_file('agload_conversion', 'Edges1', '../../etc/passwd', true); + +-- +-- Cleanup +-- SELECT drop_graph('agload_conversion', true); + +-- +-- End +-- diff --git a/src/backend/utils/load/age_load.c b/src/backend/utils/load/age_load.c index 307ec335a..c7cf0677f 100644 --- a/src/backend/utils/load/age_load.c +++ b/src/backend/utils/load/age_load.c @@ -31,6 +31,62 @@ static agtype_value *csv_value_to_agtype_value(char *csv_val); static Oid get_or_create_graph(const Name graph_name); static int32 get_or_create_label(Oid graph_oid, char *graph_name, char *label_name, char label_kind); +static char *build_safe_filename(char *name); + +#define AGE_BASE_CSV_DIRECTORY "/tmp/age/" +#define AGE_CSV_FILE_EXTENSION ".csv" + +static char *build_safe_filename(char *name) +{ + int length; + char path[PATH_MAX]; + char *resolved; + + if (name == NULL) + { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("file name cannot be NULL"))); + + } + + length = strlen(name); + + if (length == 0) + { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("file name cannot be zero length"))); + + } + + snprintf(path, sizeof(path), "%s%s", AGE_BASE_CSV_DIRECTORY, name); + + resolved = realpath(path, NULL); + + if (resolved == NULL) + { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("File or path does not exist [%s]", path))); + } + + if (strncmp(resolved, AGE_BASE_CSV_DIRECTORY, + strlen(AGE_BASE_CSV_DIRECTORY)) != 0) + { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("You can only load files located in [%s].", + AGE_BASE_CSV_DIRECTORY))); + } + + length = strlen(resolved) - 4; + if (strncmp(resolved+length, AGE_CSV_FILE_EXTENSION, + strlen(AGE_CSV_FILE_EXTENSION)) != 0) + { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("You can only load files with extension [%s].", + AGE_CSV_FILE_EXTENSION))); + } + + return resolved; +} agtype *create_empty_agtype(void) { @@ -344,7 +400,7 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) { Name graph_name; Name label_name; - text* file_path; + text* file_name; char* graph_name_str; char* label_name_str; char* file_path_str; @@ -373,7 +429,7 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) graph_name = PG_GETARG_NAME(0); label_name = PG_GETARG_NAME(1); - file_path = PG_GETARG_TEXT_P(2); + file_name = PG_GETARG_TEXT_P(2); id_field_exists = PG_GETARG_BOOL(3); load_as_agtype = PG_GETARG_BOOL(4); @@ -385,7 +441,7 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) label_name_str = AG_DEFAULT_LABEL_VERTEX; } - file_path_str = text_to_cstring(file_path); + file_path_str = build_safe_filename(text_to_cstring(file_name)); graph_oid = get_or_create_graph(graph_name); label_id = get_or_create_label(graph_oid, graph_name_str, @@ -394,6 +450,9 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) create_labels_from_csv_file(file_path_str, graph_name_str, graph_oid, label_name_str, label_id, id_field_exists, load_as_agtype); + + free(file_path_str); + PG_RETURN_VOID(); } @@ -403,7 +462,7 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) Name graph_name; Name label_name; - text* file_path; + text* file_name; char* graph_name_str; char* label_name_str; char* file_path_str; @@ -431,7 +490,7 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) graph_name = PG_GETARG_NAME(0); label_name = PG_GETARG_NAME(1); - file_path = PG_GETARG_TEXT_P(2); + file_name = PG_GETARG_TEXT_P(2); load_as_agtype = PG_GETARG_BOOL(3); graph_name_str = NameStr(*graph_name); @@ -442,7 +501,7 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) label_name_str = AG_DEFAULT_LABEL_EDGE; } - file_path_str = text_to_cstring(file_path); + file_path_str = build_safe_filename(text_to_cstring(file_name)); graph_oid = get_or_create_graph(graph_name); label_id = get_or_create_label(graph_oid, graph_name_str, @@ -450,6 +509,9 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) create_edges_from_csv_file(file_path_str, graph_name_str, graph_oid, label_name_str, label_id, load_as_agtype); + + free(file_path_str); + PG_RETURN_VOID(); } @@ -599,4 +661,4 @@ void finish_batch_insert(batch_insert_state **batch_state) pfree((*batch_state)->slots); pfree(*batch_state); *batch_state = NULL; -} \ No newline at end of file +} From dd6deb70e34e2ab75836dc6d037fde5a360d7075 Mon Sep 17 00:00:00 2001 From: Arthur Nascimento Date: Tue, 6 Jan 2026 13:36:49 -0300 Subject: [PATCH 011/100] Makefile: fix race condition on cypher_gram_def.h (#2273) The file cypher_gram.c generates cypher_gram_def.h, which is directly necessary for cypher_parser.o and cypher_keywords.o and their respective .bc files. But that direct dependency is not reflected in the Makefile, which only had the indirect dependency of .o on .c. So on high parallel builds, the .h may not have been generated by bison yet. Additionally, the .bc files should have the same dependencies as the .o files, but those are lacking. Here is an example output where the .bc file fails to build, as it was running concurrently with the bison instance that was about to finalize cypher_gram_def.h: In file included from src/backend/parser/cypher_parser.c:24: clang-17 -Wno-ignored-attributes -fno-strict-aliasing -fwrapv -fexcess-precision=standard -Wno-unused-command-line-argument -Wno-compound-token-split-by-macro -O2 -I.//src/include -I.//src/include/parser -I. -I./ -I/usr/pgsql-17/include/server -I/usr/pgsql-17/include/internal -D_GNU_SOURCE -I/usr/include -I/usr/include/libxml2 -flto=thin -emit-llvm -c -o src/backend/parser/cypher_parser.bc src/backend/parser/cypher_parser.c .//src/include/parser/cypher_gram.h:65:10: fatal error: 'parser/cypher_gram_def.h' file not found 65 | #include "parser/cypher_gram_def.h" | ^~~~~~~~~~~~~~~~~~~~~~~~~~ 1 error generated. make: *** [/usr/pgsql-17/lib/pgxs/src/makefiles/../../src/Makefile.global:1085: src/backend/parser/cypher_parser.bc] Error 1 make: *** Waiting for unfinished jobs.... gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Werror=vla -Wendif-labels -Wmissing-format-attribute -Wimplicit-fallthrough=3 -Wshadow=compatible-local -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -Wno-format-truncation -O2 -g -fmessage-length=0 -D_FORTIFY_SOURCE=2 -fstack-protector -funwind-tables -fasynchronous-unwind-tables -fPIC -fvisibility=hidden -I.//src/include -I.//src/include/parser -I. -I./ -I/usr/pgsql-17/include/server -I/usr/pgsql-17/include/internal -D_GNU_SOURCE -I/usr/include -I/usr/include/libxml2 -c -o src/backend/catalog/ag_label.o src/backend/catalog/ag_label.c /usr/bin/bison -Wno-deprecated --defines=src/include/parser/cypher_gram_def.h -o src/backend/parser/cypher_gram.c src/backend/parser/cypher_gram.y Previously, cypher_parser.o was missing the dependency, so it could start before cypher_gram_def.h was available: Considering target file 'src/backend/parser/cypher_parser.o'. File 'src/backend/parser/cypher_parser.o' does not exist. Considering target file 'src/backend/parser/cypher_parser.c'. File 'src/backend/parser/cypher_parser.c' was considered already. Considering target file 'src/backend/parser/cypher_gram.c'. File 'src/backend/parser/cypher_gram.c' was considered already. Finished prerequisites of target file 'src/backend/parser/cypher_parser.o'. Must remake target 'src/backend/parser/cypher_parser.o'. As well as cypher_parser.bc, missing the dependency on cypher_gram_def.h: Considering target file 'src/backend/parser/cypher_parser.bc'. File 'src/backend/parser/cypher_parser.bc' does not exist. Considering target file 'src/backend/parser/cypher_parser.c'. File 'src/backend/parser/cypher_parser.c' was considered already. Finished prerequisites of target file 'src/backend/parser/cypher_parser.bc'. Must remake target 'src/backend/parser/cypher_parser.bc'. Now cypher_parser.o correctly depends on cypher_gram_def.h: Considering target file 'src/backend/parser/cypher_parser.o'. File 'src/backend/parser/cypher_parser.o' does not exist. Considering target file 'src/backend/parser/cypher_parser.c'. File 'src/backend/parser/cypher_parser.c' was considered already. Considering target file 'src/backend/parser/cypher_gram.c'. File 'src/backend/parser/cypher_gram.c' was considered already. Considering target file 'src/include/parser/cypher_gram_def.h'. File 'src/include/parser/cypher_gram_def.h' was considered already. Finished prerequisites of target file 'src/backend/parser/cypher_parser.o'. Must remake target 'src/backend/parser/cypher_parser.o'. And cypher_parser.bc correctly depends on cypher_gram_def.h as well: Considering target file 'src/backend/parser/cypher_parser.bc'. File 'src/backend/parser/cypher_parser.bc' does not exist. Considering target file 'src/backend/parser/cypher_parser.c'. File 'src/backend/parser/cypher_parser.c' was considered already. Considering target file 'src/backend/parser/cypher_gram.c'. File 'src/backend/parser/cypher_gram.c' was considered already. Considering target file 'src/include/parser/cypher_gram_def.h'. File 'src/include/parser/cypher_gram_def.h' was considered already. Finished prerequisites of target file 'src/backend/parser/cypher_parser.bc'. Must remake target 'src/backend/parser/cypher_parser.bc'. --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3e73f3e68..0392d8b85 100644 --- a/Makefile +++ b/Makefile @@ -147,8 +147,10 @@ src/include/parser/cypher_gram_def.h: src/backend/parser/cypher_gram.c src/backend/parser/cypher_gram.c: BISONFLAGS += --defines=src/include/parser/cypher_gram_def.h -src/backend/parser/cypher_parser.o: src/backend/parser/cypher_gram.c -src/backend/parser/cypher_keywords.o: src/backend/parser/cypher_gram.c +src/backend/parser/cypher_parser.o: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h +src/backend/parser/cypher_parser.bc: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h +src/backend/parser/cypher_keywords.o: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h +src/backend/parser/cypher_keywords.bc: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h $(age_sql): @cat $(SQLS) > $@ From 4eeceab256f4795e6e8fb1d802935bdf403e5e93 Mon Sep 17 00:00:00 2001 From: M15terHyde <59905806+M15terHyde@users.noreply.github.com> Date: Tue, 6 Jan 2026 12:52:33 -0600 Subject: [PATCH 012/100] Revise README for Python driver updates (#2298) Updated README to from psycopg2 to psycopg3 (psycopg) --- drivers/python/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/python/README.md b/drivers/python/README.md index 749b44bfb..e64f9de67 100644 --- a/drivers/python/README.md +++ b/drivers/python/README.md @@ -28,11 +28,11 @@ AGType parser and driver support for [Apache AGE](https://age.apache.org/), grap ### Features * Unmarshal AGE result data(AGType) to Vertex, Edge, Path -* Cypher query support for Psycopg2 PostgreSQL driver (enables to use cypher queries directly) +* Cypher query support for Psycopg3 PostgreSQL driver (enables to use cypher queries directly) ### Prerequisites * over Python 3.9 -* This module runs on [psycopg2](https://www.psycopg.org/) and [antlr4-python3](https://pypi.org/project/antlr4-python3-runtime/) +* This module runs on [psycopg3](https://www.psycopg.org/) and [antlr4-python3](https://pypi.org/project/antlr4-python3-runtime/) ``` sudo apt-get update sudo apt-get install python3-dev libpq-dev @@ -80,7 +80,7 @@ SET search_path = ag_catalog, "$user", public; ``` ### Usage -* If you are not familiar with Psycopg2 driver : Go to [Jupyter Notebook : Basic Sample](samples/apache-age-basic.ipynb) +* If you are not familiar with Psycopg driver : Go to [Jupyter Notebook : Basic Sample](samples/apache-age-basic.ipynb) * Simpler way to access Apache AGE [AGE Sample](samples/apache-age-note.ipynb) in Samples. * Agtype converting samples: [Agtype Sample](samples/apache-age-agtypes.ipynb) in Samples. @@ -119,7 +119,7 @@ Here the following value required Insert From networkx directed graph into an AGE database. #### Parameters -- `connection` (psycopg2.connect): Connection object to the AGE database. +- `connection` (psycopg.connect): Connection object to the AGE database. - `G` (networkx.DiGraph): Networkx directed graph to be converted and inserted. @@ -152,7 +152,7 @@ Converts data from a Apache AGE graph database into a Networkx directed graph. #### Parameters -- `connection` (psycopg2.connect): Connection object to the PostgreSQL database. +- `connection` (psycopg.connect): Connection object to the PostgreSQL database. - `graphName` (str): Name of the graph. - `G` (None | nx.DiGraph): Optional Networkx directed graph. If provided, the data will be added to this graph. - `query` (str | None): Optional Cypher query to retrieve data from the database. @@ -167,3 +167,4 @@ Converts data from a Apache AGE graph database into a Networkx directed graph. # Call the function to convert data into a Networkx graph graph = age_to_networkx(connection, graphName="MyGraph" ) ``` + From 2e8f7ab992fc5db6cedf88cbdffc15df3a3cf932 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Fri, 9 Jan 2026 12:27:47 -0800 Subject: [PATCH 013/100] Fix Issue 2289: handle empty list in IN expression (#2294) NOTE: This PR was created with AI tools and a human. When evaluating 'x IN []' with an empty list, the transform_AEXPR_IN function would return NULL because no expressions were processed. This caused a 'cache lookup failed for type 0' error downstream. This fix adds an early check for the empty list case: - 'x IN []' returns false (nothing can be in an empty list) Additional NOTE: Cypher does not have 'NOT IN' syntax. To check if a value is NOT in a list, use 'NOT (x IN list)'. The NOT operator will invert the false from an empty list to true as expected. The fix returns a boolean constant directly, avoiding the NULL result that caused the type lookup failure. Added regression tests. modified: regress/expected/expr.out modified: regress/sql/expr.sql modified: src/backend/parser/cypher_expr.c --- regress/expected/expr.out | 72 ++++++++++++++++++++++++++++++++ regress/sql/expr.sql | 23 ++++++++++ src/backend/parser/cypher_expr.c | 30 ++++++++++++- 3 files changed, 123 insertions(+), 2 deletions(-) diff --git a/regress/expected/expr.out b/regress/expected/expr.out index 926a958d6..6d9341451 100644 --- a/regress/expected/expr.out +++ b/regress/expected/expr.out @@ -319,6 +319,50 @@ $$RETURN 1 IN [[null]]$$) AS r(c boolean); f (1 row) +-- empty list: x IN [] should always return false +SELECT * FROM cypher('expr', +$$RETURN 1 IN []$$) AS r(c boolean); + c +--- + f +(1 row) + +SELECT * FROM cypher('expr', +$$RETURN 'a' IN []$$) AS r(c boolean); + c +--- + f +(1 row) + +SELECT * FROM cypher('expr', +$$RETURN null IN []$$) AS r(c boolean); + c +--- + f +(1 row) + +SELECT * FROM cypher('expr', +$$RETURN [1,2,3] IN []$$) AS r(c boolean); + c +--- + f +(1 row) + +-- NOT (x IN []) should always return true +SELECT * FROM cypher('expr', +$$RETURN NOT (1 IN [])$$) AS r(c boolean); + c +--- + t +(1 row) + +SELECT * FROM cypher('expr', +$$RETURN NOT ('a' IN [])$$) AS r(c boolean); + c +--- + t +(1 row) + -- should error - ERROR: object of IN must be a list SELECT * FROM cypher('expr', $$RETURN null IN 'str' $$) AS r(c boolean); @@ -9155,9 +9199,37 @@ ERROR: could not find rte for x LINE 2: ...({ a0:COUNT { MATCH () WHERE CASE WHEN true THEN (x IS NULL)... ^ HINT: variable x does not exist within scope of usage +-- +-- Issue 2289: 1 IN [] causes cache lookup failed for type 0 +-- +-- Additional test cases were added above to the IN operator +-- +SELECT * FROM create_graph('issue_2289'); +NOTICE: graph "issue_2289" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('issue_2289', $$ RETURN (1 IN []) AS v $$) AS (v agtype); + v +------- + false +(1 row) + -- -- Cleanup -- +SELECT * FROM drop_graph('issue_2289', true); +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table issue_2289._ag_label_vertex +drop cascades to table issue_2289._ag_label_edge +NOTICE: graph "issue_2289" has been dropped + drop_graph +------------ + +(1 row) + SELECT * FROM drop_graph('issue_2263', true); NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table issue_2263._ag_label_vertex diff --git a/regress/sql/expr.sql b/regress/sql/expr.sql index 7bf1f26b2..445e2d237 100644 --- a/regress/sql/expr.sql +++ b/regress/sql/expr.sql @@ -157,6 +157,20 @@ SELECT * FROM cypher('expr', $$RETURN 1 in [[1]]$$) AS r(c boolean); SELECT * FROM cypher('expr', $$RETURN 1 IN [[null]]$$) AS r(c boolean); +-- empty list: x IN [] should always return false +SELECT * FROM cypher('expr', +$$RETURN 1 IN []$$) AS r(c boolean); +SELECT * FROM cypher('expr', +$$RETURN 'a' IN []$$) AS r(c boolean); +SELECT * FROM cypher('expr', +$$RETURN null IN []$$) AS r(c boolean); +SELECT * FROM cypher('expr', +$$RETURN [1,2,3] IN []$$) AS r(c boolean); +-- NOT (x IN []) should always return true +SELECT * FROM cypher('expr', +$$RETURN NOT (1 IN [])$$) AS r(c boolean); +SELECT * FROM cypher('expr', +$$RETURN NOT ('a' IN [])$$) AS r(c boolean); -- should error - ERROR: object of IN must be a list SELECT * FROM cypher('expr', $$RETURN null IN 'str' $$) AS r(c boolean); @@ -3690,9 +3704,18 @@ SELECT * FROM cypher('issue_2263', $$ CREATE x = (), ({ a0:COUNT { MATCH () WHERE CASE WHEN true THEN (x IS NULL) END RETURN 0 } }) $$) AS (out agtype); +-- +-- Issue 2289: 1 IN [] causes cache lookup failed for type 0 +-- +-- Additional test cases were added above to the IN operator +-- +SELECT * FROM create_graph('issue_2289'); +SELECT * FROM cypher('issue_2289', $$ RETURN (1 IN []) AS v $$) AS (v agtype); + -- -- Cleanup -- +SELECT * FROM drop_graph('issue_2289', true); SELECT * FROM drop_graph('issue_2263', true); SELECT * FROM drop_graph('issue_1988', true); SELECT * FROM drop_graph('issue_1953', true); diff --git a/src/backend/parser/cypher_expr.c b/src/backend/parser/cypher_expr.c index 5f4de86b9..fc0335def 100644 --- a/src/backend/parser/cypher_expr.c +++ b/src/backend/parser/cypher_expr.c @@ -600,6 +600,34 @@ static Node *transform_AEXPR_IN(cypher_parsestate *cpstate, A_Expr *a) Assert(is_ag_node(a->rexpr, cypher_list)); + rexpr = (cypher_list *)a->rexpr; + + /* + * Handle empty list case: x IN [] is always false, x NOT IN [] is always true. + * We need to check this before processing to avoid returning NULL result + * which causes "cache lookup failed for type 0" error. + */ + if (rexpr->elems == NIL || list_length((List *)rexpr->elems) == 0) + { + Datum bool_value; + Const *const_result; + + /* If operator is <> (NOT IN), result is true; otherwise (IN) result is false */ + if (strcmp(strVal(linitial(a->name)), "<>") == 0) + { + bool_value = BoolGetDatum(true); + } + else + { + bool_value = BoolGetDatum(false); + } + + const_result = makeConst(BOOLOID, -1, InvalidOid, sizeof(bool), + bool_value, false, true); + + return (Node *)const_result; + } + /* If the operator is <>, combine with AND not OR. */ if (strcmp(strVal(linitial(a->name)), "<>") == 0) { @@ -614,8 +642,6 @@ static Node *transform_AEXPR_IN(cypher_parsestate *cpstate, A_Expr *a) rexprs = rvars = rnonvars = NIL; - rexpr = (cypher_list *)a->rexpr; - foreach(l, (List *) rexpr->elems) { Node *rexpr = transform_cypher_expr_recurse(cpstate, lfirst(l)); From 7beb653303529b05391667de4204ffb4da318eeb Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Fri, 9 Jan 2026 12:55:36 -0800 Subject: [PATCH 014/100] Fix and improve index.sql regression test coverage (#2300) NOTE: This PR was created with AI tools and a human. - Remove unused copy command (leftover from deleted agload_test_graph test) - Replace broken Section 4 that referenced non-existent graph with comprehensive WHERE clause tests covering string, int, bool, and float properties with AND/OR/NOT operators - Add EXPLAIN tests to verify index usage: - Section 3: Validate GIN indices (load_city_gin_idx, load_country_gin_idx) show Bitmap Index Scan for property matching - Section 4: Validate all expression indices (city_country_code_idx, city_id_idx, city_west_coast_idx, country_life_exp_idx) show Index Scan for WHERE clause filtering All indices now have EXPLAIN verification confirming they are used as expected. modified: regress/expected/index.out modified: regress/sql/index.sql --- regress/expected/index.out | 290 ++++++++++++++++++++++++++++++++++--- regress/sql/index.sql | 174 ++++++++++++++++++++-- 2 files changed, 436 insertions(+), 28 deletions(-) diff --git a/regress/expected/index.out b/regress/expected/index.out index 3ed7b1c33..9faead660 100644 --- a/regress/expected/index.out +++ b/regress/expected/index.out @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -\! cp -r regress/age_load/data regress/instance/data/age_load LOAD 'age'; SET search_path TO ag_catalog; SET enable_mergejoin = ON; @@ -385,6 +384,19 @@ CREATE INDEX load_city_gin_idx ON cypher_index."City" USING gin (properties); CREATE INDEX load_country_gin_idx ON cypher_index."Country" USING gin (properties); +-- Verify GIN index is used for City property match +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (c:City {city_id: 1}) + RETURN c +$$) as (plan agtype); + QUERY PLAN +-------------------------------------------------------------- + Bitmap Heap Scan on "City" c + Recheck Cond: (properties @> '{"city_id": 1}'::agtype) + -> Bitmap Index Scan on load_city_gin_idx + Index Cond: (properties @> '{"city_id": 1}'::agtype) +(4 rows) + SELECT * FROM cypher('cypher_index', $$ MATCH (c:City {city_id: 1}) RETURN c @@ -418,6 +430,19 @@ $$) as (n agtype); {"id": 1970324836974597, "label": "City", "properties": {"name": "Vancouver", "city_id": 5, "west_coast": true, "country_code": "CA"}}::vertex (4 rows) +-- Verify GIN index is used for Country property match +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (c:Country {life_expectancy: 82.05}) + RETURN c +$$) as (plan agtype); + QUERY PLAN +-------------------------------------------------------------------------- + Bitmap Heap Scan on "Country" c + Recheck Cond: (properties @> '{"life_expectancy": 82.05}'::agtype) + -> Bitmap Index Scan on load_country_gin_idx + Index Cond: (properties @> '{"life_expectancy": 82.05}'::agtype) +(4 rows) + SELECT * FROM cypher('cypher_index', $$ MATCH (c:Country {life_expectancy: 82.05}) RETURN c @@ -441,26 +466,259 @@ DROP INDEX cypher_index.load_country_gin_idx; -- -- Section 4: Index use with WHERE clause -- -SELECT COUNT(*) FROM cypher('cypher_index', $$ +-- Create expression index on country_code property +CREATE INDEX city_country_code_idx ON cypher_index."City" +(ag_catalog.agtype_access_operator(properties, '"country_code"'::agtype)); +-- Verify index is used with EXPLAIN (should show Index Scan on city_country_code_idx) +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:City) + WHERE a.country_code = 'US' + RETURN a +$$) as (plan agtype); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Index Scan using city_country_code_idx on "City" a + Index Cond: (agtype_access_operator(VARIADIC ARRAY[properties, '"country_code"'::agtype]) = '"US"'::agtype) +(2 rows) + +-- Test WHERE with indexed string property +SELECT * FROM cypher('cypher_index', $$ MATCH (a:City) - WHERE a.country_code = 'RS' + WHERE a.country_code = 'US' + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + name +----------------- + "New York" + "San Fransisco" + "Los Angeles" + "Seattle" +(4 rows) + +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.country_code = 'CA' + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + name +------------- + "Vancouver" + "Toronto" + "Montreal" +(3 rows) + +-- Test WHERE with no matching results +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.country_code = 'XX' + RETURN a.name +$$) as (name agtype); + name +------ +(0 rows) + +-- Create expression index on city_id property +CREATE INDEX city_id_idx ON cypher_index."City" +(ag_catalog.agtype_access_operator(properties, '"city_id"'::agtype)); +-- Verify index is used with EXPLAIN for integer property +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:City) + WHERE a.city_id = 1 RETURN a -$$) as (n agtype); - count -------- - 0 +$$) as (plan agtype); + QUERY PLAN +------------------------------------------------------------------------------------------------------- + Index Scan using city_id_idx on "City" a + Index Cond: (agtype_access_operator(VARIADIC ARRAY[properties, '"city_id"'::agtype]) = '1'::agtype) +(2 rows) + +-- Test WHERE with indexed integer property +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.city_id = 1 + RETURN a.name +$$) as (name agtype); + name +------------ + "New York" (1 row) -CREATE INDEX CONCURRENTLY cntry_ode_idx ON cypher_index."City" -(ag_catalog.agtype_access_operator(properties, '"country_code"'::agtype)); -SELECT COUNT(*) FROM cypher('agload_test_graph', $$ +SELECT * FROM cypher('cypher_index', $$ MATCH (a:City) - WHERE a.country_code = 'RS' + WHERE a.city_id = 5 + RETURN a.name +$$) as (name agtype); + name +------------- + "Vancouver" +(1 row) + +-- Test WHERE with comparison operators on indexed property +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.city_id < 3 + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + name +----------------- + "New York" + "San Fransisco" +(2 rows) + +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.city_id >= 8 + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + name +------------- + "Monterrey" + "Tijuana" +(2 rows) + +-- Create expression index on west_coast boolean property +CREATE INDEX city_west_coast_idx ON cypher_index."City" +(ag_catalog.agtype_access_operator(properties, '"west_coast"'::agtype)); +-- Verify index is used with EXPLAIN for boolean property +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:City) + WHERE a.west_coast = true RETURN a -$$) as (n agtype); -ERROR: graph "agload_test_graph" does not exist -LINE 1: SELECT COUNT(*) FROM cypher('agload_test_graph', $$ - ^ +$$) as (plan agtype); + QUERY PLAN +------------------------------------------------------------------------------------------------------------- + Index Scan using city_west_coast_idx on "City" a + Index Cond: (agtype_access_operator(VARIADIC ARRAY[properties, '"west_coast"'::agtype]) = 'true'::agtype) +(2 rows) + +-- Test WHERE with indexed boolean property +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.west_coast = true + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + name +----------------- + "San Fransisco" + "Los Angeles" + "Seattle" + "Vancouver" +(4 rows) + +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.west_coast = false + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + name +--------------- + "New York" + "Toronto" + "Montreal" + "Mexico City" + "Monterrey" + "Tijuana" +(6 rows) + +-- Test WHERE with multiple conditions (AND) +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.country_code = 'US' AND a.west_coast = true + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + name +----------------- + "San Fransisco" + "Los Angeles" + "Seattle" +(3 rows) + +-- Test WHERE with OR conditions +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.city_id = 1 OR a.city_id = 5 + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + name +------------- + "New York" + "Vancouver" +(2 rows) + +-- Test WHERE with NOT +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE NOT a.west_coast = true AND a.country_code = 'US' + RETURN a.name +$$) as (name agtype); + name +------------ + "New York" +(1 row) + +-- Create expression index on life_expectancy for Country +CREATE INDEX country_life_exp_idx ON cypher_index."Country" +(ag_catalog.agtype_access_operator(properties, '"life_expectancy"'::agtype)); +-- Verify index is used with EXPLAIN for float property +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (c:Country) + WHERE c.life_expectancy > 80.0 + RETURN c +$$) as (plan agtype); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------ + Index Scan using country_life_exp_idx on "Country" c + Index Cond: (agtype_access_operator(VARIADIC ARRAY[properties, '"life_expectancy"'::agtype]) > '80.0'::agtype) +(2 rows) + +-- Test WHERE with float property +SELECT * FROM cypher('cypher_index', $$ + MATCH (c:Country) + WHERE c.life_expectancy > 80.0 + RETURN c.name +$$) as (name agtype); + name +---------- + "Canada" +(1 row) + +SELECT * FROM cypher('cypher_index', $$ + MATCH (c:Country) + WHERE c.life_expectancy < 76.0 + RETURN c.name +$$) as (name agtype); + name +---------- + "Mexico" +(1 row) + +-- Test WHERE in combination with pattern matching +SELECT * FROM cypher('cypher_index', $$ + MATCH (country:Country)<-[:has_city]-(city:City) + WHERE country.country_code = 'CA' + RETURN city.name + ORDER BY city.city_id +$$) as (name agtype); + name +------------- + "Vancouver" + "Toronto" + "Montreal" +(3 rows) + +-- Clean up indices +DROP INDEX cypher_index.city_country_code_idx; +DROP INDEX cypher_index.city_id_idx; +DROP INDEX cypher_index.city_west_coast_idx; +DROP INDEX cypher_index.country_life_exp_idx; -- -- General Cleanup -- @@ -478,5 +736,3 @@ NOTICE: graph "cypher_index" has been dropped (1 row) -SELECT drop_graph('agload_test_graph', true); -ERROR: graph "agload_test_graph" does not exist diff --git a/regress/sql/index.sql b/regress/sql/index.sql index d9a4331a4..96e7dd81a 100644 --- a/regress/sql/index.sql +++ b/regress/sql/index.sql @@ -17,8 +17,6 @@ * under the License. */ -\! cp -r regress/age_load/data regress/instance/data/age_load - LOAD 'age'; SET search_path TO ag_catalog; @@ -219,6 +217,11 @@ ON cypher_index."City" USING gin (properties); CREATE INDEX load_country_gin_idx ON cypher_index."Country" USING gin (properties); +-- Verify GIN index is used for City property match +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (c:City {city_id: 1}) + RETURN c +$$) as (plan agtype); SELECT * FROM cypher('cypher_index', $$ MATCH (c:City {city_id: 1}) @@ -235,6 +238,12 @@ SELECT * FROM cypher('cypher_index', $$ RETURN c $$) as (n agtype); +-- Verify GIN index is used for Country property match +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (c:Country {life_expectancy: 82.05}) + RETURN c +$$) as (plan agtype); + SELECT * FROM cypher('cypher_index', $$ MATCH (c:Country {life_expectancy: 82.05}) RETURN c @@ -250,23 +259,166 @@ DROP INDEX cypher_index.load_country_gin_idx; -- -- Section 4: Index use with WHERE clause -- -SELECT COUNT(*) FROM cypher('cypher_index', $$ +-- Create expression index on country_code property +CREATE INDEX city_country_code_idx ON cypher_index."City" +(ag_catalog.agtype_access_operator(properties, '"country_code"'::agtype)); + +-- Verify index is used with EXPLAIN (should show Index Scan on city_country_code_idx) +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:City) + WHERE a.country_code = 'US' + RETURN a +$$) as (plan agtype); + +-- Test WHERE with indexed string property +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.country_code = 'US' + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + +SELECT * FROM cypher('cypher_index', $$ MATCH (a:City) - WHERE a.country_code = 'RS' + WHERE a.country_code = 'CA' + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + +-- Test WHERE with no matching results +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.country_code = 'XX' + RETURN a.name +$$) as (name agtype); + +-- Create expression index on city_id property +CREATE INDEX city_id_idx ON cypher_index."City" +(ag_catalog.agtype_access_operator(properties, '"city_id"'::agtype)); + +-- Verify index is used with EXPLAIN for integer property +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:City) + WHERE a.city_id = 1 RETURN a -$$) as (n agtype); +$$) as (plan agtype); -CREATE INDEX CONCURRENTLY cntry_ode_idx ON cypher_index."City" -(ag_catalog.agtype_access_operator(properties, '"country_code"'::agtype)); +-- Test WHERE with indexed integer property +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.city_id = 1 + RETURN a.name +$$) as (name agtype); + +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.city_id = 5 + RETURN a.name +$$) as (name agtype); + +-- Test WHERE with comparison operators on indexed property +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.city_id < 3 + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); -SELECT COUNT(*) FROM cypher('agload_test_graph', $$ +SELECT * FROM cypher('cypher_index', $$ MATCH (a:City) - WHERE a.country_code = 'RS' + WHERE a.city_id >= 8 + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + +-- Create expression index on west_coast boolean property +CREATE INDEX city_west_coast_idx ON cypher_index."City" +(ag_catalog.agtype_access_operator(properties, '"west_coast"'::agtype)); + +-- Verify index is used with EXPLAIN for boolean property +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:City) + WHERE a.west_coast = true RETURN a -$$) as (n agtype); +$$) as (plan agtype); + +-- Test WHERE with indexed boolean property +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.west_coast = true + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.west_coast = false + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + +-- Test WHERE with multiple conditions (AND) +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.country_code = 'US' AND a.west_coast = true + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + +-- Test WHERE with OR conditions +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE a.city_id = 1 OR a.city_id = 5 + RETURN a.name + ORDER BY a.city_id +$$) as (name agtype); + +-- Test WHERE with NOT +SELECT * FROM cypher('cypher_index', $$ + MATCH (a:City) + WHERE NOT a.west_coast = true AND a.country_code = 'US' + RETURN a.name +$$) as (name agtype); + +-- Create expression index on life_expectancy for Country +CREATE INDEX country_life_exp_idx ON cypher_index."Country" +(ag_catalog.agtype_access_operator(properties, '"life_expectancy"'::agtype)); + +-- Verify index is used with EXPLAIN for float property +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (c:Country) + WHERE c.life_expectancy > 80.0 + RETURN c +$$) as (plan agtype); + +-- Test WHERE with float property +SELECT * FROM cypher('cypher_index', $$ + MATCH (c:Country) + WHERE c.life_expectancy > 80.0 + RETURN c.name +$$) as (name agtype); + +SELECT * FROM cypher('cypher_index', $$ + MATCH (c:Country) + WHERE c.life_expectancy < 76.0 + RETURN c.name +$$) as (name agtype); + +-- Test WHERE in combination with pattern matching +SELECT * FROM cypher('cypher_index', $$ + MATCH (country:Country)<-[:has_city]-(city:City) + WHERE country.country_code = 'CA' + RETURN city.name + ORDER BY city.city_id +$$) as (name agtype); + +-- Clean up indices +DROP INDEX cypher_index.city_country_code_idx; +DROP INDEX cypher_index.city_id_idx; +DROP INDEX cypher_index.city_west_coast_idx; +DROP INDEX cypher_index.country_life_exp_idx; -- -- General Cleanup -- SELECT drop_graph('cypher_index', true); -SELECT drop_graph('agload_test_graph', true); From a1f472d6f9344dc4449ac7343bf7d81a31b66f02 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Sat, 10 Jan 2026 08:37:54 -0800 Subject: [PATCH 015/100] Fix and improve index.sql addendum (#2301) NOTE: This PR was created with the help of AI tools and a human. Added additional requested regression tests - *EXPLAIN for pattern with WHERE clause *EXPLAIN for pattern with filters on both country and city modified: regress/expected/index.out modified: regress/sql/index.sql --- regress/expected/index.out | 34 ++++++++++++++++++++++++++++++++++ regress/sql/index.sql | 14 ++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/regress/expected/index.out b/regress/expected/index.out index 9faead660..745cab269 100644 --- a/regress/expected/index.out +++ b/regress/expected/index.out @@ -626,6 +626,19 @@ $$) as (name agtype); "Tijuana" (6 rows) +-- EXPLAIN for pattern with WHERE clause +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:City) + WHERE a.country_code = 'US' AND a.west_coast = true + RETURN a +$$) as (plan agtype); + QUERY PLAN +------------------------------------------------------------------------------------------------------------- + Index Scan using city_west_coast_idx on "City" a + Index Cond: (agtype_access_operator(VARIADIC ARRAY[properties, '"west_coast"'::agtype]) = 'true'::agtype) + Filter: (agtype_access_operator(VARIADIC ARRAY[properties, '"country_code"'::agtype]) = '"US"'::agtype) +(3 rows) + -- Test WHERE with multiple conditions (AND) SELECT * FROM cypher('cypher_index', $$ MATCH (a:City) @@ -700,6 +713,27 @@ $$) as (name agtype); "Mexico" (1 row) +-- EXPLAIN for pattern with filters on both country and city +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (country:Country)<-[:has_city]-(city:City) + WHERE country.country_code = 'CA' AND city.west_coast = true + RETURN city.name +$$) as (plan agtype); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------- + Nested Loop + -> Nested Loop + -> Index Scan using city_west_coast_idx on "City" city + Index Cond: (agtype_access_operator(VARIADIC ARRAY[properties, '"west_coast"'::agtype]) = 'true'::agtype) + -> Bitmap Heap Scan on has_city _age_default_alias_0 + Recheck Cond: (start_id = city.id) + -> Bitmap Index Scan on has_city_start_id_idx + Index Cond: (start_id = city.id) + -> Index Scan using "Country_pkey" on "Country" country + Index Cond: (id = _age_default_alias_0.end_id) + Filter: (agtype_access_operator(VARIADIC ARRAY[properties, '"country_code"'::agtype]) = '"CA"'::agtype) +(11 rows) + -- Test WHERE in combination with pattern matching SELECT * FROM cypher('cypher_index', $$ MATCH (country:Country)<-[:has_city]-(city:City) diff --git a/regress/sql/index.sql b/regress/sql/index.sql index 96e7dd81a..a6e075c70 100644 --- a/regress/sql/index.sql +++ b/regress/sql/index.sql @@ -357,6 +357,13 @@ SELECT * FROM cypher('cypher_index', $$ ORDER BY a.city_id $$) as (name agtype); +-- EXPLAIN for pattern with WHERE clause +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (a:City) + WHERE a.country_code = 'US' AND a.west_coast = true + RETURN a +$$) as (plan agtype); + -- Test WHERE with multiple conditions (AND) SELECT * FROM cypher('cypher_index', $$ MATCH (a:City) @@ -404,6 +411,13 @@ SELECT * FROM cypher('cypher_index', $$ RETURN c.name $$) as (name agtype); +-- EXPLAIN for pattern with filters on both country and city +SELECT * FROM cypher('cypher_index', $$ + EXPLAIN (costs off) MATCH (country:Country)<-[:has_city]-(city:City) + WHERE country.country_code = 'CA' AND city.west_coast = true + RETURN city.name +$$) as (plan agtype); + -- Test WHERE in combination with pattern matching SELECT * FROM cypher('cypher_index', $$ MATCH (country:Country)<-[:has_city]-(city:City) From c979380e9865a624ad73eef01ec84717bb817f2b Mon Sep 17 00:00:00 2001 From: Jean-Paul Abbuehl Date: Mon, 12 Jan 2026 21:10:08 +0100 Subject: [PATCH 016/100] feat: Add 32-bit platform support for graphid type (#2286) * feat: Add 32-bit platform support for graphid type This enables AGE to work on 32-bit platforms including WebAssembly (WASM). Problem: - graphid is int64 (8 bytes) with PASSEDBYVALUE - On 32-bit systems, Datum is only 4 bytes - PostgreSQL rejects pass-by-value types larger than Datum Solution: - Makefile-only change (no C code modifications) - When SIZEOF_DATUM=4 is passed to make, strip PASSEDBYVALUE from the generated SQL - If not specified, normal 64-bit behavior is preserved (PASSEDBYVALUE kept) This change is backward compatible: - 64-bit systems continue using pass-by-value - 32-bit systems now work with pass-by-reference Motivation: PGlite (PostgreSQL compiled to WebAssembly) uses 32-bit pointers and requires this patch to run AGE. Tested on: - 64-bit Linux (regression tests pass) - 32-bit WebAssembly via PGlite (all operations work) Co-authored-by: abbuehlj --- Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0392d8b85..17f2ed653 100644 --- a/Makefile +++ b/Makefile @@ -138,6 +138,10 @@ PG_CONFIG ?= pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) +# 32-bit platform support: pass SIZEOF_DATUM=4 to enable (e.g., make SIZEOF_DATUM=4) +# When SIZEOF_DATUM=4, PASSEDBYVALUE is stripped from graphid type for pass-by-reference. +# If not specified, normal 64-bit behavior is used (PASSEDBYVALUE preserved). + src/backend/parser/cypher_keywords.o: src/include/parser/cypher_kwlist_d.h src/include/parser/cypher_kwlist_d.h: src/include/parser/cypher_kwlist.h $(GEN_KEYWORDLIST_DEPS) @@ -152,8 +156,14 @@ src/backend/parser/cypher_parser.bc: src/backend/parser/cypher_gram.c src/includ src/backend/parser/cypher_keywords.o: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/backend/parser/cypher_keywords.bc: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h -$(age_sql): +# Strip PASSEDBYVALUE on 32-bit (SIZEOF_DATUM=4) for graphid pass-by-reference +$(age_sql): $(SQLS) @cat $(SQLS) > $@ +ifeq ($(SIZEOF_DATUM),4) + @echo "32-bit build: removing PASSEDBYVALUE from graphid type" + @sed 's/^ PASSEDBYVALUE,$$/ -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@ + @grep -q 'PASSEDBYVALUE removed for 32-bit' $@ || { echo "Error: PASSEDBYVALUE replacement failed in $@"; exit 1; } +endif src/backend/parser/ag_scanner.c: FLEX_NO_BACKUP=yes From b9d0982892306abff0013dd8f336e153684b02e9 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Fri, 16 Jan 2026 15:12:22 -0800 Subject: [PATCH 017/100] Optimize vertex/edge field access with direct array indexing (#2302) NOTE: This PR was created using AI tools and a human. Leverage deterministic key ordering from uniqueify_agtype_object() to access vertex/edge fields in O(1) instead of O(log n) binary search. Fields are sorted by key length, giving fixed positions: - Vertex: id(0), label(1), properties(2) - Edge: id(0), label(1), end_id(2), start_id(3), properties(4) Changes: - Add field index constants and accessor macros to agtype.h - Update age_id(), age_start_id(), age_end_id(), age_label(), age_properties() to use direct field access - Add fill_agtype_value_no_copy() for read-only scalar extraction without memory allocation - Add compare_agtype_scalar_containers() fast path for scalar comparison - Update hash_agtype_value(), equals_agtype_scalar_value(), and compare_agtype_scalar_values() to use direct field access macros - Add fast path in get_one_agtype_from_variadic_args() bypassing extract_variadic_args() for single argument case - Add comprehensive regression test (30 tests) Performance impact: Improves ORDER BY, hash joins, aggregations, and Cypher functions (id, start_id, end_id, label, properties) on vertices and edges. All previous regression tests were not impacted. Additional regression test added to enhance coverage. modified: Makefile new file: regress/expected/direct_field_access.out new file: regress/sql/direct_field_access.sql modified: src/backend/utils/adt/agtype.c modified: src/backend/utils/adt/agtype_util.c modified: src/include/utils/agtype.h --- Makefile | 3 +- regress/expected/direct_field_access.out | 535 +++++++++++++++++++++++ regress/sql/direct_field_access.sql | 319 ++++++++++++++ src/backend/utils/adt/agtype.c | 136 +++++- src/backend/utils/adt/agtype_util.c | 237 +++++++++- src/include/utils/agtype.h | 103 +++++ 6 files changed, 1304 insertions(+), 29 deletions(-) create mode 100644 regress/expected/direct_field_access.out create mode 100644 regress/sql/direct_field_access.sql diff --git a/Makefile b/Makefile index 17f2ed653..ffad7d6af 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,8 @@ REGRESS = scan \ name_validation \ jsonb_operators \ list_comprehension \ - map_projection + map_projection \ + direct_field_access ifneq ($(EXTRA_TESTS),) REGRESS += $(EXTRA_TESTS) diff --git a/regress/expected/direct_field_access.out b/regress/expected/direct_field_access.out new file mode 100644 index 000000000..0a059cdd9 --- /dev/null +++ b/regress/expected/direct_field_access.out @@ -0,0 +1,535 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Direct Field Access Optimizations Test + * + * Tests for optimizations that directly access agtype fields without + * using the full iterator machinery or binary search: + * + * 1. fill_agtype_value_no_copy() - Read-only access without memory allocation + * 2. compare_agtype_scalar_containers() - Fast path for scalar comparisons + * 3. Direct pairs[0] access for vertex/edge id comparison + * 4. Fast path in get_one_agtype_from_variadic_args() + */ +LOAD 'age'; +SET search_path TO ag_catalog; +SELECT create_graph('direct_access'); +NOTICE: graph "direct_access" has been created + create_graph +-------------- + +(1 row) + +-- +-- Section 1: Scalar Comparison Fast Path Tests +-- +-- These tests exercise the compare_agtype_scalar_containers() fast path +-- which uses fill_agtype_value_no_copy() for read-only comparisons. +-- +-- Integer comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN 1 < 2, 2 > 1, 1 = 1, 1 <> 2 +$$) AS (lt agtype, gt agtype, eq agtype, ne agtype); + lt | gt | eq | ne +------+------+------+------ + true | true | true | true +(1 row) + +SELECT * FROM cypher('direct_access', $$ + RETURN 100 < 50, 100 > 50, 100 = 100, 100 <> 100 +$$) AS (lt agtype, gt agtype, eq agtype, ne agtype); + lt | gt | eq | ne +-------+------+------+------- + false | true | true | false +(1 row) + +-- Float comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN 1.5 < 2.5, 2.5 > 1.5, 1.5 = 1.5, 1.5 <> 2.5 +$$) AS (lt agtype, gt agtype, eq agtype, ne agtype); + lt | gt | eq | ne +------+------+------+------ + true | true | true | true +(1 row) + +-- String comparisons (tests no-copy string pointer) +SELECT * FROM cypher('direct_access', $$ + RETURN 'abc' < 'abd', 'abd' > 'abc', 'abc' = 'abc', 'abc' <> 'abd' +$$) AS (lt agtype, gt agtype, eq agtype, ne agtype); + lt | gt | eq | ne +------+------+------+------ + true | true | true | true +(1 row) + +SELECT * FROM cypher('direct_access', $$ + RETURN 'hello world' < 'hello worlds', 'test' > 'TEST' +$$) AS (lt agtype, gt agtype); + lt | gt +------+------ + true | true +(1 row) + +-- Boolean comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN false < true, true > false, true = true, false <> true +$$) AS (lt agtype, gt agtype, eq agtype, ne agtype); + lt | gt | eq | ne +------+------+------+------ + true | true | true | true +(1 row) + +-- Null comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN null = null, null <> null +$$) AS (eq agtype, ne agtype); + eq | ne +----+---- + | +(1 row) + +-- Mixed numeric type comparisons (integer vs float) +SELECT * FROM cypher('direct_access', $$ + RETURN 1 < 1.5, 2.0 > 1, 1.0 = 1 +$$) AS (lt agtype, gt agtype, eq agtype); + lt | gt | eq +------+------+------ + true | true | true +(1 row) + +-- Numeric type comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN 1.234::numeric < 1.235::numeric, + 1.235::numeric > 1.234::numeric, + 1.234::numeric = 1.234::numeric +$$) AS (lt agtype, gt agtype, eq agtype); + lt | gt | eq +------+------+------ + true | true | true +(1 row) + +-- +-- Section 2: ORDER BY Tests (exercises comparison fast path) +-- +-- ORDER BY uses compare_agtype_containers_orderability which now has +-- a fast path for scalar comparisons. +-- +-- Integer ORDER BY +SELECT * FROM cypher('direct_access', $$ + UNWIND [5, 3, 8, 1, 9, 2, 7, 4, 6] AS n + RETURN n ORDER BY n +$$) AS (n agtype); + n +--- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 +(9 rows) + +SELECT * FROM cypher('direct_access', $$ + UNWIND [5, 3, 8, 1, 9, 2, 7, 4, 6] AS n + RETURN n ORDER BY n DESC +$$) AS (n agtype); + n +--- + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2 + 1 +(9 rows) + +-- String ORDER BY +SELECT * FROM cypher('direct_access', $$ + UNWIND ['banana', 'apple', 'cherry', 'date'] AS s + RETURN s ORDER BY s +$$) AS (s agtype); + s +---------- + "apple" + "banana" + "cherry" + "date" +(4 rows) + +-- Float ORDER BY +SELECT * FROM cypher('direct_access', $$ + UNWIND [3.14, 2.71, 1.41, 1.73] AS f + RETURN f ORDER BY f +$$) AS (f agtype); + f +------ + 1.41 + 1.73 + 2.71 + 3.14 +(4 rows) + +-- Boolean ORDER BY +SELECT * FROM cypher('direct_access', $$ + UNWIND [true, false, true, false] AS b + RETURN b ORDER BY b +$$) AS (b agtype); + b +------- + false + false + true + true +(4 rows) + +-- +-- Section 3: Vertex/Edge Direct ID Access Tests +-- +-- These tests exercise the direct pairs[0] access optimization for +-- extracting graphid from vertices and edges during comparison. +-- +-- Create test data +SELECT * FROM cypher('direct_access', $$ + CREATE (a:Person {name: 'Alice', age: 30}), + (b:Person {name: 'Bob', age: 25}), + (c:Person {name: 'Charlie', age: 35}), + (d:Person {name: 'Diana', age: 28}), + (e:Person {name: 'Eve', age: 32}), + (a)-[:KNOWS {since: 2020}]->(b), + (b)-[:KNOWS {since: 2019}]->(c), + (c)-[:KNOWS {since: 2021}]->(d), + (d)-[:KNOWS {since: 2018}]->(e), + (e)-[:KNOWS {since: 2022}]->(a) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- Test max() on vertices (uses compare_agtype_scalar_values with AGTV_VERTEX) +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person) + RETURN max(p) +$$) AS (max_vertex agtype); + max_vertex +---------------------------------------------------------------------------------------------- + {"id": 844424930131973, "label": "Person", "properties": {"age": 32, "name": "Eve"}}::vertex +(1 row) + +-- Test min() on vertices +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person) + RETURN min(p) +$$) AS (min_vertex agtype); + min_vertex +------------------------------------------------------------------------------------------------ + {"id": 844424930131969, "label": "Person", "properties": {"age": 30, "name": "Alice"}}::vertex +(1 row) + +-- Test max() on edges (uses compare_agtype_scalar_values with AGTV_EDGE) +SELECT * FROM cypher('direct_access', $$ + MATCH ()-[r:KNOWS]->() + RETURN max(r) +$$) AS (max_edge agtype); + max_edge +----------------------------------------------------------------------------------------------------------------------------------------- + {"id": 1125899906842629, "label": "KNOWS", "end_id": 844424930131969, "start_id": 844424930131973, "properties": {"since": 2022}}::edge +(1 row) + +-- Test min() on edges +SELECT * FROM cypher('direct_access', $$ + MATCH ()-[r:KNOWS]->() + RETURN min(r) +$$) AS (min_edge agtype); + min_edge +----------------------------------------------------------------------------------------------------------------------------------------- + {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {"since": 2020}}::edge +(1 row) + +-- ORDER BY on vertices (uses direct id comparison) +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person) + RETURN p.name ORDER BY p +$$) AS (name agtype); + name +----------- + "Alice" + "Bob" + "Charlie" + "Diana" + "Eve" +(5 rows) + +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person) + RETURN p.name ORDER BY p DESC +$$) AS (name agtype); + name +----------- + "Eve" + "Diana" + "Charlie" + "Bob" + "Alice" +(5 rows) + +-- ORDER BY on edges +SELECT * FROM cypher('direct_access', $$ + MATCH ()-[r:KNOWS]->() + RETURN r.since ORDER BY r +$$) AS (since agtype); + since +------- + 2020 + 2019 + 2021 + 2018 + 2022 +(5 rows) + +-- Vertex comparison in WHERE +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person), (b:Person) + WHERE a < b + RETURN a.name, b.name +$$) AS (a_name agtype, b_name agtype); + a_name | b_name +-----------+----------- + "Alice" | "Bob" + "Alice" | "Charlie" + "Alice" | "Diana" + "Alice" | "Eve" + "Bob" | "Charlie" + "Bob" | "Diana" + "Bob" | "Eve" + "Charlie" | "Diana" + "Charlie" | "Eve" + "Diana" | "Eve" +(10 rows) + +-- +-- Section 4: Fast Path for get_one_agtype_from_variadic_args +-- +-- These tests exercise the fast path that bypasses extract_variadic_args +-- when the argument is already agtype. +-- +-- Direct agtype comparison operators (use the fast path) +SELECT * FROM cypher('direct_access', $$ + RETURN 42 = 42, 42 <> 43, 42 < 100, 42 > 10 +$$) AS (eq agtype, ne agtype, lt agtype, gt agtype); + eq | ne | lt | gt +------+------+------+------ + true | true | true | true +(1 row) + +-- Arithmetic operators (also use the fast path) +SELECT * FROM cypher('direct_access', $$ + RETURN 10 + 5, 10 - 5, 10 * 5, 10 / 5 +$$) AS (add agtype, sub agtype, mul agtype, div agtype); + add | sub | mul | div +-----+-----+-----+----- + 15 | 5 | 50 | 2 +(1 row) + +-- String functions that take agtype args +SELECT * FROM cypher('direct_access', $$ + RETURN toUpper('hello'), toLower('WORLD'), size('test') +$$) AS (upper agtype, lower agtype, sz agtype); + upper | lower | sz +---------+---------+---- + "HELLO" | "world" | 4 +(1 row) + +-- Type checking functions +SELECT * FROM cypher('direct_access', $$ + RETURN toInteger('42'), toFloat('3.14'), toString(42) +$$) AS (int_val agtype, float_val agtype, str_val agtype); + int_val | float_val | str_val +---------+-----------+--------- + 42 | 3.14 | "42" +(1 row) + +-- +-- Section 5: Direct Field Access for Accessor Functions +-- +-- These tests exercise the direct field access macros in id(), start_id(), +-- end_id(), label(), and properties() functions. +-- +-- Test id() on vertices (uses AGTYPE_VERTEX_GET_ID macro - index 0) +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person {name: 'Alice'}) + RETURN id(p) +$$) AS (vertex_id agtype); + vertex_id +----------------- + 844424930131969 +(1 row) + +-- Test id() on edges (uses AGTYPE_EDGE_GET_ID macro - index 0) +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person {name: 'Alice'})-[r:KNOWS]->(b:Person {name: 'Bob'}) + RETURN id(r) +$$) AS (edge_id agtype); + edge_id +------------------ + 1125899906842625 +(1 row) + +-- Test start_id() on edges (uses AGTYPE_EDGE_GET_START_ID macro - index 3) +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person {name: 'Alice'})-[r:KNOWS]->(b:Person {name: 'Bob'}) + RETURN start_id(r), id(a) +$$) AS (start_id agtype, alice_id agtype); + start_id | alice_id +-----------------+----------------- + 844424930131969 | 844424930131969 +(1 row) + +-- Test end_id() on edges (uses AGTYPE_EDGE_GET_END_ID macro - index 2) +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person {name: 'Alice'})-[r:KNOWS]->(b:Person {name: 'Bob'}) + RETURN end_id(r), id(b) +$$) AS (end_id agtype, bob_id agtype); + end_id | bob_id +-----------------+----------------- + 844424930131970 | 844424930131970 +(1 row) + +-- Test label() on vertices (uses AGTYPE_VERTEX_GET_LABEL macro - index 1) +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person {name: 'Alice'}) + RETURN label(p) +$$) AS (vertex_label agtype); + vertex_label +-------------- + "Person" +(1 row) + +-- Test label() on edges (uses AGTYPE_EDGE_GET_LABEL macro - index 1) +SELECT * FROM cypher('direct_access', $$ + MATCH ()-[r:KNOWS]->() + RETURN DISTINCT label(r) +$$) AS (edge_label agtype); + edge_label +------------ + "KNOWS" +(1 row) + +-- Test properties() on vertices (uses AGTYPE_VERTEX_GET_PROPERTIES macro - index 2) +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person {name: 'Alice'}) + RETURN properties(p) +$$) AS (vertex_props agtype); + vertex_props +------------------------------ + {"age": 30, "name": "Alice"} +(1 row) + +-- Test properties() on edges (uses AGTYPE_EDGE_GET_PROPERTIES macro - index 4) +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person {name: 'Alice'})-[r:KNOWS]->(b:Person {name: 'Bob'}) + RETURN properties(r) +$$) AS (edge_props agtype); + edge_props +----------------- + {"since": 2020} +(1 row) + +-- Combined accessor test - verify all fields are accessible +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person {name: 'Alice'})-[r:KNOWS]->(b:Person) + RETURN id(a), label(a), properties(a).name, + id(r), start_id(r), end_id(r), label(r), properties(r).since, + id(b), label(b), properties(b).name +$$) AS (a_id agtype, a_label agtype, a_name agtype, + r_id agtype, r_start agtype, r_end agtype, r_label agtype, r_since agtype, + b_id agtype, b_label agtype, b_name agtype); + a_id | a_label | a_name | r_id | r_start | r_end | r_label | r_since | b_id | b_label | b_name +-----------------+----------+---------+------------------+-----------------+-----------------+---------+---------+-----------------+----------+-------- + 844424930131969 | "Person" | "Alice" | 1125899906842625 | 844424930131969 | 844424930131970 | "KNOWS" | 2020 | 844424930131970 | "Person" | "Bob" +(1 row) + +-- +-- Section 6: Mixed Comparisons and Edge Cases +-- +-- Array comparisons (should NOT use scalar fast path) +SELECT * FROM cypher('direct_access', $$ + RETURN [1,2,3] = [1,2,3], [1,2,3] < [1,2,4] +$$) AS (eq agtype, lt agtype); + eq | lt +------+------ + true | true +(1 row) + +-- Object comparisons (should NOT use scalar fast path) +SELECT * FROM cypher('direct_access', $$ + RETURN {a:1, b:2} = {a:1, b:2} +$$) AS (eq agtype); + eq +------ + true +(1 row) + +-- Large integer comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN 9223372036854775807 > 9223372036854775806, + -9223372036854775808 < -9223372036854775807 +$$) AS (big_gt agtype, neg_lt agtype); + big_gt | neg_lt +--------+-------- + true | true +(1 row) + +-- Empty string comparison +SELECT * FROM cypher('direct_access', $$ + RETURN '' < 'a', '' = '' +$$) AS (lt agtype, eq agtype); + lt | eq +------+------ + true | true +(1 row) + +-- Special float values +SELECT * FROM cypher('direct_access', $$ + RETURN 0.0 = -0.0 +$$) AS (zero_eq agtype); + zero_eq +--------- + true +(1 row) + +-- +-- Cleanup +-- +SELECT drop_graph('direct_access', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table direct_access._ag_label_vertex +drop cascades to table direct_access._ag_label_edge +drop cascades to table direct_access."Person" +drop cascades to table direct_access."KNOWS" +NOTICE: graph "direct_access" has been dropped + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/direct_field_access.sql b/regress/sql/direct_field_access.sql new file mode 100644 index 000000000..c8060be4a --- /dev/null +++ b/regress/sql/direct_field_access.sql @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Direct Field Access Optimizations Test + * + * Tests for optimizations that directly access agtype fields without + * using the full iterator machinery or binary search: + * + * 1. fill_agtype_value_no_copy() - Read-only access without memory allocation + * 2. compare_agtype_scalar_containers() - Fast path for scalar comparisons + * 3. Direct pairs[0] access for vertex/edge id comparison + * 4. Fast path in get_one_agtype_from_variadic_args() + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +SELECT create_graph('direct_access'); + +-- +-- Section 1: Scalar Comparison Fast Path Tests +-- +-- These tests exercise the compare_agtype_scalar_containers() fast path +-- which uses fill_agtype_value_no_copy() for read-only comparisons. +-- + +-- Integer comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN 1 < 2, 2 > 1, 1 = 1, 1 <> 2 +$$) AS (lt agtype, gt agtype, eq agtype, ne agtype); + +SELECT * FROM cypher('direct_access', $$ + RETURN 100 < 50, 100 > 50, 100 = 100, 100 <> 100 +$$) AS (lt agtype, gt agtype, eq agtype, ne agtype); + +-- Float comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN 1.5 < 2.5, 2.5 > 1.5, 1.5 = 1.5, 1.5 <> 2.5 +$$) AS (lt agtype, gt agtype, eq agtype, ne agtype); + +-- String comparisons (tests no-copy string pointer) +SELECT * FROM cypher('direct_access', $$ + RETURN 'abc' < 'abd', 'abd' > 'abc', 'abc' = 'abc', 'abc' <> 'abd' +$$) AS (lt agtype, gt agtype, eq agtype, ne agtype); + +SELECT * FROM cypher('direct_access', $$ + RETURN 'hello world' < 'hello worlds', 'test' > 'TEST' +$$) AS (lt agtype, gt agtype); + +-- Boolean comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN false < true, true > false, true = true, false <> true +$$) AS (lt agtype, gt agtype, eq agtype, ne agtype); + +-- Null comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN null = null, null <> null +$$) AS (eq agtype, ne agtype); + +-- Mixed numeric type comparisons (integer vs float) +SELECT * FROM cypher('direct_access', $$ + RETURN 1 < 1.5, 2.0 > 1, 1.0 = 1 +$$) AS (lt agtype, gt agtype, eq agtype); + +-- Numeric type comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN 1.234::numeric < 1.235::numeric, + 1.235::numeric > 1.234::numeric, + 1.234::numeric = 1.234::numeric +$$) AS (lt agtype, gt agtype, eq agtype); + +-- +-- Section 2: ORDER BY Tests (exercises comparison fast path) +-- +-- ORDER BY uses compare_agtype_containers_orderability which now has +-- a fast path for scalar comparisons. +-- + +-- Integer ORDER BY +SELECT * FROM cypher('direct_access', $$ + UNWIND [5, 3, 8, 1, 9, 2, 7, 4, 6] AS n + RETURN n ORDER BY n +$$) AS (n agtype); + +SELECT * FROM cypher('direct_access', $$ + UNWIND [5, 3, 8, 1, 9, 2, 7, 4, 6] AS n + RETURN n ORDER BY n DESC +$$) AS (n agtype); + +-- String ORDER BY +SELECT * FROM cypher('direct_access', $$ + UNWIND ['banana', 'apple', 'cherry', 'date'] AS s + RETURN s ORDER BY s +$$) AS (s agtype); + +-- Float ORDER BY +SELECT * FROM cypher('direct_access', $$ + UNWIND [3.14, 2.71, 1.41, 1.73] AS f + RETURN f ORDER BY f +$$) AS (f agtype); + +-- Boolean ORDER BY +SELECT * FROM cypher('direct_access', $$ + UNWIND [true, false, true, false] AS b + RETURN b ORDER BY b +$$) AS (b agtype); + +-- +-- Section 3: Vertex/Edge Direct ID Access Tests +-- +-- These tests exercise the direct pairs[0] access optimization for +-- extracting graphid from vertices and edges during comparison. +-- + +-- Create test data +SELECT * FROM cypher('direct_access', $$ + CREATE (a:Person {name: 'Alice', age: 30}), + (b:Person {name: 'Bob', age: 25}), + (c:Person {name: 'Charlie', age: 35}), + (d:Person {name: 'Diana', age: 28}), + (e:Person {name: 'Eve', age: 32}), + (a)-[:KNOWS {since: 2020}]->(b), + (b)-[:KNOWS {since: 2019}]->(c), + (c)-[:KNOWS {since: 2021}]->(d), + (d)-[:KNOWS {since: 2018}]->(e), + (e)-[:KNOWS {since: 2022}]->(a) +$$) AS (result agtype); + +-- Test max() on vertices (uses compare_agtype_scalar_values with AGTV_VERTEX) +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person) + RETURN max(p) +$$) AS (max_vertex agtype); + +-- Test min() on vertices +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person) + RETURN min(p) +$$) AS (min_vertex agtype); + +-- Test max() on edges (uses compare_agtype_scalar_values with AGTV_EDGE) +SELECT * FROM cypher('direct_access', $$ + MATCH ()-[r:KNOWS]->() + RETURN max(r) +$$) AS (max_edge agtype); + +-- Test min() on edges +SELECT * FROM cypher('direct_access', $$ + MATCH ()-[r:KNOWS]->() + RETURN min(r) +$$) AS (min_edge agtype); + +-- ORDER BY on vertices (uses direct id comparison) +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person) + RETURN p.name ORDER BY p +$$) AS (name agtype); + +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person) + RETURN p.name ORDER BY p DESC +$$) AS (name agtype); + +-- ORDER BY on edges +SELECT * FROM cypher('direct_access', $$ + MATCH ()-[r:KNOWS]->() + RETURN r.since ORDER BY r +$$) AS (since agtype); + +-- Vertex comparison in WHERE +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person), (b:Person) + WHERE a < b + RETURN a.name, b.name +$$) AS (a_name agtype, b_name agtype); + +-- +-- Section 4: Fast Path for get_one_agtype_from_variadic_args +-- +-- These tests exercise the fast path that bypasses extract_variadic_args +-- when the argument is already agtype. +-- + +-- Direct agtype comparison operators (use the fast path) +SELECT * FROM cypher('direct_access', $$ + RETURN 42 = 42, 42 <> 43, 42 < 100, 42 > 10 +$$) AS (eq agtype, ne agtype, lt agtype, gt agtype); + +-- Arithmetic operators (also use the fast path) +SELECT * FROM cypher('direct_access', $$ + RETURN 10 + 5, 10 - 5, 10 * 5, 10 / 5 +$$) AS (add agtype, sub agtype, mul agtype, div agtype); + +-- String functions that take agtype args +SELECT * FROM cypher('direct_access', $$ + RETURN toUpper('hello'), toLower('WORLD'), size('test') +$$) AS (upper agtype, lower agtype, sz agtype); + +-- Type checking functions +SELECT * FROM cypher('direct_access', $$ + RETURN toInteger('42'), toFloat('3.14'), toString(42) +$$) AS (int_val agtype, float_val agtype, str_val agtype); + +-- +-- Section 5: Direct Field Access for Accessor Functions +-- +-- These tests exercise the direct field access macros in id(), start_id(), +-- end_id(), label(), and properties() functions. +-- + +-- Test id() on vertices (uses AGTYPE_VERTEX_GET_ID macro - index 0) +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person {name: 'Alice'}) + RETURN id(p) +$$) AS (vertex_id agtype); + +-- Test id() on edges (uses AGTYPE_EDGE_GET_ID macro - index 0) +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person {name: 'Alice'})-[r:KNOWS]->(b:Person {name: 'Bob'}) + RETURN id(r) +$$) AS (edge_id agtype); + +-- Test start_id() on edges (uses AGTYPE_EDGE_GET_START_ID macro - index 3) +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person {name: 'Alice'})-[r:KNOWS]->(b:Person {name: 'Bob'}) + RETURN start_id(r), id(a) +$$) AS (start_id agtype, alice_id agtype); + +-- Test end_id() on edges (uses AGTYPE_EDGE_GET_END_ID macro - index 2) +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person {name: 'Alice'})-[r:KNOWS]->(b:Person {name: 'Bob'}) + RETURN end_id(r), id(b) +$$) AS (end_id agtype, bob_id agtype); + +-- Test label() on vertices (uses AGTYPE_VERTEX_GET_LABEL macro - index 1) +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person {name: 'Alice'}) + RETURN label(p) +$$) AS (vertex_label agtype); + +-- Test label() on edges (uses AGTYPE_EDGE_GET_LABEL macro - index 1) +SELECT * FROM cypher('direct_access', $$ + MATCH ()-[r:KNOWS]->() + RETURN DISTINCT label(r) +$$) AS (edge_label agtype); + +-- Test properties() on vertices (uses AGTYPE_VERTEX_GET_PROPERTIES macro - index 2) +SELECT * FROM cypher('direct_access', $$ + MATCH (p:Person {name: 'Alice'}) + RETURN properties(p) +$$) AS (vertex_props agtype); + +-- Test properties() on edges (uses AGTYPE_EDGE_GET_PROPERTIES macro - index 4) +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person {name: 'Alice'})-[r:KNOWS]->(b:Person {name: 'Bob'}) + RETURN properties(r) +$$) AS (edge_props agtype); + +-- Combined accessor test - verify all fields are accessible +SELECT * FROM cypher('direct_access', $$ + MATCH (a:Person {name: 'Alice'})-[r:KNOWS]->(b:Person) + RETURN id(a), label(a), properties(a).name, + id(r), start_id(r), end_id(r), label(r), properties(r).since, + id(b), label(b), properties(b).name +$$) AS (a_id agtype, a_label agtype, a_name agtype, + r_id agtype, r_start agtype, r_end agtype, r_label agtype, r_since agtype, + b_id agtype, b_label agtype, b_name agtype); + +-- +-- Section 6: Mixed Comparisons and Edge Cases +-- + +-- Array comparisons (should NOT use scalar fast path) +SELECT * FROM cypher('direct_access', $$ + RETURN [1,2,3] = [1,2,3], [1,2,3] < [1,2,4] +$$) AS (eq agtype, lt agtype); + +-- Object comparisons (should NOT use scalar fast path) +SELECT * FROM cypher('direct_access', $$ + RETURN {a:1, b:2} = {a:1, b:2} +$$) AS (eq agtype); + +-- Large integer comparisons +SELECT * FROM cypher('direct_access', $$ + RETURN 9223372036854775807 > 9223372036854775806, + -9223372036854775808 < -9223372036854775807 +$$) AS (big_gt agtype, neg_lt agtype); + +-- Empty string comparison +SELECT * FROM cypher('direct_access', $$ + RETURN '' < 'a', '' = '' +$$) AS (lt agtype, eq agtype); + +-- Special float values +SELECT * FROM cypher('direct_access', $$ + RETURN 0.0 = -0.0 +$$) AS (zero_eq agtype); + +-- +-- Cleanup +-- +SELECT drop_graph('direct_access', true); diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index 02fc3221c..f2458a30b 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -5409,10 +5409,24 @@ Datum age_id(PG_FUNCTION_ARGS) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("id() argument must be a vertex, an edge or null"))); - agtv_result = GET_AGTYPE_VALUE_OBJECT_VALUE(agtv_object, "id"); - - Assert(agtv_result != NULL); - Assert(agtv_result->type = AGTV_INTEGER); + /* + * Direct field access optimization: id is at a fixed index for both + * vertex and edge objects due to key length sorting. + */ + if (agtv_object->type == AGTV_VERTEX) + { + agtv_result = AGTYPE_VERTEX_GET_ID(agtv_object); + } + else if (agtv_object->type == AGTV_EDGE) + { + agtv_result = AGTYPE_EDGE_GET_ID(agtv_object); + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("id() unexpected argument type"))); + } PG_RETURN_POINTER(agtype_value_to_agtype(agtv_result)); } @@ -5447,10 +5461,11 @@ Datum age_start_id(PG_FUNCTION_ARGS) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("start_id() argument must be an edge or null"))); - agtv_result = GET_AGTYPE_VALUE_OBJECT_VALUE(agtv_object, "start_id"); - - Assert(agtv_result != NULL); - Assert(agtv_result->type = AGTV_INTEGER); + /* + * Direct field access optimization: start_id is at index 3 for edge + * objects due to key length sorting (id=0, label=1, end_id=2, start_id=3). + */ + agtv_result = AGTYPE_EDGE_GET_START_ID(agtv_object); PG_RETURN_POINTER(agtype_value_to_agtype(agtv_result)); } @@ -5485,10 +5500,11 @@ Datum age_end_id(PG_FUNCTION_ARGS) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("end_id() argument must be an edge or null"))); - agtv_result = GET_AGTYPE_VALUE_OBJECT_VALUE(agtv_object, "end_id"); - - Assert(agtv_result != NULL); - Assert(agtv_result->type = AGTV_INTEGER); + /* + * Direct field access optimization: end_id is at index 2 for edge + * objects due to key length sorting (id=0, label=1, end_id=2). + */ + agtv_result = AGTYPE_EDGE_GET_END_ID(agtv_object); PG_RETURN_POINTER(agtype_value_to_agtype(agtv_result)); } @@ -6038,10 +6054,25 @@ Datum age_properties(PG_FUNCTION_ARGS) errmsg("properties() argument must be a vertex, an edge or null"))); } - agtv_result = GET_AGTYPE_VALUE_OBJECT_VALUE(agtv_object, "properties"); - - Assert(agtv_result != NULL); - Assert(agtv_result->type = AGTV_OBJECT); + /* + * Direct field access optimization: properties is at index 2 for vertex + * (id=0, label=1, properties=2) and index 4 for edge (id=0, label=1, + * end_id=2, start_id=3, properties=4) due to key length sorting. + */ + if (agtv_object->type == AGTV_VERTEX) + { + agtv_result = AGTYPE_VERTEX_GET_PROPERTIES(agtv_object); + } + else if (agtv_object->type == AGTV_EDGE) + { + agtv_result = AGTYPE_EDGE_GET_PROPERTIES(agtv_object); + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("properties() unexpected argument type"))); + } PG_RETURN_POINTER(agtype_value_to_agtype(agtv_result)); } @@ -7170,8 +7201,24 @@ Datum age_label(PG_FUNCTION_ARGS) } - /* extract the label agtype value from the vertex or edge */ - label = GET_AGTYPE_VALUE_OBJECT_VALUE(agtv_value, "label"); + /* + * Direct field access optimization: label is at a fixed index for both + * vertex and edge objects due to key length sorting. + */ + if (agtv_value->type == AGTV_VERTEX) + { + label = AGTYPE_VERTEX_GET_LABEL(agtv_value); + } + else if (agtv_value->type == AGTV_EDGE) + { + label = AGTYPE_EDGE_GET_LABEL(agtv_value); + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("label() unexpected argument type"))); + } PG_RETURN_POINTER(agtype_value_to_agtype(label)); } @@ -10507,6 +10554,59 @@ agtype *get_one_agtype_from_variadic_args(FunctionCallInfo fcinfo, Oid *types = NULL; agtype *agtype_result = NULL; + /* + * Fast path optimization: For non-variadic calls where the argument + * is already an agtype, we can avoid the overhead of extract_variadic_args + * which allocates three arrays. This is the common case for most agtype + * comparison and arithmetic operators. + */ + if (!get_fn_expr_variadic(fcinfo->flinfo)) + { + int total_args = PG_NARGS(); + int actual_nargs = total_args - variadic_offset; + + /* Verify expected number of arguments */ + if (actual_nargs != expected_nargs) + { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("number of args %d does not match expected %d", + actual_nargs, expected_nargs))); + } + + /* Check for SQL NULL */ + if (PG_ARGISNULL(variadic_offset)) + { + return NULL; + } + + /* Check if the argument is already an agtype */ + if (get_fn_expr_argtype(fcinfo->flinfo, variadic_offset) == AGTYPEOID) + { + agtype_container *agtc; + + agtype_result = DATUM_GET_AGTYPE_P(PG_GETARG_DATUM(variadic_offset)); + agtc = &agtype_result->root; + + /* + * Is this a scalar (scalars are stored as one element arrays)? + * If so, test for agtype NULL. + */ + if (AGTYPE_CONTAINER_IS_SCALAR(agtc) && + AGTE_IS_NULL(agtc->children[0])) + { + return NULL; + } + + return agtype_result; + } + + /* + * Not an agtype, need to convert. Fall through to use + * extract_variadic_args for type conversion handling. + */ + } + + /* Standard path using extract_variadic_args */ nargs = extract_variadic_args(fcinfo, variadic_offset, false, &args, &types, &nulls); /* throw an error if the number of args is not the expected number */ diff --git a/src/backend/utils/adt/agtype_util.c b/src/backend/utils/adt/agtype_util.c index 01a965cdd..b39723413 100644 --- a/src/backend/utils/adt/agtype_util.c +++ b/src/backend/utils/adt/agtype_util.c @@ -41,6 +41,14 @@ #include "utils/agtype_ext.h" +/* + * Extended type header macros - must match definitions in agtype_ext.c. + * These are used for deserializing extended agtype values (INTEGER, FLOAT, + * VERTEX, EDGE, PATH) from their binary representation. + */ +#define AGT_HEADER_TYPE uint32 +#define AGT_HEADER_SIZE sizeof(AGT_HEADER_TYPE) + /* * Maximum number of elements in an array (or key/value pairs in an object). * This is limited by two things: the size of the agtentry array must fit @@ -56,6 +64,11 @@ static void fill_agtype_value(agtype_container *container, int index, char *base_addr, uint32 offset, agtype_value *result); +static void fill_agtype_value_no_copy(agtype_container *container, int index, + char *base_addr, uint32 offset, + agtype_value *result); +static int compare_agtype_scalar_containers(agtype_container *a, + agtype_container *b); static bool equals_agtype_scalar_value(agtype_value *a, agtype_value *b); static agtype *convert_to_agtype(agtype_value *val); static void convert_agtype_value(StringInfo buffer, agtentry *header, @@ -264,6 +277,24 @@ int compare_agtype_containers_orderability(agtype_container *a, agtype_iterator *itb; int res = 0; + /* + * Fast path optimization for scalar values. + * + * The most common case in ORDER BY and comparison operations is comparing + * scalar values (integers, strings, floats, etc.). For these cases, we can + * avoid the overhead of the full iterator machinery by directly extracting + * and comparing the scalar values. + * + * This provides significant performance improvement because: + * 1. We avoid allocating two agtype_iterator structures + * 2. We avoid the iterator state machine overhead + * 3. We use no-copy extraction where possible + */ + if (AGTYPE_CONTAINER_IS_SCALAR(a) && AGTYPE_CONTAINER_IS_SCALAR(b)) + { + return compare_agtype_scalar_containers(a, b); + } + ita = agtype_iterator_init(a); itb = agtype_iterator_init(b); @@ -751,6 +782,173 @@ static void fill_agtype_value(agtype_container *container, int index, } } +/* + * A helper function to fill in an agtype_value WITHOUT making deep copies. + * This is used for read-only comparison operations where the agtype_value + * will not outlive the container data. The caller MUST NOT free the + * agtype_value content or use it after the container is freed. + * + * This function provides significant performance improvements for comparison + * operations by avoiding palloc/memcpy for strings and numerics. + * + * Note: For AGTV_STRING, val.string.val points directly into container data. + * Note: For AGTV_NUMERIC, val.numeric points directly into container data. + * Note: Extended types (VERTEX, EDGE, PATH) still require deserialization, + * so they use the standard fill_agtype_value path. + */ +static void fill_agtype_value_no_copy(agtype_container *container, int index, + char *base_addr, uint32 offset, + agtype_value *result) +{ + agtentry entry = container->children[index]; + + if (AGTE_IS_NULL(entry)) + { + result->type = AGTV_NULL; + } + else if (AGTE_IS_STRING(entry)) + { + result->type = AGTV_STRING; + /* Point directly into the container data - no copy */ + result->val.string.val = base_addr + offset; + result->val.string.len = get_agtype_length(container, index); + } + else if (AGTE_IS_NUMERIC(entry)) + { + result->type = AGTV_NUMERIC; + /* Point directly into the container data - no copy */ + result->val.numeric = (Numeric)(base_addr + INTALIGN(offset)); + } + else if (AGTE_IS_AGTYPE(entry)) + { + /* + * For extended types (INTEGER, FLOAT, VERTEX, EDGE, PATH), we need + * to deserialize. INTEGER and FLOAT don't allocate, but composite + * types (VERTEX, EDGE, PATH) do. For simple scalar comparisons, + * we handle INTEGER and FLOAT directly here. + */ + char *base = base_addr + INTALIGN(offset); + AGT_HEADER_TYPE agt_header = *((AGT_HEADER_TYPE *)base); + + switch (agt_header) + { + case AGT_HEADER_INTEGER: + result->type = AGTV_INTEGER; + result->val.int_value = *((int64 *)(base + AGT_HEADER_SIZE)); + break; + + case AGT_HEADER_FLOAT: + result->type = AGTV_FLOAT; + result->val.float_value = *((float8 *)(base + AGT_HEADER_SIZE)); + break; + + default: + /* + * For VERTEX, EDGE, PATH - use standard deserialization. + * These are composite types that require full parsing. + */ + ag_deserialize_extended_type(base_addr, offset, result); + break; + } + } + else if (AGTE_IS_BOOL_TRUE(entry)) + { + result->type = AGTV_BOOL; + result->val.boolean = true; + } + else if (AGTE_IS_BOOL_FALSE(entry)) + { + result->type = AGTV_BOOL; + result->val.boolean = false; + } + else + { + Assert(AGTE_IS_CONTAINER(entry)); + result->type = AGTV_BINARY; + /* Remove alignment padding from data pointer and length */ + result->val.binary.data = + (agtype_container *)(base_addr + INTALIGN(offset)); + result->val.binary.len = get_agtype_length(container, index) - + (INTALIGN(offset) - offset); + } +} + +/* + * Fast path comparison for scalar agtype containers. + * + * This function compares two scalar containers directly without the overhead + * of the full iterator machinery. It extracts the scalar values using no-copy + * fill and compares them directly. + * + * Returns: negative if a < b, 0 if a == b, positive if a > b + */ +static int compare_agtype_scalar_containers(agtype_container *a, + agtype_container *b) +{ + agtype_value va; + agtype_value vb; + char *base_addr_a; + char *base_addr_b; + int result; + bool need_free_a = false; + bool need_free_b = false; + + Assert(AGTYPE_CONTAINER_IS_SCALAR(a)); + Assert(AGTYPE_CONTAINER_IS_SCALAR(b)); + + /* Scalars are stored as single-element arrays */ + base_addr_a = (char *)&a->children[1]; + base_addr_b = (char *)&b->children[1]; + + /* Use no-copy fill to avoid allocations for simple types */ + fill_agtype_value_no_copy(a, 0, base_addr_a, 0, &va); + fill_agtype_value_no_copy(b, 0, base_addr_b, 0, &vb); + + /* + * Check if we need to free the values after comparison. + * Only VERTEX, EDGE, and PATH types allocate memory in no-copy mode. + */ + if (va.type == AGTV_VERTEX || va.type == AGTV_EDGE || va.type == AGTV_PATH) + { + need_free_a = true; + } + if (vb.type == AGTV_VERTEX || vb.type == AGTV_EDGE || vb.type == AGTV_PATH) + { + need_free_b = true; + } + + /* + * Compare the scalar values. If types match or are numeric compatible, + * use scalar comparison. Otherwise, use type-based ordering. + */ + if ((va.type == vb.type) || + ((va.type == AGTV_INTEGER || va.type == AGTV_FLOAT || + va.type == AGTV_NUMERIC) && + (vb.type == AGTV_INTEGER || vb.type == AGTV_FLOAT || + vb.type == AGTV_NUMERIC))) + { + result = compare_agtype_scalar_values(&va, &vb); + } + else + { + /* Type-defined order */ + result = (get_type_sort_priority(va.type) < + get_type_sort_priority(vb.type)) ? -1 : 1; + } + + /* Free any allocated memory from composite types */ + if (need_free_a) + { + pfree_agtype_value_content(&va); + } + if (need_free_b) + { + pfree_agtype_value_content(&vb); + } + + return result; +} + /* * Push agtype_value into agtype_parse_state. * @@ -1597,7 +1795,8 @@ void agtype_hash_scalar_value_extended(const agtype_value *scalar_val, case AGTV_VERTEX: { graphid id; - agtype_value *id_agt = GET_AGTYPE_VALUE_OBJECT_VALUE(scalar_val, "id"); + agtype_value *id_agt; + id_agt = AGTYPE_VERTEX_GET_ID(scalar_val); id = id_agt->val.int_value; tmp = DatumGetUInt64(DirectFunctionCall2( hashint8extended, Float8GetDatum(id), UInt64GetDatum(seed))); @@ -1606,7 +1805,8 @@ void agtype_hash_scalar_value_extended(const agtype_value *scalar_val, case AGTV_EDGE: { graphid id; - agtype_value *id_agt = GET_AGTYPE_VALUE_OBJECT_VALUE(scalar_val, "id"); + agtype_value *id_agt; + id_agt = AGTYPE_EDGE_GET_ID(scalar_val); id = id_agt->val.int_value; tmp = DatumGetUInt64(DirectFunctionCall2( hashint8extended, Float8GetDatum(id), UInt64GetDatum(seed))); @@ -1704,8 +1904,8 @@ static bool equals_agtype_scalar_value(agtype_value *a, agtype_value *b) case AGTV_VERTEX: { graphid a_graphid, b_graphid; - a_graphid = a->val.object.pairs[0].value.val.int_value; - b_graphid = b->val.object.pairs[0].value.val.int_value; + a_graphid = AGTYPE_VERTEX_GET_ID(a)->val.int_value; + b_graphid = AGTYPE_VERTEX_GET_ID(b)->val.int_value; return a_graphid == b_graphid; } @@ -1790,16 +1990,33 @@ int compare_agtype_scalar_values(agtype_value *a, agtype_value *b) return compare_two_floats_orderability(a->val.float_value, b->val.float_value); case AGTV_VERTEX: - case AGTV_EDGE: { - agtype_value *a_id, *b_id; graphid a_graphid, b_graphid; - a_id = GET_AGTYPE_VALUE_OBJECT_VALUE(a, "id"); - b_id = GET_AGTYPE_VALUE_OBJECT_VALUE(b, "id"); + /* Direct field access optimization using macros defined in agtype.h. */ + a_graphid = AGTYPE_VERTEX_GET_ID(a)->val.int_value; + b_graphid = AGTYPE_VERTEX_GET_ID(b)->val.int_value; + + if (a_graphid == b_graphid) + { + return 0; + } + else if (a_graphid > b_graphid) + { + return 1; + } + else + { + return -1; + } + } + case AGTV_EDGE: + { + graphid a_graphid, b_graphid; - a_graphid = a_id->val.int_value; - b_graphid = b_id->val.int_value; + /* Direct field access optimization using macros defined in agtype.h. */ + a_graphid = AGTYPE_EDGE_GET_ID(a)->val.int_value; + b_graphid = AGTYPE_EDGE_GET_ID(b)->val.int_value; if (a_graphid == b_graphid) { diff --git a/src/include/utils/agtype.h b/src/include/utils/agtype.h index ab2ba08cc..ec9125073 100644 --- a/src/include/utils/agtype.h +++ b/src/include/utils/agtype.h @@ -322,6 +322,109 @@ enum agtype_value_type AGTV_BINARY }; +/* + * Direct field access indices for vertex and edge objects. + * + * Vertex and edge objects are serialized with keys sorted by length first, + * then lexicographically (via uniqueify_agtype_object). This means field + * positions are deterministic and can be accessed directly without binary + * search, providing O(1) access instead of O(log n). + * + * Vertex keys by length: "id"(2), "label"(5), "properties"(10) + * Edge keys by length: "id"(2), "label"(5), "end_id"(6), "start_id"(8), "properties"(10) + */ +#define VERTEX_FIELD_ID 0 +#define VERTEX_FIELD_LABEL 1 +#define VERTEX_FIELD_PROPERTIES 2 +#define VERTEX_NUM_FIELDS 3 + +#define EDGE_FIELD_ID 0 +#define EDGE_FIELD_LABEL 1 +#define EDGE_FIELD_END_ID 2 +#define EDGE_FIELD_START_ID 3 +#define EDGE_FIELD_PROPERTIES 4 +#define EDGE_NUM_FIELDS 5 + +/* + * Macros for direct field access from vertex/edge agtype_value objects. + * These avoid the binary search overhead of GET_AGTYPE_VALUE_OBJECT_VALUE. + * Validation is integrated - macros will error if field count is incorrect. + * Uses GCC statement expressions to allow validation within expressions. + */ +#define AGTYPE_VERTEX_GET_ID(v) \ + ({ \ + if ((v)->val.object.num_pairs != VERTEX_NUM_FIELDS) \ + ereport(ERROR, \ + (errcode(ERRCODE_DATA_CORRUPTED), \ + errmsg("invalid vertex structure: expected %d fields, found %d", \ + VERTEX_NUM_FIELDS, (v)->val.object.num_pairs))); \ + &(v)->val.object.pairs[VERTEX_FIELD_ID].value; \ + }) +#define AGTYPE_VERTEX_GET_LABEL(v) \ + ({ \ + if ((v)->val.object.num_pairs != VERTEX_NUM_FIELDS) \ + ereport(ERROR, \ + (errcode(ERRCODE_DATA_CORRUPTED), \ + errmsg("invalid vertex structure: expected %d fields, found %d", \ + VERTEX_NUM_FIELDS, (v)->val.object.num_pairs))); \ + &(v)->val.object.pairs[VERTEX_FIELD_LABEL].value; \ + }) +#define AGTYPE_VERTEX_GET_PROPERTIES(v) \ + ({ \ + if ((v)->val.object.num_pairs != VERTEX_NUM_FIELDS) \ + ereport(ERROR, \ + (errcode(ERRCODE_DATA_CORRUPTED), \ + errmsg("invalid vertex structure: expected %d fields, found %d", \ + VERTEX_NUM_FIELDS, (v)->val.object.num_pairs))); \ + &(v)->val.object.pairs[VERTEX_FIELD_PROPERTIES].value; \ + }) + +#define AGTYPE_EDGE_GET_ID(e) \ + ({ \ + if ((e)->val.object.num_pairs != EDGE_NUM_FIELDS) \ + ereport(ERROR, \ + (errcode(ERRCODE_DATA_CORRUPTED), \ + errmsg("invalid edge structure: expected %d fields, found %d", \ + EDGE_NUM_FIELDS, (e)->val.object.num_pairs))); \ + &(e)->val.object.pairs[EDGE_FIELD_ID].value; \ + }) +#define AGTYPE_EDGE_GET_LABEL(e) \ + ({ \ + if ((e)->val.object.num_pairs != EDGE_NUM_FIELDS) \ + ereport(ERROR, \ + (errcode(ERRCODE_DATA_CORRUPTED), \ + errmsg("invalid edge structure: expected %d fields, found %d", \ + EDGE_NUM_FIELDS, (e)->val.object.num_pairs))); \ + &(e)->val.object.pairs[EDGE_FIELD_LABEL].value; \ + }) +#define AGTYPE_EDGE_GET_END_ID(e) \ + ({ \ + if ((e)->val.object.num_pairs != EDGE_NUM_FIELDS) \ + ereport(ERROR, \ + (errcode(ERRCODE_DATA_CORRUPTED), \ + errmsg("invalid edge structure: expected %d fields, found %d", \ + EDGE_NUM_FIELDS, (e)->val.object.num_pairs))); \ + &(e)->val.object.pairs[EDGE_FIELD_END_ID].value; \ + }) +#define AGTYPE_EDGE_GET_START_ID(e) \ + ({ \ + if ((e)->val.object.num_pairs != EDGE_NUM_FIELDS) \ + ereport(ERROR, \ + (errcode(ERRCODE_DATA_CORRUPTED), \ + errmsg("invalid edge structure: expected %d fields, found %d", \ + EDGE_NUM_FIELDS, (e)->val.object.num_pairs))); \ + &(e)->val.object.pairs[EDGE_FIELD_START_ID].value; \ + }) +#define AGTYPE_EDGE_GET_PROPERTIES(e) \ + ({ \ + if ((e)->val.object.num_pairs != EDGE_NUM_FIELDS) \ + ereport(ERROR, \ + (errcode(ERRCODE_DATA_CORRUPTED), \ + errmsg("invalid edge structure: expected %d fields, found %d", \ + EDGE_NUM_FIELDS, (e)->val.object.num_pairs))); \ + &(e)->val.object.pairs[EDGE_FIELD_PROPERTIES].value; \ + }) + /* * agtype_value: In-memory representation of agtype. This is a convenient * deserialized representation, that can easily support using the "val" From 8bdeec54e898451771f4dc021a13b00781d6d1a0 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Sat, 17 Jan 2026 03:18:10 -0800 Subject: [PATCH 018/100] Upgrade Jest to v29 for node: protocol compatibility (#2307) Note: This PR was created with AI tools and a human. The pg-connection-string module (dependency of pg) now uses the node: protocol prefix for built-in modules (e.g., require('node:process')). Jest 26 does not support this syntax, causing test failures. Changes: - Upgrade jest from ^26.6.3 to ^29.7.0 - Upgrade ts-jest from ^26.5.1 to ^29.4.6 - Upgrade @types/jest from ^26.0.20 to ^29.5.14 - Update typescript to ^4.9.5 This also resolves 19 npm audit vulnerabilities (17 moderate, 2 high) that existed in the older Jest 26 dependency tree. modified: drivers/nodejs/package.json --- drivers/nodejs/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/nodejs/package.json b/drivers/nodejs/package.json index 9f88bc2ba..6be11c780 100644 --- a/drivers/nodejs/package.json +++ b/drivers/nodejs/package.json @@ -33,7 +33,7 @@ "pg": ">=6.0.0" }, "devDependencies": { - "@types/jest": "^26.0.20", + "@types/jest": "^29.5.14", "@types/pg": "^7.14.10", "@typescript-eslint/eslint-plugin": "^4.22.1", "@typescript-eslint/parser": "^4.22.1", @@ -44,8 +44,8 @@ "eslint-plugin-jest": "^24.3.6", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.3.1", - "jest": "^26.6.3", - "ts-jest": "^26.5.1", - "typescript": "^4.1.5" + "jest": "^29.7.0", + "ts-jest": "^29.4.6", + "typescript": "^4.9.5" } } From 56a92d8c1be364e07bac51665a362ca91957194b Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Sun, 18 Jan 2026 10:02:55 -0800 Subject: [PATCH 019/100] Fix Issue 1884: Ambiguous column reference (#2306) Fix Issue 1884: Ambiguous column reference and invalid AGT header errors. Note: This PR was created with AI tools and a human, or 2. This commit addresses two related bugs that occur when using SET to store graph elements (vertices, edges, paths) as property values: Issue 1884 - "column reference is ambiguous" error: When a Cypher query uses the same variable in both the SET expression RHS and the RETURN clause (e.g., SET n.prop = n RETURN n), PostgreSQL would report "column reference is ambiguous" because the variable appeared in multiple subqueries without proper qualification. Solution: The fix for this issue was already in place through the target entry naming scheme that qualifies column references. "Invalid AGT header value" offset error: When deserializing nested VERTEX, EDGE, or PATH values stored in properties, the system would fail with errors like "Invalid AGT header value: 0x00000041". This occurred because ag_serialize_extended_type() did not include alignment padding (padlen) in the agtentry length calculation for these types, while fill_agtype_value() uses INTALIGN() when reading, causing offset mismatch. Solution: Modified ag_serialize_extended_type() in agtype_ext.c to include padlen in the agtentry length for VERTEX, EDGE, and PATH cases, matching the existing pattern used for INTEGER, FLOAT, and NUMERIC types: *agtentry = AGTENTRY_IS_AGTYPE | (padlen + (AGTENTRY_OFFLENMASK & ...)); This ensures the serialized length accounts for alignment padding, allowing correct deserialization of nested graph elements. Appropriate regression tests were added to verify the fixes. Co-authored by: Zainab Saad <105385638+Zainab-Saad@users.noreply.github.com> modified: regress/expected/cypher_set.out modified: regress/sql/cypher_set.sql modified: src/backend/parser/cypher_clause.c modified: src/backend/utils/adt/agtype_ext.c --- regress/expected/cypher_set.out | 266 +++++++++++++++++++++++++++++ regress/sql/cypher_set.sql | 164 ++++++++++++++++++ src/backend/parser/cypher_clause.c | 19 ++- src/backend/utils/adt/agtype_ext.c | 8 +- 4 files changed, 451 insertions(+), 6 deletions(-) diff --git a/regress/expected/cypher_set.out b/regress/expected/cypher_set.out index 1d24a7f9b..239234ed6 100644 --- a/regress/expected/cypher_set.out +++ b/regress/expected/cypher_set.out @@ -988,6 +988,245 @@ SELECT * FROM cypher('issue_1634', $$ MATCH (u) DELETE (u) $$) AS (u agtype); --- (0 rows) +-- +-- Issue 1884: column reference is ambiguous when using same variable in +-- SET expression and RETURN clause +-- +-- These tests cover: +-- 1. "column reference is ambiguous" error when variable is used in both +-- SET expression RHS (e.g., SET n.prop = n) and RETURN clause +-- 2. "Invalid AGT header value" error caused by incorrect offset calculation +-- when nested VERTEX/EDGE/PATH values are serialized in properties +-- +-- Tests use isolated data to keep output manageable and avoid cumulative nesting +-- +SELECT * FROM create_graph('issue_1884'); +NOTICE: graph "issue_1884" has been created + create_graph +-------------- + +(1 row) + +-- ============================================================================ +-- Test Group A: Basic "column reference is ambiguous" fix (Issue 1884) +-- ============================================================================ +-- Test A1: Core issue - SET n.prop = n with RETURN n (the original bug) +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestA1 {name: 'A1'}) + SET n.self = n + RETURN n +$$) AS (result agtype); + result +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "TestA1", "properties": {"name": "A1", "self": {"id": 844424930131969, "label": "TestA1", "properties": {"name": "A1"}}::vertex}}::vertex +(1 row) + +-- Test A2: Multiple variables in SET and RETURN +SELECT * FROM cypher('issue_1884', $$ + CREATE (a:TestA2 {name: 'A'})-[e:LINK {w: 1}]->(b:TestA2 {name: 'B'}) + SET a.edge = e, b.edge = e + RETURN a, e, b +$$) AS (a agtype, e agtype, b agtype); + a | e | b +-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"id": 1125899906842625, "label": "TestA2", "properties": {"edge": {"id": 1407374883553281, "label": "LINK", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {"w": 1}}::edge, "name": "A"}}::vertex | {"id": 1407374883553281, "label": "LINK", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {"w": 1}}::edge | {"id": 1125899906842626, "label": "TestA2", "properties": {"edge": {"id": 1407374883553281, "label": "LINK", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {"w": 1}}::edge, "name": "B"}}::vertex +(1 row) + +-- Test A3: SET edge property to node reference +SELECT * FROM cypher('issue_1884', $$ + CREATE (a:TestA3 {name: 'X'})-[e:REL]->(b:TestA3 {name: 'Y'}) + SET e.src = a, e.dst = b + RETURN e +$$) AS (e agtype); + e +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + {"id": 1970324836974593, "label": "REL", "end_id": 1688849860263938, "start_id": 1688849860263937, "properties": {"dst": {"id": 1688849860263938, "label": "TestA3", "properties": {"name": "Y"}}::vertex, "src": {"id": 1688849860263937, "label": "TestA3", "properties": {"name": "X"}}::vertex}}::edge +(1 row) + +-- ============================================================================ +-- Test Group B: Nested VERTEX/EDGE/PATH serialization (offset error fix) +-- ============================================================================ +-- Test B1: Vertex nested in vertex property (tests VERTEX serialization) +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestB1 {val: 1}) + SET n.copy = n + RETURN n +$$) AS (result agtype); + result +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"id": 2251799813685249, "label": "TestB1", "properties": {"val": 1, "copy": {"id": 2251799813685249, "label": "TestB1", "properties": {"val": 1}}::vertex}}::vertex +(1 row) + +-- Verify nested vertex can be read back +SELECT * FROM cypher('issue_1884', $$ + MATCH (n:TestB1) + RETURN n.copy +$$) AS (copy agtype); + copy +------------------------------------------------------------------------------- + {"id": 2251799813685249, "label": "TestB1", "properties": {"val": 1}}::vertex +(1 row) + +-- Test B2: Edge nested in node property (tests EDGE serialization) +SELECT * FROM cypher('issue_1884', $$ + CREATE (a:TestB2 {name: 'start'})-[e:B2REL {x: 100}]->(b:TestB2 {name: 'end'}) + SET a.myEdge = e + RETURN a +$$) AS (a agtype); + a +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"id": 2533274790395905, "label": "TestB2", "properties": {"name": "start", "myEdge": {"id": 2814749767106561, "label": "B2REL", "end_id": 2533274790395906, "start_id": 2533274790395905, "properties": {"x": 100}}::edge}}::vertex +(1 row) + +-- Verify nested edge can be read back +SELECT * FROM cypher('issue_1884', $$ + MATCH (n:TestB2 {name: 'start'}) + RETURN n.myEdge +$$) AS (edge agtype); + edge +-------------------------------------------------------------------------------------------------------------------------------------- + {"id": 2814749767106561, "label": "B2REL", "end_id": 2533274790395906, "start_id": 2533274790395905, "properties": {"x": 100}}::edge +(1 row) + +-- Test B3: Path nested in node property (tests PATH serialization) +-- First create the pattern +SELECT * FROM cypher('issue_1884', $$ + CREATE (a:TestB3)-[e:B3REL]->(b:TestB3) + RETURN a +$$) AS (a agtype); + a +----------------------------------------------------------------------- + {"id": 3096224743817217, "label": "TestB3", "properties": {}}::vertex +(1 row) + +-- Then match the path and set it (MATCH only sees committed data) +SELECT * FROM cypher('issue_1884', $$ + MATCH p = (a:TestB3)-[e:B3REL]->(b:TestB3) + SET a.myPath = p + RETURN a +$$) AS (a agtype); + a +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"id": 3096224743817217, "label": "TestB3", "properties": {"myPath": [{"id": 3096224743817217, "label": "TestB3", "properties": {}}::vertex, {"id": 3377699720527873, "label": "B3REL", "end_id": 3096224743817218, "start_id": 3096224743817217, "properties": {}}::edge, {"id": 3096224743817218, "label": "TestB3", "properties": {}}::vertex]::path}}::vertex +(1 row) + +-- Verify nested path can be read back +SELECT * FROM cypher('issue_1884', $$ + MATCH (n:TestB3) + WHERE n.myPath IS NOT NULL + RETURN n.myPath +$$) AS (path agtype); + path +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + [{"id": 3096224743817217, "label": "TestB3", "properties": {}}::vertex, {"id": 3377699720527873, "label": "B3REL", "end_id": 3096224743817218, "start_id": 3096224743817217, "properties": {}}::edge, {"id": 3096224743817218, "label": "TestB3", "properties": {}}::vertex]::path +(1 row) + +-- ============================================================================ +-- Test Group C: Nested structures in arrays and maps +-- ============================================================================ +-- Test C1: Vertex in array +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestC1 {tag: 'arrtest'}) + SET n.arr = [n] + RETURN n +$$) AS (result agtype); + result +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"id": 3659174697238529, "label": "TestC1", "properties": {"arr": [{"id": 3659174697238529, "label": "TestC1", "properties": {"tag": "arrtest"}}::vertex], "tag": "arrtest"}}::vertex +(1 row) + +-- Verify array with nested vertex +SELECT * FROM cypher('issue_1884', $$ + MATCH (n:TestC1) + RETURN n.arr[0] +$$) AS (elem agtype); + elem +--------------------------------------------------------------------------------------- + {"id": 3659174697238529, "label": "TestC1", "properties": {"tag": "arrtest"}}::vertex +(1 row) + +-- Test C2: Vertex in map +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestC2 {tag: 'maptest'}) + SET n.obj = {node: n} + RETURN n +$$) AS (result agtype); + result +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"id": 3940649673949185, "label": "TestC2", "properties": {"obj": {"node": {"id": 3940649673949185, "label": "TestC2", "properties": {"tag": "maptest"}}::vertex}, "tag": "maptest"}}::vertex +(1 row) + +-- Verify map with nested vertex +SELECT * FROM cypher('issue_1884', $$ + MATCH (n:TestC2) + RETURN n.obj.node +$$) AS (node agtype); + node +--------------------------------------------------------------------------------------- + {"id": 3940649673949185, "label": "TestC2", "properties": {"tag": "maptest"}}::vertex +(1 row) + +-- ============================================================================ +-- Test Group D: MERGE and CREATE with self-reference +-- ============================================================================ +-- Test D1: MERGE with SET self-reference +SELECT * FROM cypher('issue_1884', $$ + MERGE (n:TestD1 {name: 'merged'}) + SET n.ref = n + RETURN n +$$) AS (result agtype); + result +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"id": 4222124650659841, "label": "TestD1", "properties": {"ref": {"id": 4222124650659841, "label": "TestD1", "properties": {"name": "merged"}}::vertex, "name": "merged"}}::vertex +(1 row) + +-- Test D2: CREATE with SET self-reference +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestD2 {name: 'created'}) + SET n.ref = n + RETURN n +$$) AS (result agtype); + result +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"id": 4503599627370497, "label": "TestD2", "properties": {"ref": {"id": 4503599627370497, "label": "TestD2", "properties": {"name": "created"}}::vertex, "name": "created"}}::vertex +(1 row) + +-- ============================================================================ +-- Test Group E: Functions with variable references +-- ============================================================================ +-- Test E1: id() and label() functions +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestE1 {name: 'functest'}) + SET n.myId = id(n), n.myLabel = label(n) + RETURN n +$$) AS (result agtype); + result +---------------------------------------------------------------------------------------------------------------------------------------- + {"id": 4785074604081153, "label": "TestE1", "properties": {"myId": 4785074604081153, "name": "functest", "myLabel": "TestE1"}}::vertex +(1 row) + +-- Test E2: nodes() and relationships() with path +-- First create the pattern +SELECT * FROM cypher('issue_1884', $$ + CREATE (a:TestE2)-[e:E2REL]->(b:TestE2) + RETURN a +$$) AS (a agtype); + a +----------------------------------------------------------------------- + {"id": 5066549580791809, "label": "TestE2", "properties": {}}::vertex +(1 row) + +-- Then match the path and extract nodes/relationships (MATCH only sees committed data) +SELECT * FROM cypher('issue_1884', $$ + MATCH p = (a:TestE2)-[e:E2REL]->(b:TestE2) + SET a.pathNodes = nodes(p), a.pathRels = relationships(p) + RETURN a +$$) AS (a agtype); + a +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + {"id": 5066549580791809, "label": "TestE2", "properties": {"pathRels": [{"id": 5348024557502465, "label": "E2REL", "end_id": 5066549580791810, "start_id": 5066549580791809, "properties": {}}::edge], "pathNodes": [{"id": 5066549580791809, "label": "TestE2", "properties": {}}::vertex, {"id": 5066549580791810, "label": "TestE2", "properties": {}}::vertex]}}::vertex +(1 row) + -- -- Clean up -- @@ -1038,6 +1277,33 @@ NOTICE: graph "issue_1634" has been dropped (1 row) +SELECT drop_graph('issue_1884', true); +NOTICE: drop cascades to 19 other objects +DETAIL: drop cascades to table issue_1884._ag_label_vertex +drop cascades to table issue_1884._ag_label_edge +drop cascades to table issue_1884."TestA1" +drop cascades to table issue_1884."TestA2" +drop cascades to table issue_1884."LINK" +drop cascades to table issue_1884."TestA3" +drop cascades to table issue_1884."REL" +drop cascades to table issue_1884."TestB1" +drop cascades to table issue_1884."TestB2" +drop cascades to table issue_1884."B2REL" +drop cascades to table issue_1884."TestB3" +drop cascades to table issue_1884."B3REL" +drop cascades to table issue_1884."TestC1" +drop cascades to table issue_1884."TestC2" +drop cascades to table issue_1884."TestD1" +drop cascades to table issue_1884."TestD2" +drop cascades to table issue_1884."TestE1" +drop cascades to table issue_1884."TestE2" +drop cascades to table issue_1884."E2REL" +NOTICE: graph "issue_1884" has been dropped + drop_graph +------------ + +(1 row) + -- -- End -- diff --git a/regress/sql/cypher_set.sql b/regress/sql/cypher_set.sql index a2667153d..e745d5d6e 100644 --- a/regress/sql/cypher_set.sql +++ b/regress/sql/cypher_set.sql @@ -379,6 +379,169 @@ SELECT * FROM cypher('issue_1634', $$ MERGE (v:PERSION {id: '1'}) SELECT * FROM cypher('issue_1634', $$ MATCH (u) DELETE (u) $$) AS (u agtype); +-- +-- Issue 1884: column reference is ambiguous when using same variable in +-- SET expression and RETURN clause +-- +-- These tests cover: +-- 1. "column reference is ambiguous" error when variable is used in both +-- SET expression RHS (e.g., SET n.prop = n) and RETURN clause +-- 2. "Invalid AGT header value" error caused by incorrect offset calculation +-- when nested VERTEX/EDGE/PATH values are serialized in properties +-- +-- Tests use isolated data to keep output manageable and avoid cumulative nesting +-- +SELECT * FROM create_graph('issue_1884'); + +-- ============================================================================ +-- Test Group A: Basic "column reference is ambiguous" fix (Issue 1884) +-- ============================================================================ + +-- Test A1: Core issue - SET n.prop = n with RETURN n (the original bug) +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestA1 {name: 'A1'}) + SET n.self = n + RETURN n +$$) AS (result agtype); + +-- Test A2: Multiple variables in SET and RETURN +SELECT * FROM cypher('issue_1884', $$ + CREATE (a:TestA2 {name: 'A'})-[e:LINK {w: 1}]->(b:TestA2 {name: 'B'}) + SET a.edge = e, b.edge = e + RETURN a, e, b +$$) AS (a agtype, e agtype, b agtype); + +-- Test A3: SET edge property to node reference +SELECT * FROM cypher('issue_1884', $$ + CREATE (a:TestA3 {name: 'X'})-[e:REL]->(b:TestA3 {name: 'Y'}) + SET e.src = a, e.dst = b + RETURN e +$$) AS (e agtype); + +-- ============================================================================ +-- Test Group B: Nested VERTEX/EDGE/PATH serialization (offset error fix) +-- ============================================================================ + +-- Test B1: Vertex nested in vertex property (tests VERTEX serialization) +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestB1 {val: 1}) + SET n.copy = n + RETURN n +$$) AS (result agtype); + +-- Verify nested vertex can be read back +SELECT * FROM cypher('issue_1884', $$ + MATCH (n:TestB1) + RETURN n.copy +$$) AS (copy agtype); + +-- Test B2: Edge nested in node property (tests EDGE serialization) +SELECT * FROM cypher('issue_1884', $$ + CREATE (a:TestB2 {name: 'start'})-[e:B2REL {x: 100}]->(b:TestB2 {name: 'end'}) + SET a.myEdge = e + RETURN a +$$) AS (a agtype); + +-- Verify nested edge can be read back +SELECT * FROM cypher('issue_1884', $$ + MATCH (n:TestB2 {name: 'start'}) + RETURN n.myEdge +$$) AS (edge agtype); + +-- Test B3: Path nested in node property (tests PATH serialization) +-- First create the pattern +SELECT * FROM cypher('issue_1884', $$ + CREATE (a:TestB3)-[e:B3REL]->(b:TestB3) + RETURN a +$$) AS (a agtype); + +-- Then match the path and set it (MATCH only sees committed data) +SELECT * FROM cypher('issue_1884', $$ + MATCH p = (a:TestB3)-[e:B3REL]->(b:TestB3) + SET a.myPath = p + RETURN a +$$) AS (a agtype); + +-- Verify nested path can be read back +SELECT * FROM cypher('issue_1884', $$ + MATCH (n:TestB3) + WHERE n.myPath IS NOT NULL + RETURN n.myPath +$$) AS (path agtype); + +-- ============================================================================ +-- Test Group C: Nested structures in arrays and maps +-- ============================================================================ + +-- Test C1: Vertex in array +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestC1 {tag: 'arrtest'}) + SET n.arr = [n] + RETURN n +$$) AS (result agtype); + +-- Verify array with nested vertex +SELECT * FROM cypher('issue_1884', $$ + MATCH (n:TestC1) + RETURN n.arr[0] +$$) AS (elem agtype); + +-- Test C2: Vertex in map +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestC2 {tag: 'maptest'}) + SET n.obj = {node: n} + RETURN n +$$) AS (result agtype); + +-- Verify map with nested vertex +SELECT * FROM cypher('issue_1884', $$ + MATCH (n:TestC2) + RETURN n.obj.node +$$) AS (node agtype); + +-- ============================================================================ +-- Test Group D: MERGE and CREATE with self-reference +-- ============================================================================ + +-- Test D1: MERGE with SET self-reference +SELECT * FROM cypher('issue_1884', $$ + MERGE (n:TestD1 {name: 'merged'}) + SET n.ref = n + RETURN n +$$) AS (result agtype); + +-- Test D2: CREATE with SET self-reference +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestD2 {name: 'created'}) + SET n.ref = n + RETURN n +$$) AS (result agtype); + +-- ============================================================================ +-- Test Group E: Functions with variable references +-- ============================================================================ + +-- Test E1: id() and label() functions +SELECT * FROM cypher('issue_1884', $$ + CREATE (n:TestE1 {name: 'functest'}) + SET n.myId = id(n), n.myLabel = label(n) + RETURN n +$$) AS (result agtype); + +-- Test E2: nodes() and relationships() with path +-- First create the pattern +SELECT * FROM cypher('issue_1884', $$ + CREATE (a:TestE2)-[e:E2REL]->(b:TestE2) + RETURN a +$$) AS (a agtype); + +-- Then match the path and extract nodes/relationships (MATCH only sees committed data) +SELECT * FROM cypher('issue_1884', $$ + MATCH p = (a:TestE2)-[e:E2REL]->(b:TestE2) + SET a.pathNodes = nodes(p), a.pathRels = relationships(p) + RETURN a +$$) AS (a agtype); + -- -- Clean up -- @@ -387,6 +550,7 @@ DROP FUNCTION set_test; SELECT drop_graph('cypher_set', true); SELECT drop_graph('cypher_set_1', true); SELECT drop_graph('issue_1634', true); +SELECT drop_graph('issue_1884', true); -- -- End diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index a5413bdaa..515f8c1df 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -71,6 +71,7 @@ #define AGE_VARNAME_MERGE_CLAUSE AGE_DEFAULT_VARNAME_PREFIX"merge_clause" #define AGE_VARNAME_ID AGE_DEFAULT_VARNAME_PREFIX"id" #define AGE_VARNAME_SET_CLAUSE AGE_DEFAULT_VARNAME_PREFIX"set_clause" +#define AGE_VARNAME_SET_VALUE AGE_DEFAULT_VARNAME_PREFIX"set_value" /* * In the transformation stage, we need to track @@ -1911,10 +1912,24 @@ cypher_update_information *transform_cypher_set_item_list( ((cypher_map*)set_item->expr)->keep_null = set_item->is_add; } - /* create target entry for the new property value */ + /* + * Create target entry for the new property value. + * + * We use a hidden variable name (AGE_VARNAME_SET_VALUE) for the + * SET expression value to prevent column name conflicts. This is + * necessary when the same variable is used on both the LHS and RHS + * of a SET clause (e.g., SET n.prop = n). Without this, the column + * name derived from the expression (e.g., "n") would duplicate the + * existing column name from the MATCH clause, causing a "column + * reference is ambiguous" error in subsequent clauses like RETURN. + * + * The hidden variable name will be filtered out by expand_pnsi_attrs + * when the targetlist is expanded for subsequent clauses. + */ item->prop_position = (AttrNumber)pstate->p_next_resno; target_item = transform_cypher_item(cpstate, set_item->expr, NULL, - EXPR_KIND_SELECT_TARGET, NULL, + EXPR_KIND_SELECT_TARGET, + AGE_VARNAME_SET_VALUE, false); if (nodeTag(target_item->expr) == T_Aggref) diff --git a/src/backend/utils/adt/agtype_ext.c b/src/backend/utils/adt/agtype_ext.c index 8fc6600d1..7a0ea991d 100644 --- a/src/backend/utils/adt/agtype_ext.c +++ b/src/backend/utils/adt/agtype_ext.c @@ -89,7 +89,7 @@ bool ag_serialize_extended_type(StringInfo buffer, agtentry *agtentry, object_ae += pad_buffer_to_int(buffer); *agtentry = AGTENTRY_IS_AGTYPE | - ((AGTENTRY_OFFLENMASK & (int)object_ae) + AGT_HEADER_SIZE); + (padlen + (AGTENTRY_OFFLENMASK & (int)object_ae) + AGT_HEADER_SIZE); break; } @@ -109,7 +109,7 @@ bool ag_serialize_extended_type(StringInfo buffer, agtentry *agtentry, object_ae += pad_buffer_to_int(buffer); *agtentry = AGTENTRY_IS_AGTYPE | - ((AGTENTRY_OFFLENMASK & (int)object_ae) + AGT_HEADER_SIZE); + (padlen + (AGTENTRY_OFFLENMASK & (int)object_ae) + AGT_HEADER_SIZE); break; } @@ -129,7 +129,7 @@ bool ag_serialize_extended_type(StringInfo buffer, agtentry *agtentry, object_ae += pad_buffer_to_int(buffer); *agtentry = AGTENTRY_IS_AGTYPE | - ((AGTENTRY_OFFLENMASK & (int)object_ae) + AGT_HEADER_SIZE); + (padlen + (AGTENTRY_OFFLENMASK & (int)object_ae) + AGT_HEADER_SIZE); break; } @@ -175,7 +175,7 @@ void ag_deserialize_extended_type(char *base_addr, uint32 offset, break; default: - elog(ERROR, "Invalid AGT header value."); + ereport(ERROR, (errmsg("Invalid AGT header value: 0x%08x", agt_header))); } } From b29ca5e7d2f84cfe2619eea70c4ace2cba41aa0b Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Mon, 19 Jan 2026 22:21:02 +0500 Subject: [PATCH 020/100] Replace libcsv with pg COPY for csv loading (#2310) - Commit also adds permission checks - Resolves a critical memory spike issue on loading large file - Use pg's COPY infrastructure (BeginCopyFrom, NextCopyFromRawFields) for 64KB buffered CSV parsing instead of libcsv - Add byte based flush threshold (64KB) matching COPY behavior for memory safety - Use heap_multi_insert with BulkInsertState for optimized batch inserts - Add per batch memory context to prevent memory growth during large loads - Remove libcsv dependency (libcsv.c, csv.h) - Improves loading performance by 15-20% - No previous regression tests were impacted - Added regression tests for permissions/rls Assisted-by AI --- Makefile | 1 - regress/expected/age_load.out | 189 ++++++++ regress/expected/index.out | 12 +- regress/sql/age_load.sql | 125 ++++++ regress/sql/index.sql | 2 +- src/backend/utils/load/ag_load_edges.c | 388 +++++++++-------- src/backend/utils/load/ag_load_labels.c | 381 ++++++++-------- src/backend/utils/load/age_load.c | 248 ++++++++++- src/backend/utils/load/libcsv.c | 549 ------------------------ src/include/utils/load/ag_load_edges.h | 52 +-- src/include/utils/load/ag_load_labels.h | 50 +-- src/include/utils/load/age_load.h | 27 +- src/include/utils/load/csv.h | 108 ----- 13 files changed, 1000 insertions(+), 1132 deletions(-) delete mode 100644 src/backend/utils/load/libcsv.c delete mode 100644 src/include/utils/load/csv.h diff --git a/Makefile b/Makefile index ffad7d6af..a8faa2bb8 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,6 @@ OBJS = src/backend/age.o \ src/backend/utils/load/ag_load_labels.o \ src/backend/utils/load/ag_load_edges.o \ src/backend/utils/load/age_load.o \ - src/backend/utils/load/libcsv.o \ src/backend/utils/name_validation.o \ src/backend/utils/ag_guc.o diff --git a/regress/expected/age_load.out b/regress/expected/age_load.out index 55d1ff1d6..1f76c31ce 100644 --- a/regress/expected/age_load.out +++ b/regress/expected/age_load.out @@ -454,6 +454,195 @@ NOTICE: graph "agload_conversion" has been dropped (1 row) +-- +-- Test security and permissions +-- +SELECT create_graph('agload_security'); +NOTICE: graph "agload_security" has been created + create_graph +-------------- + +(1 row) + +SELECT create_vlabel('agload_security', 'Person1'); +NOTICE: VLabel "Person1" has been created + create_vlabel +--------------- + +(1 row) + +SELECT create_vlabel('agload_security', 'Person2'); +NOTICE: VLabel "Person2" has been created + create_vlabel +--------------- + +(1 row) + +SELECT create_elabel('agload_security', 'SecEdge'); +NOTICE: ELabel "SecEdge" has been created + create_elabel +--------------- + +(1 row) + +-- +-- Test 1: File read permission (pg_read_server_files role) +-- +-- Create a user without pg_read_server_files role +CREATE USER load_test_user; +GRANT USAGE ON SCHEMA ag_catalog TO load_test_user; +-- This should fail because load_test_user doesn't have pg_read_server_files +SET ROLE load_test_user; +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); +ERROR: permission denied to LOAD from a file +DETAIL: Only roles with privileges of the "pg_read_server_files" role may LOAD from a file. +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); +ERROR: permission denied to LOAD from a file +DETAIL: Only roles with privileges of the "pg_read_server_files" role may LOAD from a file. +RESET ROLE; +-- Grant pg_read_server_files and try again - should fail on table permission now +GRANT pg_read_server_files TO load_test_user; +-- +-- Test 2: Table INSERT permission (ACL_INSERT) +-- +-- User has file read permission but no INSERT on the label table +SET ROLE load_test_user; +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); +ERROR: permission denied for table Person1 +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); +ERROR: permission denied for table SecEdge +RESET ROLE; +-- Grant INSERT permission and try again - should succeed +GRANT USAGE ON SCHEMA agload_security TO load_test_user; +GRANT INSERT ON agload_security."Person1" TO load_test_user; +GRANT INSERT ON agload_security."SecEdge" TO load_test_user; +GRANT UPDATE ON SEQUENCE agload_security."Person1_id_seq" TO load_test_user; +GRANT UPDATE ON SEQUENCE agload_security."SecEdge_id_seq" TO load_test_user; +GRANT SELECT ON ag_catalog.ag_label TO load_test_user; +GRANT SELECT ON ag_catalog.ag_graph TO load_test_user; +SET ROLE load_test_user; +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); + load_labels_from_file +----------------------- + +(1 row) + +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); + load_edges_from_file +---------------------- + +(1 row) + +RESET ROLE; +-- Verify data was loaded +SELECT COUNT(*) FROM agload_security."Person1"; + count +------- + 6 +(1 row) + +SELECT COUNT(*) FROM agload_security."SecEdge"; + count +------- + 6 +(1 row) + +-- cleanup +DELETE FROM agload_security."Person1"; +DELETE FROM agload_security."SecEdge"; +-- +-- Test 3: Row-Level Security (RLS) +-- +-- Enable RLS on the label tables +ALTER TABLE agload_security."Person1" ENABLE ROW LEVEL SECURITY; +ALTER TABLE agload_security."SecEdge" ENABLE ROW LEVEL SECURITY; +-- Switch to load_test_user +SET ROLE load_test_user; +-- Loading should fail when RLS is enabled +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); +ERROR: LOAD from file is not supported with row-level security +HINT: Use Cypher CREATE clause instead. +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); +ERROR: LOAD from file is not supported with row-level security +HINT: Use Cypher CREATE clause instead. +RESET ROLE; +-- Disable RLS and try again - should succeed +ALTER TABLE agload_security."Person1" DISABLE ROW LEVEL SECURITY; +ALTER TABLE agload_security."SecEdge" DISABLE ROW LEVEL SECURITY; +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); + load_labels_from_file +----------------------- + +(1 row) + +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); + load_edges_from_file +---------------------- + +(1 row) + +-- Verify data was loaded +SELECT COUNT(*) FROM agload_security."Person1"; + count +------- + 6 +(1 row) + +SELECT COUNT(*) FROM agload_security."SecEdge"; + count +------- + 6 +(1 row) + +-- cleanup +DELETE FROM agload_security."Person1"; +DELETE FROM agload_security."SecEdge"; +-- +-- Test 4: Constraint checking (CHECK constraint) +-- +-- Add constraint on vertex properties - fail if bool property is false +ALTER TABLE agload_security."Person1" ADD CONSTRAINT check_bool_true + CHECK ((properties->>'"bool"')::boolean = true); +-- This should fail - constraint violation +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); +ERROR: new row for relation "Person1" violates check constraint "check_bool_true" +DETAIL: Failing row contains (844424930131970, {"id": "2", "bool": "false", "__id__": 2, "string": "John", "num...). +-- Add constraint on edge properties - fail if bool property is false +ALTER TABLE agload_security."SecEdge" ADD CONSTRAINT check_bool_true + CHECK ((properties->>'"bool"')::boolean = true); +-- This should fail - some edges have bool = false +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); +ERROR: new row for relation "SecEdge" violates check constraint "check_bool_true" +DETAIL: Failing row contains (1407374883553294, 844424930131969, 1125899906842625, {"bool": "false", "string": "John", "numeric": "-2"}). +-- cleanup +ALTER TABLE agload_security."Person1" DROP CONSTRAINT check_bool_true; +ALTER TABLE agload_security."SecEdge" DROP CONSTRAINT check_bool_true; +-- +-- Cleanup +-- +REVOKE ALL ON agload_security."Person1" FROM load_test_user; +REVOKE ALL ON agload_security."SecEdge" FROM load_test_user; +REVOKE ALL ON SEQUENCE agload_security."Person1_id_seq" FROM load_test_user; +REVOKE ALL ON SEQUENCE agload_security."SecEdge_id_seq" FROM load_test_user; +REVOKE ALL ON ag_catalog.ag_label FROM load_test_user; +REVOKE ALL ON ag_catalog.ag_graph FROM load_test_user; +REVOKE ALL ON SCHEMA agload_security FROM load_test_user; +REVOKE ALL ON SCHEMA ag_catalog FROM load_test_user; +REVOKE pg_read_server_files FROM load_test_user; +DROP USER load_test_user; +SELECT drop_graph('agload_security', true); +NOTICE: drop cascades to 5 other objects +DETAIL: drop cascades to table agload_security._ag_label_vertex +drop cascades to table agload_security._ag_label_edge +drop cascades to table agload_security."Person1" +drop cascades to table agload_security."Person2" +drop cascades to table agload_security."SecEdge" +NOTICE: graph "agload_security" has been dropped + drop_graph +------------ + +(1 row) + -- -- End -- diff --git a/regress/expected/index.out b/regress/expected/index.out index 745cab269..ec62bf57d 100644 --- a/regress/expected/index.out +++ b/regress/expected/index.out @@ -264,19 +264,19 @@ $$) as (n agtype); (0 rows) -- Verify that the incices are created on id columns -SELECT indexname, indexdef FROM pg_indexes WHERE schemaname= 'cypher_index'; +SELECT indexname, indexdef FROM pg_indexes WHERE schemaname= 'cypher_index' ORDER BY 1; indexname | indexdef -----------------------------+------------------------------------------------------------------------------------------------ + City_pkey | CREATE UNIQUE INDEX "City_pkey" ON cypher_index."City" USING btree (id) + Country_pkey | CREATE UNIQUE INDEX "Country_pkey" ON cypher_index."Country" USING btree (id) + _ag_label_edge_end_id_idx | CREATE INDEX _ag_label_edge_end_id_idx ON cypher_index._ag_label_edge USING btree (end_id) _ag_label_edge_pkey | CREATE UNIQUE INDEX _ag_label_edge_pkey ON cypher_index._ag_label_edge USING btree (id) _ag_label_edge_start_id_idx | CREATE INDEX _ag_label_edge_start_id_idx ON cypher_index._ag_label_edge USING btree (start_id) - _ag_label_edge_end_id_idx | CREATE INDEX _ag_label_edge_end_id_idx ON cypher_index._ag_label_edge USING btree (end_id) _ag_label_vertex_pkey | CREATE UNIQUE INDEX _ag_label_vertex_pkey ON cypher_index._ag_label_vertex USING btree (id) - idx_pkey | CREATE UNIQUE INDEX idx_pkey ON cypher_index.idx USING btree (id) cypher_index_idx_props_uq | CREATE UNIQUE INDEX cypher_index_idx_props_uq ON cypher_index.idx USING btree (properties) - Country_pkey | CREATE UNIQUE INDEX "Country_pkey" ON cypher_index."Country" USING btree (id) - has_city_start_id_idx | CREATE INDEX has_city_start_id_idx ON cypher_index.has_city USING btree (start_id) has_city_end_id_idx | CREATE INDEX has_city_end_id_idx ON cypher_index.has_city USING btree (end_id) - City_pkey | CREATE UNIQUE INDEX "City_pkey" ON cypher_index."City" USING btree (id) + has_city_start_id_idx | CREATE INDEX has_city_start_id_idx ON cypher_index.has_city USING btree (start_id) + idx_pkey | CREATE UNIQUE INDEX idx_pkey ON cypher_index.idx USING btree (id) (10 rows) SET enable_mergejoin = ON; diff --git a/regress/sql/age_load.sql b/regress/sql/age_load.sql index cefcfb4ca..976f050af 100644 --- a/regress/sql/age_load.sql +++ b/regress/sql/age_load.sql @@ -194,6 +194,131 @@ SELECT load_edges_from_file('agload_conversion', 'Edges1', '../../etc/passwd', t -- SELECT drop_graph('agload_conversion', true); +-- +-- Test security and permissions +-- + +SELECT create_graph('agload_security'); +SELECT create_vlabel('agload_security', 'Person1'); +SELECT create_vlabel('agload_security', 'Person2'); +SELECT create_elabel('agload_security', 'SecEdge'); + +-- +-- Test 1: File read permission (pg_read_server_files role) +-- +-- Create a user without pg_read_server_files role +CREATE USER load_test_user; +GRANT USAGE ON SCHEMA ag_catalog TO load_test_user; + +-- This should fail because load_test_user doesn't have pg_read_server_files +SET ROLE load_test_user; +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); +RESET ROLE; + +-- Grant pg_read_server_files and try again - should fail on table permission now +GRANT pg_read_server_files TO load_test_user; + +-- +-- Test 2: Table INSERT permission (ACL_INSERT) +-- +-- User has file read permission but no INSERT on the label table +SET ROLE load_test_user; +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); +RESET ROLE; + +-- Grant INSERT permission and try again - should succeed +GRANT USAGE ON SCHEMA agload_security TO load_test_user; +GRANT INSERT ON agload_security."Person1" TO load_test_user; +GRANT INSERT ON agload_security."SecEdge" TO load_test_user; +GRANT UPDATE ON SEQUENCE agload_security."Person1_id_seq" TO load_test_user; +GRANT UPDATE ON SEQUENCE agload_security."SecEdge_id_seq" TO load_test_user; +GRANT SELECT ON ag_catalog.ag_label TO load_test_user; +GRANT SELECT ON ag_catalog.ag_graph TO load_test_user; + +SET ROLE load_test_user; +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); +RESET ROLE; + +-- Verify data was loaded +SELECT COUNT(*) FROM agload_security."Person1"; +SELECT COUNT(*) FROM agload_security."SecEdge"; + +-- cleanup +DELETE FROM agload_security."Person1"; +DELETE FROM agload_security."SecEdge"; + +-- +-- Test 3: Row-Level Security (RLS) +-- + +-- Enable RLS on the label tables +ALTER TABLE agload_security."Person1" ENABLE ROW LEVEL SECURITY; +ALTER TABLE agload_security."SecEdge" ENABLE ROW LEVEL SECURITY; + +-- Switch to load_test_user +SET ROLE load_test_user; + +-- Loading should fail when RLS is enabled +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); + +RESET ROLE; + +-- Disable RLS and try again - should succeed +ALTER TABLE agload_security."Person1" DISABLE ROW LEVEL SECURITY; +ALTER TABLE agload_security."SecEdge" DISABLE ROW LEVEL SECURITY; + +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); + +-- Verify data was loaded +SELECT COUNT(*) FROM agload_security."Person1"; +SELECT COUNT(*) FROM agload_security."SecEdge"; + +-- cleanup +DELETE FROM agload_security."Person1"; +DELETE FROM agload_security."SecEdge"; + +-- +-- Test 4: Constraint checking (CHECK constraint) +-- + +-- Add constraint on vertex properties - fail if bool property is false +ALTER TABLE agload_security."Person1" ADD CONSTRAINT check_bool_true + CHECK ((properties->>'"bool"')::boolean = true); + +-- This should fail - constraint violation +SELECT load_labels_from_file('agload_security', 'Person1', 'age_load/conversion_vertices.csv', true); + +-- Add constraint on edge properties - fail if bool property is false +ALTER TABLE agload_security."SecEdge" ADD CONSTRAINT check_bool_true + CHECK ((properties->>'"bool"')::boolean = true); + +-- This should fail - some edges have bool = false +SELECT load_edges_from_file('agload_security', 'SecEdge', 'age_load/conversion_edges.csv'); + +-- cleanup +ALTER TABLE agload_security."Person1" DROP CONSTRAINT check_bool_true; +ALTER TABLE agload_security."SecEdge" DROP CONSTRAINT check_bool_true; + +-- +-- Cleanup +-- +REVOKE ALL ON agload_security."Person1" FROM load_test_user; +REVOKE ALL ON agload_security."SecEdge" FROM load_test_user; +REVOKE ALL ON SEQUENCE agload_security."Person1_id_seq" FROM load_test_user; +REVOKE ALL ON SEQUENCE agload_security."SecEdge_id_seq" FROM load_test_user; +REVOKE ALL ON ag_catalog.ag_label FROM load_test_user; +REVOKE ALL ON ag_catalog.ag_graph FROM load_test_user; +REVOKE ALL ON SCHEMA agload_security FROM load_test_user; +REVOKE ALL ON SCHEMA ag_catalog FROM load_test_user; +REVOKE pg_read_server_files FROM load_test_user; +DROP USER load_test_user; +SELECT drop_graph('agload_security', true); + -- -- End -- diff --git a/regress/sql/index.sql b/regress/sql/index.sql index a6e075c70..d4a4b24a4 100644 --- a/regress/sql/index.sql +++ b/regress/sql/index.sql @@ -165,7 +165,7 @@ SELECT * FROM cypher('cypher_index', $$ $$) as (n agtype); -- Verify that the incices are created on id columns -SELECT indexname, indexdef FROM pg_indexes WHERE schemaname= 'cypher_index'; +SELECT indexname, indexdef FROM pg_indexes WHERE schemaname= 'cypher_index' ORDER BY 1; SET enable_mergejoin = ON; SET enable_hashjoin = OFF; diff --git a/src/backend/utils/load/ag_load_edges.c b/src/backend/utils/load/ag_load_edges.c index 931c6e0dc..c05bf3352 100644 --- a/src/backend/utils/load/ag_load_edges.c +++ b/src/backend/utils/load/ag_load_edges.c @@ -16,50 +16,30 @@ * specific language governing permissions and limitations * under the License. */ - #include "postgres.h" -#include "utils/load/ag_load_edges.h" -#include "utils/load/csv.h" +#include "access/heapam.h" +#include "access/table.h" +#include "catalog/namespace.h" +#include "commands/copy.h" +#include "executor/executor.h" +#include "nodes/makefuncs.h" +#include "parser/parse_node.h" +#include "utils/memutils.h" +#include "utils/rel.h" -void edge_field_cb(void *field, size_t field_len, void *data) -{ - - csv_edge_reader *cr = (csv_edge_reader*)data; - if (cr->error) - { - cr->error = 1; - ereport(NOTICE,(errmsg("There is some unknown error"))); - } - - /* check for space to store this field */ - if (cr->cur_field == cr->alloc) - { - cr->alloc *= 2; - cr->fields = repalloc_check(cr->fields, sizeof(char *) * cr->alloc); - cr->fields_len = repalloc_check(cr->header, sizeof(size_t *) * cr->alloc); - if (cr->fields == NULL) - { - cr->error = 1; - ereport(ERROR, - (errmsg("field_cb: failed to reallocate %zu bytes\n", - sizeof(char *) * cr->alloc))); - } - } - cr->fields_len[cr->cur_field] = field_len; - cr->curr_row_length += field_len; - cr->fields[cr->cur_field] = pnstrdup((char*)field, field_len); - cr->cur_field += 1; -} +#include "utils/load/ag_load_edges.h" -/* Parser calls this function when it detects end of a row */ -void edge_row_cb(int delim __attribute__((unused)), void *data) +/* + * Process a single edge row from COPY's raw fields. + * Edge CSV format: start_id, start_vertex_type, end_id, end_vertex_type, [properties...] + */ +static void process_edge_row(char **fields, int nfields, + char **header, int header_count, + int label_id, Oid label_seq_relid, + Oid graph_oid, bool load_as_agtype, + batch_insert_state *batch_state) { - - csv_edge_reader *cr = (csv_edge_reader*)data; - batch_insert_state *batch_state = cr->batch_state; - - size_t i, n_fields; int64 start_id_int; graphid start_vertex_graph_id; int start_vertex_type_id; @@ -72,104 +52,92 @@ void edge_row_cb(int delim __attribute__((unused)), void *data) int64 entry_id; TupleTableSlot *slot; - n_fields = cr->cur_field; + char *start_vertex_type; + char *end_vertex_type; + agtype *edge_properties; - if (cr->row == 0) - { - cr->header_num = cr->cur_field; - cr->header_row_length = cr->curr_row_length; - cr->header_len = (size_t* )palloc(sizeof(size_t *) * cr->cur_field); - cr->header = palloc((sizeof (char*) * cr->cur_field)); + /* Generate edge ID */ + entry_id = nextval_internal(label_seq_relid, true); + edge_id = make_graphid(label_id, entry_id); - for (i = 0; icur_field; i++) - { - cr->header_len[i] = cr->fields_len[i]; - cr->header[i] = pnstrdup(cr->fields[i], cr->header_len[i]); - } - } - else - { - entry_id = nextval_internal(cr->label_seq_relid, true); - edge_id = make_graphid(cr->label_id, entry_id); - - start_id_int = strtol(cr->fields[0], NULL, 10); - start_vertex_type_id = get_label_id(cr->fields[1], cr->graph_oid); - end_id_int = strtol(cr->fields[2], NULL, 10); - end_vertex_type_id = get_label_id(cr->fields[3], cr->graph_oid); - - start_vertex_graph_id = make_graphid(start_vertex_type_id, start_id_int); - end_vertex_graph_id = make_graphid(end_vertex_type_id, end_id_int); - - /* Get the appropriate slot from the batch state */ - slot = batch_state->slots[batch_state->num_tuples]; - - /* Clear the slots contents */ - ExecClearTuple(slot); - - /* Fill the values in the slot */ - slot->tts_values[0] = GRAPHID_GET_DATUM(edge_id); - slot->tts_values[1] = GRAPHID_GET_DATUM(start_vertex_graph_id); - slot->tts_values[2] = GRAPHID_GET_DATUM(end_vertex_graph_id); - slot->tts_values[3] = AGTYPE_P_GET_DATUM( - create_agtype_from_list_i( - cr->header, cr->fields, - n_fields, 4, cr->load_as_agtype)); - slot->tts_isnull[0] = false; - slot->tts_isnull[1] = false; - slot->tts_isnull[2] = false; - slot->tts_isnull[3] = false; - - /* Make the slot as containing virtual tuple */ - ExecStoreVirtualTuple(slot); - batch_state->num_tuples++; - - if (batch_state->num_tuples >= batch_state->max_tuples) - { - /* Insert the batch when it is full (i.e. BATCH_SIZE) */ - insert_batch(batch_state); - batch_state->num_tuples = 0; - } - } + /* Trim whitespace from vertex type names */ + start_vertex_type = trim_whitespace(fields[1]); + end_vertex_type = trim_whitespace(fields[3]); - for (i = 0; i < n_fields; ++i) - { - pfree_if_not_null(cr->fields[i]); - } + /* Parse start vertex info */ + start_id_int = strtol(fields[0], NULL, 10); + start_vertex_type_id = get_label_id(start_vertex_type, graph_oid); - if (cr->error) - { - ereport(NOTICE,(errmsg("THere is some error"))); - } + /* Parse end vertex info */ + end_id_int = strtol(fields[2], NULL, 10); + end_vertex_type_id = get_label_id(end_vertex_type, graph_oid); - cr->cur_field = 0; - cr->curr_row_length = 0; - cr->row += 1; -} + /* Create graphids for start and end vertices */ + start_vertex_graph_id = make_graphid(start_vertex_type_id, start_id_int); + end_vertex_graph_id = make_graphid(end_vertex_type_id, end_id_int); -static int is_space(unsigned char c) -{ - if (c == CSV_SPACE || c == CSV_TAB) - { - return 1; - } - else + /* Get the appropriate slot from the batch state */ + slot = batch_state->slots[batch_state->num_tuples]; + + /* Clear the slots contents */ + ExecClearTuple(slot); + + /* Build the agtype properties */ + edge_properties = create_agtype_from_list_i(header, fields, + nfields, 4, load_as_agtype); + + /* Fill the values in the slot */ + slot->tts_values[0] = GRAPHID_GET_DATUM(edge_id); + slot->tts_values[1] = GRAPHID_GET_DATUM(start_vertex_graph_id); + slot->tts_values[2] = GRAPHID_GET_DATUM(end_vertex_graph_id); + slot->tts_values[3] = AGTYPE_P_GET_DATUM(edge_properties); + slot->tts_isnull[0] = false; + slot->tts_isnull[1] = false; + slot->tts_isnull[2] = false; + slot->tts_isnull[3] = false; + + /* Make the slot as containing virtual tuple */ + ExecStoreVirtualTuple(slot); + + batch_state->buffered_bytes += VARSIZE(edge_properties); + batch_state->num_tuples++; + + /* Insert the batch when tuple count OR byte threshold is reached */ + if (batch_state->num_tuples >= BATCH_SIZE || + batch_state->buffered_bytes >= MAX_BUFFERED_BYTES) { - return 0; + insert_batch(batch_state); + batch_state->num_tuples = 0; + batch_state->buffered_bytes = 0; } } -static int is_term(unsigned char c) +/* + * Create COPY options for CSV parsing. + * Returns a List of DefElem nodes. + */ +static List *create_copy_options(void) { - if (c == CSV_CR || c == CSV_LF) - { - return 1; - } - else - { - return 0; - } + List *options = NIL; + + /* FORMAT csv */ + options = lappend(options, + makeDefElem("format", + (Node *) makeString("csv"), + -1)); + + /* HEADER false - we'll read the header ourselves */ + options = lappend(options, + makeDefElem("header", + (Node *) makeBoolean(false), + -1)); + + return options; } +/* + * Load edges from CSV file using pg's COPY infrastructure. + */ int create_edges_from_csv_file(char *file_path, char *graph_name, Oid graph_oid, @@ -177,79 +145,133 @@ int create_edges_from_csv_file(char *file_path, int label_id, bool load_as_agtype) { + Relation label_rel; + Oid label_relid; + CopyFromState cstate; + List *copy_options; + ParseState *pstate; + char **fields; + int nfields; + char **header = NULL; + int header_count = 0; + bool is_first_row = true; + char *label_seq_name; + Oid label_seq_relid; + batch_insert_state *batch_state = NULL; + MemoryContext batch_context; + MemoryContext old_context; + + /* Create a memory context for batch processing - reset after each batch */ + batch_context = AllocSetContextCreate(CurrentMemoryContext, + "AGE CSV Edge Load Batch Context", + ALLOCSET_DEFAULT_SIZES); + + /* Get the label relation */ + label_relid = get_label_relation(label_name, graph_oid); + label_rel = table_open(label_relid, RowExclusiveLock); + + /* Get sequence info */ + label_seq_name = get_label_seq_relation_name(label_name); + label_seq_relid = get_relname_relid(label_seq_name, graph_oid); + + /* Initialize the batch insert state */ + init_batch_insert(&batch_state, label_name, graph_oid); + + /* Create COPY options for CSV parsing */ + copy_options = create_copy_options(); + + /* Create a minimal ParseState for BeginCopyFrom */ + pstate = make_parsestate(NULL); - FILE *fp; - struct csv_parser p; - char buf[1024]; - size_t bytes_read; - unsigned char options = 0; - csv_edge_reader cr; - char *label_seq_name; - - if (csv_init(&p, options) != 0) + PG_TRY(); { - ereport(ERROR, - (errmsg("Failed to initialize csv parser\n"))); - } - - p.malloc_func = palloc; - p.realloc_func = repalloc_check; - p.free_func = pfree_if_not_null; + /* + * Initialize COPY FROM state. + * We pass the label relation but will only use NextCopyFromRawFields + * which returns raw parsed strings without type conversion. + */ + cstate = BeginCopyFrom(pstate, + label_rel, + NULL, /* whereClause */ + file_path, + false, /* is_program */ + NULL, /* data_source_cb */ + NIL, /* attnamelist */ + copy_options); + + /* + * Process rows using COPY's csv parsing. + * NextCopyFromRawFields uses 64KB buffers internally. + */ + while (NextCopyFromRawFields(cstate, &fields, &nfields)) + { + if (is_first_row) + { + int i; - csv_set_space_func(&p, is_space); - csv_set_term_func(&p, is_term); + /* First row is the header - save column names (in main context) */ + header_count = nfields; + header = (char **) palloc(sizeof(char *) * nfields); - fp = fopen(file_path, "rb"); - if (!fp) - { - ereport(ERROR, - (errmsg("Failed to open %s\n", file_path))); - } + for (i = 0; i < nfields; i++) + { + /* Trim whitespace from header fields */ + header[i] = trim_whitespace(fields[i]); + } - PG_TRY(); - { - label_seq_name = get_label_seq_relation_name(label_name); - - memset((void*)&cr, 0, sizeof(csv_edge_reader)); - cr.alloc = 128; - cr.fields = palloc(sizeof(char *) * cr.alloc); - cr.fields_len = palloc(sizeof(size_t *) * cr.alloc); - cr.header_row_length = 0; - cr.curr_row_length = 0; - cr.graph_name = graph_name; - cr.graph_oid = graph_oid; - cr.label_name = label_name; - cr.label_id = label_id; - cr.label_seq_relid = get_relname_relid(label_seq_name, graph_oid); - cr.load_as_agtype = load_as_agtype; - - /* Initialize the batch insert state */ - init_batch_insert(&cr.batch_state, label_name, graph_oid); - - while ((bytes_read=fread(buf, 1, 1024, fp)) > 0) - { - if (csv_parse(&p, buf, bytes_read, edge_field_cb, - edge_row_cb, &cr) != bytes_read) + is_first_row = false; + } + else { - ereport(ERROR, (errmsg("Error while parsing file: %s\n", - csv_strerror(csv_error(&p))))); + /* Switch to batch context for row processing */ + old_context = MemoryContextSwitchTo(batch_context); + + /* Data row - process it */ + process_edge_row(fields, nfields, + header, header_count, + label_id, label_seq_relid, + graph_oid, load_as_agtype, + batch_state); + + /* Switch back to main context */ + MemoryContextSwitchTo(old_context); + + /* Reset batch context after each batch to free memory */ + if (batch_state->num_tuples == 0) + { + MemoryContextReset(batch_context); + } } } - csv_fini(&p, edge_field_cb, edge_row_cb, &cr); - /* Finish any remaining batch inserts */ - finish_batch_insert(&cr.batch_state); + finish_batch_insert(&batch_state); + MemoryContextReset(batch_context); - if (ferror(fp)) - { - ereport(ERROR, (errmsg("Error while reading file %s\n", file_path))); - } + /* Clean up COPY state */ + EndCopyFrom(cstate); } PG_FINALLY(); { - fclose(fp); - csv_free(&p); + /* Free header if allocated */ + if (header != NULL) + { + int i; + for (i = 0; i < header_count; i++) + { + pfree(header[i]); + } + pfree(header); + } + + /* Close the relation */ + table_close(label_rel, RowExclusiveLock); + + /* Delete batch context */ + MemoryContextDelete(batch_context); + + /* Free parse state */ + free_parsestate(pstate); } PG_END_TRY(); diff --git a/src/backend/utils/load/ag_load_labels.c b/src/backend/utils/load/ag_load_labels.c index 1e86bbda4..5b11f68b8 100644 --- a/src/backend/utils/load/ag_load_labels.c +++ b/src/backend/utils/load/ag_load_labels.c @@ -17,155 +17,114 @@ * under the License. */ #include "postgres.h" -#include "executor/spi.h" + +#include "access/heapam.h" +#include "access/table.h" #include "catalog/namespace.h" +#include "commands/copy.h" #include "executor/executor.h" +#include "nodes/makefuncs.h" +#include "parser/parse_node.h" +#include "utils/memutils.h" +#include "utils/rel.h" #include "utils/load/ag_load_labels.h" -#include "utils/load/csv.h" - -void vertex_field_cb(void *field, size_t field_len, void *data) -{ - - csv_vertex_reader *cr = (csv_vertex_reader *) data; - - if (cr->error) - { - cr->error = 1; - ereport(NOTICE,(errmsg("There is some unknown error"))); - } - - /* check for space to store this field */ - if (cr->cur_field == cr->alloc) - { - cr->alloc *= 2; - cr->fields = repalloc_check(cr->fields, sizeof(char *) * cr->alloc); - cr->fields_len = repalloc_check(cr->header, sizeof(size_t *) * cr->alloc); - if (cr->fields == NULL) - { - cr->error = 1; - ereport(ERROR, - (errmsg("field_cb: failed to reallocate %zu bytes\n", - sizeof(char *) * cr->alloc))); - } - } - cr->fields_len[cr->cur_field] = field_len; - cr->curr_row_length += field_len; - cr->fields[cr->cur_field] = pnstrdup((char *) field, field_len); - cr->cur_field += 1; -} -void vertex_row_cb(int delim __attribute__((unused)), void *data) +/* + * Process a single vertex row from COPY's raw fields. + * Vertex CSV format: [id,] [properties...] + */ +static void process_vertex_row(char **fields, int nfields, + char **header, int header_count, + int label_id, Oid label_seq_relid, + bool id_field_exists, bool load_as_agtype, + int64 *curr_seq_num, + batch_insert_state *batch_state) { - csv_vertex_reader *cr = (csv_vertex_reader*)data; - batch_insert_state *batch_state = cr->batch_state; - size_t i, n_fields; graphid vertex_id; int64 entry_id; TupleTableSlot *slot; + agtype *vertex_properties; - n_fields = cr->cur_field; - - if (cr->row == 0) + /* Generate or use provided entry_id */ + if (id_field_exists) { - cr->header_num = cr->cur_field; - cr->header_row_length = cr->curr_row_length; - cr->header_len = (size_t* )palloc(sizeof(size_t *) * cr->cur_field); - cr->header = palloc((sizeof (char*) * cr->cur_field)); - - for (i = 0; icur_field; i++) + entry_id = strtol(fields[0], NULL, 10); + if (entry_id > *curr_seq_num) { - cr->header_len[i] = cr->fields_len[i]; - cr->header[i] = pnstrdup(cr->fields[i], cr->header_len[i]); + /* This is needed to ensure the sequence is up-to-date */ + DirectFunctionCall2(setval_oid, + ObjectIdGetDatum(label_seq_relid), + Int64GetDatum(entry_id)); + *curr_seq_num = entry_id; } } else { - if (cr->id_field_exists) - { - entry_id = strtol(cr->fields[0], NULL, 10); - if (entry_id > cr->curr_seq_num) - { - DirectFunctionCall2(setval_oid, - ObjectIdGetDatum(cr->label_seq_relid), - Int64GetDatum(entry_id)); - cr->curr_seq_num = entry_id; - } - } - else - { - entry_id = nextval_internal(cr->label_seq_relid, true); - } + entry_id = nextval_internal(label_seq_relid, true); + } - vertex_id = make_graphid(cr->label_id, entry_id); + vertex_id = make_graphid(label_id, entry_id); - /* Get the appropriate slot from the batch state */ - slot = batch_state->slots[batch_state->num_tuples]; + /* Get the appropriate slot from the batch state */ + slot = batch_state->slots[batch_state->num_tuples]; - /* Clear the slots contents */ - ExecClearTuple(slot); + /* Clear the slots contents */ + ExecClearTuple(slot); - /* Fill the values in the slot */ - slot->tts_values[0] = GRAPHID_GET_DATUM(vertex_id); - slot->tts_values[1] = AGTYPE_P_GET_DATUM( - create_agtype_from_list(cr->header, cr->fields, - n_fields, entry_id, - cr->load_as_agtype)); - slot->tts_isnull[0] = false; - slot->tts_isnull[1] = false; + /* Build the agtype properties */ + vertex_properties = create_agtype_from_list(header, fields, + nfields, entry_id, + load_as_agtype); - /* Make the slot as containing virtual tuple */ - ExecStoreVirtualTuple(slot); + /* Fill the values in the slot */ + slot->tts_values[0] = GRAPHID_GET_DATUM(vertex_id); + slot->tts_values[1] = AGTYPE_P_GET_DATUM(vertex_properties); + slot->tts_isnull[0] = false; + slot->tts_isnull[1] = false; - batch_state->num_tuples++; + /* Make the slot as containing virtual tuple */ + ExecStoreVirtualTuple(slot); - if (batch_state->num_tuples >= batch_state->max_tuples) - { - /* Insert the batch when it is full (i.e. BATCH_SIZE) */ - insert_batch(batch_state); - batch_state->num_tuples = 0; - } - } + batch_state->buffered_bytes += VARSIZE(vertex_properties); + batch_state->num_tuples++; - for (i = 0; i < n_fields; ++i) + /* Insert the batch when tuple count OR byte threshold is reached */ + if (batch_state->num_tuples >= BATCH_SIZE || + batch_state->buffered_bytes >= MAX_BUFFERED_BYTES) { - pfree_if_not_null(cr->fields[i]); + insert_batch(batch_state); + batch_state->num_tuples = 0; + batch_state->buffered_bytes = 0; } - - if (cr->error) - { - ereport(NOTICE,(errmsg("THere is some error"))); - } - - cr->cur_field = 0; - cr->curr_row_length = 0; - cr->row += 1; } -static int is_space(unsigned char c) +/* + * Create COPY options for csv parsing. + * Returns a List of DefElem nodes. + */ +static List *create_copy_options(void) { - if (c == CSV_SPACE || c == CSV_TAB) - { - return 1; - } - else - { - return 0; - } + List *options = NIL; -} -static int is_term(unsigned char c) -{ - if (c == CSV_CR || c == CSV_LF) - { - return 1; - } - else - { - return 0; - } + /* FORMAT csv */ + options = lappend(options, + makeDefElem("format", + (Node *) makeString("csv"), + -1)); + + /* HEADER false - we'll read the header ourselves */ + options = lappend(options, + makeDefElem("header", + (Node *) makeBoolean(false), + -1)); + + return options; } +/* + * Load vertex labels from csv file using pg's COPY infrastructure. + */ int create_labels_from_csv_file(char *file_path, char *graph_name, Oid graph_oid, @@ -174,96 +133,146 @@ int create_labels_from_csv_file(char *file_path, bool id_field_exists, bool load_as_agtype) { - - FILE *fp; - struct csv_parser p; - char buf[1024]; - size_t bytes_read; - unsigned char options = 0; - csv_vertex_reader cr; - char *label_seq_name; - - if (csv_init(&p, options) != 0) + Relation label_rel; + Oid label_relid; + CopyFromState cstate; + List *copy_options; + ParseState *pstate; + char **fields; + int nfields; + char **header = NULL; + int header_count = 0; + bool is_first_row = true; + char *label_seq_name; + Oid label_seq_relid; + int64 curr_seq_num = 0; + batch_insert_state *batch_state = NULL; + MemoryContext batch_context; + MemoryContext old_context; + + /* Create a memory context for batch processing - reset after each batch */ + batch_context = AllocSetContextCreate(CurrentMemoryContext, + "AGE CSV Load Batch Context", + ALLOCSET_DEFAULT_SIZES); + + /* Get the label relation */ + label_relid = get_label_relation(label_name, graph_oid); + label_rel = table_open(label_relid, RowExclusiveLock); + + /* Get sequence info */ + label_seq_name = get_label_seq_relation_name(label_name); + label_seq_relid = get_relname_relid(label_seq_name, graph_oid); + + if (id_field_exists) { - ereport(ERROR, - (errmsg("Failed to initialize csv parser\n"))); + /* + * Set the curr_seq_num since we will need it to compare with + * incoming entry_id. + */ + curr_seq_num = nextval_internal(label_seq_relid, true); } - p.malloc_func = palloc; - p.realloc_func = repalloc_check; - p.free_func = pfree_if_not_null; + /* Initialize the batch insert state */ + init_batch_insert(&batch_state, label_name, graph_oid); - csv_set_space_func(&p, is_space); - csv_set_term_func(&p, is_term); + /* Create COPY options for CSV parsing */ + copy_options = create_copy_options(); - fp = fopen(file_path, "rb"); - if (!fp) - { - ereport(ERROR, - (errmsg("Failed to open %s\n", file_path))); - } + /* Create a minimal ParseState for BeginCopyFrom */ + pstate = make_parsestate(NULL); PG_TRY(); { - label_seq_name = get_label_seq_relation_name(label_name); - - memset((void*)&cr, 0, sizeof(csv_vertex_reader)); - - cr.alloc = 2048; - cr.fields = palloc(sizeof(char *) * cr.alloc); - cr.fields_len = palloc(sizeof(size_t *) * cr.alloc); - cr.header_row_length = 0; - cr.curr_row_length = 0; - cr.graph_name = graph_name; - cr.graph_oid = graph_oid; - cr.label_name = label_name; - cr.label_id = label_id; - cr.id_field_exists = id_field_exists; - cr.label_seq_relid = get_relname_relid(label_seq_name, graph_oid); - cr.load_as_agtype = load_as_agtype; - - if (cr.id_field_exists) + /* + * Initialize COPY FROM state. + * We pass the label relation but will only use NextCopyFromRawFields + * which returns raw parsed strings without type conversion. + */ + cstate = BeginCopyFrom(pstate, + label_rel, + NULL, /* whereClause */ + file_path, + false, /* is_program */ + NULL, /* data_source_cb */ + NIL, /* attnamelist - NULL means all columns */ + copy_options); + + /* + * Process rows using COPY's csv parsing. + * NextCopyFromRawFields uses 64KB buffers internally. + */ + while (NextCopyFromRawFields(cstate, &fields, &nfields)) { - /* - * Set the curr_seq_num since we will need it to compare with - * incoming entry_id. - * - * We cant use currval because it will error out if nextval was - * not called before in the session. - */ - cr.curr_seq_num = nextval_internal(cr.label_seq_relid, true); - } + if (is_first_row) + { + int i; - /* Initialize the batch insert state */ - init_batch_insert(&cr.batch_state, label_name, graph_oid); + /* First row is the header - save column names (in main context) */ + header_count = nfields; + header = (char **) palloc(sizeof(char *) * nfields); - while ((bytes_read=fread(buf, 1, 1024, fp)) > 0) - { - if (csv_parse(&p, buf, bytes_read, vertex_field_cb, - vertex_row_cb, &cr) != bytes_read) + for (i = 0; i < nfields; i++) + { + /* Trim whitespace from header fields */ + header[i] = trim_whitespace(fields[i]); + } + + is_first_row = false; + } + else { - ereport(ERROR, (errmsg("Error while parsing file: %s\n", - csv_strerror(csv_error(&p))))); + /* Switch to batch context for row processing */ + old_context = MemoryContextSwitchTo(batch_context); + + /* Data row - process it */ + process_vertex_row(fields, nfields, + header, header_count, + label_id, label_seq_relid, + id_field_exists, load_as_agtype, + &curr_seq_num, + batch_state); + + /* Switch back to main context */ + MemoryContextSwitchTo(old_context); + + /* Reset batch context after each batch to free memory */ + if (batch_state->num_tuples == 0) + { + MemoryContextReset(batch_context); + } } } - csv_fini(&p, vertex_field_cb, vertex_row_cb, &cr); - /* Finish any remaining batch inserts */ - finish_batch_insert(&cr.batch_state); + finish_batch_insert(&batch_state); + MemoryContextReset(batch_context); - if (ferror(fp)) - { - ereport(ERROR, (errmsg("Error while reading file %s\n", - file_path))); - } + /* Clean up COPY state */ + EndCopyFrom(cstate); } PG_FINALLY(); { - fclose(fp); - csv_free(&p); + /* Free header if allocated */ + if (header != NULL) + { + int i; + for (i = 0; i < header_count; i++) + { + pfree(header[i]); + } + pfree(header); + } + + /* Close the relation */ + table_close(label_rel, RowExclusiveLock); + + /* Delete batch context */ + MemoryContextDelete(batch_context); + + /* Free parse state */ + free_parsestate(pstate); } PG_END_TRY(); return EXIT_SUCCESS; -} \ No newline at end of file +} diff --git a/src/backend/utils/load/age_load.c b/src/backend/utils/load/age_load.c index c7cf0677f..f9634668c 100644 --- a/src/backend/utils/load/age_load.c +++ b/src/backend/utils/load/age_load.c @@ -18,24 +18,81 @@ */ #include "postgres.h" + +#include "access/heapam.h" +#include "access/table.h" +#include "access/tableam.h" +#include "access/xact.h" #include "catalog/indexing.h" +#include "catalog/pg_authid.h" #include "executor/executor.h" +#include "miscadmin.h" +#include "nodes/parsenodes.h" +#include "parser/parse_relation.h" +#include "utils/acl.h" #include "utils/json.h" +#include "utils/rel.h" +#include "utils/rls.h" #include "utils/load/ag_load_edges.h" #include "utils/load/ag_load_labels.h" #include "utils/load/age_load.h" -#include "utils/rel.h" static agtype_value *csv_value_to_agtype_value(char *csv_val); static Oid get_or_create_graph(const Name graph_name); static int32 get_or_create_label(Oid graph_oid, char *graph_name, char *label_name, char label_kind); static char *build_safe_filename(char *name); +static void check_file_read_permission(void); +static void check_table_permissions(Oid relid); +static void check_rls_for_load(Oid relid); #define AGE_BASE_CSV_DIRECTORY "/tmp/age/" #define AGE_CSV_FILE_EXTENSION ".csv" +/* + * Trim leading and trailing whitespace from a string. + * Returns a newly allocated string with whitespace removed. + * Returns empty string for NULL input. + */ +char *trim_whitespace(const char *str) +{ + const char *start; + const char *end; + size_t len; + + if (str == NULL) + { + return pstrdup(""); + } + + /* Find first non-whitespace character */ + start = str; + while (*start && (*start == ' ' || *start == '\t' || + *start == '\n' || *start == '\r')) + { + start++; + } + + /* If string is all whitespace, return empty string */ + if (*start == '\0') + { + return pstrdup(""); + } + + /* Find last non-whitespace character */ + end = str + strlen(str) - 1; + while (end > start && (*end == ' ' || *end == '\t' || + *end == '\n' || *end == '\r')) + { + end--; + } + + /* Copy the trimmed string */ + len = end - start + 1; + return pnstrdup(start, len); +} + static char *build_safe_filename(char *name) { int length; @@ -88,6 +145,51 @@ static char *build_safe_filename(char *name) return resolved; } +/* + * Check if the current user has permission to read server files. + * Only users with the pg_read_server_files role can load from files. + */ +static void check_file_read_permission(void) +{ + if (!has_privs_of_role(GetUserId(), ROLE_PG_READ_SERVER_FILES)) + { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to LOAD from a file"), + errdetail("Only roles with privileges of the \"%s\" role may LOAD from a file.", + "pg_read_server_files"))); + } +} + +/* + * Check if the current user has INSERT permission on the target table. + */ +static void check_table_permissions(Oid relid) +{ + AclResult aclresult; + + aclresult = pg_class_aclcheck(relid, GetUserId(), ACL_INSERT); + if (aclresult != ACLCHECK_OK) + { + aclcheck_error(aclresult, OBJECT_TABLE, get_rel_name(relid)); + } +} + +/* + * Check if RLS is enabled on the target table. + * CSV loading is not supported with row-level security. + */ +static void check_rls_for_load(Oid relid) +{ + if (check_enable_rls(relid, InvalidOid, true) == RLS_ENABLED) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("LOAD from file is not supported with row-level security"), + errhint("Use Cypher CREATE clause instead."))); + } +} + agtype *create_empty_agtype(void) { agtype* out; @@ -118,6 +220,14 @@ static agtype_value *csv_value_to_agtype_value(char *csv_val) char *new_csv_val; agtype_value *res; + /* Handle NULL or empty input - return null agtype value */ + if (csv_val == NULL || csv_val[0] == '\0') + { + res = palloc(sizeof(agtype_value)); + res->type = AGTV_NULL; + return res; + } + if (!json_validate(cstring_to_text(csv_val), false, false)) { /* wrap the string with double-quote */ @@ -175,18 +285,40 @@ agtype *create_agtype_from_list(char **header, char **fields, size_t fields_len, for (i = 0; itype = AGTV_STRING; + value_agtype->val.string.len = 0; + value_agtype->val.string.val = pstrdup(""); + } + else + { + value_agtype = string_to_agtype_value(trimmed_value); + } } result.res = push_agtype_value(&result.parse_state, @@ -228,18 +360,40 @@ agtype* create_agtype_from_list_i(char **header, char **fields, for (i = start_index; i < fields_len; i++) { + char *trimmed_value; + + /* Skip empty header fields (e.g., from trailing commas) */ + if (header[i] == NULL || header[i][0] == '\0') + { + continue; + } + key_agtype = string_to_agtype_value(header[i]); result.res = push_agtype_value(&result.parse_state, WAGT_KEY, key_agtype); + /* Trim whitespace from field value */ + trimmed_value = trim_whitespace(fields[i]); + if (load_as_agtype) { - value_agtype = csv_value_to_agtype_value(fields[i]); + value_agtype = csv_value_to_agtype_value(trimmed_value); } else { - value_agtype = string_to_agtype_value(fields[i]); + /* Handle empty field values */ + if (trimmed_value[0] == '\0') + { + value_agtype = palloc(sizeof(agtype_value)); + value_agtype->type = AGTV_STRING; + value_agtype->val.string.len = 0; + value_agtype->val.string.val = pstrdup(""); + } + else + { + value_agtype = string_to_agtype_value(trimmed_value); + } } result.res = push_agtype_value(&result.parse_state, @@ -362,11 +516,24 @@ void insert_batch(batch_insert_state *batch_state) List *result; int i; + /* Check constraints for each tuple before inserting */ + if (batch_state->resultRelInfo->ri_RelationDesc->rd_att->constr) + { + for (i = 0; i < batch_state->num_tuples; i++) + { + ExecConstraints(batch_state->resultRelInfo, + batch_state->slots[i], + batch_state->estate); + } + } + /* Insert the tuples */ heap_multi_insert(batch_state->resultRelInfo->ri_RelationDesc, batch_state->slots, batch_state->num_tuples, - GetCurrentCommandId(true), 0, NULL); - + GetCurrentCommandId(true), + TABLE_INSERT_SKIP_FSM, /* Skip free space map for bulk */ + batch_state->bistate); /* Use bulk insert state */ + /* Insert index entries for the tuples */ if (batch_state->resultRelInfo->ri_NumIndices > 0) { @@ -405,6 +572,7 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) char* label_name_str; char* file_path_str; Oid graph_oid; + Oid label_relid; int32 label_id; bool id_field_exists; bool load_as_agtype; @@ -427,6 +595,9 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) errmsg("file path must not be NULL"))); } + /* Check file read permission first */ + check_file_read_permission(); + graph_name = PG_GETARG_NAME(0); label_name = PG_GETARG_NAME(1); file_name = PG_GETARG_TEXT_P(2); @@ -447,6 +618,11 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) label_id = get_or_create_label(graph_oid, graph_name_str, label_name_str, LABEL_KIND_VERTEX); + /* Get the label relation and check permissions */ + label_relid = get_label_relation(label_name_str, graph_oid); + check_table_permissions(label_relid); + check_rls_for_load(label_relid); + create_labels_from_csv_file(file_path_str, graph_name_str, graph_oid, label_name_str, label_id, id_field_exists, load_as_agtype); @@ -459,7 +635,6 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) PG_FUNCTION_INFO_V1(load_edges_from_file); Datum load_edges_from_file(PG_FUNCTION_ARGS) { - Name graph_name; Name label_name; text* file_name; @@ -467,6 +642,7 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) char* label_name_str; char* file_path_str; Oid graph_oid; + Oid label_relid; int32 label_id; bool load_as_agtype; @@ -488,6 +664,9 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) errmsg("file path must not be NULL"))); } + /* Check file read permission first */ + check_file_read_permission(); + graph_name = PG_GETARG_NAME(0); label_name = PG_GETARG_NAME(1); file_name = PG_GETARG_TEXT_P(2); @@ -507,6 +686,11 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) label_id = get_or_create_label(graph_oid, graph_name_str, label_name_str, LABEL_KIND_EDGE); + /* Get the label relation and check permissions */ + label_relid = get_label_relation(label_name_str, graph_oid); + check_table_permissions(label_relid); + check_rls_for_load(label_relid); + create_edges_from_csv_file(file_path_str, graph_name_str, graph_oid, label_name_str, label_id, load_as_agtype); @@ -597,19 +781,42 @@ void init_batch_insert(batch_insert_state **batch_state, Oid relid; EState *estate; ResultRelInfo *resultRelInfo; + RangeTblEntry *rte; + RTEPermissionInfo *perminfo; + List *range_table = NIL; + List *perminfos = NIL; int i; - /* Open the relation */ + /* Get the relation OID */ relid = get_label_relation(label_name, graph_oid); - relation = table_open(relid, RowExclusiveLock); /* Initialize executor state */ estate = CreateExecutorState(); - /* Initialize resultRelInfo */ + /* Create range table entry for ExecConstraints */ + rte = makeNode(RangeTblEntry); + rte->rtekind = RTE_RELATION; + rte->relid = relid; + rte->relkind = RELKIND_RELATION; + rte->rellockmode = RowExclusiveLock; + rte->perminfoindex = 1; + range_table = list_make1(rte); + + /* Create permission info */ + perminfo = makeNode(RTEPermissionInfo); + perminfo->relid = relid; + perminfo->requiredPerms = ACL_INSERT; + perminfos = list_make1(perminfo); + + /* Initialize range table in executor state */ + ExecInitRangeTable(estate, range_table, perminfos); + + /* Initialize resultRelInfo - this opens the relation */ resultRelInfo = makeNode(ResultRelInfo); - InitResultRelInfo(resultRelInfo, relation, 1, NULL, estate->es_instrument); - estate->es_result_relations = &resultRelInfo; + ExecInitResultRelation(estate, resultRelInfo, 1); + + /* Get relation from resultRelInfo (opened by ExecInitResultRelation) */ + relation = resultRelInfo->ri_RelationDesc; /* Open the indices */ ExecOpenIndices(resultRelInfo, false); @@ -619,8 +826,9 @@ void init_batch_insert(batch_insert_state **batch_state, (*batch_state)->slots = palloc(sizeof(TupleTableSlot *) * BATCH_SIZE); (*batch_state)->estate = estate; (*batch_state)->resultRelInfo = resultRelInfo; - (*batch_state)->max_tuples = BATCH_SIZE; (*batch_state)->num_tuples = 0; + (*batch_state)->buffered_bytes = 0; + (*batch_state)->bistate = GetBulkInsertState(); /* Create slots */ for (i = 0; i < BATCH_SIZE; i++) @@ -651,12 +859,14 @@ void finish_batch_insert(batch_insert_state **batch_state) ExecDropSingleTupleTableSlot((*batch_state)->slots[i]); } - /* Clean up, close the indices and relation */ - ExecCloseIndices((*batch_state)->resultRelInfo); - table_close((*batch_state)->resultRelInfo->ri_RelationDesc, - RowExclusiveLock); + /* Free BulkInsertState */ + FreeBulkInsertState((*batch_state)->bistate); + + /* Close result relations and range table relations */ + ExecCloseResultRelations((*batch_state)->estate); + ExecCloseRangeTableRelations((*batch_state)->estate); - /* Clean up batch state */ + /* Clean up executor state */ FreeExecutorState((*batch_state)->estate); pfree((*batch_state)->slots); pfree(*batch_state); diff --git a/src/backend/utils/load/libcsv.c b/src/backend/utils/load/libcsv.c deleted file mode 100644 index f0e8b46be..000000000 --- a/src/backend/utils/load/libcsv.c +++ /dev/null @@ -1,549 +0,0 @@ -/* -libcsv - parse and write csv data -Copyright (C) 2008 Robert Gamble - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -*/ - -#include - -#if __STDC_VERSION__ >= 199901L -# include -#else - /* C89 doesn't have stdint.h or SIZE_MAX */ -# define SIZE_MAX ((size_t)-1) -#endif - -#include "utils/load/csv.h" - -#define VERSION "3.0.3" - -#define ROW_NOT_BEGUN 0 -#define FIELD_NOT_BEGUN 1 -#define FIELD_BEGUN 2 -#define FIELD_MIGHT_HAVE_ENDED 3 - -/* - Explanation of states - ROW_NOT_BEGUN There have not been any fields encountered for this row - FIELD_NOT_BEGUN There have been fields but we are currently not in one - FIELD_BEGUN We are in a field - FIELD_MIGHT_HAVE_ENDED - We encountered a double quote inside a quoted field, the - field is either ended or the quote is literal -*/ - -#define MEM_BLK_SIZE 128 - -#define SUBMIT_FIELD(p) \ - do { \ - if (!quoted) \ - entry_pos -= spaces; \ - if (p->options & CSV_APPEND_NULL) \ - ((p)->entry_buf[entry_pos]) = '\0'; \ - if (cb1 && (p->options & CSV_EMPTY_IS_NULL) && !quoted && entry_pos == 0) \ - cb1(NULL, entry_pos, data); \ - else if (cb1) \ - cb1(p->entry_buf, entry_pos, data); \ - pstate = FIELD_NOT_BEGUN; \ - entry_pos = quoted = spaces = 0; \ - } while (0) - -#define SUBMIT_ROW(p, c) \ - do { \ - if (cb2) \ - cb2(c, data); \ - pstate = ROW_NOT_BEGUN; \ - entry_pos = quoted = spaces = 0; \ - } while (0) - -#define SUBMIT_CHAR(p, c) ((p)->entry_buf[entry_pos++] = (c)) - -static const char *csv_errors[] = {"success", - "error parsing data while strict checking enabled", - "memory exhausted while increasing buffer size", - "data size too large", - "invalid status code"}; - -int -csv_error(const struct csv_parser *p) -{ - assert(p && "received null csv_parser"); - - /* Return the current status of the parser */ - return p->status; -} - -const char * -csv_strerror(int status) -{ - /* Return a textual description of status */ - if (status >= CSV_EINVALID || status < 0) - return csv_errors[CSV_EINVALID]; - else - return csv_errors[status]; -} - -int -csv_get_opts(const struct csv_parser *p) -{ - /* Return the currently set options of parser */ - if (p == NULL) - return -1; - - return p->options; -} - -int -csv_set_opts(struct csv_parser *p, unsigned char options) -{ - /* Set the options */ - if (p == NULL) - return -1; - - p->options = options; - return 0; -} - -int -csv_init(struct csv_parser *p, unsigned char options) -{ - /* Initialize a csv_parser object returns 0 on success, -1 on error */ - if (p == NULL) - return -1; - - p->entry_buf = NULL; - p->pstate = ROW_NOT_BEGUN; - p->quoted = 0; - p->spaces = 0; - p->entry_pos = 0; - p->entry_size = 0; - p->status = 0; - p->options = options; - p->quote_char = CSV_QUOTE; - p->delim_char = CSV_COMMA; - p->is_space = NULL; - p->is_term = NULL; - p->blk_size = MEM_BLK_SIZE; - p->malloc_func = NULL; - p->realloc_func = realloc; - p->free_func = free; - - return 0; -} - -void -csv_free(struct csv_parser *p) -{ - /* Free the entry_buffer of csv_parser object */ - if (p == NULL) - return; - - if (p->entry_buf && p->free_func) - p->free_func(p->entry_buf); - - p->entry_buf = NULL; - p->entry_size = 0; - - return; -} - -int -csv_fini(struct csv_parser *p, void (*cb1)(void *, size_t, void *), void (*cb2)(int c, void *), void *data) -{ - int quoted; - int pstate; - size_t spaces; - size_t entry_pos; - - if (p == NULL) - return -1; - - /* Finalize parsing. Needed, for example, when file does not end in a newline */ - quoted = p->quoted; - pstate = p->pstate; - spaces = p->spaces; - entry_pos = p->entry_pos; - - if ((pstate == FIELD_BEGUN) && p->quoted && (p->options & CSV_STRICT) && (p->options & CSV_STRICT_FINI)) { - /* Current field is quoted, no end-quote was seen, and CSV_STRICT_FINI is set */ - p->status = CSV_EPARSE; - return -1; - } - - switch (pstate) { - case FIELD_MIGHT_HAVE_ENDED: - p->entry_pos -= p->spaces + 1; /* get rid of spaces and original quote */ - entry_pos = p->entry_pos; - /*lint -fallthrough */ - case FIELD_NOT_BEGUN: - case FIELD_BEGUN: - /* Unnecessary: - quoted = p->quoted, pstate = p->pstate; - spaces = p->spaces, entry_pos = p->entry_pos; - */ - SUBMIT_FIELD(p); - SUBMIT_ROW(p, -1); - break; - case ROW_NOT_BEGUN: /* Already ended properly */ - ; - } - - /* Reset parser */ - p->spaces = p->quoted = p->entry_pos = p->status = 0; - p->pstate = ROW_NOT_BEGUN; - - return 0; -} - -void -csv_set_delim(struct csv_parser *p, unsigned char c) -{ - /* Set the delimiter */ - if (p) p->delim_char = c; -} - -void -csv_set_quote(struct csv_parser *p, unsigned char c) -{ - /* Set the quote character */ - if (p) p->quote_char = c; -} - -unsigned char -csv_get_delim(const struct csv_parser *p) -{ - assert(p && "received null csv_parser"); - - /* Get the delimiter */ - return p->delim_char; -} - -unsigned char -csv_get_quote(const struct csv_parser *p) -{ - assert(p && "received null csv_parser"); - - /* Get the quote character */ - return p->quote_char; -} - -void -csv_set_space_func(struct csv_parser *p, int (*f)(unsigned char)) -{ - /* Set the space function */ - if (p) p->is_space = f; -} - -void -csv_set_term_func(struct csv_parser *p, int (*f)(unsigned char)) -{ - /* Set the term function */ - if (p) p->is_term = f; -} - -void -csv_set_realloc_func(struct csv_parser *p, void *(*f)(void *, size_t)) -{ - /* Set the realloc function used to increase buffer size */ - if (p && f) p->realloc_func = f; -} - -void -csv_set_free_func(struct csv_parser *p, void (*f)(void *)) -{ - /* Set the free function used to free the buffer */ - if (p && f) p->free_func = f; -} - -void -csv_set_blk_size(struct csv_parser *p, size_t size) -{ - /* Set the block size used to increment buffer size */ - if (p) p->blk_size = size; -} - -size_t -csv_get_buffer_size(const struct csv_parser *p) -{ - /* Get the size of the entry buffer */ - if (p) - return p->entry_size; - return 0; -} - -static int -csv_increase_buffer(struct csv_parser *p) -{ - size_t to_add; - void *vp; - - if (p == NULL) return 0; - if (p->realloc_func == NULL) return 0; - - /* Increase the size of the entry buffer. Attempt to increase size by - * p->blk_size, if this is larger than SIZE_MAX try to increase current - * buffer size to SIZE_MAX. If allocation fails, try to allocate halve - * the size and try again until successful or increment size is zero. - */ - - to_add = p->blk_size; - - if ( p->entry_size >= SIZE_MAX - to_add ) - to_add = SIZE_MAX - p->entry_size; - - if (!to_add) { - p->status = CSV_ETOOBIG; - return -1; - } - - while ((vp = p->realloc_func(p->entry_buf, p->entry_size + to_add)) == NULL) { - to_add /= 2; - if (!to_add) { - p->status = CSV_ENOMEM; - return -1; - } - } - - /* Update entry buffer pointer and entry_size if successful */ - p->entry_buf = vp; - p->entry_size += to_add; - return 0; -} - -size_t -csv_parse(struct csv_parser *p, const void *s, size_t len, void (*cb1)(void *, size_t, void *), void (*cb2)(int c, void *), void *data) -{ - unsigned const char *us = s; /* Access input data as array of unsigned char */ - unsigned char c; /* The character we are currently processing */ - size_t pos = 0; /* The number of characters we have processed in this call */ - - /* Store key fields into local variables for performance */ - unsigned char delim = p->delim_char; - unsigned char quote = p->quote_char; - int (*is_space)(unsigned char) = p->is_space; - int (*is_term)(unsigned char) = p->is_term; - int quoted = p->quoted; - int pstate = p->pstate; - size_t spaces = p->spaces; - size_t entry_pos = p->entry_pos; - - - if (!p->entry_buf && pos < len) { - /* Buffer hasn't been allocated yet and len > 0 */ - if (csv_increase_buffer(p) != 0) { - p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; - return pos; - } - } - - while (pos < len) { - /* Check memory usage, increase buffer if necessary */ - if (entry_pos == ((p->options & CSV_APPEND_NULL) ? p->entry_size - 1 : p->entry_size) ) { - if (csv_increase_buffer(p) != 0) { - p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; - return pos; - } - } - - c = us[pos++]; - - switch (pstate) { - case ROW_NOT_BEGUN: - case FIELD_NOT_BEGUN: - if ((is_space ? is_space(c) : c == CSV_SPACE || c == CSV_TAB) && c!=delim) { /* Space or Tab */ - continue; - } else if (is_term ? is_term(c) : c == CSV_CR || c == CSV_LF) { /* Carriage Return or Line Feed */ - if (pstate == FIELD_NOT_BEGUN) { - SUBMIT_FIELD(p); - SUBMIT_ROW(p, c); - } else { /* ROW_NOT_BEGUN */ - /* Don't submit empty rows by default */ - if (p->options & CSV_REPALL_NL) { - SUBMIT_ROW(p, c); - } - } - continue; - } else if (c == delim) { /* Comma */ - SUBMIT_FIELD(p); - break; - } else if (c == quote) { /* Quote */ - pstate = FIELD_BEGUN; - quoted = 1; - } else { /* Anything else */ - pstate = FIELD_BEGUN; - quoted = 0; - SUBMIT_CHAR(p, c); - } - break; - case FIELD_BEGUN: - if (c == quote) { /* Quote */ - if (quoted) { - SUBMIT_CHAR(p, c); - pstate = FIELD_MIGHT_HAVE_ENDED; - } else { - /* STRICT ERROR - double quote inside non-quoted field */ - if (p->options & CSV_STRICT) { - p->status = CSV_EPARSE; - p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; - return pos-1; - } - SUBMIT_CHAR(p, c); - spaces = 0; - } - } else if (c == delim) { /* Comma */ - if (quoted) { - SUBMIT_CHAR(p, c); - } else { - SUBMIT_FIELD(p); - } - } else if (is_term ? is_term(c) : c == CSV_CR || c == CSV_LF) { /* Carriage Return or Line Feed */ - if (!quoted) { - SUBMIT_FIELD(p); - SUBMIT_ROW(p, c); - } else { - SUBMIT_CHAR(p, c); - } - } else if (!quoted && (is_space? is_space(c) : c == CSV_SPACE || c == CSV_TAB)) { /* Tab or space for non-quoted field */ - SUBMIT_CHAR(p, c); - spaces++; - } else { /* Anything else */ - SUBMIT_CHAR(p, c); - spaces = 0; - } - break; - case FIELD_MIGHT_HAVE_ENDED: - /* This only happens when a quote character is encountered in a quoted field */ - if (c == delim) { /* Comma */ - entry_pos -= spaces + 1; /* get rid of spaces and original quote */ - SUBMIT_FIELD(p); - } else if (is_term ? is_term(c) : c == CSV_CR || c == CSV_LF) { /* Carriage Return or Line Feed */ - entry_pos -= spaces + 1; /* get rid of spaces and original quote */ - SUBMIT_FIELD(p); - SUBMIT_ROW(p, c); - } else if (is_space ? is_space(c) : c == CSV_SPACE || c == CSV_TAB) { /* Space or Tab */ - SUBMIT_CHAR(p, c); - spaces++; - } else if (c == quote) { /* Quote */ - if (spaces) { - /* STRICT ERROR - unescaped double quote */ - if (p->options & CSV_STRICT) { - p->status = CSV_EPARSE; - p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; - return pos-1; - } - spaces = 0; - SUBMIT_CHAR(p, c); - } else { - /* Two quotes in a row */ - pstate = FIELD_BEGUN; - } - } else { /* Anything else */ - /* STRICT ERROR - unescaped double quote */ - if (p->options & CSV_STRICT) { - p->status = CSV_EPARSE; - p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; - return pos-1; - } - pstate = FIELD_BEGUN; - spaces = 0; - SUBMIT_CHAR(p, c); - } - break; - default: - break; - } - } - p->quoted = quoted, p->pstate = pstate, p->spaces = spaces, p->entry_pos = entry_pos; - return pos; -} - -size_t -csv_write (void *dest, size_t dest_size, const void *src, size_t src_size) -{ - return csv_write2(dest, dest_size, src, src_size, CSV_QUOTE); -} - -int -csv_fwrite (FILE *fp, const void *src, size_t src_size) -{ - return csv_fwrite2(fp, src, src_size, CSV_QUOTE); -} - -size_t -csv_write2 (void *dest, size_t dest_size, const void *src, size_t src_size, unsigned char quote) -{ - unsigned char *cdest = dest; - const unsigned char *csrc = src; - size_t chars = 0; - - if (src == NULL) - return 0; - - if (dest == NULL) - dest_size = 0; - - if (dest_size > 0) - *cdest++ = quote; - chars++; - - while (src_size) { - if (*csrc == quote) { - if (dest_size > chars) - *cdest++ = quote; - if (chars < SIZE_MAX) chars++; - } - if (dest_size > chars) - *cdest++ = *csrc; - if (chars < SIZE_MAX) chars++; - src_size--; - csrc++; - } - - if (dest_size > chars) - *cdest = quote; - if (chars < SIZE_MAX) chars++; - - return chars; -} - -int -csv_fwrite2 (FILE *fp, const void *src, size_t src_size, unsigned char quote) -{ - const unsigned char *csrc = src; - - if (fp == NULL || src == NULL) - return 0; - - if (fputc(quote, fp) == EOF) - return EOF; - - while (src_size) { - if (*csrc == quote) { - if (fputc(quote, fp) == EOF) - return EOF; - } - if (fputc(*csrc, fp) == EOF) - return EOF; - src_size--; - csrc++; - } - - if (fputc(quote, fp) == EOF) { - return EOF; - } - - return 0; -} diff --git a/src/include/utils/load/ag_load_edges.h b/src/include/utils/load/ag_load_edges.h index df663b1dd..4db00d93a 100644 --- a/src/include/utils/load/ag_load_edges.h +++ b/src/include/utils/load/ag_load_edges.h @@ -17,42 +17,28 @@ * under the License. */ -#include "access/heapam.h" -#include "utils/load/age_load.h" - #ifndef AG_LOAD_EDGES_H #define AG_LOAD_EDGES_H -typedef struct { - size_t row; - char **header; - size_t *header_len; - size_t header_num; - char **fields; - size_t *fields_len; - size_t alloc; - size_t cur_field; - int error; - size_t header_row_length; - size_t curr_row_length; - char *graph_name; - Oid graph_oid; - char *label_name; - int label_id; - Oid label_seq_relid; - char *start_vertex; - char *end_vertex; - bool load_as_agtype; - batch_insert_state *batch_state; -} csv_edge_reader; - - -void edge_field_cb(void *field, size_t field_len, void *data); -void edge_row_cb(int delim __attribute__((unused)), void *data); +#include "utils/load/age_load.h" +/* + * Load edges from a CSV file using pg's COPY infrastructure. + * + * CSV format: start_id, start_vertex_type, end_id, end_vertex_type, [properties...] + * + * Parameters: + * file_path - Path to the CSV file (must be in /tmp/age/) + * graph_name - Name of the graph + * graph_oid - OID of the graph + * label_name - Name of the edge label + * label_id - ID of the label + * load_as_agtype - If true, parse CSV values as agtype (JSON-like) + * + * Returns EXIT_SUCCESS on success. + */ int create_edges_from_csv_file(char *file_path, char *graph_name, Oid graph_oid, - char *label_name, int label_id, - bool load_as_agtype); - -#endif /*AG_LOAD_EDGES_H */ + char *label_name, int label_id, + bool load_as_agtype); +#endif /* AG_LOAD_EDGES_H */ diff --git a/src/include/utils/load/ag_load_labels.h b/src/include/utils/load/ag_load_labels.h index b8ed1572e..c3d517f30 100644 --- a/src/include/utils/load/ag_load_labels.h +++ b/src/include/utils/load/ag_load_labels.h @@ -17,46 +17,26 @@ * under the License. */ - #ifndef AG_LOAD_LABELS_H #define AG_LOAD_LABELS_H -#include "access/heapam.h" #include "utils/load/age_load.h" -struct counts { - long unsigned fields; - long unsigned allvalues; - long unsigned rows; -}; - -typedef struct { - size_t row; - char **header; - size_t *header_len; - size_t header_num; - char **fields; - size_t *fields_len; - size_t alloc; - size_t cur_field; - int error; - size_t header_row_length; - size_t curr_row_length; - char *graph_name; - Oid graph_oid; - char *label_name; - int label_id; - Oid label_seq_relid; - bool id_field_exists; - bool load_as_agtype; - int curr_seq_num; - batch_insert_state *batch_state; -} csv_vertex_reader; - - -void vertex_field_cb(void *field, size_t field_len, void *data); -void vertex_row_cb(int delim __attribute__((unused)), void *data); - +/* + * Load vertex labels from a CSV file using pg's COPY infrastructure. + * CSV format: [id,] [properties...] + * + * Parameters: + * file_path - Path to the CSV file (must be in /tmp/age/) + * graph_name - Name of the graph + * graph_oid - OID of the graph + * label_name - Name of the vertex label + * label_id - ID of the label + * id_field_exists - If true, first CSV column contains the vertex ID + * load_as_agtype - If true, parse CSV values as agtype (JSON-like) + * + * Returns EXIT_SUCCESS on success. + */ int create_labels_from_csv_file(char *file_path, char *graph_name, Oid graph_oid, char *label_name, int label_id, bool id_field_exists, bool load_as_agtype); diff --git a/src/include/utils/load/age_load.h b/src/include/utils/load/age_load.h index 72f11493d..6573c79f3 100644 --- a/src/include/utils/load/age_load.h +++ b/src/include/utils/load/age_load.h @@ -17,6 +17,10 @@ * under the License. */ +#ifndef AG_LOAD_H +#define AG_LOAD_H + +#include "access/heapam.h" #include "commands/sequence.h" #include "utils/builtins.h" #include "utils/lsyscache.h" @@ -27,10 +31,8 @@ #include "commands/graph_commands.h" #include "utils/ag_cache.h" -#ifndef AGE_ENTITY_CREATOR_H -#define AGE_ENTITY_CREATOR_H - #define BATCH_SIZE 1000 +#define MAX_BUFFERED_BYTES 65535 /* 64KB, same as pg COPY */ typedef struct batch_insert_state { @@ -38,26 +40,29 @@ typedef struct batch_insert_state ResultRelInfo *resultRelInfo; TupleTableSlot **slots; int num_tuples; - int max_tuples; + size_t buffered_bytes; + BulkInsertState bistate; } batch_insert_state; -agtype* create_empty_agtype(void); - -agtype* create_agtype_from_list(char **header, char **fields, +agtype *create_empty_agtype(void); +agtype *create_agtype_from_list(char **header, char **fields, size_t fields_len, int64 vertex_id, bool load_as_agtype); -agtype* create_agtype_from_list_i(char **header, char **fields, +agtype *create_agtype_from_list_i(char **header, char **fields, size_t fields_len, size_t start_index, bool load_as_agtype); + void insert_vertex_simple(Oid graph_oid, char *label_name, graphid vertex_id, agtype *vertex_properties); void insert_edge_simple(Oid graph_oid, char *label_name, graphid edge_id, graphid start_id, graphid end_id, - agtype* end_properties); -void insert_batch(batch_insert_state *batch_state); + agtype *edge_properties); void init_batch_insert(batch_insert_state **batch_state, char *label_name, Oid graph_oid); +void insert_batch(batch_insert_state *batch_state); void finish_batch_insert(batch_insert_state **batch_state); -#endif /* AGE_ENTITY_CREATOR_H */ +char *trim_whitespace(const char *str); + +#endif /* AG_LOAD_H */ diff --git a/src/include/utils/load/csv.h b/src/include/utils/load/csv.h deleted file mode 100644 index 062536977..000000000 --- a/src/include/utils/load/csv.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Created by Shoaib on 12/5/2021. -*/ - -/* -libcsv - parse and write csv data -Copyright (C) 2008-2021 Robert Gamble -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -*/ - -#ifndef LIBCSV_H__ -#define LIBCSV_H__ -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#define CSV_MAJOR 3 -#define CSV_MINOR 0 -#define CSV_RELEASE 3 - -/* Error Codes */ -#define CSV_SUCCESS 0 -#define CSV_EPARSE 1 /* Parse error in strict mode */ -#define CSV_ENOMEM 2 /* Out of memory while increasing buffer size */ -#define CSV_ETOOBIG 3 /* Buffer larger than SIZE_MAX needed */ -#define CSV_EINVALID 4 /* Invalid code,should never be received from csv_error*/ - - -/* parser options */ -#define CSV_STRICT 1 /* enable strict mode */ -#define CSV_REPALL_NL 2 /* report all unquoted carriage returns and linefeeds */ -#define CSV_STRICT_FINI 4 /* causes csv_fini to return CSV_EPARSE if last - field is quoted and doesn't contain ending - quote */ -#define CSV_APPEND_NULL 8 /* Ensure that all fields are null-terminated */ -#define CSV_EMPTY_IS_NULL 16 /* Pass null pointer to cb1 function when - empty, unquoted fields are encountered */ - - -/* Character values */ -#define CSV_TAB 0x09 -#define CSV_SPACE 0x20 -#define CSV_CR 0x0d -#define CSV_LF 0x0a -#define CSV_COMMA 0x2c -#define CSV_QUOTE 0x22 - -struct csv_parser { - int pstate; /* Parser state */ - int quoted; /* Is the current field a quoted field? */ - size_t spaces; /* Number of continuous spaces after quote or in a non-quoted field */ - unsigned char * entry_buf; /* Entry buffer */ - size_t entry_pos; /* Current position in entry_buf (and current size of entry) */ - size_t entry_size; /* Size of entry buffer */ - int status; /* Operation status */ - unsigned char options; - unsigned char quote_char; - unsigned char delim_char; - int (*is_space)(unsigned char); - int (*is_term)(unsigned char); - size_t blk_size; - void *(*malloc_func)(size_t); /* not used */ - void *(*realloc_func)(void *, size_t); /* function used to allocate buffer memory */ - void (*free_func)(void *); /* function used to free buffer memory */ -}; - -/* Function Prototypes */ -int csv_init(struct csv_parser *p, unsigned char options); -int csv_fini(struct csv_parser *p, void (*cb1)(void *, size_t, void *), void (*cb2)(int, void *), void *data); -void csv_free(struct csv_parser *p); -int csv_error(const struct csv_parser *p); -const char * csv_strerror(int error); -size_t csv_parse(struct csv_parser *p, const void *s, size_t len, void (*cb1)(void *, size_t, void *), void (*cb2)(int, void *), void *data); -size_t csv_write(void *dest, size_t dest_size, const void *src, size_t src_size); -int csv_fwrite(FILE *fp, const void *src, size_t src_size); -size_t csv_write2(void *dest, size_t dest_size, const void *src, size_t src_size, unsigned char quote); -int csv_fwrite2(FILE *fp, const void *src, size_t src_size, unsigned char quote); -int csv_get_opts(const struct csv_parser *p); -int csv_set_opts(struct csv_parser *p, unsigned char options); -void csv_set_delim(struct csv_parser *p, unsigned char c); -void csv_set_quote(struct csv_parser *p, unsigned char c); -unsigned char csv_get_delim(const struct csv_parser *p); -unsigned char csv_get_quote(const struct csv_parser *p); -void csv_set_space_func(struct csv_parser *p, int (*f)(unsigned char)); -void csv_set_term_func(struct csv_parser *p, int (*f)(unsigned char)); -void csv_set_realloc_func(struct csv_parser *p, void *(*)(void *, size_t)); -void csv_set_free_func(struct csv_parser *p, void (*)(void *)); -void csv_set_blk_size(struct csv_parser *p, size_t); -size_t csv_get_buffer_size(const struct csv_parser *p); - -#ifdef __cplusplus -} -#endif - -#endif From 1702ae075de6897e9555fa527060f622304a1d93 Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Tue, 20 Jan 2026 23:44:19 +0500 Subject: [PATCH 021/100] Add RLS support and fix permission checks (#2309) - Previously, age only set ACL_SELECT and ACL_INSERT in RTEPermissionInfo, bypassing pg's privilege checking for DELETE and UPDATE operations. - Additionally, RLS policies were not enforced because AGE uses CMD_SELECT for all Cypher queries, causing the rewriter to skip RLS policy application. Permission fixes: - Add ACL_DELETE permission flag for DELETE clause operations - Add ACL_UPDATE permission flag for SET/REMOVE clause operations - Recursively search RTEs including subqueries for permission info RLS support: - Implemented at executor level because age transforms all cypher queries to CMD_SELECT, so pg's rewriter never adds RLS policies for INSERT/UPDATE/DELETE operations. There isnt an appropriate rewriter hook to modify this behavior, so we do it in executor instead. - Add setup_wcos() to apply WITH CHECK policies at execution time for CREATE, SET, and MERGE operations - Add setup_security_quals() and check_security_quals() to apply USING policies for UPDATE and DELETE operations - USING policies silently filter rows (matching pg behavior) - WITH CHECK policies raise errors on violation - DETACH DELETE raises error if edge RLS blocks deletion to prevent dangling edges - Add permission checks and rls in startnode/endnode functions - Add regression tests Assisted-by AI --- Makefile | 3 +- regress/expected/security.out | 1657 ++++++++++++++++++++++++++ regress/sql/security.sql | 1451 ++++++++++++++++++++++ src/backend/executor/cypher_create.c | 8 + src/backend/executor/cypher_delete.c | 99 +- src/backend/executor/cypher_merge.c | 10 +- src/backend/executor/cypher_set.c | 89 +- src/backend/executor/cypher_utils.c | 780 ++++++++++++ src/backend/parser/cypher_clause.c | 103 ++ src/backend/utils/adt/agtype.c | 27 +- src/include/executor/cypher_utils.h | 22 + 11 files changed, 4243 insertions(+), 6 deletions(-) create mode 100644 regress/expected/security.out create mode 100644 regress/sql/security.sql diff --git a/Makefile b/Makefile index a8faa2bb8..2d2912571 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,8 @@ REGRESS = scan \ jsonb_operators \ list_comprehension \ map_projection \ - direct_field_access + direct_field_access \ + security ifneq ($(EXTRA_TESTS),) REGRESS += $(EXTRA_TESTS) diff --git a/regress/expected/security.out b/regress/expected/security.out new file mode 100644 index 000000000..59e58cb05 --- /dev/null +++ b/regress/expected/security.out @@ -0,0 +1,1657 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +-- +-- Test Privileges +-- +-- +-- Setup: Create test graph and data as superuser +-- +SELECT create_graph('security_test'); +NOTICE: graph "security_test" has been created + create_graph +-------------- + +(1 row) + +-- Create test vertices +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Alice', age: 30}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Bob', age: 25}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('security_test', $$ + CREATE (:Document {title: 'Secret', content: 'classified'}) +$$) AS (a agtype); + a +--- +(0 rows) + +-- Create test edges +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Alice'}), (b:Person {name: 'Bob'}) + CREATE (a)-[:KNOWS {since: 2020}]->(b) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Alice'}), (d:Document) + CREATE (a)-[:OWNS]->(d) +$$) AS (a agtype); + a +--- +(0 rows) + +-- +-- Create test roles with different permission levels +-- +-- Role with only SELECT (read-only) +CREATE ROLE security_test_readonly LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_readonly; +GRANT SELECT ON ALL TABLES IN SCHEMA security_test TO security_test_readonly; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_readonly; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_readonly; +-- Role with SELECT and INSERT +CREATE ROLE security_test_insert LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_insert; +GRANT SELECT, INSERT ON ALL TABLES IN SCHEMA security_test TO security_test_insert; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_insert; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_insert; +-- Grant sequence usage for ID generation +GRANT USAGE ON ALL SEQUENCES IN SCHEMA security_test TO security_test_insert; +-- Role with SELECT and UPDATE +CREATE ROLE security_test_update LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_update; +GRANT SELECT, UPDATE ON ALL TABLES IN SCHEMA security_test TO security_test_update; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_update; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_update; +-- Role with SELECT and DELETE +CREATE ROLE security_test_delete LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_delete; +GRANT SELECT, DELETE ON ALL TABLES IN SCHEMA security_test TO security_test_delete; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_delete; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_delete; +CREATE ROLE security_test_detach_delete LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_detach_delete; +GRANT SELECT ON ALL TABLES IN SCHEMA security_test TO security_test_detach_delete; +GRANT DELETE ON security_test."Person" TO security_test_detach_delete; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_detach_delete; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_detach_delete; +-- Role with all permissions +CREATE ROLE security_test_full LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_full; +GRANT ALL ON ALL TABLES IN SCHEMA security_test TO security_test_full; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_full; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_full; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA security_test TO security_test_full; +-- Role with NO SELECT on graph tables (to test read failures) +CREATE ROLE security_test_noread LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_noread; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_noread; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_noread; +-- No SELECT on security_test tables +-- ============================================================================ +-- PART 1: SELECT Permission Tests - Failure Cases (No Read Permission) +-- ============================================================================ +SET ROLE security_test_noread; +-- Test: MATCH on vertices should fail without SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person) RETURN p.name +$$) AS (name agtype); +ERROR: permission denied for table Person +-- Test: MATCH on edges should fail without SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH ()-[k:KNOWS]->() RETURN k +$$) AS (k agtype); +ERROR: permission denied for table _ag_label_vertex +-- Test: MATCH with path should fail +SELECT * FROM cypher('security_test', $$ + MATCH (a)-[e]->(b) RETURN a, e, b +$$) AS (a agtype, e agtype, b agtype); +ERROR: permission denied for table _ag_label_vertex +RESET ROLE; +-- Create role with SELECT only on base label tables, not child labels +-- NOTE: PostgreSQL inheritance allows access to child table rows when querying +-- through a parent table. This is expected behavior - SELECT on _ag_label_vertex +-- allows reading all vertices (including Person, Document) via inheritance. +CREATE ROLE security_test_base_only LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_base_only; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_base_only; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_base_only; +-- Only grant SELECT on base tables, NOT on Person, Document, KNOWS, OWNS +GRANT SELECT ON security_test._ag_label_vertex TO security_test_base_only; +GRANT SELECT ON security_test._ag_label_edge TO security_test_base_only; +SET ROLE security_test_base_only; +-- Test: MATCH (n) succeeds because PostgreSQL inheritance allows access to child rows +-- when querying through parent table. Permission on _ag_label_vertex grants read +-- access to all vertices via inheritance hierarchy. +SELECT * FROM cypher('security_test', $$ + MATCH (n) RETURN n +$$) AS (n agtype); + n +------------------------------------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "Person", "properties": {"age": 30, "name": "Alice"}}::vertex + {"id": 844424930131970, "label": "Person", "properties": {"age": 25, "name": "Bob"}}::vertex + {"id": 1125899906842625, "label": "Document", "properties": {"title": "Secret", "content": "classified"}}::vertex +(3 rows) + +-- Test: MATCH ()-[e]->() succeeds via inheritance (same reason as above) +SELECT * FROM cypher('security_test', $$ + MATCH ()-[e]->() RETURN e +$$) AS (e agtype); + e +----------------------------------------------------------------------------------------------------------------------------------------- + {"id": 1407374883553281, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {"since": 2020}}::edge + {"id": 1688849860263937, "label": "OWNS", "end_id": 1125899906842625, "start_id": 844424930131969, "properties": {}}::edge +(2 rows) + +-- ============================================================================ +-- PART 2: SELECT Permission Tests - Success Cases (Read-Only Role) +-- ============================================================================ +RESET ROLE; +SET ROLE security_test_readonly; +-- Test: MATCH should succeed with SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +--------- + "Alice" + "Bob" +(2 rows) + +-- Test: MATCH with edges should succeed +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person)-[k:KNOWS]->(b:Person) + RETURN a.name, b.name +$$) AS (a agtype, b agtype); + a | b +---------+------- + "Alice" | "Bob" +(1 row) + +-- Test: MATCH across multiple labels should succeed +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person)-[:OWNS]->(d:Document) + RETURN p.name, d.title +$$) AS (person agtype, doc agtype); + person | doc +---------+---------- + "Alice" | "Secret" +(1 row) + +-- ============================================================================ +-- PART 3: INSERT Permission Tests (CREATE clause) +-- ============================================================================ +-- Test: CREATE should fail with only SELECT permission +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Charlie'}) +$$) AS (a agtype); +ERROR: permission denied for table Person +-- Test: CREATE edge should fail +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Alice'}), (b:Person {name: 'Bob'}) + CREATE (a)-[:FRIENDS]->(b) +$$) AS (a agtype); +ERROR: permission denied for schema security_test +LINE 1: SELECT * FROM cypher('security_test', $$ + ^ +RESET ROLE; +SET ROLE security_test_insert; +-- Test: CREATE vertex should succeed with INSERT permission +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Charlie', age: 35}) +$$) AS (a agtype); + a +--- +(0 rows) + +-- Test: CREATE edge should succeed with INSERT permission +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Charlie'}), (b:Person {name: 'Alice'}) + CREATE (a)-[:KNOWS {since: 2023}]->(b) +$$) AS (a agtype); + a +--- +(0 rows) + +-- Verify the inserts worked +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'}) RETURN p.name, p.age +$$) AS (name agtype, age agtype); + name | age +-----------+----- + "Charlie" | 35 +(1 row) + +-- ============================================================================ +-- PART 4: UPDATE Permission Tests (SET clause) +-- ============================================================================ +RESET ROLE; +SET ROLE security_test_readonly; +-- Test: SET should fail with only SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Alice'}) + SET p.age = 31 + RETURN p +$$) AS (p agtype); +ERROR: permission denied for table Person +-- Test: SET on edge should fail +SELECT * FROM cypher('security_test', $$ + MATCH ()-[k:KNOWS]->() + SET k.since = 2021 + RETURN k +$$) AS (k agtype); +ERROR: permission denied for table KNOWS +RESET ROLE; +SET ROLE security_test_update; +-- Test: SET should succeed with UPDATE permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Alice'}) + SET p.age = 31 + RETURN p.name, p.age +$$) AS (name agtype, age agtype); + name | age +---------+----- + "Alice" | 31 +(1 row) + +-- Test: SET on edge should succeed +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Alice'})-[k:KNOWS]->(b:Person {name: 'Bob'}) + SET k.since = 2019 + RETURN k.since +$$) AS (since agtype); + since +------- + 2019 +(1 row) + +-- Test: SET with map update should succeed +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Bob'}) + SET p += {hobby: 'reading'} + RETURN p.name, p.hobby +$$) AS (name agtype, hobby agtype); + name | hobby +-------+----------- + "Bob" | "reading" +(1 row) + +-- ============================================================================ +-- PART 5: UPDATE Permission Tests (REMOVE clause) +-- ============================================================================ +RESET ROLE; +SET ROLE security_test_readonly; +-- Test: REMOVE should fail with only SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Bob'}) + REMOVE p.hobby + RETURN p +$$) AS (p agtype); +ERROR: permission denied for table Person +RESET ROLE; +SET ROLE security_test_update; +-- Test: REMOVE should succeed with UPDATE permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Bob'}) + REMOVE p.hobby + RETURN p.name, p.hobby +$$) AS (name agtype, hobby agtype); + name | hobby +-------+------- + "Bob" | +(1 row) + +-- ============================================================================ +-- PART 6: DELETE Permission Tests +-- ============================================================================ +RESET ROLE; +SET ROLE security_test_readonly; +-- Test: DELETE should fail with only SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'}) + DELETE p +$$) AS (a agtype); +ERROR: permission denied for table Person +RESET ROLE; +SET ROLE security_test_update; +-- Test: DELETE should fail with only UPDATE permission (need DELETE) +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'}) + DELETE p +$$) AS (a agtype); +ERROR: permission denied for table Person +RESET ROLE; +SET ROLE security_test_delete; +-- Test: DELETE vertex should succeed with DELETE permission +-- First delete the edge connected to Charlie +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'})-[k:KNOWS]->() + DELETE k +$$) AS (a agtype); + a +--- +(0 rows) + +-- Now delete the vertex +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'}) + DELETE p +$$) AS (a agtype); + a +--- +(0 rows) + +-- Verify deletion +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'}) RETURN p +$$) AS (p agtype); + p +--- +(0 rows) + +-- ============================================================================ +-- PART 7: DETACH DELETE Tests +-- ============================================================================ +RESET ROLE; +-- Create a new vertex with edge for DETACH DELETE test +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Dave', age: 40}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Alice'}), (d:Person {name: 'Dave'}) + CREATE (a)-[:KNOWS {since: 2022}]->(d) +$$) AS (a agtype); + a +--- +(0 rows) + +SET ROLE security_test_detach_delete; +-- Test: DETACH DELETE should fail without DELETE on edge table +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Dave'}) + DETACH DELETE p +$$) AS (a agtype); +ERROR: permission denied for table KNOWS +RESET ROLE; +GRANT DELETE ON security_test."KNOWS" TO security_test_detach_delete; +SET ROLE security_test_detach_delete; +-- Test: DETACH DELETE should succeed now when user has DELETE on both vertex and edge tables +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Dave'}) + DETACH DELETE p +$$) AS (a agtype); + a +--- +(0 rows) + +-- Verify deletion +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Dave'}) RETURN p +$$) AS (p agtype); + p +--- +(0 rows) + +-- ============================================================================ +-- PART 8: MERGE Permission Tests +-- ============================================================================ +RESET ROLE; +SET ROLE security_test_readonly; +-- Test: MERGE that would create should fail without INSERT +SELECT * FROM cypher('security_test', $$ + MERGE (p:Person {name: 'Eve'}) + RETURN p +$$) AS (p agtype); +ERROR: permission denied for table Person +RESET ROLE; +SET ROLE security_test_insert; +-- Test: MERGE that creates should succeed with INSERT permission +SELECT * FROM cypher('security_test', $$ + MERGE (p:Person {name: 'Eve', age: 28}) + RETURN p.name, p.age +$$) AS (name agtype, age agtype); + name | age +-------+----- + "Eve" | 28 +(1 row) + +-- Test: MERGE that matches existing should succeed (only needs SELECT) +SELECT * FROM cypher('security_test', $$ + MERGE (p:Person {name: 'Eve'}) + RETURN p.name +$$) AS (name agtype); + name +------- + "Eve" +(1 row) + +-- ============================================================================ +-- PART 9: Full Permission Role Tests +-- ============================================================================ +RESET ROLE; +SET ROLE security_test_full; +-- Full permission role should be able to do everything +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Frank', age: 50}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Frank'}) + SET p.age = 51 + RETURN p.name, p.age +$$) AS (name agtype, age agtype); + name | age +---------+----- + "Frank" | 51 +(1 row) + +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Frank'}) + DELETE p +$$) AS (a agtype); + a +--- +(0 rows) + +-- ============================================================================ +-- PART 10: Permission on Specific Labels +-- ============================================================================ +RESET ROLE; +-- Create a role with permission only on Person label, not Document +CREATE ROLE security_test_person_only LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_person_only; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_person_only; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA ag_catalog TO security_test_person_only; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_person_only; +-- Only grant permissions on Person table +GRANT SELECT, INSERT, UPDATE, DELETE ON security_test."Person" TO security_test_person_only; +GRANT SELECT ON security_test."KNOWS" TO security_test_person_only; +GRANT SELECT ON security_test._ag_label_vertex TO security_test_person_only; +GRANT SELECT ON security_test._ag_label_edge TO security_test_person_only; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA security_test TO security_test_person_only; +SET ROLE security_test_person_only; +-- Test: Operations on Person should succeed +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Alice'}) RETURN p.name +$$) AS (name agtype); + name +--------- + "Alice" +(1 row) + +-- Test: SELECT on Document should fail (no permission) +SELECT * FROM cypher('security_test', $$ + MATCH (d:Document) RETURN d.title +$$) AS (title agtype); +ERROR: permission denied for table Document +-- Test: CREATE Document should fail (no permission on Document table) +SELECT * FROM cypher('security_test', $$ + CREATE (:Document {title: 'New Doc'}) +$$) AS (a agtype); +ERROR: permission denied for table Document +-- ============================================================================ +-- PART 11: Function EXECUTE Permission Tests +-- ============================================================================ +RESET ROLE; +-- Create role with no function execute permissions +CREATE ROLE security_test_noexec LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_noexec; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_noexec; +-- Revoke execute from PUBLIC on functions we want to test +REVOKE EXECUTE ON FUNCTION ag_catalog.create_graph(name) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION ag_catalog.drop_graph(name, boolean) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION ag_catalog.create_vlabel(cstring, cstring) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION ag_catalog.create_elabel(cstring, cstring) FROM PUBLIC; +SET ROLE security_test_noexec; +-- Test: create_graph should fail without EXECUTE permission +SELECT create_graph('unauthorized_graph'); +ERROR: permission denied for function create_graph +-- Test: drop_graph should fail without EXECUTE permission +SELECT drop_graph('security_test', true); +ERROR: permission denied for function drop_graph +-- Test: create_vlabel should fail without EXECUTE permission +SELECT create_vlabel('security_test', 'NewLabel'); +ERROR: permission denied for function create_vlabel +-- Test: create_elabel should fail without EXECUTE permission +SELECT create_elabel('security_test', 'NewEdge'); +ERROR: permission denied for function create_elabel +RESET ROLE; +-- Grant execute on specific function and test +GRANT EXECUTE ON FUNCTION ag_catalog.create_vlabel(cstring, cstring) TO security_test_noexec; +SET ROLE security_test_noexec; +-- Test: create_vlabel should now get past execute check (will fail on schema permission instead) +SELECT create_vlabel('security_test', 'TestLabel'); +ERROR: permission denied for schema security_test +-- Test: create_graph should still fail with execute permission denied +SELECT create_graph('unauthorized_graph'); +ERROR: permission denied for function create_graph +RESET ROLE; +-- Restore execute permissions to PUBLIC +GRANT EXECUTE ON FUNCTION ag_catalog.create_graph(name) TO PUBLIC; +GRANT EXECUTE ON FUNCTION ag_catalog.drop_graph(name, boolean) TO PUBLIC; +GRANT EXECUTE ON FUNCTION ag_catalog.create_vlabel(cstring, cstring) TO PUBLIC; +GRANT EXECUTE ON FUNCTION ag_catalog.create_elabel(cstring, cstring) TO PUBLIC; +-- ============================================================================ +-- PART 12: startNode/endNode Permission Tests +-- ============================================================================ +-- Create role with SELECT on base tables but NOT on Person label +CREATE ROLE security_test_edge_only LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_edge_only; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_edge_only; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_edge_only; +GRANT SELECT ON security_test."KNOWS" TO security_test_edge_only; +GRANT SELECT ON security_test._ag_label_edge TO security_test_edge_only; +GRANT SELECT ON security_test._ag_label_vertex TO security_test_edge_only; +-- Note: NOT granting SELECT on security_test."Person" +SET ROLE security_test_edge_only; +-- Test: endNode fails without SELECT on Person table +SELECT * FROM cypher('security_test', $$ + MATCH ()-[e:KNOWS]->() + RETURN endNode(e) +$$) AS (end_vertex agtype); +ERROR: permission denied for table Person +-- Test: startNode fails without SELECT on Person table +SELECT * FROM cypher('security_test', $$ + MATCH ()-[e:KNOWS]->() + RETURN startNode(e) +$$) AS (start_vertex agtype); +ERROR: permission denied for table Person +RESET ROLE; +-- Grant SELECT on Person and verify success +GRANT SELECT ON security_test."Person" TO security_test_edge_only; +SET ROLE security_test_edge_only; +-- Test: Should now succeed with SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH ()-[e:KNOWS]->() + RETURN startNode(e).name, endNode(e).name +$$) AS (start_name agtype, end_name agtype); + start_name | end_name +------------+---------- + "Alice" | "Bob" +(1 row) + +RESET ROLE; +-- ============================================================================ +-- Cleanup +-- ============================================================================ +RESET ROLE; +-- Drop all owned objects and privileges for each role, then drop the role +DROP OWNED BY security_test_noread CASCADE; +DROP ROLE security_test_noread; +DROP OWNED BY security_test_base_only CASCADE; +DROP ROLE security_test_base_only; +DROP OWNED BY security_test_readonly CASCADE; +DROP ROLE security_test_readonly; +DROP OWNED BY security_test_insert CASCADE; +DROP ROLE security_test_insert; +DROP OWNED BY security_test_update CASCADE; +DROP ROLE security_test_update; +DROP OWNED BY security_test_delete CASCADE; +DROP ROLE security_test_delete; +DROP OWNED BY security_test_detach_delete CASCADE; +DROP ROLE security_test_detach_delete; +DROP OWNED BY security_test_full CASCADE; +DROP ROLE security_test_full; +DROP OWNED BY security_test_person_only CASCADE; +DROP ROLE security_test_person_only; +DROP OWNED BY security_test_noexec CASCADE; +DROP ROLE security_test_noexec; +DROP OWNED BY security_test_edge_only CASCADE; +DROP ROLE security_test_edge_only; +-- Drop test graph +SELECT drop_graph('security_test', true); +NOTICE: drop cascades to 6 other objects +DETAIL: drop cascades to table security_test._ag_label_vertex +drop cascades to table security_test._ag_label_edge +drop cascades to table security_test."Person" +drop cascades to table security_test."Document" +drop cascades to table security_test."KNOWS" +drop cascades to table security_test."OWNS" +NOTICE: graph "security_test" has been dropped + drop_graph +------------ + +(1 row) + +-- +-- Row-Level Security (RLS) Tests +-- +-- +-- Setup: Create test graph, data and roles for RLS tests +-- +SELECT create_graph('rls_graph'); +NOTICE: graph "rls_graph" has been created + create_graph +-------------- + +(1 row) + +-- Create test roles +CREATE ROLE rls_user1 LOGIN; +CREATE ROLE rls_user2 LOGIN; +CREATE ROLE rls_admin LOGIN BYPASSRLS; -- Role that bypasses RLS +-- Create base test data FIRST (as superuser) - this creates the label tables +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'Alice', owner: 'rls_user1', department: 'Engineering', level: 1}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'Bob', owner: 'rls_user2', department: 'Engineering', level: 2}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'Charlie', owner: 'rls_user1', department: 'Sales', level: 1}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'Diana', owner: 'rls_user2', department: 'Sales', level: 3}) +$$) AS (a agtype); + a +--- +(0 rows) + +-- Create a second vertex label for multi-label tests +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Document {title: 'Public Doc', classification: 'public', owner: 'rls_user1'}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Document {title: 'Secret Doc', classification: 'secret', owner: 'rls_user2'}) +$$) AS (a agtype); + a +--- +(0 rows) + +-- Create edges +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'}), (b:Person {name: 'Bob'}) + CREATE (a)-[:KNOWS {since: 2020, strength: 'weak'}]->(b) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Charlie'}), (b:Person {name: 'Diana'}) + CREATE (a)-[:KNOWS {since: 2021, strength: 'strong'}]->(b) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'}), (b:Person {name: 'Charlie'}) + CREATE (a)-[:KNOWS {since: 2022, strength: 'strong'}]->(b) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'}), (d:Document {title: 'Public Doc'}) + CREATE (a)-[:AUTHORED]->(d) +$$) AS (a agtype); + a +--- +(0 rows) + +-- Grant permissions AFTER creating tables (so Person, Document, KNOWS, AUTHORED exist) +GRANT USAGE ON SCHEMA rls_graph TO rls_user1, rls_user2, rls_admin; +GRANT ALL ON ALL TABLES IN SCHEMA rls_graph TO rls_user1, rls_user2, rls_admin; +GRANT USAGE ON SCHEMA ag_catalog TO rls_user1, rls_user2, rls_admin; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA rls_graph TO rls_user1, rls_user2, rls_admin; +-- ============================================================================ +-- PART 1: Vertex SELECT Policies (USING clause) +-- ============================================================================ +-- Enable RLS on Person label +ALTER TABLE rls_graph."Person" ENABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."Person" FORCE ROW LEVEL SECURITY; +-- 1.1: Basic ownership filtering +CREATE POLICY person_select_own ON rls_graph."Person" + FOR SELECT + USING (properties->>'"owner"' = current_user); +-- Test as rls_user1 - should only see Alice and Charlie (owned by rls_user1) +SET ROLE rls_user1; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +----------- + "Alice" + "Charlie" +(2 rows) + +-- Test as rls_user2 - should only see Bob and Diana (owned by rls_user2) +SET ROLE rls_user2; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +--------- + "Bob" + "Diana" +(2 rows) + +RESET ROLE; +-- 1.2: Default deny - no permissive policies means no access +DROP POLICY person_select_own ON rls_graph."Person"; +-- With no policies, RLS blocks all access +SET ROLE rls_user1; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +------ +(0 rows) + +RESET ROLE; +-- ============================================================================ +-- PART 2: Vertex INSERT Policies (WITH CHECK) - CREATE +-- ============================================================================ +-- Allow SELECT for all (so we can verify results) +CREATE POLICY person_select_all ON rls_graph."Person" + FOR SELECT USING (true); +-- 2.1: Basic WITH CHECK - users can only insert rows they own +CREATE POLICY person_insert_own ON rls_graph."Person" + FOR INSERT + WITH CHECK (properties->>'"owner"' = current_user); +-- Test as rls_user1 - should succeed (owner matches current_user) +SET ROLE rls_user1; +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'User1Created', owner: 'rls_user1', department: 'Test', level: 1}) +$$) AS (a agtype); + a +--- +(0 rows) + +-- Test as rls_user1 - should FAIL (owner doesn't match current_user) +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'User1Fake', owner: 'rls_user2', department: 'Test', level: 1}) +$$) AS (a agtype); +ERROR: new row violates row-level security policy for table "Person" +RESET ROLE; +-- Verify only User1Created was created +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Test' RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +---------------- + "User1Created" +(1 row) + +-- 2.2: Default deny for INSERT - no INSERT policy blocks all inserts +DROP POLICY person_insert_own ON rls_graph."Person"; +SET ROLE rls_user1; +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'ShouldFail', owner: 'rls_user1', department: 'Blocked', level: 1}) +$$) AS (a agtype); +ERROR: new row violates row-level security policy for table "Person" +RESET ROLE; +-- Verify nothing was created in Blocked department +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Blocked' RETURN p.name +$$) AS (name agtype); + name +------ +(0 rows) + +-- cleanup +DROP POLICY person_select_all ON rls_graph."Person"; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Test' DELETE p +$$) AS (a agtype); + a +--- +(0 rows) + +-- ============================================================================ +-- PART 3: Vertex UPDATE Policies - SET +-- ============================================================================ +CREATE POLICY person_select_all ON rls_graph."Person" + FOR SELECT USING (true); +-- 3.1: USING clause only - filter which rows can be updated +CREATE POLICY person_update_using ON rls_graph."Person" + FOR UPDATE + USING (properties->>'"owner"' = current_user); +SET ROLE rls_user1; +-- Should succeed - rls_user1 owns Alice +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) SET p.updated = true RETURN p.name, p.updated +$$) AS (name agtype, updated agtype); + name | updated +---------+--------- + "Alice" | true +(1 row) + +-- Should silently skip - rls_user1 doesn't own Bob (USING filters it out) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Bob'}) SET p.updated = true RETURN p.name, p.updated +$$) AS (name agtype, updated agtype); + name | updated +------+--------- +(0 rows) + +RESET ROLE; +-- Verify Alice was updated, Bob was not +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.name IN ['Alice', 'Bob'] RETURN p.name, p.updated ORDER BY p.name +$$) AS (name agtype, updated agtype); + name | updated +---------+--------- + "Alice" | true + "Bob" | +(2 rows) + +-- 3.2: WITH CHECK clause - validate new values +DROP POLICY person_update_using ON rls_graph."Person"; +CREATE POLICY person_update_check ON rls_graph."Person" + FOR UPDATE + USING (true) -- Can update any row + WITH CHECK (properties->>'"owner"' = current_user); -- But new value must keep owner +SET ROLE rls_user1; +-- Should succeed - modifying property but keeping owner +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) SET p.verified = true RETURN p.name, p.verified +$$) AS (name agtype, verified agtype); + name | verified +---------+---------- + "Alice" | true +(1 row) + +-- Should FAIL - trying to change owner to someone else +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) SET p.owner = 'rls_user2' RETURN p.owner +$$) AS (owner agtype); +ERROR: new row violates row-level security policy for table "Person" +RESET ROLE; +-- Verify owner wasn't changed +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) RETURN p.owner +$$) AS (owner agtype); + owner +------------- + "rls_user1" +(1 row) + +-- 3.3: Both USING and WITH CHECK together +DROP POLICY person_update_check ON rls_graph."Person"; +CREATE POLICY person_update_both ON rls_graph."Person" + FOR UPDATE + USING (properties->>'"owner"' = current_user) + WITH CHECK (properties->>'"owner"' = current_user); +SET ROLE rls_user1; +-- Should succeed - owns Alice, keeping owner +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) SET p.status = 'active' RETURN p.name, p.status +$$) AS (name agtype, status agtype); + name | status +---------+---------- + "Alice" | "active" +(1 row) + +-- Should silently skip - doesn't own Bob (USING filters) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Bob'}) SET p.status = 'active' RETURN p.name, p.status +$$) AS (name agtype, status agtype); + name | status +------+-------- +(0 rows) + +RESET ROLE; +-- ============================================================================ +-- PART 4: Vertex UPDATE Policies - REMOVE +-- ============================================================================ +-- Keep existing update policy, test REMOVE operation +SET ROLE rls_user1; +-- Should succeed - owns Alice +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) REMOVE p.status RETURN p.name, p.status +$$) AS (name agtype, status agtype); + name | status +---------+-------- + "Alice" | +(1 row) + +-- Should silently skip - doesn't own Bob +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Bob'}) REMOVE p.department RETURN p.name, p.department +$$) AS (name agtype, dept agtype); + name | dept +------+------ +(0 rows) + +RESET ROLE; +-- Verify Bob still has department +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Bob'}) RETURN p.department +$$) AS (dept agtype); + dept +--------------- + "Engineering" +(1 row) + +-- cleanup +DROP POLICY person_select_all ON rls_graph."Person"; +DROP POLICY person_update_both ON rls_graph."Person"; +-- ============================================================================ +-- PART 5: Vertex DELETE Policies +-- ============================================================================ +CREATE POLICY person_select_all ON rls_graph."Person" + FOR SELECT USING (true); +-- Create test data for delete tests +CREATE POLICY person_insert_all ON rls_graph."Person" + FOR INSERT WITH CHECK (true); +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'DeleteTest1', owner: 'rls_user1', department: 'DeleteTest', level: 1}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'DeleteTest2', owner: 'rls_user2', department: 'DeleteTest', level: 1}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'DeleteTest3', owner: 'rls_user1', department: 'DeleteTest', level: 1}) +$$) AS (a agtype); + a +--- +(0 rows) + +DROP POLICY person_insert_all ON rls_graph."Person"; +-- 5.1: Basic USING filtering for DELETE +CREATE POLICY person_delete_own ON rls_graph."Person" + FOR DELETE + USING (properties->>'"owner"' = current_user); +SET ROLE rls_user1; +-- Should succeed - owns DeleteTest1 +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DeleteTest1'}) DELETE p +$$) AS (a agtype); + a +--- +(0 rows) + +-- Should silently skip - doesn't own DeleteTest2 +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DeleteTest2'}) DELETE p +$$) AS (a agtype); + a +--- +(0 rows) + +RESET ROLE; +-- Verify DeleteTest1 deleted, DeleteTest2 still exists +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'DeleteTest' RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +--------------- + "DeleteTest2" + "DeleteTest3" +(2 rows) + +-- 5.2: Default deny for DELETE - no policy blocks all deletes +DROP POLICY person_delete_own ON rls_graph."Person"; +SET ROLE rls_user1; +-- Should silently skip - no DELETE policy means default deny +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DeleteTest3'}) DELETE p +$$) AS (a agtype); + a +--- +(0 rows) + +RESET ROLE; +-- Verify DeleteTest3 still exists +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DeleteTest3'}) RETURN p.name +$$) AS (name agtype); + name +--------------- + "DeleteTest3" +(1 row) + +-- cleanup +DROP POLICY person_select_all ON rls_graph."Person"; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'DeleteTest' DELETE p +$$) AS (a agtype); + a +--- +(0 rows) + +-- ============================================================================ +-- PART 6: MERGE Policies +-- ============================================================================ +CREATE POLICY person_select_all ON rls_graph."Person" + FOR SELECT USING (true); +CREATE POLICY person_insert_own ON rls_graph."Person" + FOR INSERT + WITH CHECK (properties->>'"owner"' = current_user); +-- 6.1: MERGE creating new vertex - INSERT policy applies +SET ROLE rls_user1; +-- Should succeed - creating with correct owner +SELECT * FROM cypher('rls_graph', $$ + MERGE (p:Person {name: 'MergeNew1', owner: 'rls_user1', department: 'Merge', level: 1}) + RETURN p.name +$$) AS (name agtype); + name +------------- + "MergeNew1" +(1 row) + +-- Should FAIL - creating with wrong owner +SELECT * FROM cypher('rls_graph', $$ + MERGE (p:Person {name: 'MergeNew2', owner: 'rls_user2', department: 'Merge', level: 1}) + RETURN p.name +$$) AS (name agtype); +ERROR: new row violates row-level security policy for table "Person" +RESET ROLE; +-- 6.2: MERGE matching existing - only SELECT needed +SET ROLE rls_user1; +-- Should succeed - Alice exists and SELECT allowed +SELECT * FROM cypher('rls_graph', $$ + MERGE (p:Person {name: 'Alice'}) + RETURN p.name, p.owner +$$) AS (name agtype, owner agtype); + name | owner +---------+------------- + "Alice" | "rls_user1" +(1 row) + +RESET ROLE; +-- Verify only MergeNew1 was created +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Merge' RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +------------- + "MergeNew1" +(1 row) + +-- cleanup +DROP POLICY person_select_all ON rls_graph."Person"; +DROP POLICY person_insert_own ON rls_graph."Person"; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Merge' DELETE p +$$) AS (a agtype); + a +--- +(0 rows) + +-- ============================================================================ +-- PART 7: Edge SELECT Policies +-- ============================================================================ +-- Disable vertex RLS, enable edge RLS +ALTER TABLE rls_graph."Person" DISABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."KNOWS" ENABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."KNOWS" FORCE ROW LEVEL SECURITY; +-- Policy: Only see edges from 2021 or later +CREATE POLICY knows_select_recent ON rls_graph."KNOWS" + FOR SELECT + USING ((properties->>'"since"')::int >= 2021); +SET ROLE rls_user1; +-- Should only see 2021 and 2022 edges (not 2020) +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS]->() RETURN k.since ORDER BY k.since +$$) AS (since agtype); + since +------- + 2021 + 2022 +(2 rows) + +RESET ROLE; +-- ============================================================================ +-- PART 8: Edge INSERT Policies (CREATE edge) +-- ============================================================================ +DROP POLICY knows_select_recent ON rls_graph."KNOWS"; +CREATE POLICY knows_select_all ON rls_graph."KNOWS" + FOR SELECT USING (true); +-- Policy: Can only create edges with strength = 'strong' +CREATE POLICY knows_insert_strong ON rls_graph."KNOWS" + FOR INSERT + WITH CHECK (properties->>'"strength"' = 'strong'); +SET ROLE rls_user1; +-- Should succeed - strength is 'strong' +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Bob'}), (b:Person {name: 'Diana'}) + CREATE (a)-[:KNOWS {since: 2023, strength: 'strong'}]->(b) +$$) AS (a agtype); + a +--- +(0 rows) + +-- Should FAIL - strength is 'weak' +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Diana'}), (b:Person {name: 'Alice'}) + CREATE (a)-[:KNOWS {since: 2023, strength: 'weak'}]->(b) +$$) AS (a agtype); +ERROR: new row violates row-level security policy for table "KNOWS" +RESET ROLE; +-- Verify only strong edge was created +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS]->() WHERE k.since = 2023 RETURN k.strength ORDER BY k.strength +$$) AS (strength agtype); + strength +---------- + "strong" +(1 row) + +-- cleanup +DROP POLICY knows_insert_strong ON rls_graph."KNOWS"; +-- ============================================================================ +-- PART 9: Edge UPDATE Policies (SET on edge) +-- ============================================================================ +-- Policy: Can only update edges with strength = 'strong' +CREATE POLICY knows_update_strong ON rls_graph."KNOWS" + FOR UPDATE + USING (properties->>'"strength"' = 'strong') + WITH CHECK (properties->>'"strength"' = 'strong'); +SET ROLE rls_user1; +-- Should succeed - edge has strength 'strong' +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS {since: 2021}]->() SET k.notes = 'updated' RETURN k.since, k.notes +$$) AS (since agtype, notes agtype); + since | notes +-------+----------- + 2021 | "updated" +(1 row) + +-- Should silently skip - edge has strength 'weak' +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS {since: 2020}]->() SET k.notes = 'updated' RETURN k.since, k.notes +$$) AS (since agtype, notes agtype); + since | notes +-------+------- +(0 rows) + +RESET ROLE; +-- Verify only 2021 edge was updated +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS]->() WHERE k.since IN [2020, 2021] RETURN k.since, k.notes ORDER BY k.since +$$) AS (since agtype, notes agtype); + since | notes +-------+----------- + 2020 | + 2021 | "updated" +(2 rows) + +-- cleanup +DROP POLICY knows_select_all ON rls_graph."KNOWS"; +DROP POLICY knows_update_strong ON rls_graph."KNOWS"; +-- ============================================================================ +-- PART 10: Edge DELETE Policies +-- ============================================================================ +CREATE POLICY knows_select_all ON rls_graph."KNOWS" + FOR SELECT USING (true); +-- Create test edges for delete +CREATE POLICY knows_insert_all ON rls_graph."KNOWS" + FOR INSERT WITH CHECK (true); +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Bob'}), (b:Person {name: 'Charlie'}) + CREATE (a)-[:KNOWS {since: 2018, strength: 'weak'}]->(b) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Diana'}), (b:Person {name: 'Charlie'}) + CREATE (a)-[:KNOWS {since: 2019, strength: 'strong'}]->(b) +$$) AS (a agtype); + a +--- +(0 rows) + +DROP POLICY knows_insert_all ON rls_graph."KNOWS"; +-- Policy: Can only delete edges with strength = 'weak' +CREATE POLICY knows_delete_weak ON rls_graph."KNOWS" + FOR DELETE + USING (properties->>'"strength"' = 'weak'); +SET ROLE rls_user1; +-- Should succeed - edge has strength 'weak' +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS {since: 2018}]->() DELETE k +$$) AS (a agtype); + a +--- +(0 rows) + +-- Should silently skip - edge has strength 'strong' +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS {since: 2019}]->() DELETE k +$$) AS (a agtype); + a +--- +(0 rows) + +RESET ROLE; +-- Verify 2018 edge deleted, 2019 edge still exists +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS]->() WHERE k.since IN [2018, 2019] RETURN k.since ORDER BY k.since +$$) AS (since agtype); + since +------- + 2019 +(1 row) + +-- cleanup +DROP POLICY knows_delete_weak ON rls_graph."KNOWS"; +-- ============================================================================ +-- PART 11: DETACH DELETE +-- ============================================================================ +-- Re-enable Person RLS +ALTER TABLE rls_graph."Person" ENABLE ROW LEVEL SECURITY; +CREATE POLICY person_all ON rls_graph."Person" + FOR ALL USING (true) WITH CHECK (true); +-- Create test data with a protected edge +CREATE POLICY knows_insert_all ON rls_graph."KNOWS" + FOR INSERT WITH CHECK (true); +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'DetachTest1', owner: 'test', department: 'Detach', level: 1}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'DetachTest2', owner: 'test', department: 'Detach', level: 1}) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'DetachTest1'}), (b:Person {name: 'DetachTest2'}) + CREATE (a)-[:KNOWS {since: 2010, strength: 'protected'}]->(b) +$$) AS (a agtype); + a +--- +(0 rows) + +DROP POLICY knows_insert_all ON rls_graph."KNOWS"; +-- Policy: Cannot delete edges with strength = 'protected' +CREATE POLICY knows_delete_not_protected ON rls_graph."KNOWS" + FOR DELETE + USING (properties->>'"strength"' != 'protected'); +SET ROLE rls_user1; +-- Should ERROR - DETACH DELETE cannot silently skip (would leave dangling edge) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DetachTest1'}) DETACH DELETE p +$$) AS (a agtype); +ERROR: cannot delete edge due to row-level security policy on "KNOWS" +HINT: DETACH DELETE requires permission to delete all connected edges. +RESET ROLE; +-- Verify vertex still exists (delete was blocked) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DetachTest1'}) RETURN p.name +$$) AS (name agtype); + name +--------------- + "DetachTest1" +(1 row) + +-- cleanup +DROP POLICY person_all ON rls_graph."Person"; +DROP POLICY knows_select_all ON rls_graph."KNOWS"; +DROP POLICY knows_delete_not_protected ON rls_graph."KNOWS"; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Detach' DETACH DELETE p +$$) AS (a agtype); + a +--- +(0 rows) + +-- ============================================================================ +-- PART 12: Multiple Labels in Single Query +-- ============================================================================ +-- Enable RLS on Document too +ALTER TABLE rls_graph."Document" ENABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."Document" FORCE ROW LEVEL SECURITY; +-- Policy: Users see their own Person records +CREATE POLICY person_own ON rls_graph."Person" + FOR SELECT + USING (properties->>'"owner"' = current_user); +-- Policy: Users see only public documents +CREATE POLICY doc_public ON rls_graph."Document" + FOR SELECT + USING (properties->>'"classification"' = 'public'); +SET ROLE rls_user1; +-- Should only see Alice and Charlie (Person) with Public Doc (Document) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +----------- + "Alice" + "Charlie" +(2 rows) + +SELECT * FROM cypher('rls_graph', $$ + MATCH (d:Document) RETURN d.title ORDER BY d.title +$$) AS (title agtype); + title +-------------- + "Public Doc" +(1 row) + +-- Combined query - should respect both policies +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person)-[:AUTHORED]->(d:Document) + RETURN p.name, d.title +$$) AS (person agtype, doc agtype); + person | doc +---------+-------------- + "Alice" | "Public Doc" +(1 row) + +RESET ROLE; +-- ============================================================================ +-- PART 13: Permissive vs Restrictive Policies +-- ============================================================================ +DROP POLICY person_own ON rls_graph."Person"; +DROP POLICY doc_public ON rls_graph."Document"; +ALTER TABLE rls_graph."Document" DISABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."KNOWS" DISABLE ROW LEVEL SECURITY; +-- 13.1: Multiple permissive policies (OR logic) +CREATE POLICY person_permissive_own ON rls_graph."Person" + AS PERMISSIVE FOR SELECT + USING (properties->>'"owner"' = current_user); +CREATE POLICY person_permissive_eng ON rls_graph."Person" + AS PERMISSIVE FOR SELECT + USING (properties->>'"department"' = 'Engineering'); +SET ROLE rls_user1; +-- Should see: Alice (own), Charlie (own), Bob (Engineering) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department IN ['Engineering', 'Sales'] + RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +----------- + "Alice" + "Bob" + "Charlie" +(3 rows) + +RESET ROLE; +-- 13.2: Add restrictive policy (AND with permissive) +CREATE POLICY person_restrictive_level ON rls_graph."Person" + AS RESTRICTIVE FOR SELECT + USING ((properties->>'"level"')::int <= 2); +SET ROLE rls_user1; +-- Should see: Alice (own, level 1), Bob (Engineering, level 2), Charlie (own, level 1) +-- Diana (level 3) blocked by restrictive +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name, p.level ORDER BY p.name +$$) AS (name agtype, level agtype); + name | level +-----------+------- + "Alice" | 1 + "Bob" | 2 + "Charlie" | 1 +(3 rows) + +RESET ROLE; +-- 13.3: Multiple restrictive policies (all must pass) +CREATE POLICY person_restrictive_sales ON rls_graph."Person" + AS RESTRICTIVE FOR SELECT + USING (properties->>'"department"' != 'Sales'); +SET ROLE rls_user1; +-- Should see: Alice (own, level 1, not Sales), Bob (Engineering, level 2, not Sales) +-- Charlie blocked by Sales restriction +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +--------- + "Alice" + "Bob" +(2 rows) + +RESET ROLE; +-- ============================================================================ +-- PART 14: BYPASSRLS Role and Superuser Behavior +-- ============================================================================ +DROP POLICY person_permissive_own ON rls_graph."Person"; +DROP POLICY person_permissive_eng ON rls_graph."Person"; +DROP POLICY person_restrictive_level ON rls_graph."Person"; +DROP POLICY person_restrictive_sales ON rls_graph."Person"; +-- Restrictive policy that blocks most access +CREATE POLICY person_very_restrictive ON rls_graph."Person" + FOR SELECT + USING (properties->>'"name"' = 'Nobody'); +-- 14.1: Regular user sees nothing +SET ROLE rls_user1; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +------ +(0 rows) + +RESET ROLE; +-- 14.2: BYPASSRLS role sees everything +SET ROLE rls_admin; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +----------- + "Alice" + "Bob" + "Charlie" + "Diana" +(4 rows) + +RESET ROLE; +-- 14.3: Superuser sees everything (implicit bypass) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + name +----------- + "Alice" + "Bob" + "Charlie" + "Diana" +(4 rows) + +-- ============================================================================ +-- PART 15: Complex Multi-Operation Queries +-- ============================================================================ +DROP POLICY person_very_restrictive ON rls_graph."Person"; +CREATE POLICY person_select_all ON rls_graph."Person" + FOR SELECT USING (true); +CREATE POLICY person_insert_own ON rls_graph."Person" + FOR INSERT + WITH CHECK (properties->>'"owner"' = current_user); +CREATE POLICY person_update_own ON rls_graph."Person" + FOR UPDATE + USING (properties->>'"owner"' = current_user) + WITH CHECK (properties->>'"owner"' = current_user); +-- 15.1: MATCH + CREATE in one query +SET ROLE rls_user1; +-- Should succeed - creating with correct owner +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'}) + CREATE (a)-[:KNOWS]->(:Person {name: 'NewFromMatch', owner: 'rls_user1', department: 'Complex', level: 1}) +$$) AS (a agtype); + a +--- +(0 rows) + +RESET ROLE; +-- Verify creation +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'NewFromMatch'}) RETURN p.name, p.owner +$$) AS (name agtype, owner agtype); + name | owner +----------------+------------- + "NewFromMatch" | "rls_user1" +(1 row) + +-- 15.2: MATCH + SET in one query +SET ROLE rls_user1; +-- Should succeed on Alice (own), skip Bob (not own) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.name IN ['Alice', 'Bob'] + SET p.complexTest = true + RETURN p.name, p.complexTest +$$) AS (name agtype, test agtype); + name | test +---------+------ + "Alice" | true +(1 row) + +RESET ROLE; +-- Verify only Alice was updated +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.name IN ['Alice', 'Bob'] + RETURN p.name, p.complexTest ORDER BY p.name +$$) AS (name agtype, test agtype); + name | test +---------+------ + "Alice" | true + "Bob" | +(2 rows) + +-- cleanup +DROP POLICY IF EXISTS person_select_all ON rls_graph."Person"; +DROP POLICY IF EXISTS person_insert_own ON rls_graph."Person"; +DROP POLICY IF EXISTS person_update_own ON rls_graph."Person"; +-- ============================================================================ +-- PART 16: startNode/endNode RLS Enforcement +-- ============================================================================ +ALTER TABLE rls_graph."Person" DISABLE ROW LEVEL SECURITY; +-- Enable RLS on Person with restrictive policy +ALTER TABLE rls_graph."Person" ENABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."Person" FORCE ROW LEVEL SECURITY; +-- Policy: users can only see their own Person records +CREATE POLICY person_own ON rls_graph."Person" + FOR SELECT + USING (properties->>'"owner"' = current_user); +-- Enable edge access for testing +ALTER TABLE rls_graph."KNOWS" ENABLE ROW LEVEL SECURITY; +CREATE POLICY knows_all ON rls_graph."KNOWS" + FOR SELECT USING (true); +-- 16.1: startNode blocked by RLS - should error +SET ROLE rls_user1; +-- rls_user1 can see the edge (Alice->Bob) but cannot see Bob (owned by rls_user2) +-- endNode should error because Bob is blocked by RLS +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'})-[e:KNOWS]->(b) + RETURN endNode(e) +$$) AS (end_vertex agtype); +ERROR: access to vertex 844424930131970 denied by row-level security policy on "Person" +-- 16.2: endNode blocked by RLS - should error +-- rls_user1 cannot see Bob, so startNode on an edge starting from Bob should error +SET ROLE rls_user2; +-- rls_user2 can see Bob but not Alice (owned by rls_user1) +-- startNode should error because Alice is blocked by RLS +SELECT * FROM cypher('rls_graph', $$ + MATCH (a)-[e:KNOWS]->(b:Person {name: 'Bob'}) + RETURN startNode(e) +$$) AS (start_vertex agtype); +ERROR: access to vertex 844424930131969 denied by row-level security policy on "Person" +-- 16.3: startNode/endNode succeed when RLS allows access +SET ROLE rls_user1; +-- Alice->Charlie edge: rls_user1 owns both, should succeed +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'})-[e:KNOWS]->(c:Person {name: 'Charlie'}) + RETURN startNode(e).name, endNode(e).name +$$) AS (start_name agtype, end_name agtype); + start_name | end_name +------------+----------- + "Alice" | "Charlie" +(1 row) + +RESET ROLE; +-- cleanup +DROP POLICY person_own ON rls_graph."Person"; +DROP POLICY knows_all ON rls_graph."KNOWS"; +ALTER TABLE rls_graph."KNOWS" DISABLE ROW LEVEL SECURITY; +-- ============================================================================ +-- RLS CLEANUP +-- ============================================================================ +RESET ROLE; +-- Disable RLS on all tables +ALTER TABLE rls_graph."Person" DISABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."Document" DISABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."KNOWS" DISABLE ROW LEVEL SECURITY; +-- Drop roles +DROP OWNED BY rls_user1 CASCADE; +DROP ROLE rls_user1; +DROP OWNED BY rls_user2 CASCADE; +DROP ROLE rls_user2; +DROP OWNED BY rls_admin CASCADE; +DROP ROLE rls_admin; +-- Drop test graph +SELECT drop_graph('rls_graph', true); +NOTICE: drop cascades to 6 other objects +DETAIL: drop cascades to table rls_graph._ag_label_vertex +drop cascades to table rls_graph._ag_label_edge +drop cascades to table rls_graph."Person" +drop cascades to table rls_graph."Document" +drop cascades to table rls_graph."KNOWS" +drop cascades to table rls_graph."AUTHORED" +NOTICE: graph "rls_graph" has been dropped + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/security.sql b/regress/sql/security.sql new file mode 100644 index 000000000..344dd23d4 --- /dev/null +++ b/regress/sql/security.sql @@ -0,0 +1,1451 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +-- +-- Test Privileges +-- + +-- +-- Setup: Create test graph and data as superuser +-- +SELECT create_graph('security_test'); + +-- Create test vertices +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Alice', age: 30}) +$$) AS (a agtype); + +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Bob', age: 25}) +$$) AS (a agtype); + +SELECT * FROM cypher('security_test', $$ + CREATE (:Document {title: 'Secret', content: 'classified'}) +$$) AS (a agtype); + +-- Create test edges +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Alice'}), (b:Person {name: 'Bob'}) + CREATE (a)-[:KNOWS {since: 2020}]->(b) +$$) AS (a agtype); + +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Alice'}), (d:Document) + CREATE (a)-[:OWNS]->(d) +$$) AS (a agtype); + +-- +-- Create test roles with different permission levels +-- + +-- Role with only SELECT (read-only) +CREATE ROLE security_test_readonly LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_readonly; +GRANT SELECT ON ALL TABLES IN SCHEMA security_test TO security_test_readonly; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_readonly; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_readonly; + +-- Role with SELECT and INSERT +CREATE ROLE security_test_insert LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_insert; +GRANT SELECT, INSERT ON ALL TABLES IN SCHEMA security_test TO security_test_insert; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_insert; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_insert; +-- Grant sequence usage for ID generation +GRANT USAGE ON ALL SEQUENCES IN SCHEMA security_test TO security_test_insert; + +-- Role with SELECT and UPDATE +CREATE ROLE security_test_update LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_update; +GRANT SELECT, UPDATE ON ALL TABLES IN SCHEMA security_test TO security_test_update; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_update; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_update; + +-- Role with SELECT and DELETE +CREATE ROLE security_test_delete LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_delete; +GRANT SELECT, DELETE ON ALL TABLES IN SCHEMA security_test TO security_test_delete; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_delete; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_delete; + +CREATE ROLE security_test_detach_delete LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_detach_delete; +GRANT SELECT ON ALL TABLES IN SCHEMA security_test TO security_test_detach_delete; +GRANT DELETE ON security_test."Person" TO security_test_detach_delete; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_detach_delete; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_detach_delete; + +-- Role with all permissions +CREATE ROLE security_test_full LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_full; +GRANT ALL ON ALL TABLES IN SCHEMA security_test TO security_test_full; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_full; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_full; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA security_test TO security_test_full; + +-- Role with NO SELECT on graph tables (to test read failures) +CREATE ROLE security_test_noread LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_noread; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_noread; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_noread; +-- No SELECT on security_test tables + +-- ============================================================================ +-- PART 1: SELECT Permission Tests - Failure Cases (No Read Permission) +-- ============================================================================ + +SET ROLE security_test_noread; + +-- Test: MATCH on vertices should fail without SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person) RETURN p.name +$$) AS (name agtype); + +-- Test: MATCH on edges should fail without SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH ()-[k:KNOWS]->() RETURN k +$$) AS (k agtype); + +-- Test: MATCH with path should fail +SELECT * FROM cypher('security_test', $$ + MATCH (a)-[e]->(b) RETURN a, e, b +$$) AS (a agtype, e agtype, b agtype); + +RESET ROLE; + +-- Create role with SELECT only on base label tables, not child labels +-- NOTE: PostgreSQL inheritance allows access to child table rows when querying +-- through a parent table. This is expected behavior - SELECT on _ag_label_vertex +-- allows reading all vertices (including Person, Document) via inheritance. +CREATE ROLE security_test_base_only LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_base_only; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_base_only; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_base_only; +-- Only grant SELECT on base tables, NOT on Person, Document, KNOWS, OWNS +GRANT SELECT ON security_test._ag_label_vertex TO security_test_base_only; +GRANT SELECT ON security_test._ag_label_edge TO security_test_base_only; + +SET ROLE security_test_base_only; + +-- Test: MATCH (n) succeeds because PostgreSQL inheritance allows access to child rows +-- when querying through parent table. Permission on _ag_label_vertex grants read +-- access to all vertices via inheritance hierarchy. +SELECT * FROM cypher('security_test', $$ + MATCH (n) RETURN n +$$) AS (n agtype); + +-- Test: MATCH ()-[e]->() succeeds via inheritance (same reason as above) +SELECT * FROM cypher('security_test', $$ + MATCH ()-[e]->() RETURN e +$$) AS (e agtype); + +-- ============================================================================ +-- PART 2: SELECT Permission Tests - Success Cases (Read-Only Role) +-- ============================================================================ + +RESET ROLE; +SET ROLE security_test_readonly; + +-- Test: MATCH should succeed with SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +-- Test: MATCH with edges should succeed +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person)-[k:KNOWS]->(b:Person) + RETURN a.name, b.name +$$) AS (a agtype, b agtype); + +-- Test: MATCH across multiple labels should succeed +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person)-[:OWNS]->(d:Document) + RETURN p.name, d.title +$$) AS (person agtype, doc agtype); + +-- ============================================================================ +-- PART 3: INSERT Permission Tests (CREATE clause) +-- ============================================================================ + +-- Test: CREATE should fail with only SELECT permission +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Charlie'}) +$$) AS (a agtype); + +-- Test: CREATE edge should fail +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Alice'}), (b:Person {name: 'Bob'}) + CREATE (a)-[:FRIENDS]->(b) +$$) AS (a agtype); + +RESET ROLE; +SET ROLE security_test_insert; + +-- Test: CREATE vertex should succeed with INSERT permission +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Charlie', age: 35}) +$$) AS (a agtype); + +-- Test: CREATE edge should succeed with INSERT permission +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Charlie'}), (b:Person {name: 'Alice'}) + CREATE (a)-[:KNOWS {since: 2023}]->(b) +$$) AS (a agtype); + +-- Verify the inserts worked +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'}) RETURN p.name, p.age +$$) AS (name agtype, age agtype); + +-- ============================================================================ +-- PART 4: UPDATE Permission Tests (SET clause) +-- ============================================================================ + +RESET ROLE; +SET ROLE security_test_readonly; + +-- Test: SET should fail with only SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Alice'}) + SET p.age = 31 + RETURN p +$$) AS (p agtype); + +-- Test: SET on edge should fail +SELECT * FROM cypher('security_test', $$ + MATCH ()-[k:KNOWS]->() + SET k.since = 2021 + RETURN k +$$) AS (k agtype); + +RESET ROLE; +SET ROLE security_test_update; + +-- Test: SET should succeed with UPDATE permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Alice'}) + SET p.age = 31 + RETURN p.name, p.age +$$) AS (name agtype, age agtype); + +-- Test: SET on edge should succeed +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Alice'})-[k:KNOWS]->(b:Person {name: 'Bob'}) + SET k.since = 2019 + RETURN k.since +$$) AS (since agtype); + +-- Test: SET with map update should succeed +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Bob'}) + SET p += {hobby: 'reading'} + RETURN p.name, p.hobby +$$) AS (name agtype, hobby agtype); + +-- ============================================================================ +-- PART 5: UPDATE Permission Tests (REMOVE clause) +-- ============================================================================ + +RESET ROLE; +SET ROLE security_test_readonly; + +-- Test: REMOVE should fail with only SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Bob'}) + REMOVE p.hobby + RETURN p +$$) AS (p agtype); + +RESET ROLE; +SET ROLE security_test_update; + +-- Test: REMOVE should succeed with UPDATE permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Bob'}) + REMOVE p.hobby + RETURN p.name, p.hobby +$$) AS (name agtype, hobby agtype); + +-- ============================================================================ +-- PART 6: DELETE Permission Tests +-- ============================================================================ + +RESET ROLE; +SET ROLE security_test_readonly; + +-- Test: DELETE should fail with only SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'}) + DELETE p +$$) AS (a agtype); + +RESET ROLE; +SET ROLE security_test_update; + +-- Test: DELETE should fail with only UPDATE permission (need DELETE) +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'}) + DELETE p +$$) AS (a agtype); + +RESET ROLE; +SET ROLE security_test_delete; + +-- Test: DELETE vertex should succeed with DELETE permission +-- First delete the edge connected to Charlie +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'})-[k:KNOWS]->() + DELETE k +$$) AS (a agtype); + +-- Now delete the vertex +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'}) + DELETE p +$$) AS (a agtype); + +-- Verify deletion +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Charlie'}) RETURN p +$$) AS (p agtype); + +-- ============================================================================ +-- PART 7: DETACH DELETE Tests +-- ============================================================================ + +RESET ROLE; + +-- Create a new vertex with edge for DETACH DELETE test +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Dave', age: 40}) +$$) AS (a agtype); + +SELECT * FROM cypher('security_test', $$ + MATCH (a:Person {name: 'Alice'}), (d:Person {name: 'Dave'}) + CREATE (a)-[:KNOWS {since: 2022}]->(d) +$$) AS (a agtype); + +SET ROLE security_test_detach_delete; + +-- Test: DETACH DELETE should fail without DELETE on edge table +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Dave'}) + DETACH DELETE p +$$) AS (a agtype); + +RESET ROLE; +GRANT DELETE ON security_test."KNOWS" TO security_test_detach_delete; +SET ROLE security_test_detach_delete; + +-- Test: DETACH DELETE should succeed now when user has DELETE on both vertex and edge tables +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Dave'}) + DETACH DELETE p +$$) AS (a agtype); + +-- Verify deletion +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Dave'}) RETURN p +$$) AS (p agtype); + +-- ============================================================================ +-- PART 8: MERGE Permission Tests +-- ============================================================================ + +RESET ROLE; +SET ROLE security_test_readonly; + +-- Test: MERGE that would create should fail without INSERT +SELECT * FROM cypher('security_test', $$ + MERGE (p:Person {name: 'Eve'}) + RETURN p +$$) AS (p agtype); + +RESET ROLE; +SET ROLE security_test_insert; + +-- Test: MERGE that creates should succeed with INSERT permission +SELECT * FROM cypher('security_test', $$ + MERGE (p:Person {name: 'Eve', age: 28}) + RETURN p.name, p.age +$$) AS (name agtype, age agtype); + +-- Test: MERGE that matches existing should succeed (only needs SELECT) +SELECT * FROM cypher('security_test', $$ + MERGE (p:Person {name: 'Eve'}) + RETURN p.name +$$) AS (name agtype); + +-- ============================================================================ +-- PART 9: Full Permission Role Tests +-- ============================================================================ + +RESET ROLE; +SET ROLE security_test_full; + +-- Full permission role should be able to do everything +SELECT * FROM cypher('security_test', $$ + CREATE (:Person {name: 'Frank', age: 50}) +$$) AS (a agtype); + +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Frank'}) + SET p.age = 51 + RETURN p.name, p.age +$$) AS (name agtype, age agtype); + +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Frank'}) + DELETE p +$$) AS (a agtype); + +-- ============================================================================ +-- PART 10: Permission on Specific Labels +-- ============================================================================ + +RESET ROLE; + +-- Create a role with permission only on Person label, not Document +CREATE ROLE security_test_person_only LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_person_only; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_person_only; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA ag_catalog TO security_test_person_only; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_person_only; +-- Only grant permissions on Person table +GRANT SELECT, INSERT, UPDATE, DELETE ON security_test."Person" TO security_test_person_only; +GRANT SELECT ON security_test."KNOWS" TO security_test_person_only; +GRANT SELECT ON security_test._ag_label_vertex TO security_test_person_only; +GRANT SELECT ON security_test._ag_label_edge TO security_test_person_only; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA security_test TO security_test_person_only; + +SET ROLE security_test_person_only; + +-- Test: Operations on Person should succeed +SELECT * FROM cypher('security_test', $$ + MATCH (p:Person {name: 'Alice'}) RETURN p.name +$$) AS (name agtype); + +-- Test: SELECT on Document should fail (no permission) +SELECT * FROM cypher('security_test', $$ + MATCH (d:Document) RETURN d.title +$$) AS (title agtype); + +-- Test: CREATE Document should fail (no permission on Document table) +SELECT * FROM cypher('security_test', $$ + CREATE (:Document {title: 'New Doc'}) +$$) AS (a agtype); + +-- ============================================================================ +-- PART 11: Function EXECUTE Permission Tests +-- ============================================================================ + +RESET ROLE; + +-- Create role with no function execute permissions +CREATE ROLE security_test_noexec LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_noexec; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_noexec; + +-- Revoke execute from PUBLIC on functions we want to test +REVOKE EXECUTE ON FUNCTION ag_catalog.create_graph(name) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION ag_catalog.drop_graph(name, boolean) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION ag_catalog.create_vlabel(cstring, cstring) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION ag_catalog.create_elabel(cstring, cstring) FROM PUBLIC; + +SET ROLE security_test_noexec; + +-- Test: create_graph should fail without EXECUTE permission +SELECT create_graph('unauthorized_graph'); + +-- Test: drop_graph should fail without EXECUTE permission +SELECT drop_graph('security_test', true); + +-- Test: create_vlabel should fail without EXECUTE permission +SELECT create_vlabel('security_test', 'NewLabel'); + +-- Test: create_elabel should fail without EXECUTE permission +SELECT create_elabel('security_test', 'NewEdge'); + +RESET ROLE; + +-- Grant execute on specific function and test +GRANT EXECUTE ON FUNCTION ag_catalog.create_vlabel(cstring, cstring) TO security_test_noexec; + +SET ROLE security_test_noexec; + +-- Test: create_vlabel should now get past execute check (will fail on schema permission instead) +SELECT create_vlabel('security_test', 'TestLabel'); + +-- Test: create_graph should still fail with execute permission denied +SELECT create_graph('unauthorized_graph'); + +RESET ROLE; + +-- Restore execute permissions to PUBLIC +GRANT EXECUTE ON FUNCTION ag_catalog.create_graph(name) TO PUBLIC; +GRANT EXECUTE ON FUNCTION ag_catalog.drop_graph(name, boolean) TO PUBLIC; +GRANT EXECUTE ON FUNCTION ag_catalog.create_vlabel(cstring, cstring) TO PUBLIC; +GRANT EXECUTE ON FUNCTION ag_catalog.create_elabel(cstring, cstring) TO PUBLIC; + +-- ============================================================================ +-- PART 12: startNode/endNode Permission Tests +-- ============================================================================ + +-- Create role with SELECT on base tables but NOT on Person label +CREATE ROLE security_test_edge_only LOGIN; +GRANT USAGE ON SCHEMA security_test TO security_test_edge_only; +GRANT USAGE ON SCHEMA ag_catalog TO security_test_edge_only; +GRANT SELECT ON ALL TABLES IN SCHEMA ag_catalog TO security_test_edge_only; +GRANT SELECT ON security_test."KNOWS" TO security_test_edge_only; +GRANT SELECT ON security_test._ag_label_edge TO security_test_edge_only; +GRANT SELECT ON security_test._ag_label_vertex TO security_test_edge_only; +-- Note: NOT granting SELECT on security_test."Person" + +SET ROLE security_test_edge_only; + +-- Test: endNode fails without SELECT on Person table +SELECT * FROM cypher('security_test', $$ + MATCH ()-[e:KNOWS]->() + RETURN endNode(e) +$$) AS (end_vertex agtype); + +-- Test: startNode fails without SELECT on Person table +SELECT * FROM cypher('security_test', $$ + MATCH ()-[e:KNOWS]->() + RETURN startNode(e) +$$) AS (start_vertex agtype); + +RESET ROLE; + +-- Grant SELECT on Person and verify success +GRANT SELECT ON security_test."Person" TO security_test_edge_only; + +SET ROLE security_test_edge_only; + +-- Test: Should now succeed with SELECT permission +SELECT * FROM cypher('security_test', $$ + MATCH ()-[e:KNOWS]->() + RETURN startNode(e).name, endNode(e).name +$$) AS (start_name agtype, end_name agtype); + +RESET ROLE; + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +RESET ROLE; + +-- Drop all owned objects and privileges for each role, then drop the role +DROP OWNED BY security_test_noread CASCADE; +DROP ROLE security_test_noread; + +DROP OWNED BY security_test_base_only CASCADE; +DROP ROLE security_test_base_only; + +DROP OWNED BY security_test_readonly CASCADE; +DROP ROLE security_test_readonly; + +DROP OWNED BY security_test_insert CASCADE; +DROP ROLE security_test_insert; + +DROP OWNED BY security_test_update CASCADE; +DROP ROLE security_test_update; + +DROP OWNED BY security_test_delete CASCADE; +DROP ROLE security_test_delete; + +DROP OWNED BY security_test_detach_delete CASCADE; +DROP ROLE security_test_detach_delete; + +DROP OWNED BY security_test_full CASCADE; +DROP ROLE security_test_full; + +DROP OWNED BY security_test_person_only CASCADE; +DROP ROLE security_test_person_only; + +DROP OWNED BY security_test_noexec CASCADE; +DROP ROLE security_test_noexec; + +DROP OWNED BY security_test_edge_only CASCADE; +DROP ROLE security_test_edge_only; + +-- Drop test graph +SELECT drop_graph('security_test', true); + +-- +-- Row-Level Security (RLS) Tests +-- + +-- +-- Setup: Create test graph, data and roles for RLS tests +-- +SELECT create_graph('rls_graph'); + +-- Create test roles +CREATE ROLE rls_user1 LOGIN; +CREATE ROLE rls_user2 LOGIN; +CREATE ROLE rls_admin LOGIN BYPASSRLS; -- Role that bypasses RLS + +-- Create base test data FIRST (as superuser) - this creates the label tables +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'Alice', owner: 'rls_user1', department: 'Engineering', level: 1}) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'Bob', owner: 'rls_user2', department: 'Engineering', level: 2}) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'Charlie', owner: 'rls_user1', department: 'Sales', level: 1}) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'Diana', owner: 'rls_user2', department: 'Sales', level: 3}) +$$) AS (a agtype); + +-- Create a second vertex label for multi-label tests +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Document {title: 'Public Doc', classification: 'public', owner: 'rls_user1'}) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Document {title: 'Secret Doc', classification: 'secret', owner: 'rls_user2'}) +$$) AS (a agtype); + +-- Create edges +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'}), (b:Person {name: 'Bob'}) + CREATE (a)-[:KNOWS {since: 2020, strength: 'weak'}]->(b) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Charlie'}), (b:Person {name: 'Diana'}) + CREATE (a)-[:KNOWS {since: 2021, strength: 'strong'}]->(b) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'}), (b:Person {name: 'Charlie'}) + CREATE (a)-[:KNOWS {since: 2022, strength: 'strong'}]->(b) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'}), (d:Document {title: 'Public Doc'}) + CREATE (a)-[:AUTHORED]->(d) +$$) AS (a agtype); + +-- Grant permissions AFTER creating tables (so Person, Document, KNOWS, AUTHORED exist) +GRANT USAGE ON SCHEMA rls_graph TO rls_user1, rls_user2, rls_admin; +GRANT ALL ON ALL TABLES IN SCHEMA rls_graph TO rls_user1, rls_user2, rls_admin; +GRANT USAGE ON SCHEMA ag_catalog TO rls_user1, rls_user2, rls_admin; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA rls_graph TO rls_user1, rls_user2, rls_admin; + +-- ============================================================================ +-- PART 1: Vertex SELECT Policies (USING clause) +-- ============================================================================ + +-- Enable RLS on Person label +ALTER TABLE rls_graph."Person" ENABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."Person" FORCE ROW LEVEL SECURITY; + +-- 1.1: Basic ownership filtering +CREATE POLICY person_select_own ON rls_graph."Person" + FOR SELECT + USING (properties->>'"owner"' = current_user); + +-- Test as rls_user1 - should only see Alice and Charlie (owned by rls_user1) +SET ROLE rls_user1; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +-- Test as rls_user2 - should only see Bob and Diana (owned by rls_user2) +SET ROLE rls_user2; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +RESET ROLE; + +-- 1.2: Default deny - no permissive policies means no access +DROP POLICY person_select_own ON rls_graph."Person"; + +-- With no policies, RLS blocks all access +SET ROLE rls_user1; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +RESET ROLE; + +-- ============================================================================ +-- PART 2: Vertex INSERT Policies (WITH CHECK) - CREATE +-- ============================================================================ + +-- Allow SELECT for all (so we can verify results) +CREATE POLICY person_select_all ON rls_graph."Person" + FOR SELECT USING (true); + +-- 2.1: Basic WITH CHECK - users can only insert rows they own +CREATE POLICY person_insert_own ON rls_graph."Person" + FOR INSERT + WITH CHECK (properties->>'"owner"' = current_user); + +-- Test as rls_user1 - should succeed (owner matches current_user) +SET ROLE rls_user1; +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'User1Created', owner: 'rls_user1', department: 'Test', level: 1}) +$$) AS (a agtype); + +-- Test as rls_user1 - should FAIL (owner doesn't match current_user) +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'User1Fake', owner: 'rls_user2', department: 'Test', level: 1}) +$$) AS (a agtype); + +RESET ROLE; + +-- Verify only User1Created was created +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Test' RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +-- 2.2: Default deny for INSERT - no INSERT policy blocks all inserts +DROP POLICY person_insert_own ON rls_graph."Person"; + +SET ROLE rls_user1; +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'ShouldFail', owner: 'rls_user1', department: 'Blocked', level: 1}) +$$) AS (a agtype); +RESET ROLE; + +-- Verify nothing was created in Blocked department +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Blocked' RETURN p.name +$$) AS (name agtype); + +-- cleanup +DROP POLICY person_select_all ON rls_graph."Person"; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Test' DELETE p +$$) AS (a agtype); + +-- ============================================================================ +-- PART 3: Vertex UPDATE Policies - SET +-- ============================================================================ + +CREATE POLICY person_select_all ON rls_graph."Person" + FOR SELECT USING (true); + +-- 3.1: USING clause only - filter which rows can be updated +CREATE POLICY person_update_using ON rls_graph."Person" + FOR UPDATE + USING (properties->>'"owner"' = current_user); + +SET ROLE rls_user1; + +-- Should succeed - rls_user1 owns Alice +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) SET p.updated = true RETURN p.name, p.updated +$$) AS (name agtype, updated agtype); + +-- Should silently skip - rls_user1 doesn't own Bob (USING filters it out) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Bob'}) SET p.updated = true RETURN p.name, p.updated +$$) AS (name agtype, updated agtype); + +RESET ROLE; + +-- Verify Alice was updated, Bob was not +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.name IN ['Alice', 'Bob'] RETURN p.name, p.updated ORDER BY p.name +$$) AS (name agtype, updated agtype); + +-- 3.2: WITH CHECK clause - validate new values +DROP POLICY person_update_using ON rls_graph."Person"; + +CREATE POLICY person_update_check ON rls_graph."Person" + FOR UPDATE + USING (true) -- Can update any row + WITH CHECK (properties->>'"owner"' = current_user); -- But new value must keep owner + +SET ROLE rls_user1; + +-- Should succeed - modifying property but keeping owner +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) SET p.verified = true RETURN p.name, p.verified +$$) AS (name agtype, verified agtype); + +-- Should FAIL - trying to change owner to someone else +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) SET p.owner = 'rls_user2' RETURN p.owner +$$) AS (owner agtype); + +RESET ROLE; + +-- Verify owner wasn't changed +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) RETURN p.owner +$$) AS (owner agtype); + +-- 3.3: Both USING and WITH CHECK together +DROP POLICY person_update_check ON rls_graph."Person"; + +CREATE POLICY person_update_both ON rls_graph."Person" + FOR UPDATE + USING (properties->>'"owner"' = current_user) + WITH CHECK (properties->>'"owner"' = current_user); + +SET ROLE rls_user1; + +-- Should succeed - owns Alice, keeping owner +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) SET p.status = 'active' RETURN p.name, p.status +$$) AS (name agtype, status agtype); + +-- Should silently skip - doesn't own Bob (USING filters) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Bob'}) SET p.status = 'active' RETURN p.name, p.status +$$) AS (name agtype, status agtype); + +RESET ROLE; + +-- ============================================================================ +-- PART 4: Vertex UPDATE Policies - REMOVE +-- ============================================================================ + +-- Keep existing update policy, test REMOVE operation + +SET ROLE rls_user1; + +-- Should succeed - owns Alice +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Alice'}) REMOVE p.status RETURN p.name, p.status +$$) AS (name agtype, status agtype); + +-- Should silently skip - doesn't own Bob +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Bob'}) REMOVE p.department RETURN p.name, p.department +$$) AS (name agtype, dept agtype); + +RESET ROLE; + +-- Verify Bob still has department +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'Bob'}) RETURN p.department +$$) AS (dept agtype); + +-- cleanup +DROP POLICY person_select_all ON rls_graph."Person"; +DROP POLICY person_update_both ON rls_graph."Person"; + +-- ============================================================================ +-- PART 5: Vertex DELETE Policies +-- ============================================================================ + +CREATE POLICY person_select_all ON rls_graph."Person" + FOR SELECT USING (true); + +-- Create test data for delete tests +CREATE POLICY person_insert_all ON rls_graph."Person" + FOR INSERT WITH CHECK (true); + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'DeleteTest1', owner: 'rls_user1', department: 'DeleteTest', level: 1}) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'DeleteTest2', owner: 'rls_user2', department: 'DeleteTest', level: 1}) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'DeleteTest3', owner: 'rls_user1', department: 'DeleteTest', level: 1}) +$$) AS (a agtype); + +DROP POLICY person_insert_all ON rls_graph."Person"; + +-- 5.1: Basic USING filtering for DELETE +CREATE POLICY person_delete_own ON rls_graph."Person" + FOR DELETE + USING (properties->>'"owner"' = current_user); + +SET ROLE rls_user1; + +-- Should succeed - owns DeleteTest1 +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DeleteTest1'}) DELETE p +$$) AS (a agtype); + +-- Should silently skip - doesn't own DeleteTest2 +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DeleteTest2'}) DELETE p +$$) AS (a agtype); + +RESET ROLE; + +-- Verify DeleteTest1 deleted, DeleteTest2 still exists +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'DeleteTest' RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +-- 5.2: Default deny for DELETE - no policy blocks all deletes +DROP POLICY person_delete_own ON rls_graph."Person"; + +SET ROLE rls_user1; + +-- Should silently skip - no DELETE policy means default deny +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DeleteTest3'}) DELETE p +$$) AS (a agtype); + +RESET ROLE; + +-- Verify DeleteTest3 still exists +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DeleteTest3'}) RETURN p.name +$$) AS (name agtype); + +-- cleanup +DROP POLICY person_select_all ON rls_graph."Person"; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'DeleteTest' DELETE p +$$) AS (a agtype); + +-- ============================================================================ +-- PART 6: MERGE Policies +-- ============================================================================ + +CREATE POLICY person_select_all ON rls_graph."Person" + FOR SELECT USING (true); + +CREATE POLICY person_insert_own ON rls_graph."Person" + FOR INSERT + WITH CHECK (properties->>'"owner"' = current_user); + +-- 6.1: MERGE creating new vertex - INSERT policy applies +SET ROLE rls_user1; + +-- Should succeed - creating with correct owner +SELECT * FROM cypher('rls_graph', $$ + MERGE (p:Person {name: 'MergeNew1', owner: 'rls_user1', department: 'Merge', level: 1}) + RETURN p.name +$$) AS (name agtype); + +-- Should FAIL - creating with wrong owner +SELECT * FROM cypher('rls_graph', $$ + MERGE (p:Person {name: 'MergeNew2', owner: 'rls_user2', department: 'Merge', level: 1}) + RETURN p.name +$$) AS (name agtype); + +RESET ROLE; + +-- 6.2: MERGE matching existing - only SELECT needed +SET ROLE rls_user1; + +-- Should succeed - Alice exists and SELECT allowed +SELECT * FROM cypher('rls_graph', $$ + MERGE (p:Person {name: 'Alice'}) + RETURN p.name, p.owner +$$) AS (name agtype, owner agtype); + +RESET ROLE; + +-- Verify only MergeNew1 was created +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Merge' RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +-- cleanup +DROP POLICY person_select_all ON rls_graph."Person"; +DROP POLICY person_insert_own ON rls_graph."Person"; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Merge' DELETE p +$$) AS (a agtype); + +-- ============================================================================ +-- PART 7: Edge SELECT Policies +-- ============================================================================ + +-- Disable vertex RLS, enable edge RLS +ALTER TABLE rls_graph."Person" DISABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."KNOWS" ENABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."KNOWS" FORCE ROW LEVEL SECURITY; + +-- Policy: Only see edges from 2021 or later +CREATE POLICY knows_select_recent ON rls_graph."KNOWS" + FOR SELECT + USING ((properties->>'"since"')::int >= 2021); + +SET ROLE rls_user1; + +-- Should only see 2021 and 2022 edges (not 2020) +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS]->() RETURN k.since ORDER BY k.since +$$) AS (since agtype); + +RESET ROLE; + +-- ============================================================================ +-- PART 8: Edge INSERT Policies (CREATE edge) +-- ============================================================================ + +DROP POLICY knows_select_recent ON rls_graph."KNOWS"; + +CREATE POLICY knows_select_all ON rls_graph."KNOWS" + FOR SELECT USING (true); + +-- Policy: Can only create edges with strength = 'strong' +CREATE POLICY knows_insert_strong ON rls_graph."KNOWS" + FOR INSERT + WITH CHECK (properties->>'"strength"' = 'strong'); + +SET ROLE rls_user1; + +-- Should succeed - strength is 'strong' +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Bob'}), (b:Person {name: 'Diana'}) + CREATE (a)-[:KNOWS {since: 2023, strength: 'strong'}]->(b) +$$) AS (a agtype); + +-- Should FAIL - strength is 'weak' +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Diana'}), (b:Person {name: 'Alice'}) + CREATE (a)-[:KNOWS {since: 2023, strength: 'weak'}]->(b) +$$) AS (a agtype); + +RESET ROLE; + +-- Verify only strong edge was created +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS]->() WHERE k.since = 2023 RETURN k.strength ORDER BY k.strength +$$) AS (strength agtype); + +-- cleanup +DROP POLICY knows_insert_strong ON rls_graph."KNOWS"; + +-- ============================================================================ +-- PART 9: Edge UPDATE Policies (SET on edge) +-- ============================================================================ + +-- Policy: Can only update edges with strength = 'strong' +CREATE POLICY knows_update_strong ON rls_graph."KNOWS" + FOR UPDATE + USING (properties->>'"strength"' = 'strong') + WITH CHECK (properties->>'"strength"' = 'strong'); + +SET ROLE rls_user1; + +-- Should succeed - edge has strength 'strong' +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS {since: 2021}]->() SET k.notes = 'updated' RETURN k.since, k.notes +$$) AS (since agtype, notes agtype); + +-- Should silently skip - edge has strength 'weak' +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS {since: 2020}]->() SET k.notes = 'updated' RETURN k.since, k.notes +$$) AS (since agtype, notes agtype); + +RESET ROLE; + +-- Verify only 2021 edge was updated +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS]->() WHERE k.since IN [2020, 2021] RETURN k.since, k.notes ORDER BY k.since +$$) AS (since agtype, notes agtype); + +-- cleanup +DROP POLICY knows_select_all ON rls_graph."KNOWS"; +DROP POLICY knows_update_strong ON rls_graph."KNOWS"; + +-- ============================================================================ +-- PART 10: Edge DELETE Policies +-- ============================================================================ + +CREATE POLICY knows_select_all ON rls_graph."KNOWS" + FOR SELECT USING (true); + +-- Create test edges for delete +CREATE POLICY knows_insert_all ON rls_graph."KNOWS" + FOR INSERT WITH CHECK (true); + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Bob'}), (b:Person {name: 'Charlie'}) + CREATE (a)-[:KNOWS {since: 2018, strength: 'weak'}]->(b) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Diana'}), (b:Person {name: 'Charlie'}) + CREATE (a)-[:KNOWS {since: 2019, strength: 'strong'}]->(b) +$$) AS (a agtype); + +DROP POLICY knows_insert_all ON rls_graph."KNOWS"; + +-- Policy: Can only delete edges with strength = 'weak' +CREATE POLICY knows_delete_weak ON rls_graph."KNOWS" + FOR DELETE + USING (properties->>'"strength"' = 'weak'); + +SET ROLE rls_user1; + +-- Should succeed - edge has strength 'weak' +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS {since: 2018}]->() DELETE k +$$) AS (a agtype); + +-- Should silently skip - edge has strength 'strong' +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS {since: 2019}]->() DELETE k +$$) AS (a agtype); + +RESET ROLE; + +-- Verify 2018 edge deleted, 2019 edge still exists +SELECT * FROM cypher('rls_graph', $$ + MATCH ()-[k:KNOWS]->() WHERE k.since IN [2018, 2019] RETURN k.since ORDER BY k.since +$$) AS (since agtype); + +-- cleanup +DROP POLICY knows_delete_weak ON rls_graph."KNOWS"; + +-- ============================================================================ +-- PART 11: DETACH DELETE +-- ============================================================================ + +-- Re-enable Person RLS +ALTER TABLE rls_graph."Person" ENABLE ROW LEVEL SECURITY; +CREATE POLICY person_all ON rls_graph."Person" + FOR ALL USING (true) WITH CHECK (true); + +-- Create test data with a protected edge +CREATE POLICY knows_insert_all ON rls_graph."KNOWS" + FOR INSERT WITH CHECK (true); + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'DetachTest1', owner: 'test', department: 'Detach', level: 1}) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + CREATE (:Person {name: 'DetachTest2', owner: 'test', department: 'Detach', level: 1}) +$$) AS (a agtype); + +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'DetachTest1'}), (b:Person {name: 'DetachTest2'}) + CREATE (a)-[:KNOWS {since: 2010, strength: 'protected'}]->(b) +$$) AS (a agtype); + +DROP POLICY knows_insert_all ON rls_graph."KNOWS"; + +-- Policy: Cannot delete edges with strength = 'protected' +CREATE POLICY knows_delete_not_protected ON rls_graph."KNOWS" + FOR DELETE + USING (properties->>'"strength"' != 'protected'); + +SET ROLE rls_user1; + +-- Should ERROR - DETACH DELETE cannot silently skip (would leave dangling edge) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DetachTest1'}) DETACH DELETE p +$$) AS (a agtype); + +RESET ROLE; + +-- Verify vertex still exists (delete was blocked) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'DetachTest1'}) RETURN p.name +$$) AS (name agtype); + +-- cleanup +DROP POLICY person_all ON rls_graph."Person"; +DROP POLICY knows_select_all ON rls_graph."KNOWS"; +DROP POLICY knows_delete_not_protected ON rls_graph."KNOWS"; +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department = 'Detach' DETACH DELETE p +$$) AS (a agtype); + +-- ============================================================================ +-- PART 12: Multiple Labels in Single Query +-- ============================================================================ + +-- Enable RLS on Document too +ALTER TABLE rls_graph."Document" ENABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."Document" FORCE ROW LEVEL SECURITY; + +-- Policy: Users see their own Person records +CREATE POLICY person_own ON rls_graph."Person" + FOR SELECT + USING (properties->>'"owner"' = current_user); + +-- Policy: Users see only public documents +CREATE POLICY doc_public ON rls_graph."Document" + FOR SELECT + USING (properties->>'"classification"' = 'public'); + +SET ROLE rls_user1; + +-- Should only see Alice and Charlie (Person) with Public Doc (Document) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +SELECT * FROM cypher('rls_graph', $$ + MATCH (d:Document) RETURN d.title ORDER BY d.title +$$) AS (title agtype); + +-- Combined query - should respect both policies +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person)-[:AUTHORED]->(d:Document) + RETURN p.name, d.title +$$) AS (person agtype, doc agtype); + +RESET ROLE; + +-- ============================================================================ +-- PART 13: Permissive vs Restrictive Policies +-- ============================================================================ + +DROP POLICY person_own ON rls_graph."Person"; +DROP POLICY doc_public ON rls_graph."Document"; + +ALTER TABLE rls_graph."Document" DISABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."KNOWS" DISABLE ROW LEVEL SECURITY; + +-- 13.1: Multiple permissive policies (OR logic) +CREATE POLICY person_permissive_own ON rls_graph."Person" + AS PERMISSIVE FOR SELECT + USING (properties->>'"owner"' = current_user); + +CREATE POLICY person_permissive_eng ON rls_graph."Person" + AS PERMISSIVE FOR SELECT + USING (properties->>'"department"' = 'Engineering'); + +SET ROLE rls_user1; + +-- Should see: Alice (own), Charlie (own), Bob (Engineering) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.department IN ['Engineering', 'Sales'] + RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +RESET ROLE; + +-- 13.2: Add restrictive policy (AND with permissive) +CREATE POLICY person_restrictive_level ON rls_graph."Person" + AS RESTRICTIVE FOR SELECT + USING ((properties->>'"level"')::int <= 2); + +SET ROLE rls_user1; + +-- Should see: Alice (own, level 1), Bob (Engineering, level 2), Charlie (own, level 1) +-- Diana (level 3) blocked by restrictive +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name, p.level ORDER BY p.name +$$) AS (name agtype, level agtype); + +RESET ROLE; + +-- 13.3: Multiple restrictive policies (all must pass) +CREATE POLICY person_restrictive_sales ON rls_graph."Person" + AS RESTRICTIVE FOR SELECT + USING (properties->>'"department"' != 'Sales'); + +SET ROLE rls_user1; + +-- Should see: Alice (own, level 1, not Sales), Bob (Engineering, level 2, not Sales) +-- Charlie blocked by Sales restriction +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +RESET ROLE; + +-- ============================================================================ +-- PART 14: BYPASSRLS Role and Superuser Behavior +-- ============================================================================ + +DROP POLICY person_permissive_own ON rls_graph."Person"; +DROP POLICY person_permissive_eng ON rls_graph."Person"; +DROP POLICY person_restrictive_level ON rls_graph."Person"; +DROP POLICY person_restrictive_sales ON rls_graph."Person"; + +-- Restrictive policy that blocks most access +CREATE POLICY person_very_restrictive ON rls_graph."Person" + FOR SELECT + USING (properties->>'"name"' = 'Nobody'); + +-- 14.1: Regular user sees nothing +SET ROLE rls_user1; + +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +RESET ROLE; + +-- 14.2: BYPASSRLS role sees everything +SET ROLE rls_admin; + +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +RESET ROLE; + +-- 14.3: Superuser sees everything (implicit bypass) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) RETURN p.name ORDER BY p.name +$$) AS (name agtype); + +-- ============================================================================ +-- PART 15: Complex Multi-Operation Queries +-- ============================================================================ + +DROP POLICY person_very_restrictive ON rls_graph."Person"; + +CREATE POLICY person_select_all ON rls_graph."Person" + FOR SELECT USING (true); + +CREATE POLICY person_insert_own ON rls_graph."Person" + FOR INSERT + WITH CHECK (properties->>'"owner"' = current_user); + +CREATE POLICY person_update_own ON rls_graph."Person" + FOR UPDATE + USING (properties->>'"owner"' = current_user) + WITH CHECK (properties->>'"owner"' = current_user); + +-- 15.1: MATCH + CREATE in one query +SET ROLE rls_user1; + +-- Should succeed - creating with correct owner +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'}) + CREATE (a)-[:KNOWS]->(:Person {name: 'NewFromMatch', owner: 'rls_user1', department: 'Complex', level: 1}) +$$) AS (a agtype); + +RESET ROLE; + +-- Verify creation +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person {name: 'NewFromMatch'}) RETURN p.name, p.owner +$$) AS (name agtype, owner agtype); + +-- 15.2: MATCH + SET in one query +SET ROLE rls_user1; + +-- Should succeed on Alice (own), skip Bob (not own) +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.name IN ['Alice', 'Bob'] + SET p.complexTest = true + RETURN p.name, p.complexTest +$$) AS (name agtype, test agtype); + +RESET ROLE; + +-- Verify only Alice was updated +SELECT * FROM cypher('rls_graph', $$ + MATCH (p:Person) WHERE p.name IN ['Alice', 'Bob'] + RETURN p.name, p.complexTest ORDER BY p.name +$$) AS (name agtype, test agtype); + +-- cleanup +DROP POLICY IF EXISTS person_select_all ON rls_graph."Person"; +DROP POLICY IF EXISTS person_insert_own ON rls_graph."Person"; +DROP POLICY IF EXISTS person_update_own ON rls_graph."Person"; + +-- ============================================================================ +-- PART 16: startNode/endNode RLS Enforcement +-- ============================================================================ + +ALTER TABLE rls_graph."Person" DISABLE ROW LEVEL SECURITY; + +-- Enable RLS on Person with restrictive policy +ALTER TABLE rls_graph."Person" ENABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."Person" FORCE ROW LEVEL SECURITY; + +-- Policy: users can only see their own Person records +CREATE POLICY person_own ON rls_graph."Person" + FOR SELECT + USING (properties->>'"owner"' = current_user); + +-- Enable edge access for testing +ALTER TABLE rls_graph."KNOWS" ENABLE ROW LEVEL SECURITY; +CREATE POLICY knows_all ON rls_graph."KNOWS" + FOR SELECT USING (true); + +-- 16.1: startNode blocked by RLS - should error +SET ROLE rls_user1; + +-- rls_user1 can see the edge (Alice->Bob) but cannot see Bob (owned by rls_user2) +-- endNode should error because Bob is blocked by RLS +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'})-[e:KNOWS]->(b) + RETURN endNode(e) +$$) AS (end_vertex agtype); + +-- 16.2: endNode blocked by RLS - should error +-- rls_user1 cannot see Bob, so startNode on an edge starting from Bob should error +SET ROLE rls_user2; + +-- rls_user2 can see Bob but not Alice (owned by rls_user1) +-- startNode should error because Alice is blocked by RLS +SELECT * FROM cypher('rls_graph', $$ + MATCH (a)-[e:KNOWS]->(b:Person {name: 'Bob'}) + RETURN startNode(e) +$$) AS (start_vertex agtype); + +-- 16.3: startNode/endNode succeed when RLS allows access +SET ROLE rls_user1; + +-- Alice->Charlie edge: rls_user1 owns both, should succeed +SELECT * FROM cypher('rls_graph', $$ + MATCH (a:Person {name: 'Alice'})-[e:KNOWS]->(c:Person {name: 'Charlie'}) + RETURN startNode(e).name, endNode(e).name +$$) AS (start_name agtype, end_name agtype); + +RESET ROLE; + +-- cleanup +DROP POLICY person_own ON rls_graph."Person"; +DROP POLICY knows_all ON rls_graph."KNOWS"; +ALTER TABLE rls_graph."KNOWS" DISABLE ROW LEVEL SECURITY; + +-- ============================================================================ +-- RLS CLEANUP +-- ============================================================================ + +RESET ROLE; + +-- Disable RLS on all tables +ALTER TABLE rls_graph."Person" DISABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."Document" DISABLE ROW LEVEL SECURITY; +ALTER TABLE rls_graph."KNOWS" DISABLE ROW LEVEL SECURITY; + +-- Drop roles +DROP OWNED BY rls_user1 CASCADE; +DROP ROLE rls_user1; + +DROP OWNED BY rls_user2 CASCADE; +DROP ROLE rls_user2; + +DROP OWNED BY rls_admin CASCADE; +DROP ROLE rls_admin; + +-- Drop test graph +SELECT drop_graph('rls_graph', true); diff --git a/src/backend/executor/cypher_create.c b/src/backend/executor/cypher_create.c index 2091ea29c..a90c2a196 100644 --- a/src/backend/executor/cypher_create.c +++ b/src/backend/executor/cypher_create.c @@ -19,6 +19,8 @@ #include "postgres.h" +#include "utils/rls.h" + #include "catalog/ag_label.h" #include "executor/cypher_executor.h" #include "executor/cypher_utils.h" @@ -120,6 +122,12 @@ static void begin_cypher_create(CustomScanState *node, EState *estate, cypher_node->prop_expr_state = ExecInitExpr(cypher_node->prop_expr, (PlanState *)node); } + + /* Setup RLS WITH CHECK policies if RLS is enabled */ + if (check_enable_rls(rel->rd_id, InvalidOid, true) == RLS_ENABLED) + { + setup_wcos(cypher_node->resultRelInfo, estate, node, CMD_INSERT); + } } } diff --git a/src/backend/executor/cypher_delete.c b/src/backend/executor/cypher_delete.c index 5f9aa561d..4766c6e7a 100644 --- a/src/backend/executor/cypher_delete.c +++ b/src/backend/executor/cypher_delete.c @@ -19,8 +19,11 @@ #include "postgres.h" -#include "storage/bufmgr.h" #include "common/hashfn.h" +#include "miscadmin.h" +#include "storage/bufmgr.h" +#include "utils/acl.h" +#include "utils/rls.h" #include "catalog/ag_label.h" #include "executor/cypher_executor.h" @@ -370,6 +373,16 @@ static void process_delete_list(CustomScanState *node) ExprContext *econtext = css->css.ss.ps.ps_ExprContext; TupleTableSlot *scanTupleSlot = econtext->ecxt_scantuple; EState *estate = node->ss.ps.state; + HTAB *qual_cache = NULL; + HASHCTL hashctl; + + /* Hash table for caching compiled security quals per label */ + MemSet(&hashctl, 0, sizeof(hashctl)); + hashctl.keysize = sizeof(Oid); + hashctl.entrysize = sizeof(RLSCacheEntry); + hashctl.hcxt = CurrentMemoryContext; + qual_cache = hash_create("delete_qual_cache", 8, &hashctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); foreach(lc, css->delete_data->delete_items) { @@ -382,6 +395,7 @@ static void process_delete_list(CustomScanState *node) char *label_name; Integer *pos; int entity_position; + Oid relid; item = lfirst(lc); @@ -400,6 +414,7 @@ static void process_delete_list(CustomScanState *node) label_name = pnstrdup(label->val.string.val, label->val.string.len); resultRelInfo = create_entity_result_rel_info(estate, css->delete_data->graph_name, label_name); + relid = RelationGetRelid(resultRelInfo->ri_RelationDesc); /* * Setup the scan key to require the id field on-disc to match the @@ -447,6 +462,36 @@ static void process_delete_list(CustomScanState *node) continue; } + /* Check RLS security quals (USING policy) before delete */ + if (check_enable_rls(relid, InvalidOid, true) == RLS_ENABLED) + { + RLSCacheEntry *entry; + bool found; + + /* Get cached security quals and slot for this label */ + entry = hash_search(qual_cache, &relid, HASH_ENTER, &found); + if (!found) + { + entry->qualExprs = setup_security_quals(resultRelInfo, estate, + node, CMD_DELETE); + entry->slot = ExecInitExtraTupleSlot( + estate, RelationGetDescr(resultRelInfo->ri_RelationDesc), + &TTSOpsHeapTuple); + entry->withCheckOptions = NIL; + entry->withCheckOptionExprs = NIL; + } + + ExecStoreHeapTuple(heap_tuple, entry->slot, false); + + /* Silently skip if USING policy filters out this row */ + if (!check_security_quals(entry->qualExprs, entry->slot, econtext)) + { + table_endscan(scan_desc); + destroy_entity_result_rel_info(resultRelInfo); + continue; + } + } + /* * For vertices, we insert the vertex ID in the hashtable * vertex_id_htab. This hashtable is used later to process @@ -466,6 +511,9 @@ static void process_delete_list(CustomScanState *node) table_endscan(scan_desc); destroy_entity_result_rel_info(resultRelInfo); } + + /* Clean up the cache */ + hash_destroy(qual_cache); } /* @@ -489,9 +537,14 @@ static void check_for_connected_edges(CustomScanState *node) TableScanDesc scan_desc; HeapTuple tuple; TupleTableSlot *slot; + Oid relid; + bool rls_enabled = false; + List *qualExprs = NIL; + ExprContext *econtext = NULL; resultRelInfo = create_entity_result_rel_info(estate, graph_name, label_name); + relid = RelationGetRelid(resultRelInfo->ri_RelationDesc); estate->es_snapshot->curcid = GetCurrentCommandId(false); estate->es_output_cid = GetCurrentCommandId(false); scan_desc = table_beginscan(resultRelInfo->ri_RelationDesc, @@ -500,6 +553,22 @@ static void check_for_connected_edges(CustomScanState *node) estate, RelationGetDescr(resultRelInfo->ri_RelationDesc), &TTSOpsHeapTuple); + /* + * For DETACH DELETE with RLS enabled, compile the security qual + * expressions once per label for efficient evaluation. + */ + if (css->delete_data->detach) + { + /* Setup RLS security quals for this label */ + if (check_enable_rls(relid, InvalidOid, true) == RLS_ENABLED) + { + rls_enabled = true; + econtext = css->css.ss.ps.ps_ExprContext; + qualExprs = setup_security_quals(resultRelInfo, estate, node, + CMD_DELETE); + } + } + /* for each row */ while (true) { @@ -537,6 +606,34 @@ static void check_for_connected_edges(CustomScanState *node) { if (css->delete_data->detach) { + AclResult aclresult; + + /* Check that the user has DELETE permission on the edge table */ + aclresult = pg_class_aclcheck(relid, GetUserId(), ACL_DELETE); + if (aclresult != ACLCHECK_OK) + { + aclcheck_error(aclresult, OBJECT_TABLE, label_name); + } + + /* Check RLS security quals (USING policy) before delete */ + if (rls_enabled) + { + /* + * For DETACH DELETE, error out if edge RLS check fails. + * Unlike normal DELETE which silently skips, we cannot + * silently skip edges here as it would leave dangling + * edges pointing to deleted vertices. + */ + if (!check_security_quals(qualExprs, slot, econtext)) + { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot delete edge due to row-level security policy on \"%s\"", + label_name), + errhint("DETACH DELETE requires permission to delete all connected edges."))); + } + } + delete_entity(estate, resultRelInfo, tuple); } else diff --git a/src/backend/executor/cypher_merge.c b/src/backend/executor/cypher_merge.c index 9136825ab..e0d6c78e8 100644 --- a/src/backend/executor/cypher_merge.c +++ b/src/backend/executor/cypher_merge.c @@ -19,10 +19,12 @@ #include "postgres.h" +#include "utils/datum.h" +#include "utils/rls.h" + #include "catalog/ag_label.h" #include "executor/cypher_executor.h" #include "executor/cypher_utils.h" -#include "utils/datum.h" /* * The following structure is used to hold a single vertex or edge component @@ -180,6 +182,12 @@ static void begin_cypher_merge(CustomScanState *node, EState *estate, cypher_node->prop_expr_state = ExecInitExpr(cypher_node->prop_expr, (PlanState *)node); } + + /* Setup RLS WITH CHECK policies if RLS is enabled */ + if (check_enable_rls(rel->rd_id, InvalidOid, true) == RLS_ENABLED) + { + setup_wcos(cypher_node->resultRelInfo, estate, node, CMD_INSERT); + } } /* diff --git a/src/backend/executor/cypher_set.c b/src/backend/executor/cypher_set.c index d1837fb16..40cf2b232 100644 --- a/src/backend/executor/cypher_set.c +++ b/src/backend/executor/cypher_set.c @@ -19,7 +19,10 @@ #include "postgres.h" +#include "common/hashfn.h" +#include "executor/executor.h" #include "storage/bufmgr.h" +#include "utils/rls.h" #include "executor/cypher_executor.h" #include "executor/cypher_utils.h" @@ -125,6 +128,13 @@ static HeapTuple update_entity_tuple(ResultRelInfo *resultRelInfo, ExecConstraints(resultRelInfo, elemTupleSlot, estate); } + /* Check RLS WITH CHECK policies if configured */ + if (resultRelInfo->ri_WithCheckOptions != NIL) + { + ExecWithCheckOptions(WCO_RLS_UPDATE_CHECK, resultRelInfo, + elemTupleSlot, estate); + } + result = table_tuple_update(resultRelInfo->ri_RelationDesc, &tuple->t_self, elemTupleSlot, cid, estate->es_snapshot, @@ -355,9 +365,20 @@ static void process_update_list(CustomScanState *node) EState *estate = css->css.ss.ps.state; int *luindex = NULL; int lidx = 0; + HTAB *qual_cache = NULL; + HASHCTL hashctl; /* allocate an array to hold the last update index of each 'entity' */ luindex = palloc0(sizeof(int) * scanTupleSlot->tts_nvalid); + + /* Hash table for caching compiled security quals per label */ + MemSet(&hashctl, 0, sizeof(hashctl)); + hashctl.keysize = sizeof(Oid); + hashctl.entrysize = sizeof(RLSCacheEntry); + hashctl.hcxt = CurrentMemoryContext; + qual_cache = hash_create("update_qual_cache", 8, &hashctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + /* * Iterate through the SET items list and store the loop index of each * 'entity' update. As there is only one entry for each entity, this will @@ -505,6 +526,38 @@ static void process_update_list(CustomScanState *node) estate, RelationGetDescr(resultRelInfo->ri_RelationDesc), &TTSOpsHeapTuple); + /* Setup RLS policies if RLS is enabled */ + if (check_enable_rls(resultRelInfo->ri_RelationDesc->rd_id, + InvalidOid, true) == RLS_ENABLED) + { + Oid relid = RelationGetRelid(resultRelInfo->ri_RelationDesc); + RLSCacheEntry *entry; + bool found; + + /* Get cached RLS state for this label, or set it up */ + entry = hash_search(qual_cache, &relid, HASH_ENTER, &found); + if (!found) + { + /* Setup WITH CHECK policies */ + setup_wcos(resultRelInfo, estate, node, CMD_UPDATE); + entry->withCheckOptions = resultRelInfo->ri_WithCheckOptions; + entry->withCheckOptionExprs = resultRelInfo->ri_WithCheckOptionExprs; + + /* Setup security quals */ + entry->qualExprs = setup_security_quals(resultRelInfo, estate, + node, CMD_UPDATE); + entry->slot = ExecInitExtraTupleSlot( + estate, RelationGetDescr(resultRelInfo->ri_RelationDesc), + &TTSOpsHeapTuple); + } + else + { + /* Use cached WCOs */ + resultRelInfo->ri_WithCheckOptions = entry->withCheckOptions; + resultRelInfo->ri_WithCheckOptionExprs = entry->withCheckOptionExprs; + } + } + /* * Now that we have the updated properties, create a either a vertex or * edge Datum for the in-memory update, and setup the tupleTableSlot @@ -580,8 +633,36 @@ static void process_update_list(CustomScanState *node) */ if (HeapTupleIsValid(heap_tuple)) { - heap_tuple = update_entity_tuple(resultRelInfo, slot, estate, - heap_tuple); + bool should_update = true; + Oid relid = RelationGetRelid(resultRelInfo->ri_RelationDesc); + + /* Check RLS security quals (USING policy) before update */ + if (check_enable_rls(relid, InvalidOid, true) == RLS_ENABLED) + { + RLSCacheEntry *entry; + + /* Entry was already created earlier when setting up WCOs */ + entry = hash_search(qual_cache, &relid, HASH_FIND, NULL); + if (!entry) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("missing RLS cache entry for relation %u", + relid))); + } + + ExecStoreHeapTuple(heap_tuple, entry->slot, false); + should_update = check_security_quals(entry->qualExprs, + entry->slot, + econtext); + } + + /* Silently skip if USING policy filters out this row */ + if (should_update) + { + heap_tuple = update_entity_tuple(resultRelInfo, slot, estate, + heap_tuple); + } } /* close the ScanDescription */ table_endscan(scan_desc); @@ -595,6 +676,10 @@ static void process_update_list(CustomScanState *node) /* increment loop index */ lidx++; } + + /* Clean up the cache */ + hash_destroy(qual_cache); + /* free our lookup array */ pfree_if_not_null(luindex); } diff --git a/src/backend/executor/cypher_utils.c b/src/backend/executor/cypher_utils.c index c8d568831..eff829925 100644 --- a/src/backend/executor/cypher_utils.c +++ b/src/backend/executor/cypher_utils.c @@ -24,14 +24,36 @@ #include "postgres.h" +#include "executor/executor.h" +#include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" +#include "rewrite/rewriteManip.h" +#include "rewrite/rowsecurity.h" +#include "utils/acl.h" +#include "utils/rls.h" #include "catalog/ag_label.h" #include "commands/label_commands.h" #include "executor/cypher_utils.h" #include "utils/ag_cache.h" +/* RLS helper function declarations */ +static void get_policies_for_relation(Relation relation, CmdType cmd, + Oid user_id, List **permissive_policies, + List **restrictive_policies); +static void add_with_check_options(Relation rel, int rt_index, WCOKind kind, + List *permissive_policies, + List *restrictive_policies, + List **withCheckOptions, bool *hasSubLinks, + bool force_using); +static void add_security_quals(int rt_index, List *permissive_policies, + List *restrictive_policies, + List **securityQuals, bool *hasSubLinks); +static void sort_policies_by_name(List *policies); +static int row_security_policy_cmp(const ListCell *a, const ListCell *b); +static bool check_role_for_policy(ArrayType *policy_roles, Oid user_id); + /* * Given the graph name and the label name, create a ResultRelInfo for the table * those two variables represent. Open the Indices too. @@ -255,6 +277,13 @@ HeapTuple insert_entity_tuple_cid(ResultRelInfo *resultRelInfo, ExecConstraints(resultRelInfo, elemTupleSlot, estate); } + /* Check RLS WITH CHECK policies if configured */ + if (resultRelInfo->ri_WithCheckOptions != NIL) + { + ExecWithCheckOptions(WCO_RLS_INSERT_CHECK, resultRelInfo, + elemTupleSlot, estate); + } + /* Insert the tuple normally */ table_tuple_insert(resultRelInfo->ri_RelationDesc, elemTupleSlot, cid, 0, NULL); @@ -268,3 +297,754 @@ HeapTuple insert_entity_tuple_cid(ResultRelInfo *resultRelInfo, return tuple; } + +/* + * setup_wcos + * + * WithCheckOptions are added during the rewrite phase, but since AGE uses + * CMD_SELECT for all queries, WCOs don't get added for CREATE/SET/MERGE + * operations. This function compensates by adding WCOs at execution time. + * + * Based on PostgreSQL's row security implementation in rowsecurity.c + */ +void setup_wcos(ResultRelInfo *resultRelInfo, EState *estate, + CustomScanState *node, CmdType cmd) +{ + List *permissive_policies; + List *restrictive_policies; + List *withCheckOptions = NIL; + List *wcoExprs = NIL; + ListCell *lc; + Relation rel; + Oid user_id; + int rt_index; + WCOKind wco_kind; + bool hasSubLinks = false; + + /* Determine the WCO kind based on command type */ + if (cmd == CMD_INSERT) + { + wco_kind = WCO_RLS_INSERT_CHECK; + } + else if (cmd == CMD_UPDATE) + { + wco_kind = WCO_RLS_UPDATE_CHECK; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg_internal("unexpected command type for setup_wcos"))); + } + + rel = resultRelInfo->ri_RelationDesc; + + /* + * Use rt_index=1 since we're evaluating policies against a single relation. + * Policy quals are stored with varno=1, and we set ecxt_scantuple to the + * tuple we want to check, so keeping varno=1 is correct. + */ + rt_index = 1; + user_id = GetUserId(); + + /* Get the policies for the specified command type */ + get_policies_for_relation(rel, cmd, user_id, + &permissive_policies, + &restrictive_policies); + + /* Build WithCheckOptions from the policies */ + add_with_check_options(rel, rt_index, wco_kind, + permissive_policies, + restrictive_policies, + &withCheckOptions, + &hasSubLinks, + false); + + /* Compile the WCO expressions */ + foreach(lc, withCheckOptions) + { + WithCheckOption *wco = lfirst_node(WithCheckOption, lc); + ExprState *wcoExpr; + + /* Ensure qual is a List for ExecInitQual */ + if (!IsA(wco->qual, List)) + { + wco->qual = (Node *) list_make1(wco->qual); + } + + wcoExpr = ExecInitQual((List *) wco->qual, (PlanState *) node); + wcoExprs = lappend(wcoExprs, wcoExpr); + } + + /* Set up the ResultRelInfo with WCOs */ + resultRelInfo->ri_WithCheckOptions = withCheckOptions; + resultRelInfo->ri_WithCheckOptionExprs = wcoExprs; +} + +/* + * get_policies_for_relation + * + * Returns lists of permissive and restrictive policies to be applied to the + * specified relation, based on the command type and role. + * + * This includes any policies added by extensions. + * + * Copied from PostgreSQL's src/backend/rewrite/rowsecurity.c + */ +static void +get_policies_for_relation(Relation relation, CmdType cmd, Oid user_id, + List **permissive_policies, + List **restrictive_policies) +{ + ListCell *item; + + *permissive_policies = NIL; + *restrictive_policies = NIL; + + /* No policies if RLS descriptor is not present */ + if (relation->rd_rsdesc == NULL) + { + return; + } + + /* First find all internal policies for the relation. */ + foreach(item, relation->rd_rsdesc->policies) + { + bool cmd_matches = false; + RowSecurityPolicy *policy = (RowSecurityPolicy *) lfirst(item); + + /* Always add ALL policies, if they exist. */ + if (policy->polcmd == '*') + { + cmd_matches = true; + } + else + { + /* Check whether the policy applies to the specified command type */ + switch (cmd) + { + case CMD_SELECT: + if (policy->polcmd == ACL_SELECT_CHR) + { + cmd_matches = true; + } + break; + case CMD_INSERT: + if (policy->polcmd == ACL_INSERT_CHR) + { + cmd_matches = true; + } + break; + case CMD_UPDATE: + if (policy->polcmd == ACL_UPDATE_CHR) + { + cmd_matches = true; + } + break; + case CMD_DELETE: + if (policy->polcmd == ACL_DELETE_CHR) + { + cmd_matches = true; + } + break; + case CMD_MERGE: + /* + * We do not support a separate policy for MERGE command. + * Instead it derives from the policies defined for other + * commands. + */ + break; + default: + elog(ERROR, "unrecognized policy command type %d", + (int) cmd); + break; + } + } + + /* + * Add this policy to the relevant list of policies if it applies to + * the specified role. + */ + if (cmd_matches && check_role_for_policy(policy->roles, user_id)) + { + if (policy->permissive) + { + *permissive_policies = lappend(*permissive_policies, policy); + } + else + { + *restrictive_policies = lappend(*restrictive_policies, policy); + } + } + } + + /* + * We sort restrictive policies by name so that any WCOs they generate are + * checked in a well-defined order. + */ + sort_policies_by_name(*restrictive_policies); + + /* + * Then add any permissive or restrictive policies defined by extensions. + * These are simply appended to the lists of internal policies, if they + * apply to the specified role. + */ + if (row_security_policy_hook_restrictive) + { + List *hook_policies = + (*row_security_policy_hook_restrictive) (cmd, relation); + + /* + * As with built-in restrictive policies, we sort any hook-provided + * restrictive policies by name also. Note that we also intentionally + * always check all built-in restrictive policies, in name order, + * before checking restrictive policies added by hooks, in name order. + */ + sort_policies_by_name(hook_policies); + + foreach(item, hook_policies) + { + RowSecurityPolicy *policy = (RowSecurityPolicy *) lfirst(item); + + if (check_role_for_policy(policy->roles, user_id)) + { + *restrictive_policies = lappend(*restrictive_policies, policy); + } + } + } + + if (row_security_policy_hook_permissive) + { + List *hook_policies = + (*row_security_policy_hook_permissive) (cmd, relation); + + foreach(item, hook_policies) + { + RowSecurityPolicy *policy = (RowSecurityPolicy *) lfirst(item); + + if (check_role_for_policy(policy->roles, user_id)) + { + *permissive_policies = lappend(*permissive_policies, policy); + } + } + } +} + +/* + * add_with_check_options + * + * Add WithCheckOptions of the specified kind to check that new records + * added by an INSERT or UPDATE are consistent with the specified RLS + * policies. Normally new data must satisfy the WITH CHECK clauses from the + * policies. If a policy has no explicit WITH CHECK clause, its USING clause + * is used instead. In the special case of an UPDATE arising from an + * INSERT ... ON CONFLICT DO UPDATE, existing records are first checked using + * a WCO_RLS_CONFLICT_CHECK WithCheckOption, which always uses the USING + * clauses from RLS policies. + * + * New WCOs are added to withCheckOptions, and hasSubLinks is set to true if + * any of the check clauses added contain sublink subqueries. + * + * Copied from PostgreSQL's src/backend/rewrite/rowsecurity.c + */ +static void +add_with_check_options(Relation rel, + int rt_index, + WCOKind kind, + List *permissive_policies, + List *restrictive_policies, + List **withCheckOptions, + bool *hasSubLinks, + bool force_using) +{ + ListCell *item; + List *permissive_quals = NIL; + +#define QUAL_FOR_WCO(policy) \ + ( !force_using && \ + (policy)->with_check_qual != NULL ? \ + (policy)->with_check_qual : (policy)->qual ) + + /* + * First collect up the permissive policy clauses, similar to + * add_security_quals. + */ + foreach(item, permissive_policies) + { + RowSecurityPolicy *policy = (RowSecurityPolicy *) lfirst(item); + Expr *qual = QUAL_FOR_WCO(policy); + + if (qual != NULL) + { + permissive_quals = lappend(permissive_quals, copyObject(qual)); + *hasSubLinks |= policy->hassublinks; + } + } + + /* + * There must be at least one permissive qual found or no rows are allowed + * to be added. This is the same as in add_security_quals. + * + * If there are no permissive_quals then we fall through and return a + * single 'false' WCO, preventing all new rows. + */ + if (permissive_quals != NIL) + { + /* + * Add a single WithCheckOption for all the permissive policy clauses, + * combining them together using OR. This check has no policy name, + * since if the check fails it means that no policy granted permission + * to perform the update, rather than any particular policy being + * violated. + */ + WithCheckOption *wco; + + wco = makeNode(WithCheckOption); + wco->kind = kind; + wco->relname = pstrdup(RelationGetRelationName(rel)); + wco->polname = NULL; + wco->cascaded = false; + + if (list_length(permissive_quals) == 1) + { + wco->qual = (Node *) linitial(permissive_quals); + } + else + { + wco->qual = (Node *) makeBoolExpr(OR_EXPR, permissive_quals, -1); + } + + ChangeVarNodes(wco->qual, 1, rt_index, 0); + + *withCheckOptions = list_append_unique(*withCheckOptions, wco); + + /* + * Now add WithCheckOptions for each of the restrictive policy clauses + * (which will be combined together using AND). We use a separate + * WithCheckOption for each restrictive policy to allow the policy + * name to be included in error reports if the policy is violated. + */ + foreach(item, restrictive_policies) + { + RowSecurityPolicy *policy = (RowSecurityPolicy *) lfirst(item); + Expr *qual = QUAL_FOR_WCO(policy); + + if (qual != NULL) + { + qual = copyObject(qual); + ChangeVarNodes((Node *) qual, 1, rt_index, 0); + + wco = makeNode(WithCheckOption); + wco->kind = kind; + wco->relname = pstrdup(RelationGetRelationName(rel)); + wco->polname = pstrdup(policy->policy_name); + wco->qual = (Node *) qual; + wco->cascaded = false; + + *withCheckOptions = list_append_unique(*withCheckOptions, wco); + *hasSubLinks |= policy->hassublinks; + } + } + } + else + { + /* + * If there were no policy clauses to check new data, add a single + * always-false WCO (a default-deny policy). + */ + WithCheckOption *wco; + + wco = makeNode(WithCheckOption); + wco->kind = kind; + wco->relname = pstrdup(RelationGetRelationName(rel)); + wco->polname = NULL; + wco->qual = (Node *) makeConst(BOOLOID, -1, InvalidOid, + sizeof(bool), BoolGetDatum(false), + false, true); + wco->cascaded = false; + + *withCheckOptions = lappend(*withCheckOptions, wco); + } +} + +/* + * sort_policies_by_name + * + * This is only used for restrictive policies, ensuring that any + * WithCheckOptions they generate are applied in a well-defined order. + * This is not necessary for permissive policies, since they are all combined + * together using OR into a single WithCheckOption check. + * + * Copied from PostgreSQL's src/backend/rewrite/rowsecurity.c + */ +static void +sort_policies_by_name(List *policies) +{ + list_sort(policies, row_security_policy_cmp); +} + +/* + * list_sort comparator to sort RowSecurityPolicy entries by name + * + * Copied from PostgreSQL's src/backend/rewrite/rowsecurity.c + */ +static int +row_security_policy_cmp(const ListCell *a, const ListCell *b) +{ + const RowSecurityPolicy *pa = (const RowSecurityPolicy *) lfirst(a); + const RowSecurityPolicy *pb = (const RowSecurityPolicy *) lfirst(b); + + /* Guard against NULL policy names from extensions */ + if (pa->policy_name == NULL) + { + return pb->policy_name == NULL ? 0 : 1; + } + if (pb->policy_name == NULL) + { + return -1; + } + + return strcmp(pa->policy_name, pb->policy_name); +} + +/* + * check_role_for_policy - + * determines if the policy should be applied for the current role + * + * Copied from PostgreSQL's src/backend/rewrite/rowsecurity.c + */ +static bool +check_role_for_policy(ArrayType *policy_roles, Oid user_id) +{ + int i; + Oid *roles = (Oid *) ARR_DATA_PTR(policy_roles); + + /* Quick fall-thru for policies applied to all roles */ + if (roles[0] == ACL_ID_PUBLIC) + { + return true; + } + + for (i = 0; i < ARR_DIMS(policy_roles)[0]; i++) + { + if (has_privs_of_role(user_id, roles[i])) + { + return true; + } + } + + return false; +} + +/* + * add_security_quals + * + * Add security quals to enforce the specified RLS policies, restricting + * access to existing data in a table. If there are no policies controlling + * access to the table, then all access is prohibited --- i.e., an implicit + * default-deny policy is used. + * + * New security quals are added to securityQuals, and hasSubLinks is set to + * true if any of the quals added contain sublink subqueries. + * + * Copied from PostgreSQL's src/backend/rewrite/rowsecurity.c + */ +static void +add_security_quals(int rt_index, + List *permissive_policies, + List *restrictive_policies, + List **securityQuals, + bool *hasSubLinks) +{ + ListCell *item; + List *permissive_quals = NIL; + Expr *rowsec_expr; + + /* + * First collect up the permissive quals. If we do not find any + * permissive policies then no rows are visible (this is handled below). + */ + foreach(item, permissive_policies) + { + RowSecurityPolicy *policy = (RowSecurityPolicy *) lfirst(item); + + if (policy->qual != NULL) + { + permissive_quals = lappend(permissive_quals, + copyObject(policy->qual)); + *hasSubLinks |= policy->hassublinks; + } + } + + /* + * We must have permissive quals, always, or no rows are visible. + * + * If we do not, then we simply return a single 'false' qual which results + * in no rows being visible. + */ + if (permissive_quals != NIL) + { + /* + * We now know that permissive policies exist, so we can now add + * security quals based on the USING clauses from the restrictive + * policies. Since these need to be combined together using AND, we + * can just add them one at a time. + */ + foreach(item, restrictive_policies) + { + RowSecurityPolicy *policy = (RowSecurityPolicy *) lfirst(item); + Expr *qual; + + if (policy->qual != NULL) + { + qual = copyObject(policy->qual); + ChangeVarNodes((Node *) qual, 1, rt_index, 0); + + *securityQuals = list_append_unique(*securityQuals, qual); + *hasSubLinks |= policy->hassublinks; + } + } + + /* + * Then add a single security qual combining together the USING + * clauses from all the permissive policies using OR. + */ + if (list_length(permissive_quals) == 1) + { + rowsec_expr = (Expr *) linitial(permissive_quals); + } + else + { + rowsec_expr = makeBoolExpr(OR_EXPR, permissive_quals, -1); + } + + ChangeVarNodes((Node *) rowsec_expr, 1, rt_index, 0); + *securityQuals = list_append_unique(*securityQuals, rowsec_expr); + } + else + { + /* + * A permissive policy must exist for rows to be visible at all. + * Therefore, if there were no permissive policies found, return a + * single always-false clause. + */ + *securityQuals = lappend(*securityQuals, + makeConst(BOOLOID, -1, InvalidOid, + sizeof(bool), BoolGetDatum(false), + false, true)); + } +} + +/* + * setup_security_quals + * + * Security quals (USING policies) are added during the rewrite phase, but + * since AGE uses CMD_SELECT for all queries, they don't get added for + * UPDATE/DELETE operations. This function sets up security quals at + * execution time to be evaluated against each tuple before modification. + * + * Returns a list of compiled ExprState for the security quals. + */ +List * +setup_security_quals(ResultRelInfo *resultRelInfo, EState *estate, + CustomScanState *node, CmdType cmd) +{ + List *permissive_policies; + List *restrictive_policies; + List *securityQuals = NIL; + List *qualExprs = NIL; + ListCell *lc; + Relation rel; + Oid user_id; + int rt_index; + bool hasSubLinks = false; + + /* Only UPDATE and DELETE have security quals */ + if (cmd != CMD_UPDATE && cmd != CMD_DELETE) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg_internal("unexpected command type for setup_security_quals"))); + } + + rel = resultRelInfo->ri_RelationDesc; + + /* If no RLS policies exist, return empty list */ + if (rel->rd_rsdesc == NULL) + { + return NIL; + } + + /* + * Use rt_index=1 since we're evaluating policies against a single relation. + * Policy quals are stored with varno=1, and we set ecxt_scantuple to the + * tuple we want to check, so keeping varno=1 is correct. + */ + rt_index = 1; + user_id = GetUserId(); + + /* Get the policies for the specified command type */ + get_policies_for_relation(rel, cmd, user_id, + &permissive_policies, + &restrictive_policies); + + /* Build security quals from the policies */ + add_security_quals(rt_index, permissive_policies, restrictive_policies, + &securityQuals, &hasSubLinks); + + /* Compile the security qual expressions */ + foreach(lc, securityQuals) + { + Expr *qual = (Expr *) lfirst(lc); + ExprState *qualExpr; + + /* Ensure qual is a List for ExecInitQual */ + if (!IsA(qual, List)) + { + qual = (Expr *) list_make1(qual); + } + + qualExpr = ExecInitQual((List *) qual, (PlanState *) node); + qualExprs = lappend(qualExprs, qualExpr); + } + + return qualExprs; +} + +/* + * check_security_quals + * + * Evaluate security quals against a tuple. Returns true if all quals pass + * (row can be modified), false if any qual fails (row should be silently + * skipped). + * + * This matches PostgreSQL's behavior where USING expressions for UPDATE/DELETE + * silently filter rows rather than raising errors. + */ +bool +check_security_quals(List *qualExprs, TupleTableSlot *slot, + ExprContext *econtext) +{ + ListCell *lc; + TupleTableSlot *saved_scantuple; + bool result = true; + + if (qualExprs == NIL) + { + return true; + } + + /* Save and set up the scan tuple for expression evaluation */ + saved_scantuple = econtext->ecxt_scantuple; + econtext->ecxt_scantuple = slot; + + foreach(lc, qualExprs) + { + ExprState *qualExpr = (ExprState *) lfirst(lc); + + if (!ExecQual(qualExpr, econtext)) + { + result = false; + break; + } + } + + econtext->ecxt_scantuple = saved_scantuple; + return result; +} + +/* + * check_rls_for_tuple + * + * Check RLS policies for a tuple without needing full executor context. + * Used by standalone functions like startNode()/endNode() that access + * tables directly. + * + * Returns true if the tuple passes RLS checks (or if RLS is not enabled), + * false if the tuple should be filtered out. + */ +bool +check_rls_for_tuple(Relation rel, HeapTuple tuple, CmdType cmd) +{ + List *permissive_policies; + List *restrictive_policies; + List *securityQuals = NIL; + ListCell *lc; + Oid user_id; + bool hasSubLinks = false; + bool result = true; + EState *estate; + ExprContext *econtext; + TupleTableSlot *slot; + + /* If RLS is not enabled, tuple passes */ + if (check_enable_rls(RelationGetRelid(rel), InvalidOid, true) != RLS_ENABLED) + { + return true; + } + + /* If no RLS policies exist on the relation, tuple passes */ + if (rel->rd_rsdesc == NULL) + { + return true; + } + + /* Get the policies for the specified command type */ + user_id = GetUserId(); + get_policies_for_relation(rel, cmd, user_id, + &permissive_policies, + &restrictive_policies); + + /* Build security quals from the policies (use rt_index=1) */ + add_security_quals(1, permissive_policies, restrictive_policies, + &securityQuals, &hasSubLinks); + + /* If no quals, tuple passes */ + if (securityQuals == NIL) + { + return true; + } + + /* Create minimal execution environment */ + estate = CreateExecutorState(); + econtext = CreateExprContext(estate); + + /* Create tuple slot and store the tuple */ + slot = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsHeapTuple); + ExecStoreHeapTuple(tuple, slot, false); + econtext->ecxt_scantuple = slot; + + /* Compile and evaluate each qual */ + foreach(lc, securityQuals) + { + Expr *qual = (Expr *) lfirst(lc); + ExprState *qualExpr; + List *qualList; + + /* ExecPrepareQual expects a List */ + if (!IsA(qual, List)) + { + qualList = list_make1(qual); + } + else + { + qualList = (List *) qual; + } + + /* Use ExecPrepareQual for standalone expression evaluation */ + qualExpr = ExecPrepareQual(qualList, estate); + + if (!ExecQual(qualExpr, econtext)) + { + result = false; + break; + } + } + + /* Clean up */ + ExecDropSingleTupleTableSlot(slot); + FreeExprContext(econtext, true); + FreeExecutorState(estate); + + return result; +} diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 515f8c1df..7b636a3d4 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -346,6 +346,100 @@ static bool isa_special_VLE_case(cypher_path *path); static ParseNamespaceItem *find_pnsi(cypher_parsestate *cpstate, char *varname); static bool has_list_comp_or_subquery(Node *expr, void *context); +/* + * Add required permissions to the RTEPermissionInfo for a relation. + * Recursively searches through RTEs including subqueries. + */ +static bool +add_rte_permissions_recurse(List *rtable, List *rteperminfos, + Oid relid, AclMode permissions) +{ + ListCell *lc; + + /* First check the perminfos at this level */ + foreach(lc, rteperminfos) + { + RTEPermissionInfo *perminfo = lfirst(lc); + + if (perminfo->relid == relid) + { + perminfo->requiredPerms |= permissions; + return true; + } + } + + /* Then recurse into subqueries */ + foreach(lc, rtable) + { + RangeTblEntry *rte = lfirst(lc); + + if (rte->rtekind == RTE_SUBQUERY && rte->subquery != NULL) + { + if (add_rte_permissions_recurse(rte->subquery->rtable, + rte->subquery->rteperminfos, + relid, permissions)) + { + return true; + } + } + } + + return false; +} + +/* + * Add required permissions to the RTEPermissionInfo for a relation. + * Searches through p_rteperminfos and subqueries for a matching relOid + * and adds the specified permissions to requiredPerms. + */ +static void +add_rte_permissions(ParseState *pstate, Oid relid, AclMode permissions) +{ + add_rte_permissions_recurse(pstate->p_rtable, pstate->p_rteperminfos, + relid, permissions); +} + +/* + * Add required permissions to the label table for a given entity variable. + * Looks up the entity by variable name, extracts its label, and adds + * the specified permissions to the corresponding RTEPermissionInfo. + */ +static void +add_entity_permissions(cypher_parsestate *cpstate, char *var_name, + AclMode permissions) +{ + ParseState *pstate = (ParseState *)cpstate; + transform_entity *entity; + char *label = NULL; + Oid relid; + + entity = find_variable(cpstate, var_name); + if (entity == NULL) + { + return; + } + + if (entity->type == ENT_VERTEX) + { + label = entity->entity.node->label; + } + else if (entity->type == ENT_EDGE) + { + label = entity->entity.rel->label; + } + + if (label == NULL) + { + return; + } + + relid = get_label_relation(label, cpstate->graph_oid); + if (OidIsValid(relid)) + { + add_rte_permissions(pstate, relid, permissions); + } +} + /* * transform a cypher_clause */ @@ -1561,6 +1655,9 @@ static List *transform_cypher_delete_item_list(cypher_parsestate *cpstate, parser_errposition(pstate, col->location))); } + /* Add ACL_DELETE permission to the entity's label table */ + add_entity_permissions(cpstate, val->sval, ACL_DELETE); + add_volatile_wrapper_to_target_entry(query->targetList, resno); pos = makeInteger(resno); @@ -1726,6 +1823,9 @@ cypher_update_information *transform_cypher_remove_item_list( parser_errposition(pstate, set_item->location))); } + /* Add ACL_UPDATE permission to the entity's label table */ + add_entity_permissions(cpstate, variable_name, ACL_UPDATE); + add_volatile_wrapper_to_target_entry(query->targetList, item->entity_position); @@ -1903,6 +2003,9 @@ cypher_update_information *transform_cypher_set_item_list( parser_errposition(pstate, set_item->location))); } + /* Add ACL_UPDATE permission to the entity's label table */ + add_entity_permissions(cpstate, variable_name, ACL_UPDATE); + add_volatile_wrapper_to_target_entry(query->targetList, item->entity_position); diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index f2458a30b..c552727d8 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -44,7 +44,10 @@ #include "libpq/pqformat.h" #include "miscadmin.h" #include "parser/parse_coerce.h" +#include "nodes/nodes.h" +#include "utils/acl.h" #include "utils/builtins.h" +#include "executor/cypher_utils.h" #include "utils/float.h" #include "utils/lsyscache.h" #include "utils/snapmgr.h" @@ -5625,15 +5628,24 @@ static Datum get_vertex(const char *graph, const char *vertex_label, HeapTuple tuple; TupleDesc tupdesc; Datum id, properties, result; + AclResult aclresult; /* get the specific graph namespace (schema) */ Oid graph_namespace_oid = get_namespace_oid(graph, false); /* get the specific vertex label table (schema.vertex_label) */ Oid vertex_label_table_oid = get_relname_relid(vertex_label, - graph_namespace_oid); + graph_namespace_oid); /* get the active snapshot */ Snapshot snapshot = GetActiveSnapshot(); + /* check for SELECT permission on the table */ + aclresult = pg_class_aclcheck(vertex_label_table_oid, GetUserId(), + ACL_SELECT); + if (aclresult != ACLCHECK_OK) + { + aclcheck_error(aclresult, OBJECT_TABLE, vertex_label); + } + /* initialize the scan key */ ScanKeyInit(&scan_keys[0], 1, BTEqualStrategyNumber, F_OIDEQ, Int64GetDatum(graphid)); @@ -5646,11 +5658,24 @@ static Datum get_vertex(const char *graph, const char *vertex_label, /* bail if the tuple isn't valid */ if (!HeapTupleIsValid(tuple)) { + table_endscan(scan_desc); + table_close(graph_vertex_label, ShareLock); ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("graphid %lu does not exist", graphid))); } + /* Check RLS policies - error if filtered out */ + if (!check_rls_for_tuple(graph_vertex_label, tuple, CMD_SELECT)) + { + table_endscan(scan_desc); + table_close(graph_vertex_label, ShareLock); + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("access to vertex %lu denied by row-level security policy on \"%s\"", + graphid, vertex_label))); + } + /* get the tupdesc - we don't need to release this one */ tupdesc = RelationGetDescr(graph_vertex_label); /* bail if the number of columns differs */ diff --git a/src/include/executor/cypher_utils.h b/src/include/executor/cypher_utils.h index 0798f153c..fc4067455 100644 --- a/src/include/executor/cypher_utils.h +++ b/src/include/executor/cypher_utils.h @@ -21,6 +21,7 @@ #define AG_CYPHER_UTILS_H #include "access/heapam.h" +#include "nodes/execnodes.h" #include "nodes/cypher_nodes.h" #include "utils/agtype.h" @@ -127,4 +128,25 @@ HeapTuple insert_entity_tuple_cid(ResultRelInfo *resultRelInfo, TupleTableSlot *elemTupleSlot, EState *estate, CommandId cid); +/* RLS support */ +void setup_wcos(ResultRelInfo *resultRelInfo, EState *estate, + CustomScanState *node, CmdType cmd); +List *setup_security_quals(ResultRelInfo *resultRelInfo, EState *estate, + CustomScanState *node, CmdType cmd); +bool check_security_quals(List *qualExprs, TupleTableSlot *slot, + ExprContext *econtext); +bool check_rls_for_tuple(Relation rel, HeapTuple tuple, CmdType cmd); + +/* Hash table entry for caching RLS state per label */ +typedef struct RLSCacheEntry +{ + Oid relid; /* hash key */ + /* Security quals (USING policies) for UPDATE/DELETE */ + List *qualExprs; + TupleTableSlot *slot; /* slot for old tuple (RLS check) */ + /* WCOs - used only in SET */ + List *withCheckOptions; + List *withCheckOptionExprs; +} RLSCacheEntry; + #endif From b3219fd4228cc261df06070e37512c76b8c07678 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Wed, 21 Jan 2026 10:01:57 -0800 Subject: [PATCH 022/100] Master to PostgreSQL version 18 (#2315) * Updated CI, Labeler, Docker, and branch security files for PG18 (#2246) Updated the CI and Docker files for the PG18 Updated the labeler and branch security files for PG18. Some of these only apply to the master branch but are updated for consistency. modified: .asf.yaml modified: .github/labeler.yml modified: .github/workflows/go-driver.yml modified: .github/workflows/installcheck.yaml modified: .github/workflows/jdbc-driver.yaml modified: .github/workflows/nodejs-driver.yaml modified: .github/workflows/python-driver.yaml modified: docker/Dockerfile modified: docker/Dockerfile.dev modified: drivers/docker-compose.yml * PG18 port for AGE (#2251) * [PG18 port][Set1] Fix header dependencies and use TupleDescAttr macro - Include executor/executor.h for PG18 header reorganization and use TupleDescAttr() accessor macro instead of direct attrs[] access. * [PG18 port][Set2] Adapt to expandRTE signature change and pg_noreturn - Add VarReturningType parameter to expandRTE() calls using VAR_RETURNING_DEFAULT. - Replace pg_attribute_noreturn() with pg_noreturn prefix specifier. * [PG18 port][Set3] Fix double ExecOpenIndices call for PG18 compatibility - PG18 enforces stricter assertions in ExecOpenIndices, requiring ri_IndexRelationDescs to be NULL when called. - In update_entity_tuple(), indices may already be opened by the caller (create_entity_result_rel_info), causing assertion failures. - Add a check to only open indices if not already open, and track ownership with a boolean flag to ensure we only close what we opened. - Found when regression tests failed with assertions, which this change resolves. * [PG18 port][Set4] Update regression test expected output for ordering PG18's implementation changes result in different row ordering for queries without explicit ORDER BY clauses. Update expected output files to reflect the new ordering while maintaining identical result content. * [PG18 port][Set5] Address review comments - coding standard fix Note: Assisted by GitHub Copilot Agent mode. * Fix DockerHub build warning messages (#2252) PR fixes build warning messages on DockerHub and on my local build. No regression tests needed. modified: src/include/nodes/ag_nodes.h modified: src/include/optimizer/cypher_createplan.h modified: src/include/optimizer/cypher_pathnode.h modified: tools/gen_keywordlist.pl --------- Co-authored-by: Krishnakumar R (KK) <65895020+kk-src@users.noreply.github.com> --- .asf.yaml | 4 +++ .github/labeler.yml | 3 +++ .github/workflows/installcheck.yaml | 33 ++++++++++++----------- README.md | 10 +++---- docker/Dockerfile | 12 ++++----- docker/Dockerfile.dev | 4 +-- regress/expected/cypher_match.out | 30 ++++++++++----------- src/backend/catalog/ag_label.c | 1 + src/backend/executor/cypher_create.c | 1 + src/backend/executor/cypher_delete.c | 5 ++-- src/backend/executor/cypher_merge.c | 1 + src/backend/executor/cypher_set.c | 28 ++++++++++++++----- src/backend/parser/cypher_clause.c | 4 +-- src/backend/utils/adt/agtype_parser.c | 8 +++--- src/backend/utils/load/age_load.c | 2 +- src/include/nodes/ag_nodes.h | 3 +++ src/include/optimizer/cypher_createplan.h | 7 +++++ src/include/optimizer/cypher_pathnode.h | 7 +++++ tools/gen_keywordlist.pl | 1 + 19 files changed, 104 insertions(+), 60 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 8a0149a04..be18a25c1 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -50,6 +50,10 @@ github: required_pull_request_reviews: required_approving_review_count: 1 + PG18: + required_pull_request_reviews: + required_approving_review_count: 1 + PG17: required_pull_request_reviews: required_approving_review_count: 1 diff --git a/.github/labeler.yml b/.github/labeler.yml index 6baa297c5..f860c2b19 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -19,5 +19,8 @@ PG16: PG17: - base-branch: 'PG17' +PG18: +- base-branch: 'PG18' + master: - base-branch: 'master' diff --git a/.github/workflows/installcheck.yaml b/.github/workflows/installcheck.yaml index da266f0da..dcb2057df 100644 --- a/.github/workflows/installcheck.yaml +++ b/.github/workflows/installcheck.yaml @@ -3,6 +3,7 @@ name: Build / Regression on: push: branches: [ "master" ] + pull_request: branches: [ "master" ] @@ -11,53 +12,53 @@ jobs: runs-on: ubuntu-latest steps: - - name: Get latest commit id of PostgreSQL 17 + - name: Get latest commit id of PostgreSQL 18 run: | - echo "PG_COMMIT_HASH=$(git ls-remote https://git.postgresql.org/git/postgresql.git refs/heads/REL_17_STABLE | awk '{print $1}')" >> $GITHUB_ENV + echo "PG_COMMIT_HASH=$(git ls-remote https://git.postgresql.org/git/postgresql.git refs/heads/REL_18_STABLE | awk '{print $1}')" >> $GITHUB_ENV - - name: Cache PostgreSQL 17 + - name: Cache PostgreSQL 18 uses: actions/cache@v3 - id: pg17cache + id: pg18cache with: - path: ~/pg17 - key: ${{ runner.os }}-v1-pg17-${{ env.PG_COMMIT_HASH }} + path: ~/pg18 + key: ${{ runner.os }}-v1-pg18-${{ env.PG_COMMIT_HASH }} - name: Install necessary dependencies run: | sudo apt-get update sudo apt-get install -y build-essential libreadline-dev zlib1g-dev flex bison - - name: Install PostgreSQL 17 and some extensions - if: steps.pg17cache.outputs.cache-hit != 'true' + - name: Install PostgreSQL 18 and some extensions + if: steps.pg18cache.outputs.cache-hit != 'true' run: | - git clone --depth 1 --branch REL_17_STABLE https://git.postgresql.org/git/postgresql.git ~/pg17source - cd ~/pg17source - ./configure --prefix=$HOME/pg17 CFLAGS="-std=gnu99 -ggdb -O0" --enable-cassert + git clone --depth 1 --branch REL_18_STABLE https://git.postgresql.org/git/postgresql.git ~/pg18source + cd ~/pg18source + ./configure --prefix=$HOME/pg18 CFLAGS="-std=gnu99 -ggdb -O0" --enable-cassert make install -j$(nproc) > /dev/null cd contrib cd fuzzystrmatch - make PG_CONFIG=$HOME/pg17/bin/pg_config install -j$(nproc) > /dev/null + make PG_CONFIG=$HOME/pg18/bin/pg_config install -j$(nproc) > /dev/null cd ../pg_trgm - make PG_CONFIG=$HOME/pg17/bin/pg_config install -j$(nproc) > /dev/null + make PG_CONFIG=$HOME/pg18/bin/pg_config install -j$(nproc) > /dev/null - uses: actions/checkout@v3 - name: Build AGE id: build run: | - make PG_CONFIG=$HOME/pg17/bin/pg_config install -j$(nproc) + make PG_CONFIG=$HOME/pg18/bin/pg_config install -j$(nproc) - name: Pull and build pgvector id: pgvector run: | git clone https://github.com/pgvector/pgvector.git cd pgvector - make PG_CONFIG=$HOME/pg17/bin/pg_config install -j$(nproc) > /dev/null + make PG_CONFIG=$HOME/pg18/bin/pg_config install -j$(nproc) > /dev/null - name: Regression tests id: regression_tests run: | - make PG_CONFIG=$HOME/pg17/bin/pg_config installcheck EXTRA_TESTS="pgvector fuzzystrmatch pg_trgm" + make PG_CONFIG=$HOME/pg18/bin/pg_config installcheck EXTRA_TESTS="pgvector fuzzystrmatch pg_trgm" continue-on-error: true - name: Dump regression test errors diff --git a/README.md b/README.md index 9f99f6ddb..d6c4c1d39 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@   - - + +   @@ -125,7 +125,7 @@ Apache AGE is intended to be simple to install and run. It can be installed with  Install PostgreSQL -You will need to install an AGE compatible version of Postgres, for now AGE supports Postgres 11, 12, 13, 14, 15, 16 & 17. Supporting the latest versions is on AGE roadmap. +You will need to install an AGE compatible version of Postgres, for now AGE supports Postgres 11, 12, 13, 14, 15, 16, 17, & 18. Supporting the latest versions is on AGE roadmap.

 Installation via Package Manager @@ -143,7 +143,7 @@ sudo apt install postgresql  Installation From Source Code

-You can download the Postgres source code and install your own instance of Postgres. You can read instructions on how to install from source code for different versions on the official Postgres Website. +You can download the Postgres source code and install your own instance of Postgres. You can read instructions on how to install from source code for different versions on the official Postgres Website. @@ -152,7 +152,7 @@ You can download the Postgres Clone the github repository or download the download an official release. -Run the pg_config utility and check the version of PostgreSQL. Currently, only PostgreSQL versions 11, 12, 13, 14, 15, 16 & 17 are supported. If you have any other version of Postgres, you will need to install PostgreSQL version 11, 12, 13, 14, 15, 16 & 17. +Run the pg_config utility and check the version of PostgreSQL. Currently, only PostgreSQL versions 11, 12, 13, 14, 15, 16, 17, & 18 are supported. If you have any other version of Postgres, you will need to install PostgreSQL version 11, 12, 13, 14, 15, 16, 17, & 18.
```bash diff --git a/docker/Dockerfile b/docker/Dockerfile index 0436dc8f9..3eb17c798 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,14 +17,14 @@ # # Build stage: Install necessary development tools for compilation and installation -FROM postgres:17 AS build +FROM postgres:18 AS build RUN apt-get update \ && apt-get install -y --no-install-recommends --no-install-suggests \ bison \ build-essential \ flex \ - postgresql-server-dev-17 + postgresql-server-dev-18 COPY . /age @@ -34,7 +34,7 @@ RUN make && make install # Final stage: Create a final image by copying the files created in the build stage -FROM postgres:17 +FROM postgres:18 RUN apt-get update \ && apt-get install -y --no-install-recommends --no-install-suggests \ @@ -48,9 +48,9 @@ ENV LANG=en_US.UTF-8 ENV LC_COLLATE=en_US.UTF-8 ENV LC_CTYPE=en_US.UTF-8 -COPY --from=build /usr/lib/postgresql/17/lib/age.so /usr/lib/postgresql/17/lib/ -COPY --from=build /usr/share/postgresql/17/extension/age--1.6.0.sql /usr/share/postgresql/17/extension/ -COPY --from=build /usr/share/postgresql/17/extension/age.control /usr/share/postgresql/17/extension/ +COPY --from=build /usr/lib/postgresql/18/lib/age.so /usr/lib/postgresql/18/lib/ +COPY --from=build /usr/share/postgresql/18/extension/age--1.6.0.sql /usr/share/postgresql/18/extension/ +COPY --from=build /usr/share/postgresql/18/extension/age.control /usr/share/postgresql/18/extension/ COPY docker/docker-entrypoint-initdb.d/00-create-extension-age.sql /docker-entrypoint-initdb.d/00-create-extension-age.sql CMD ["postgres", "-c", "shared_preload_libraries=age"] diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 48b2db3ed..e02c21fc4 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -17,14 +17,14 @@ # -FROM postgres:17 +FROM postgres:18 RUN apt-get update RUN apt-get install --assume-yes --no-install-recommends --no-install-suggests \ bison \ build-essential \ flex \ - postgresql-server-dev-17 \ + postgresql-server-dev-18 \ locales ENV LANG=en_US.UTF-8 diff --git a/regress/expected/cypher_match.out b/regress/expected/cypher_match.out index a0e284beb..ea425e463 100644 --- a/regress/expected/cypher_match.out +++ b/regress/expected/cypher_match.out @@ -348,10 +348,10 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (i agtype); i ---------------------------------------------------------------------------------- - {"id": 1688849860263939, "label": "v2", "properties": {"id": "end"}}::vertex - {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex + {"id": 1688849860263939, "label": "v2", "properties": {"id": "end"}}::vertex {"id": 1688849860263937, "label": "v2", "properties": {"id": "initial"}}::vertex + {"id": 1688849860263938, "label": "v2", "properties": {"id": "middle"}}::vertex (4 rows) SELECT * FROM cypher('cypher_match', $$ @@ -537,18 +537,18 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (i agtype, b agtype, c agtype); i | b | c ---+-----------+----------- - | "end" | "middle" - 0 | "end" | "middle" - 1 | "end" | "middle" | "middle" | "end" 0 | "middle" | "end" 1 | "middle" | "end" - | "middle" | "initial" - 0 | "middle" | "initial" - 1 | "middle" | "initial" + | "end" | "middle" + 0 | "end" | "middle" + 1 | "end" | "middle" | "initial" | "middle" 0 | "initial" | "middle" 1 | "initial" | "middle" + | "middle" | "initial" + 0 | "middle" | "initial" + 1 | "middle" | "initial" (12 rows) SELECT * FROM cypher('cypher_match', $$ @@ -558,18 +558,18 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (i agtype, c agtype); i | c ---+----------- - | "middle" - 0 | "middle" - 1 | "middle" | "end" 0 | "end" 1 | "end" - | "initial" - 0 | "initial" - 1 | "initial" | "middle" 0 | "middle" 1 | "middle" + | "middle" + 0 | "middle" + 1 | "middle" + | "initial" + 0 | "initial" + 1 | "initial" (12 rows) -- @@ -2421,8 +2421,8 @@ SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relation SELECT * FROM cypher('cypher_match', $$ MATCH p=(a {name:a.name})-[u {relationship: u.relationship}]->(b {age:b.age}) RETURN p $$) as (a agtype); a ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 281474976710659, "label": "", "properties": {"age": 3, "name": "orphan"}}::vertex, {"id": 4785074604081154, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710659, "properties": {"years": 4, "relationship": "enemies"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path [{"id": 281474976710661, "label": "", "properties": {"age": 4, "name": "T"}}::vertex, {"id": 4785074604081153, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710661, "properties": {"years": 3, "relationship": "friends"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path + [{"id": 281474976710659, "label": "", "properties": {"age": 3, "name": "orphan"}}::vertex, {"id": 4785074604081154, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710659, "properties": {"years": 4, "relationship": "enemies"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path (2 rows) SELECT * FROM cypher('cypher_match', $$ CREATE () WITH * MATCH (x{n0:x.n1}) RETURN 0 $$) as (a agtype); diff --git a/src/backend/catalog/ag_label.c b/src/backend/catalog/ag_label.c index b6dcf77a3..54c31ef36 100644 --- a/src/backend/catalog/ag_label.c +++ b/src/backend/catalog/ag_label.c @@ -21,6 +21,7 @@ #include "access/genam.h" #include "catalog/indexing.h" +#include "executor/executor.h" #include "nodes/makefuncs.h" #include "utils/builtins.h" #include "utils/lsyscache.h" diff --git a/src/backend/executor/cypher_create.c b/src/backend/executor/cypher_create.c index a90c2a196..495eb3a08 100644 --- a/src/backend/executor/cypher_create.c +++ b/src/backend/executor/cypher_create.c @@ -19,6 +19,7 @@ #include "postgres.h" +#include "executor/executor.h" #include "utils/rls.h" #include "catalog/ag_label.h" diff --git a/src/backend/executor/cypher_delete.c b/src/backend/executor/cypher_delete.c index 4766c6e7a..0b486ad5e 100644 --- a/src/backend/executor/cypher_delete.c +++ b/src/backend/executor/cypher_delete.c @@ -19,9 +19,10 @@ #include "postgres.h" +#include "executor/executor.h" +#include "storage/bufmgr.h" #include "common/hashfn.h" #include "miscadmin.h" -#include "storage/bufmgr.h" #include "utils/acl.h" #include "utils/rls.h" @@ -260,7 +261,7 @@ static agtype_value *extract_entity(CustomScanState *node, tupleDescriptor = scanTupleSlot->tts_tupleDescriptor; /* type checking, make sure the entity is an agtype vertex or edge */ - if (tupleDescriptor->attrs[entity_position -1].atttypid != AGTYPEOID) + if (TupleDescAttr(tupleDescriptor, entity_position -1)->atttypid != AGTYPEOID) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("DELETE clause can only delete agtype"))); diff --git a/src/backend/executor/cypher_merge.c b/src/backend/executor/cypher_merge.c index e0d6c78e8..a1bb4686c 100644 --- a/src/backend/executor/cypher_merge.c +++ b/src/backend/executor/cypher_merge.c @@ -19,6 +19,7 @@ #include "postgres.h" +#include "executor/executor.h" #include "utils/datum.h" #include "utils/rls.h" diff --git a/src/backend/executor/cypher_set.c b/src/backend/executor/cypher_set.c index 40cf2b232..a1063af32 100644 --- a/src/backend/executor/cypher_set.c +++ b/src/backend/executor/cypher_set.c @@ -105,6 +105,7 @@ static HeapTuple update_entity_tuple(ResultRelInfo *resultRelInfo, TM_Result result; CommandId cid = GetCurrentCommandId(true); ResultRelInfo **saved_resultRels = estate->es_result_relations; + bool close_indices = false; estate->es_result_relations = &resultRelInfo; @@ -116,7 +117,16 @@ static HeapTuple update_entity_tuple(ResultRelInfo *resultRelInfo, if (lock_result == TM_Ok) { - ExecOpenIndices(resultRelInfo, false); + /* + * Open indices if not already open. The resultRelInfo may already + * have indices opened by the caller (e.g., create_entity_result_rel_info), + * so only open if needed and track that we did so for cleanup. + */ + if (resultRelInfo->ri_IndexRelationDescs == NULL) + { + ExecOpenIndices(resultRelInfo, false); + close_indices = true; + } ExecStoreVirtualTuple(elemTupleSlot); tuple = ExecFetchSlotHeapTuple(elemTupleSlot, true, NULL); tuple->t_self = old_tuple->t_self; @@ -151,7 +161,10 @@ static HeapTuple update_entity_tuple(ResultRelInfo *resultRelInfo, errmsg("tuple to be updated was already modified"))); } - ExecCloseIndices(resultRelInfo); + if (close_indices) + { + ExecCloseIndices(resultRelInfo); + } estate->es_result_relations = saved_resultRels; return tuple; @@ -170,7 +183,10 @@ static HeapTuple update_entity_tuple(ResultRelInfo *resultRelInfo, (update_indexes == TU_Summarizing)); } - ExecCloseIndices(resultRelInfo); + if (close_indices) + { + ExecCloseIndices(resultRelInfo); + } } else if (lock_result == TM_SelfModified) { @@ -320,7 +336,7 @@ static void update_all_paths(CustomScanState *node, graphid id, agtype_value *original_entity_value; /* skip nulls */ - if (scanTupleSlot->tts_tupleDescriptor->attrs[i].atttypid != AGTYPEOID) + if (TupleDescAttr(scanTupleSlot->tts_tupleDescriptor, i)->atttypid != AGTYPEOID) { continue; } @@ -435,7 +451,7 @@ static void process_update_list(CustomScanState *node) continue; } - if (scanTupleSlot->tts_tupleDescriptor->attrs[update_item->entity_position -1].atttypid != AGTYPEOID) + if (TupleDescAttr(scanTupleSlot->tts_tupleDescriptor, update_item->entity_position -1)->atttypid != AGTYPEOID) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -669,7 +685,7 @@ static void process_update_list(CustomScanState *node) } estate->es_snapshot->curcid = cid; - /* close relation */ + /* close relation */ ExecCloseIndices(resultRelInfo); table_close(resultRelInfo->ri_RelationDesc, RowExclusiveLock); diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 7b636a3d4..991e3f785 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -2729,9 +2729,9 @@ static void get_res_cols(ParseState *pstate, ParseNamespaceItem *l_pnsi, List *colnames = NIL; List *colvars = NIL; - expandRTE(l_pnsi->p_rte, l_pnsi->p_rtindex, 0, -1, false, + expandRTE(l_pnsi->p_rte, l_pnsi->p_rtindex, 0, VAR_RETURNING_DEFAULT, -1, false, &l_colnames, &l_colvars); - expandRTE(r_pnsi->p_rte, r_pnsi->p_rtindex, 0, -1, false, + expandRTE(r_pnsi->p_rte, r_pnsi->p_rtindex, 0, VAR_RETURNING_DEFAULT, -1, false, &r_colnames, &r_colvars); /* add in all colnames and colvars from the l_rte. */ diff --git a/src/backend/utils/adt/agtype_parser.c b/src/backend/utils/adt/agtype_parser.c index c485cb925..40fc8d8c5 100644 --- a/src/backend/utils/adt/agtype_parser.c +++ b/src/backend/utils/adt/agtype_parser.c @@ -74,11 +74,9 @@ static void parse_object(agtype_lex_context *lex, agtype_sem_action *sem); static void parse_array_element(agtype_lex_context *lex, agtype_sem_action *sem); static void parse_array(agtype_lex_context *lex, agtype_sem_action *sem); -static void report_parse_error(agtype_parse_context ctx, - agtype_lex_context *lex) - pg_attribute_noreturn(); -static void report_invalid_token(agtype_lex_context *lex) - pg_attribute_noreturn(); +static pg_noreturn void report_parse_error(agtype_parse_context ctx, + agtype_lex_context *lex); +static pg_noreturn void report_invalid_token(agtype_lex_context *lex); static int report_agtype_context(agtype_lex_context *lex); static char *extract_mb_char(char *s); diff --git a/src/backend/utils/load/age_load.c b/src/backend/utils/load/age_load.c index f9634668c..e4f10d7e4 100644 --- a/src/backend/utils/load/age_load.c +++ b/src/backend/utils/load/age_load.c @@ -809,7 +809,7 @@ void init_batch_insert(batch_insert_state **batch_state, perminfos = list_make1(perminfo); /* Initialize range table in executor state */ - ExecInitRangeTable(estate, range_table, perminfos); + ExecInitRangeTable(estate, range_table, perminfos, NULL); /* Initialize resultRelInfo - this opens the relation */ resultRelInfo = makeNode(ResultRelInfo); diff --git a/src/include/nodes/ag_nodes.h b/src/include/nodes/ag_nodes.h index f0cc22043..121832c01 100644 --- a/src/include/nodes/ag_nodes.h +++ b/src/include/nodes/ag_nodes.h @@ -78,6 +78,9 @@ typedef enum ag_node_tag cypher_merge_information_t } ag_node_tag; +extern const char *node_names[]; +extern const ExtensibleNodeMethods node_methods[]; + void register_ag_nodes(void); ExtensibleNode *_new_ag_node(Size size, ag_node_tag tag); diff --git a/src/include/optimizer/cypher_createplan.h b/src/include/optimizer/cypher_createplan.h index 2d5d2e698..ab01f2b58 100644 --- a/src/include/optimizer/cypher_createplan.h +++ b/src/include/optimizer/cypher_createplan.h @@ -20,6 +20,8 @@ #ifndef AG_CYPHER_CREATEPLAN_H #define AG_CYPHER_CREATEPLAN_H +#include "nodes/extensible.h" + Plan *plan_cypher_create_path(PlannerInfo *root, RelOptInfo *rel, CustomPath *best_path, List *tlist, List *clauses, List *custom_plans); @@ -36,4 +38,9 @@ Plan *plan_cypher_merge_path(PlannerInfo *root, RelOptInfo *rel, CustomPath *best_path, List *tlist, List *clauses, List *custom_plans); +extern const CustomScanMethods cypher_create_plan_methods; +extern const CustomScanMethods cypher_set_plan_methods; +extern const CustomScanMethods cypher_delete_plan_methods; +extern const CustomScanMethods cypher_merge_plan_methods; + #endif diff --git a/src/include/optimizer/cypher_pathnode.h b/src/include/optimizer/cypher_pathnode.h index 75c2b07de..676832029 100644 --- a/src/include/optimizer/cypher_pathnode.h +++ b/src/include/optimizer/cypher_pathnode.h @@ -20,6 +20,8 @@ #ifndef AG_CYPHER_PATHNODE_H #define AG_CYPHER_PATHNODE_H +#include "nodes/extensible.h" + #define CREATE_PATH_NAME "Cypher Create" #define SET_PATH_NAME "Cypher Set" #define DELETE_PATH_NAME "Cypher Delete" @@ -34,4 +36,9 @@ CustomPath *create_cypher_delete_path(PlannerInfo *root, RelOptInfo *rel, CustomPath *create_cypher_merge_path(PlannerInfo *root, RelOptInfo *rel, List *custom_private); +extern const CustomPathMethods cypher_create_path_methods; +extern const CustomPathMethods cypher_set_path_methods; +extern const CustomPathMethods cypher_delete_path_methods; +extern const CustomPathMethods cypher_merge_path_methods; + #endif diff --git a/tools/gen_keywordlist.pl b/tools/gen_keywordlist.pl index 499300433..58e66db8e 100755 --- a/tools/gen_keywordlist.pl +++ b/tools/gen_keywordlist.pl @@ -112,6 +112,7 @@ #define %s_H #include "common/kwlookup.h" +#include "parser/cypher_keywords.h" EOM From 6287af85f291e10d048c37ffd22b902950779af7 Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Thu, 22 Jan 2026 21:50:23 +0500 Subject: [PATCH 023/100] Fix upgrade script for 1.6.0 to 1.7.0 (#2320) - Added index creation for existing labels Assisted-by AI --- age--1.6.0--y.y.y.sql | 97 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/age--1.6.0--y.y.y.sql b/age--1.6.0--y.y.y.sql index 2d693a433..c5a31b373 100644 --- a/age--1.6.0--y.y.y.sql +++ b/age--1.6.0--y.y.y.sql @@ -51,3 +51,100 @@ CREATE FUNCTION ag_catalog._ag_enforce_edge_uniqueness4(graphid, graphid, graphi STABLE PARALLEL SAFE as 'MODULE_PATHNAME'; + +-- Create indexes on id columns for existing labels +-- Vertex labels get PRIMARY KEY on id, Edge labels get indexes on start_id/end_id +DO $$ +DECLARE + label_rec record; + schema_name text; + table_name text; + idx_exists boolean; + pk_exists boolean; + idx_name text; +BEGIN + FOR label_rec IN + SELECT l.relation, l.kind + FROM ag_catalog.ag_label l + LOOP + SELECT n.nspname, c.relname INTO schema_name, table_name + FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.oid = label_rec.relation; + + IF label_rec.kind = 'e' THEN + -- Edge: check/create index on start_id + SELECT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + JOIN pg_am am ON am.oid = c.relam + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = i.indkey[0] + WHERE i.indrelid = label_rec.relation + AND a.attname = 'start_id' + AND i.indpred IS NULL -- not a partial index + AND i.indexprs IS NULL -- not an expression index + AND am.amname = 'btree' -- btree access method + ) INTO idx_exists; + + IF NOT idx_exists THEN + EXECUTE format('CREATE INDEX %I ON %I.%I USING btree (start_id)', + table_name || '_start_id_idx', schema_name, table_name); + END IF; + + -- Edge: check/create index on end_id + SELECT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + JOIN pg_am am ON am.oid = c.relam + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = i.indkey[0] + WHERE i.indrelid = label_rec.relation + AND a.attname = 'end_id' + AND i.indpred IS NULL -- not a partial index + AND i.indexprs IS NULL -- not an expression index + AND am.amname = 'btree' -- btree access method + ) INTO idx_exists; + + IF NOT idx_exists THEN + EXECUTE format('CREATE INDEX %I ON %I.%I USING btree (end_id)', + table_name || '_end_id_idx', schema_name, table_name); + END IF; + ELSE + -- Vertex: check/create PRIMARY KEY on id + SELECT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = label_rec.relation AND contype = 'p' + ) INTO pk_exists; + + IF NOT pk_exists THEN + -- Check if a usable unique index on id already exists + SELECT c.relname INTO idx_name + FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + JOIN pg_am am ON am.oid = c.relam + WHERE i.indrelid = label_rec.relation + AND i.indisunique = true + AND i.indpred IS NULL -- not a partial index + AND i.indexprs IS NULL -- not an expression index + AND am.amname = 'btree' -- btree access method + AND i.indnkeyatts = 1 -- single column index + AND EXISTS ( + SELECT 1 FROM pg_attribute a + WHERE a.attrelid = i.indrelid + AND a.attnum = i.indkey[0] + AND a.attname = 'id' + ) + LIMIT 1; + + IF idx_name IS NOT NULL THEN + -- Reuse existing unique index for primary key + EXECUTE format('ALTER TABLE %I.%I ADD CONSTRAINT %I PRIMARY KEY USING INDEX %I', + schema_name, table_name, table_name || '_pkey', idx_name); + ELSE + -- Create new primary key + EXECUTE format('ALTER TABLE %I.%I ADD PRIMARY KEY (id)', + schema_name, table_name); + END IF; + END IF; + END IF; + END LOOP; +END $$; From 858747c7e2a414651b4f8e76558fdce1bbdc0af8 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Thu, 22 Jan 2026 09:58:14 -0800 Subject: [PATCH 024/100] Advance master branch to Apache AGE version 1.7.0 (#2316) Updated the following files to advance the Apache AGE version to 1.7.0 modified: Makefile modified: README.md modified: RELEASE renamed: age--1.6.0--y.y.y.sql -> age--1.6.0--1.7.0.sql new file: age--1.7.0--y.y.y.sql modified: age.control modified: docker/Dockerfile deleted: age--1.5.0--1.6.0.sql --- Makefile | 2 +- README.md | 6 +- RELEASE | 125 +++------ age--1.5.0--1.6.0.sql | 256 ------------------ ...-1.6.0--y.y.y.sql => age--1.6.0--1.7.0.sql | 10 +- age--1.7.0--y.y.y.sql | 32 +++ age.control | 2 +- docker/Dockerfile | 2 +- 8 files changed, 78 insertions(+), 357 deletions(-) delete mode 100644 age--1.5.0--1.6.0.sql rename age--1.6.0--y.y.y.sql => age--1.6.0--1.7.0.sql (92%) create mode 100644 age--1.7.0--y.y.y.sql diff --git a/Makefile b/Makefile index 2d2912571..394951ca0 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ MODULE_big = age -age_sql = age--1.6.0.sql +age_sql = age--1.7.0.sql OBJS = src/backend/age.o \ src/backend/catalog/ag_catalog.o \ diff --git a/README.md b/README.md index d6c4c1d39..dad7f1032 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@   - +   @@ -125,7 +125,7 @@ Apache AGE is intended to be simple to install and run. It can be installed with  Install PostgreSQL -You will need to install an AGE compatible version of Postgres, for now AGE supports Postgres 11, 12, 13, 14, 15, 16, 17, & 18. Supporting the latest versions is on AGE roadmap. +You will need to install an AGE compatible version of Postgres, for now AGE supports Postgres 11, 12, 13, 14, 15, 16, 17 & 18. Supporting the latest versions is on AGE roadmap.

 Installation via Package Manager @@ -152,7 +152,7 @@ You can download the Postgres Clone the github repository or download the download an official release. -Run the pg_config utility and check the version of PostgreSQL. Currently, only PostgreSQL versions 11, 12, 13, 14, 15, 16, 17, & 18 are supported. If you have any other version of Postgres, you will need to install PostgreSQL version 11, 12, 13, 14, 15, 16, 17, & 18. +Run the pg_config utility and check the version of PostgreSQL. Currently, only PostgreSQL versions 11, 12, 13, 14, 15, 16, 17 & 18 are supported. If you have any other version of Postgres, you will need to install PostgreSQL version 11, 12, 13, 14, 15, 16, 17 & 18.
```bash diff --git a/RELEASE b/RELEASE index 4525eab2f..4a5eb45f3 100644 --- a/RELEASE +++ b/RELEASE @@ -15,91 +15,44 @@ # specific language governing permissions and limitations # under the License. -Release Notes for Apache AGE release 1.6.0 for master branch (currently PG17) +Release Notes for Apache AGE release 1.7.0 for master branch (currently PG18) -Apache AGE 1.6.0 - Release Notes +# +# WARNING! +# +# Please note the upgrade script (age--1.6.0--1.7.0.sql) may take a while to +# complete for large graphs, due to creation of indexes for existing labels. +# +# WARNING! +# + +Apache AGE 1.7.0 - Release Notes + +Master to PostgreSQL version 18 (#2315) +Add RLS support and fix permission checks (#2309) +Replace libcsv with pg COPY for csv loading (#2310) +Fix Issue 1884: Ambiguous column reference (#2306) +Upgrade Jest to v29 for node: protocol compatibility (#2307) +Optimize vertex/edge field access with direct array indexing (#2302) +feat: Add 32-bit platform support for graphid type (#2286) +Fix and improve index.sql addendum (#2301) +Fix and improve index.sql regression test coverage (#2300) +Fix Issue 2289: handle empty list in IN expression (#2294) +Revise README for Python driver updates (#2298) +Makefile: fix race condition on cypher_gram_def.h (#2273) +Restrict age_load commands (#2274) +Migrate python driver configuration to pyproject.toml (#2272) +Convert string to raw string to remove invalid escape sequence warning (#2267) +Update grammar file for maintainability (#2270) +Fix ORDER BY alias resolution with AS in Cypher queries (#2269) +Fix possible memory and file descriptors leaks (#2258) +Adjust 'could not find rte for' ERROR message (#2266) +Fix Issue 2256: segmentation fault when calling coalesce function (#2259) +Add index on id columns (#2117) +Fix issue 2245 - Creating more than 41 vlabels causes crash in drop_graph (#2248) +Fix issue 2243 - Regression in string concatenation (#2244) +Add fast functions for checking edge uniqueness (#2227) +Bump gopkg.in/yaml.v3 from 3.0.0 to 3.0.1 in /drivers/golang (#2212) +Update branch security (#2219) +Fix issue with CALL/YIELD for user defined and qualified functions. (#2217) - Fix issue 2205: left doesn't catch overflow (#2207) - Fix issue 2201: unexpected empty string behavior (#2203) - Add support for operators in cypher query (#2172) - Reimplement list comprehension (#2169) - Update labeler.yml (#2186) - Fix issue with the CI build and labeler (#2183) - Fix CSV import for edge with one property (#2175) - Remove stale bot and update .asf.yaml settings (#2171) - Prevent object access hook from accesing not installed namespace (#2161) - Fix CI build errors caused by missing dependencies (#2163) - Add support for external extensions (#2088) - Fix issue 2093: pfree() called with a NULL pointer (#2095) - Fix issue 1955 - List comprehension in WHERE clause (#2094) - Add support for fuzzystrmatch and other external extensions (#2083) - Fix memory leaks in functions part 1 (#2066) - Issue 1996 - Add agtype to json typecast (#2075) - Fix issue 2046: Memory leak during btree(agtype) (#2060) - Refactor Dockerfile to use multi-stage builds (#2004) - Revamp age csv loader (#2044) - Fix issue 2020: Memory leak (#2028) - Fix Issue 1907: SET on MERGE not storing edge properties (#2019) - Add EmitWarningsOnPlaceholders("age") (#1997) - Fix Issue 1988: How to update a property which is a keyword (#2005) - Fix obsolete docker-compose command in CIs (#2007) - Fix issue 1986: Failure creating label name close to MAX_LABEL_NAME_LEN (#1989) - Fix issue 1953 - PG Boolean used as AGTYPE object (#1959) - Add graph_exists function (#1958) - Fix issue 1956 - null key name passed. (#1957) - docs: add link to .NET open-source driver (#1938) - Add the `load_from_plugins` parameter in the Python driver to load AGE from the $libdir/plugins directory (#1935) - Fix issue 1910: Server crashes when using exists(vle path) (#1924) - Converted single line comments to multiline (#1908) - Add function is_valid_label_name (#1911) (#1912) - Fixes small typos in the python driver's README.md file (#1909) - Fix agtype_build_map to allow more than 50 pairs (#1901) - Agtype hash cmp (#1893) - Order some regression tests for stability on big-endian (#1892) - Update github stale action (#1891) - Make CALL YIELD grammar more precise (#1852) - Fix issue 1878 - Regression test failure with delete global graphs (#1881) - Corrected typos and grammatical errors in apache-age-basic.ipynb (#879) - Add workflow for stale issues and PRs (#1872) - Implement Returnless Unions in Subqueries (#1803) - [CI] Update docker image tags (#1865) - Add branch protection rules in .asf.yaml (#1854) - [CI] Update labeler github action (#1851) - Fix error using list comprehension with WITH * (#1838) - python driver psycopg3 (#1793) - Minor VLE and agtype_eq/ne performance updates (#1808) - Fix issue 1767: CREATE TABLE AS SELECT * FROM cypher, errors (#1799) - Implement Constraints on Subqueries (#1751) - Fix connection string in Python Driver (#1757) - Added integer conversion in toBoolean functions (#1199) - Update README.md (#1756) - Add the command graph_stats and improve VLE messaging for load (#1750) - Add helpful messages to the VLE subsystem (#1742) - Update README.md (#1728) - Added Networkx Support in python driver (#1716) - Remove duplicate check (#1740) - Fix Issue 1709 - MERGE creates incomplete vertices after the first one (#1721) - Fix shift/reduce conflict in grammar (#1719) - Fix Issue 1691 - MERGE incorrectly creates multiple vertices (#1718) - Implement map projection (#1710) - Add hooks for multi-arch builds on dockerhub (#1683) - Sample code for AGE-JDBC driver (#390) - Allow agtype_build_map to use AGTYPE keys (#1666) - Implement EXISTS Subquery (#1562) - Fix Issue 1634: Setting all properties with map object causes error (#1637) - Fix Issue 1630: MERGE using array not working in some cases (#1636) - Implement list comprehension (#1610) - Update Go installation and add in parser files (#1582) - Add optional parameter '=' in property constraints (#1516) - Fix unsorted output of some queries in the cypher_match test (#1507) - Update age_load to make property value conversion optional (#1525) - Update the Go driver documentation, Linux installer, and CI (#1527) - Fix json serialization error in Python Driver (#1228) - Add template for upgrading between versions of Apache AGE (#1506) - Update age_load to load scalar property values with appropriate type (#1519) - Fix apache#1513 - Invalid variable reuse in CREATE and MERGE clause (#1515) - Clean up #included files in src/include directories (#1518) - Bump gopkg.in/yaml.v3 in /drivers/golang (#1202) - Clean up #included files in catalog & commands directories (#1514) - Clean up #included files in nodes, executor, & optimizer directories (#1509) - Correct cleanup of age--x.x.x.sql files (#1505) diff --git a/age--1.5.0--1.6.0.sql b/age--1.5.0--1.6.0.sql deleted file mode 100644 index ede715c11..000000000 --- a/age--1.5.0--1.6.0.sql +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - ---* This is a TEMPLATE for upgrading from the previous version of Apache AGE ---* Please adjust the below ALTER EXTENSION to reflect the -- correct version it ---* is upgrading to. - --- This will only work within a major version of PostgreSQL, not across --- major versions. - ---* Please add all additions, deletions, and modifications to the end of this ---* file. We need to keep the order of these changes. ---* REMOVE ALL LINES ABOVE, and this one, that start with --* - --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "ALTER EXTENSION age UPDATE TO '1.6.0'" to load this file. \quit - -CREATE FUNCTION ag_catalog.agtype_contains_top_level(agtype, agtype) - RETURNS boolean - LANGUAGE c - IMMUTABLE -RETURNS NULL ON NULL INPUT -PARALLEL SAFE -AS 'MODULE_PATHNAME'; - -CREATE OPERATOR @>> ( - LEFTARG = agtype, - RIGHTARG = agtype, - FUNCTION = ag_catalog.agtype_contains_top_level, - COMMUTATOR = '<<@', - RESTRICT = matchingsel, - JOIN = matchingjoinsel -); - -CREATE FUNCTION ag_catalog.agtype_contained_by_top_level(agtype, agtype) - RETURNS boolean - LANGUAGE c - IMMUTABLE -RETURNS NULL ON NULL INPUT -PARALLEL SAFE -AS 'MODULE_PATHNAME'; - -CREATE OPERATOR <<@ ( - LEFTARG = agtype, - RIGHTARG = agtype, - FUNCTION = ag_catalog.agtype_contained_by_top_level, - COMMUTATOR = '@>>', - RESTRICT = matchingsel, - JOIN = matchingjoinsel -); - -/* - * We have to drop and recreate the operators, because - * commutator is not modifiable using ALTER OPERATOR. - */ -ALTER EXTENSION age - DROP OPERATOR ? (agtype, agtype); -ALTER EXTENSION age - DROP OPERATOR ? (agtype, text); -ALTER EXTENSION age - DROP OPERATOR ?| (agtype, agtype); -ALTER EXTENSION age - DROP OPERATOR ?| (agtype, text[]); -ALTER EXTENSION age - DROP OPERATOR ?& (agtype, agtype); -ALTER EXTENSION age - DROP OPERATOR ?& (agtype, text[]); -ALTER EXTENSION age - DROP OPERATOR @> (agtype, agtype); -ALTER EXTENSION age - DROP OPERATOR <@ (agtype, agtype); - -DROP OPERATOR ? (agtype, agtype), ? (agtype, text), - ?| (agtype, agtype), ?| (agtype, text[]), - ?& (agtype, agtype), ?& (agtype, text[]), - @> (agtype, agtype), <@ (agtype, agtype); - -CREATE OPERATOR ? ( - LEFTARG = agtype, - RIGHTARG = agtype, - FUNCTION = ag_catalog.agtype_exists_agtype, - RESTRICT = matchingsel, - JOIN = matchingjoinsel -); - -CREATE OPERATOR ? ( - LEFTARG = agtype, - RIGHTARG = text, - FUNCTION = ag_catalog.agtype_exists, - RESTRICT = matchingsel, - JOIN = matchingjoinsel -); - -CREATE OPERATOR ?| ( - LEFTARG = agtype, - RIGHTARG = agtype, - FUNCTION = ag_catalog.agtype_exists_any_agtype, - RESTRICT = matchingsel, - JOIN = matchingjoinsel -); - -CREATE OPERATOR ?| ( - LEFTARG = agtype, - RIGHTARG = text[], - FUNCTION = ag_catalog.agtype_exists_any, - RESTRICT = matchingsel, - JOIN = matchingjoinsel -); - -CREATE OPERATOR ?& ( - LEFTARG = agtype, - RIGHTARG = agtype, - FUNCTION = ag_catalog.agtype_exists_all_agtype, - RESTRICT = matchingsel, - JOIN = matchingjoinsel -); - -CREATE OPERATOR ?& ( - LEFTARG = agtype, - RIGHTARG = text[], - FUNCTION = ag_catalog.agtype_exists_all, - RESTRICT = matchingsel, - JOIN = matchingjoinsel -); - -CREATE OPERATOR @> ( - LEFTARG = agtype, - RIGHTARG = agtype, - FUNCTION = ag_catalog.agtype_contains, - COMMUTATOR = '<@', - RESTRICT = matchingsel, - JOIN = matchingjoinsel -); - -CREATE OPERATOR <@ ( - LEFTARG = agtype, - RIGHTARG = agtype, - FUNCTION = ag_catalog.agtype_contained_by, - COMMUTATOR = '@>', - RESTRICT = matchingsel, - JOIN = matchingjoinsel -); - -/* - * Since there is no option to add or drop operator from class, - * we have to drop and recreate the whole operator class. - * Reference: https://www.postgresql.org/docs/current/sql-alteropclass.html - */ - -ALTER EXTENSION age - DROP OPERATOR CLASS ag_catalog.gin_agtype_ops USING gin; - -DROP OPERATOR CLASS ag_catalog.gin_agtype_ops USING gin; -DROP OPERATOR FAMILY ag_catalog.gin_agtype_ops USING gin; - -CREATE OPERATOR CLASS ag_catalog.gin_agtype_ops -DEFAULT FOR TYPE agtype USING gin AS - OPERATOR 7 @>(agtype, agtype), - OPERATOR 8 <@(agtype, agtype), - OPERATOR 9 ?(agtype, agtype), - OPERATOR 10 ?|(agtype, agtype), - OPERATOR 11 ?&(agtype, agtype), - OPERATOR 12 @>>(agtype, agtype), - OPERATOR 13 <<@(agtype, agtype), - FUNCTION 1 ag_catalog.gin_compare_agtype(text,text), - FUNCTION 2 ag_catalog.gin_extract_agtype(agtype, internal), - FUNCTION 3 ag_catalog.gin_extract_agtype_query(agtype, internal, int2, - internal, internal), - FUNCTION 4 ag_catalog.gin_consistent_agtype(internal, int2, agtype, int4, - internal, internal), - FUNCTION 6 ag_catalog.gin_triconsistent_agtype(internal, int2, agtype, int4, - internal, internal, internal), -STORAGE text; - --- this function went from variadic "any" to just "any" type -CREATE OR REPLACE FUNCTION ag_catalog.age_tostring("any") - RETURNS agtype - LANGUAGE c - IMMUTABLE -RETURNS NULL ON NULL INPUT -PARALLEL SAFE -AS 'MODULE_PATHNAME'; - --- this is a new function for graph statistics -CREATE FUNCTION ag_catalog.age_graph_stats(agtype) - RETURNS agtype - LANGUAGE c - STABLE -PARALLEL SAFE -AS 'MODULE_PATHNAME'; - -CREATE FUNCTION ag_catalog.graph_exists(graph_name name) - RETURNS agtype - LANGUAGE c - AS 'MODULE_PATHNAME', 'age_graph_exists'; - -CREATE FUNCTION ag_catalog.age_is_valid_label_name(agtype) - RETURNS boolean - LANGUAGE c - IMMUTABLE -PARALLEL SAFE -AS 'MODULE_PATHNAME'; - -CREATE OR REPLACE FUNCTION ag_catalog.create_vlabel(graph_name cstring, label_name cstring) - RETURNS void - LANGUAGE c - AS 'MODULE_PATHNAME'; - -CREATE OR REPLACE FUNCTION ag_catalog.create_elabel(graph_name cstring, label_name cstring) - RETURNS void - LANGUAGE c - AS 'MODULE_PATHNAME'; - -CREATE FUNCTION ag_catalog.agtype_to_json(agtype) - RETURNS json - LANGUAGE c - IMMUTABLE -RETURNS NULL ON NULL INPUT -PARALLEL SAFE -AS 'MODULE_PATHNAME'; - -CREATE CAST (agtype AS json) - WITH FUNCTION ag_catalog.agtype_to_json(agtype); - -CREATE FUNCTION ag_catalog.agtype_array_to_agtype(agtype[]) - RETURNS agtype - LANGUAGE c - IMMUTABLE -RETURNS NULL ON NULL INPUT -PARALLEL SAFE -AS 'MODULE_PATHNAME'; - -CREATE CAST (agtype[] AS agtype) - WITH FUNCTION ag_catalog.agtype_array_to_agtype(agtype[]); - -CREATE OPERATOR =~ ( - LEFTARG = agtype, - RIGHTARG = agtype, - FUNCTION = ag_catalog.age_eq_tilde -); diff --git a/age--1.6.0--y.y.y.sql b/age--1.6.0--1.7.0.sql similarity index 92% rename from age--1.6.0--y.y.y.sql rename to age--1.6.0--1.7.0.sql index c5a31b373..768cb6f1b 100644 --- a/age--1.6.0--y.y.y.sql +++ b/age--1.6.0--1.7.0.sql @@ -17,19 +17,11 @@ * under the License. */ ---* This is a TEMPLATE for upgrading from the previous version of Apache AGE ---* Please adjust the below ALTER EXTENSION to reflect the -- correct version it ---* is upgrading to. - -- This will only work within a major version of PostgreSQL, not across -- major versions. -- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "ALTER EXTENSION age UPDATE TO '1.X.0'" to load this file. \quit - ---* Please add all additions, deletions, and modifications to the end of this ---* file. We need to keep the order of these changes. ---* REMOVE ALL LINES ABOVE, and this one, that start with --* +\echo Use "ALTER EXTENSION age UPDATE TO '1.7.0'" to load this file. \quit CREATE FUNCTION ag_catalog._ag_enforce_edge_uniqueness2(graphid, graphid) RETURNS bool diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql new file mode 100644 index 000000000..c95bb0002 --- /dev/null +++ b/age--1.7.0--y.y.y.sql @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +--* This is a TEMPLATE for upgrading from the previous version of Apache AGE +--* Please adjust the below ALTER EXTENSION to reflect the -- correct version it +--* is upgrading to. + +-- This will only work within a major version of PostgreSQL, not across +-- major versions. + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION age UPDATE TO '1.X.0'" to load this file. \quit + +--* Please add all additions, deletions, and modifications to the end of this +--* file. We need to keep the order of these changes. +--* REMOVE ALL LINES ABOVE, and this one, that start with --* diff --git a/age.control b/age.control index 8bd40360f..57f9fbc12 100644 --- a/age.control +++ b/age.control @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -default_version = '1.6.0' +default_version = '1.7.0' comment = 'AGE database extension' module_pathname = '$libdir/age' diff --git a/docker/Dockerfile b/docker/Dockerfile index 3eb17c798..abdf0ccd0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -49,7 +49,7 @@ ENV LC_COLLATE=en_US.UTF-8 ENV LC_CTYPE=en_US.UTF-8 COPY --from=build /usr/lib/postgresql/18/lib/age.so /usr/lib/postgresql/18/lib/ -COPY --from=build /usr/share/postgresql/18/extension/age--1.6.0.sql /usr/share/postgresql/18/extension/ +COPY --from=build /usr/share/postgresql/18/extension/age--1.7.0.sql /usr/share/postgresql/18/extension/ COPY --from=build /usr/share/postgresql/18/extension/age.control /usr/share/postgresql/18/extension/ COPY docker/docker-entrypoint-initdb.d/00-create-extension-age.sql /docker-entrypoint-initdb.d/00-create-extension-age.sql From 5fe2121a06f692efc9a8c3d77b57c67b874e1b69 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Tue, 10 Feb 2026 10:32:29 -0800 Subject: [PATCH 025/100] Add pg_upgrade support functions for PostgreSQL (#2326) Add pg_upgrade support functions for PostgreSQL for major version upgrades NOTE: This PR was created with AI tools and a human. The ag_graph.namespace column uses the regnamespace type, which pg_upgrade cannot handle in user tables. This commit adds four SQL functions to enable seamless PostgreSQL major version upgrades while preserving all graph data. New functions in ag_catalog: - age_prepare_pg_upgrade(): Converts namespace from regnamespace to oid, creates backup table with graph-to-namespace mappings (stores nspname directly to avoid quoting issues) - age_finish_pg_upgrade(): Remaps stale OIDs after upgrade, restores regnamespace type, invalidates AGE caches while preserving schema ownership - age_revert_pg_upgrade_changes(): Cancels preparation if upgrade is aborted - age_pg_upgrade_status(): Returns current upgrade readiness status Usage: 1. Before pg_upgrade: SELECT age_prepare_pg_upgrade(); 2. Run pg_upgrade as normal 3. After pg_upgrade: SELECT age_finish_pg_upgrade(); Key implementation details: - Uses transaction-level advisory locks (pg_advisory_xact_lock) for safety - Preserves original schema ownership during cache invalidation - Validates all backup rows are mapped before proceeding - Handles zero-graph edge case gracefully - Handles insufficient privileges gracefully with informative notices - Backup table deleted only after all steps succeed Files changed: - sql/age_pg_upgrade.sql: New file with function implementations - sql/sql_files: Added age_pg_upgrade entry - age--1.7.0--y.y.y.sql: Added functions for extension upgrades All regression tests pass. --- age--1.7.0--y.y.y.sql | 378 ++++++++++++++++++++++++++++++++ sql/age_pg_upgrade.sql | 475 +++++++++++++++++++++++++++++++++++++++++ sql/sql_files | 1 + 3 files changed, 854 insertions(+) create mode 100644 sql/age_pg_upgrade.sql diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index c95bb0002..4f8c54a30 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -30,3 +30,381 @@ --* Please add all additions, deletions, and modifications to the end of this --* file. We need to keep the order of these changes. --* REMOVE ALL LINES ABOVE, and this one, that start with --* + +-- +-- pg_upgrade support functions +-- +-- These functions help users upgrade PostgreSQL major versions using pg_upgrade +-- while preserving Apache AGE graph data. +-- + +CREATE FUNCTION ag_catalog.age_prepare_pg_upgrade() + RETURNS void + LANGUAGE plpgsql + SET search_path = ag_catalog, pg_catalog + AS $function$ +DECLARE + graph_count integer; +BEGIN + -- Check if namespace column is already oid type (already prepared) + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'ag_catalog' + AND table_name = 'ag_graph' + AND column_name = 'namespace' + AND data_type = 'oid' + ) THEN + RAISE NOTICE 'Database already prepared for pg_upgrade (namespace is oid type).'; + RETURN; + END IF; + + -- Drop existing backup table if it exists (from a previous failed attempt) + DROP TABLE IF EXISTS public._age_pg_upgrade_backup; + + -- Create backup table with graph names mapped to namespace names + -- We store nspname directly (not regnamespace::text) to avoid quoting issues + -- Names survive pg_upgrade while OIDs don't + CREATE TABLE public._age_pg_upgrade_backup AS + SELECT + g.graphid AS old_graphid, + g.name AS graph_name, + n.nspname AS namespace_name + FROM ag_catalog.ag_graph g + JOIN pg_namespace n ON n.oid = g.namespace::oid; + + SELECT count(*) INTO graph_count FROM public._age_pg_upgrade_backup; + + RAISE NOTICE 'Created backup table public._age_pg_upgrade_backup with % graph(s)', graph_count; + + -- Even with zero graphs, we still need to convert the column type + -- because the regnamespace type itself blocks pg_upgrade + + -- Drop the existing regnamespace-based index + DROP INDEX IF EXISTS ag_catalog.ag_graph_namespace_index; + + -- Convert namespace column from regnamespace to oid + ALTER TABLE ag_catalog.ag_graph + ALTER COLUMN namespace TYPE oid USING namespace::oid; + + -- Recreate the index with oid type + CREATE UNIQUE INDEX ag_graph_namespace_index + ON ag_catalog.ag_graph USING btree (namespace); + + -- Create a view for backward-compatible display of namespace as schema name + CREATE OR REPLACE VIEW ag_catalog.ag_graph_view AS + SELECT graphid, name, namespace::regnamespace AS namespace + FROM ag_catalog.ag_graph; + + RAISE NOTICE 'Successfully prepared database for pg_upgrade.'; + RAISE NOTICE 'The ag_graph.namespace column has been converted from regnamespace to oid.'; + RAISE NOTICE 'You can now run pg_upgrade.'; + RAISE NOTICE 'After pg_upgrade completes, run: SELECT age_finish_pg_upgrade();'; +END; +$function$; + +COMMENT ON FUNCTION ag_catalog.age_prepare_pg_upgrade() IS +'Prepares an AGE database for pg_upgrade by converting ag_graph.namespace from regnamespace to oid type. Run this before pg_upgrade.'; + +CREATE FUNCTION ag_catalog.age_finish_pg_upgrade() + RETURNS void + LANGUAGE plpgsql + SET search_path = ag_catalog, pg_catalog + AS $function$ +DECLARE + mapping_count integer; + updated_labels integer; + updated_graphs integer; +BEGIN + -- Check if backup table exists + IF NOT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = '_age_pg_upgrade_backup' + ) THEN + RAISE EXCEPTION 'Backup table public._age_pg_upgrade_backup not found. ' + 'Did you run age_prepare_pg_upgrade() before pg_upgrade?'; + END IF; + + -- Check if namespace column is oid type (was properly prepared) + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'ag_catalog' + AND table_name = 'ag_graph' + AND column_name = 'namespace' + AND data_type = 'oid' + ) THEN + RAISE EXCEPTION 'ag_graph.namespace is not oid type. ' + 'Did you run age_prepare_pg_upgrade() before pg_upgrade?'; + END IF; + + -- Create temporary mapping table with old and new OIDs + CREATE TEMP TABLE _graphid_mapping AS + SELECT + b.old_graphid, + b.graph_name, + n.oid AS new_graphid + FROM public._age_pg_upgrade_backup b + JOIN pg_namespace n ON n.nspname = b.namespace_name; + + GET DIAGNOSTICS mapping_count = ROW_COUNT; + + -- Verify all backup rows were mapped (detect missing schemas) + DECLARE + backup_count integer; + BEGIN + SELECT count(*) INTO backup_count FROM public._age_pg_upgrade_backup; + IF mapping_count < backup_count THEN + RAISE EXCEPTION 'Only % of % graphs could be mapped. Some schema names may have changed or been dropped.', + mapping_count, backup_count; + END IF; + END; + + -- Handle zero-graph case (still need to restore schema) + IF mapping_count = 0 THEN + RAISE NOTICE 'No graphs to remap (empty backup table).'; + DROP TABLE _graphid_mapping; + -- Skip to schema restoration + ELSE + RAISE NOTICE 'Found % graph(s) to remap', mapping_count; + + -- Temporarily drop foreign key constraint + ALTER TABLE ag_catalog.ag_label DROP CONSTRAINT IF EXISTS fk_graph_oid; + + -- Update ag_label.graph references to use new OIDs + UPDATE ag_catalog.ag_label l + SET graph = m.new_graphid + FROM _graphid_mapping m + WHERE l.graph = m.old_graphid; + + GET DIAGNOSTICS updated_labels = ROW_COUNT; + RAISE NOTICE 'Updated % label record(s)', updated_labels; + + -- Update ag_graph.graphid and ag_graph.namespace to new OIDs + UPDATE ag_catalog.ag_graph g + SET graphid = m.new_graphid, + namespace = m.new_graphid + FROM _graphid_mapping m + WHERE g.graphid = m.old_graphid; + + GET DIAGNOSTICS updated_graphs = ROW_COUNT; + RAISE NOTICE 'Updated % graph record(s)', updated_graphs; + + -- Restore foreign key constraint + ALTER TABLE ag_catalog.ag_label + ADD CONSTRAINT fk_graph_oid + FOREIGN KEY(graph) REFERENCES ag_catalog.ag_graph(graphid); + + -- Clean up temporary mapping table + DROP TABLE _graphid_mapping; + + RAISE NOTICE 'Successfully completed pg_upgrade OID remapping.'; + END IF; + + -- + -- Restore original schema (revert namespace to regnamespace) + -- + RAISE NOTICE 'Restoring original schema...'; + + -- Drop the view (no longer needed with regnamespace) + DROP VIEW IF EXISTS ag_catalog.ag_graph_view; + + -- Drop the existing oid-based index + DROP INDEX IF EXISTS ag_catalog.ag_graph_namespace_index; + + -- Convert namespace column back to regnamespace + ALTER TABLE ag_catalog.ag_graph + ALTER COLUMN namespace TYPE regnamespace USING namespace::regnamespace; + + -- Recreate the index with regnamespace type + CREATE UNIQUE INDEX ag_graph_namespace_index + ON ag_catalog.ag_graph USING btree (namespace); + + RAISE NOTICE 'Successfully restored ag_graph.namespace to regnamespace type.'; + + -- + -- Invalidate AGE's internal caches by touching each graph's namespace + -- AGE registers a syscache callback on NAMESPACEOID, so altering a schema + -- triggers cache invalidation. This ensures cypher queries work immediately + -- without requiring a session reconnect. + -- + -- We use xact-level advisory lock (auto-released at transaction end) + -- and preserve original schema ownership. + -- + RAISE NOTICE 'Invalidating AGE caches...'; + PERFORM pg_catalog.pg_advisory_xact_lock(hashtext('age_finish_pg_upgrade')); + DECLARE + graph_rec RECORD; + cache_invalidated boolean := false; + BEGIN + FOR graph_rec IN + SELECT n.nspname AS ns_name, r.rolname AS owner_name + FROM ag_catalog.ag_graph g + JOIN pg_namespace n ON n.oid = g.namespace + JOIN pg_roles r ON r.oid = n.nspowner + LOOP + BEGIN + -- Touch schema by changing owner to current_user then back to original + -- This triggers cache invalidation without permanently changing ownership + EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); + EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); + cache_invalidated := true; + EXCEPTION WHEN insufficient_privilege THEN + -- If we can't change ownership, skip this schema + -- The cache will be invalidated on first use anyway + RAISE NOTICE 'Could not invalidate cache for schema % (insufficient privileges)', graph_rec.ns_name; + END; + END LOOP; + IF NOT cache_invalidated AND (SELECT count(*) FROM ag_catalog.ag_graph) > 0 THEN + RAISE NOTICE 'Cache invalidation skipped. You may need to reconnect for cypher queries to work.'; + END IF; + END; + + -- Now that all steps succeeded, clean up the backup table + DROP TABLE IF EXISTS public._age_pg_upgrade_backup; + + RAISE NOTICE ''; + RAISE NOTICE 'pg_upgrade complete. All graph data has been preserved.'; +END; +$function$; + +COMMENT ON FUNCTION ag_catalog.age_finish_pg_upgrade() IS +'Completes pg_upgrade by remapping stale OIDs and restoring the original schema. Run this after pg_upgrade.'; + +CREATE FUNCTION ag_catalog.age_revert_pg_upgrade_changes() + RETURNS void + LANGUAGE plpgsql + SET search_path = ag_catalog, pg_catalog + AS $function$ +BEGIN + -- Check if namespace column is oid type (needs reverting) + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'ag_catalog' + AND table_name = 'ag_graph' + AND column_name = 'namespace' + AND data_type = 'oid' + ) THEN + RAISE NOTICE 'ag_graph.namespace is already regnamespace type. Nothing to revert.'; + RETURN; + END IF; + + -- Drop the view (no longer needed with regnamespace) + DROP VIEW IF EXISTS ag_catalog.ag_graph_view; + + -- Drop the existing oid-based index + DROP INDEX IF EXISTS ag_catalog.ag_graph_namespace_index; + + -- Convert namespace column back to regnamespace + ALTER TABLE ag_catalog.ag_graph + ALTER COLUMN namespace TYPE regnamespace USING namespace::regnamespace; + + -- Recreate the index with regnamespace type + CREATE UNIQUE INDEX ag_graph_namespace_index + ON ag_catalog.ag_graph USING btree (namespace); + + -- + -- Invalidate AGE's internal caches by touching each graph's namespace + -- We use xact-level advisory lock and preserve original ownership + -- + PERFORM pg_catalog.pg_advisory_xact_lock(hashtext('age_revert_pg_upgrade')); + DECLARE + graph_rec RECORD; + BEGIN + FOR graph_rec IN + SELECT n.nspname AS ns_name, r.rolname AS owner_name + FROM ag_catalog.ag_graph g + JOIN pg_namespace n ON n.oid = g.namespace + JOIN pg_roles r ON r.oid = n.nspowner + LOOP + BEGIN + -- Touch schema by changing owner to current_user then back to original + EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); + EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); + EXCEPTION WHEN insufficient_privilege THEN + RAISE NOTICE 'Could not invalidate cache for schema % (insufficient privileges)', graph_rec.ns_name; + END; + END LOOP; + END; + + RAISE NOTICE 'Successfully reverted ag_graph.namespace to regnamespace type.'; + RAISE NOTICE ''; + RAISE NOTICE 'Upgrade preparation has been cancelled.'; + RAISE NOTICE 'You may want to drop the backup table: DROP TABLE IF EXISTS public._age_pg_upgrade_backup;'; +END; +$function$; + +COMMENT ON FUNCTION ag_catalog.age_revert_pg_upgrade_changes() IS +'Reverts schema changes if you need to cancel after age_prepare_pg_upgrade() but before pg_upgrade. Not needed after age_finish_pg_upgrade().'; + +CREATE FUNCTION ag_catalog.age_pg_upgrade_status() + RETURNS TABLE ( + status text, + namespace_type text, + graph_count bigint, + backup_exists boolean, + message text + ) + LANGUAGE plpgsql + SET search_path = ag_catalog, pg_catalog + AS $function$ +DECLARE + ns_type text; + g_count bigint; + backup_exists boolean; +BEGIN + -- Get namespace column type + SELECT data_type INTO ns_type + FROM information_schema.columns + WHERE table_schema = 'ag_catalog' + AND table_name = 'ag_graph' + AND column_name = 'namespace'; + + -- Get graph count + SELECT count(*) INTO g_count FROM ag_catalog.ag_graph; + + -- Check for backup table + SELECT EXISTS( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = '_age_pg_upgrade_backup' + ) INTO backup_exists; + + -- Determine status and message + IF ns_type = 'regnamespace' AND NOT backup_exists THEN + -- Normal state - ready for use, needs prep before pg_upgrade + RETURN QUERY SELECT + 'NORMAL'::text, + ns_type, + g_count, + backup_exists, + 'Run SELECT age_prepare_pg_upgrade(); before pg_upgrade'::text; + ELSIF ns_type = 'regnamespace' AND backup_exists THEN + -- Unusual state - backup exists but schema wasn't converted + RETURN QUERY SELECT + 'WARNING'::text, + ns_type, + g_count, + backup_exists, + 'Backup table exists but schema not converted. Run age_prepare_pg_upgrade() again.'::text; + ELSIF ns_type = 'oid' AND backup_exists THEN + -- Prepared and ready for pg_upgrade, or awaiting finish after pg_upgrade + RETURN QUERY SELECT + 'PREPARED - AWAITING FINISH'::text, + ns_type, + g_count, + backup_exists, + 'After pg_upgrade, run SELECT age_finish_pg_upgrade();'::text; + ELSE + -- oid type without backup - manually converted or partial state + RETURN QUERY SELECT + 'CONVERTED'::text, + ns_type, + g_count, + backup_exists, + 'Namespace is oid type. If upgrading, ensure backup table exists.'::text; + END IF; +END; +$function$; + +COMMENT ON FUNCTION ag_catalog.age_pg_upgrade_status() IS +'Returns the current pg_upgrade readiness status of the AGE installation.'; diff --git a/sql/age_pg_upgrade.sql b/sql/age_pg_upgrade.sql new file mode 100644 index 000000000..42a06ecd6 --- /dev/null +++ b/sql/age_pg_upgrade.sql @@ -0,0 +1,475 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +-- +-- pg_upgrade support functions +-- +-- These functions help users upgrade PostgreSQL major versions using pg_upgrade +-- while preserving Apache AGE graph data. The ag_graph.namespace column uses +-- the regnamespace type which is not supported by pg_upgrade. These functions +-- temporarily convert the schema to be pg_upgrade compatible, then restore it +-- to the original state after the upgrade completes. +-- +-- Usage: +-- 1. Before pg_upgrade: SELECT age_prepare_pg_upgrade(); +-- 2. Run pg_upgrade as normal +-- 3. After pg_upgrade: SELECT age_finish_pg_upgrade(); +-- +-- To cancel an upgrade after preparation (before running pg_upgrade): +-- SELECT age_revert_pg_upgrade_changes(); +-- + +-- +-- age_prepare_pg_upgrade() +-- +-- Prepares an AGE database for pg_upgrade by converting the ag_graph.namespace +-- column from regnamespace to oid type. This is necessary because pg_upgrade +-- does not support the regnamespace type in user tables. +-- +-- This function: +-- 1. Creates a backup table with graph name to namespace name mappings +-- 2. Drops the existing namespace index +-- 3. Converts the namespace column from regnamespace to oid +-- 4. Recreates the namespace index with oid type +-- 5. Creates a view for backward-compatible namespace display +-- +-- Returns: void +-- Side effects: Modifies ag_graph table structure +-- +CREATE FUNCTION ag_catalog.age_prepare_pg_upgrade() + RETURNS void + LANGUAGE plpgsql + SET search_path = ag_catalog, pg_catalog + AS $function$ +DECLARE + graph_count integer; +BEGIN + -- Check if namespace column is already oid type (already prepared) + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'ag_catalog' + AND table_name = 'ag_graph' + AND column_name = 'namespace' + AND data_type = 'oid' + ) THEN + RAISE NOTICE 'Database already prepared for pg_upgrade (namespace is oid type).'; + RETURN; + END IF; + + -- Drop existing backup table if it exists (from a previous failed attempt) + DROP TABLE IF EXISTS public._age_pg_upgrade_backup; + + -- Create backup table with graph names mapped to namespace names + -- We store nspname directly (not regnamespace::text) to avoid quoting issues + -- Names survive pg_upgrade while OIDs don't + CREATE TABLE public._age_pg_upgrade_backup AS + SELECT + g.graphid AS old_graphid, + g.name AS graph_name, + n.nspname AS namespace_name + FROM ag_catalog.ag_graph g + JOIN pg_namespace n ON n.oid = g.namespace::oid; + + SELECT count(*) INTO graph_count FROM public._age_pg_upgrade_backup; + + RAISE NOTICE 'Created backup table public._age_pg_upgrade_backup with % graph(s)', graph_count; + + -- Even with zero graphs, we still need to convert the column type + -- because the regnamespace type itself blocks pg_upgrade + + -- Drop the existing regnamespace-based index + DROP INDEX IF EXISTS ag_catalog.ag_graph_namespace_index; + + -- Convert namespace column from regnamespace to oid + ALTER TABLE ag_catalog.ag_graph + ALTER COLUMN namespace TYPE oid USING namespace::oid; + + -- Recreate the index with oid type + CREATE UNIQUE INDEX ag_graph_namespace_index + ON ag_catalog.ag_graph USING btree (namespace); + + -- Create a view for backward-compatible display of namespace as schema name + CREATE OR REPLACE VIEW ag_catalog.ag_graph_view AS + SELECT graphid, name, namespace::regnamespace AS namespace + FROM ag_catalog.ag_graph; + + RAISE NOTICE 'Successfully prepared database for pg_upgrade.'; + RAISE NOTICE 'The ag_graph.namespace column has been converted from regnamespace to oid.'; + RAISE NOTICE 'You can now run pg_upgrade.'; + RAISE NOTICE 'After pg_upgrade completes, run: SELECT age_finish_pg_upgrade();'; +END; +$function$; + +COMMENT ON FUNCTION ag_catalog.age_prepare_pg_upgrade() IS +'Prepares an AGE database for pg_upgrade by converting ag_graph.namespace from regnamespace to oid type. Run this before pg_upgrade.'; + +-- +-- age_finish_pg_upgrade() +-- +-- Completes the pg_upgrade process by remapping stale OIDs in ag_graph and +-- ag_label tables to their new values, then restores the original schema. +-- After pg_upgrade, the namespace OIDs stored in these tables no longer match +-- the actual pg_namespace OIDs because PostgreSQL assigns new OIDs to schemas +-- during the upgrade. +-- +-- This function: +-- 1. Reads the backup table created by age_prepare_pg_upgrade() +-- 2. Looks up new namespace OIDs by schema name +-- 3. Updates ag_label.graph references +-- 4. Updates ag_graph.graphid and ag_graph.namespace +-- 5. Restores namespace column to regnamespace type +-- 6. Cleans up the backup table and view +-- 7. Invalidates AGE caches to ensure cypher queries work immediately +-- +-- Returns: void +-- Side effects: Updates OIDs in ag_graph and ag_label tables, restores schema +-- +CREATE FUNCTION ag_catalog.age_finish_pg_upgrade() + RETURNS void + LANGUAGE plpgsql + SET search_path = ag_catalog, pg_catalog + AS $function$ +DECLARE + mapping_count integer; + updated_labels integer; + updated_graphs integer; +BEGIN + -- Check if backup table exists + IF NOT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = '_age_pg_upgrade_backup' + ) THEN + RAISE EXCEPTION 'Backup table public._age_pg_upgrade_backup not found. ' + 'Did you run age_prepare_pg_upgrade() before pg_upgrade?'; + END IF; + + -- Check if namespace column is oid type (was properly prepared) + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'ag_catalog' + AND table_name = 'ag_graph' + AND column_name = 'namespace' + AND data_type = 'oid' + ) THEN + RAISE EXCEPTION 'ag_graph.namespace is not oid type. ' + 'Did you run age_prepare_pg_upgrade() before pg_upgrade?'; + END IF; + + -- Create temporary mapping table with old and new OIDs + CREATE TEMP TABLE _graphid_mapping AS + SELECT + b.old_graphid, + b.graph_name, + n.oid AS new_graphid + FROM public._age_pg_upgrade_backup b + JOIN pg_namespace n ON n.nspname = b.namespace_name; + + GET DIAGNOSTICS mapping_count = ROW_COUNT; + + -- Verify all backup rows were mapped (detect missing schemas) + DECLARE + backup_count integer; + BEGIN + SELECT count(*) INTO backup_count FROM public._age_pg_upgrade_backup; + IF mapping_count < backup_count THEN + RAISE EXCEPTION 'Only % of % graphs could be mapped. Some schema names may have changed or been dropped.', + mapping_count, backup_count; + END IF; + END; + + -- Handle zero-graph case (still need to restore schema) + IF mapping_count = 0 THEN + RAISE NOTICE 'No graphs to remap (empty backup table).'; + DROP TABLE _graphid_mapping; + -- Skip to schema restoration + ELSE + RAISE NOTICE 'Found % graph(s) to remap', mapping_count; + + -- Temporarily drop foreign key constraint + ALTER TABLE ag_catalog.ag_label DROP CONSTRAINT IF EXISTS fk_graph_oid; + + -- Update ag_label.graph references to use new OIDs + UPDATE ag_catalog.ag_label l + SET graph = m.new_graphid + FROM _graphid_mapping m + WHERE l.graph = m.old_graphid; + + GET DIAGNOSTICS updated_labels = ROW_COUNT; + RAISE NOTICE 'Updated % label record(s)', updated_labels; + + -- Update ag_graph.graphid and ag_graph.namespace to new OIDs + UPDATE ag_catalog.ag_graph g + SET graphid = m.new_graphid, + namespace = m.new_graphid + FROM _graphid_mapping m + WHERE g.graphid = m.old_graphid; + + GET DIAGNOSTICS updated_graphs = ROW_COUNT; + RAISE NOTICE 'Updated % graph record(s)', updated_graphs; + + -- Restore foreign key constraint + ALTER TABLE ag_catalog.ag_label + ADD CONSTRAINT fk_graph_oid + FOREIGN KEY(graph) REFERENCES ag_catalog.ag_graph(graphid); + + -- Clean up temporary mapping table + DROP TABLE _graphid_mapping; + + RAISE NOTICE 'Successfully completed pg_upgrade OID remapping.'; + END IF; + + -- + -- Restore original schema (revert namespace to regnamespace) + -- + RAISE NOTICE 'Restoring original schema...'; + + -- Drop the view (no longer needed with regnamespace) + DROP VIEW IF EXISTS ag_catalog.ag_graph_view; + + -- Drop the existing oid-based index + DROP INDEX IF EXISTS ag_catalog.ag_graph_namespace_index; + + -- Convert namespace column back to regnamespace + ALTER TABLE ag_catalog.ag_graph + ALTER COLUMN namespace TYPE regnamespace USING namespace::regnamespace; + + -- Recreate the index with regnamespace type + CREATE UNIQUE INDEX ag_graph_namespace_index + ON ag_catalog.ag_graph USING btree (namespace); + + RAISE NOTICE 'Successfully restored ag_graph.namespace to regnamespace type.'; + + -- + -- Invalidate AGE's internal caches by touching each graph's namespace + -- AGE registers a syscache callback on NAMESPACEOID, so altering a schema + -- triggers cache invalidation. This ensures cypher queries work immediately + -- without requiring a session reconnect. + -- + -- We use xact-level advisory lock (auto-released at transaction end) + -- and preserve original schema ownership. + -- + RAISE NOTICE 'Invalidating AGE caches...'; + PERFORM pg_catalog.pg_advisory_xact_lock(hashtext('age_finish_pg_upgrade')); + DECLARE + graph_rec RECORD; + cache_invalidated boolean := false; + BEGIN + FOR graph_rec IN + SELECT n.nspname AS ns_name, r.rolname AS owner_name + FROM ag_catalog.ag_graph g + JOIN pg_namespace n ON n.oid = g.namespace + JOIN pg_roles r ON r.oid = n.nspowner + LOOP + BEGIN + -- Touch schema by changing owner to current_user then back to original + -- This triggers cache invalidation without permanently changing ownership + EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); + EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); + cache_invalidated := true; + EXCEPTION WHEN insufficient_privilege THEN + -- If we can't change ownership, skip this schema + -- The cache will be invalidated on first use anyway + RAISE NOTICE 'Could not invalidate cache for schema % (insufficient privileges)', graph_rec.ns_name; + END; + END LOOP; + IF NOT cache_invalidated AND (SELECT count(*) FROM ag_catalog.ag_graph) > 0 THEN + RAISE NOTICE 'Cache invalidation skipped. You may need to reconnect for cypher queries to work.'; + END IF; + END; + + -- Now that all steps succeeded, clean up the backup table + DROP TABLE IF EXISTS public._age_pg_upgrade_backup; + + RAISE NOTICE ''; + RAISE NOTICE 'pg_upgrade complete. All graph data has been preserved.'; +END; +$function$; + +COMMENT ON FUNCTION ag_catalog.age_finish_pg_upgrade() IS +'Completes pg_upgrade by remapping stale OIDs and restoring the original schema. Run this after pg_upgrade.'; + +-- +-- age_revert_pg_upgrade_changes() +-- +-- Reverts the schema changes made by age_prepare_pg_upgrade() if you need to +-- cancel the upgrade process before running pg_upgrade. This restores the +-- namespace column to its original regnamespace type. +-- +-- NOTE: This function is NOT needed after age_finish_pg_upgrade(), which +-- automatically restores the original schema. Use this only if you called +-- age_prepare_pg_upgrade() but decided not to proceed with pg_upgrade. +-- +-- This function: +-- 1. Drops the ag_graph_view (no longer needed) +-- 2. Drops the oid-based namespace index +-- 3. Converts namespace column back to regnamespace +-- 4. Recreates the namespace index with regnamespace type +-- 5. Invalidates AGE caches to ensure cypher queries work immediately +-- 6. Does NOT clean up the backup table (manual cleanup may be needed) +-- +-- Returns: void +-- Side effects: Modifies ag_graph table structure +-- +CREATE FUNCTION ag_catalog.age_revert_pg_upgrade_changes() + RETURNS void + LANGUAGE plpgsql + SET search_path = ag_catalog, pg_catalog + AS $function$ +BEGIN + -- Check if namespace column is oid type (needs reverting) + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'ag_catalog' + AND table_name = 'ag_graph' + AND column_name = 'namespace' + AND data_type = 'oid' + ) THEN + RAISE NOTICE 'ag_graph.namespace is already regnamespace type. Nothing to revert.'; + RETURN; + END IF; + + -- Drop the view (no longer needed with regnamespace) + DROP VIEW IF EXISTS ag_catalog.ag_graph_view; + + -- Drop the existing oid-based index + DROP INDEX IF EXISTS ag_catalog.ag_graph_namespace_index; + + -- Convert namespace column back to regnamespace + ALTER TABLE ag_catalog.ag_graph + ALTER COLUMN namespace TYPE regnamespace USING namespace::regnamespace; + + -- Recreate the index with regnamespace type + CREATE UNIQUE INDEX ag_graph_namespace_index + ON ag_catalog.ag_graph USING btree (namespace); + + -- + -- Invalidate AGE's internal caches by touching each graph's namespace + -- We use xact-level advisory lock and preserve original ownership + -- + PERFORM pg_catalog.pg_advisory_xact_lock(hashtext('age_revert_pg_upgrade')); + DECLARE + graph_rec RECORD; + BEGIN + FOR graph_rec IN + SELECT n.nspname AS ns_name, r.rolname AS owner_name + FROM ag_catalog.ag_graph g + JOIN pg_namespace n ON n.oid = g.namespace + JOIN pg_roles r ON r.oid = n.nspowner + LOOP + BEGIN + -- Touch schema by changing owner to current_user then back to original + EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); + EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); + EXCEPTION WHEN insufficient_privilege THEN + RAISE NOTICE 'Could not invalidate cache for schema % (insufficient privileges)', graph_rec.ns_name; + END; + END LOOP; + END; + + RAISE NOTICE 'Successfully reverted ag_graph.namespace to regnamespace type.'; + RAISE NOTICE ''; + RAISE NOTICE 'Upgrade preparation has been cancelled.'; + RAISE NOTICE 'You may want to drop the backup table: DROP TABLE IF EXISTS public._age_pg_upgrade_backup;'; +END; +$function$; + +COMMENT ON FUNCTION ag_catalog.age_revert_pg_upgrade_changes() IS +'Reverts schema changes if you need to cancel after age_prepare_pg_upgrade() but before pg_upgrade. Not needed after age_finish_pg_upgrade().'; + +-- +-- age_pg_upgrade_status() +-- +-- Returns the current pg_upgrade readiness status of the AGE installation. +-- Useful for checking whether the database needs preparation before pg_upgrade. +-- +-- Returns: TABLE with status information +-- +CREATE FUNCTION ag_catalog.age_pg_upgrade_status() + RETURNS TABLE ( + status text, + namespace_type text, + graph_count bigint, + backup_exists boolean, + message text + ) + LANGUAGE plpgsql + SET search_path = ag_catalog, pg_catalog + AS $function$ +DECLARE + ns_type text; + g_count bigint; + backup_exists boolean; +BEGIN + -- Get namespace column type + SELECT data_type INTO ns_type + FROM information_schema.columns + WHERE table_schema = 'ag_catalog' + AND table_name = 'ag_graph' + AND column_name = 'namespace'; + + -- Get graph count + SELECT count(*) INTO g_count FROM ag_catalog.ag_graph; + + -- Check for backup table + SELECT EXISTS( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = '_age_pg_upgrade_backup' + ) INTO backup_exists; + + -- Determine status and message + IF ns_type = 'regnamespace' AND NOT backup_exists THEN + -- Normal state - ready for use, needs prep before pg_upgrade + RETURN QUERY SELECT + 'NORMAL'::text, + ns_type, + g_count, + backup_exists, + 'Run SELECT age_prepare_pg_upgrade(); before pg_upgrade'::text; + ELSIF ns_type = 'regnamespace' AND backup_exists THEN + -- Unusual state - backup exists but schema wasn't converted + RETURN QUERY SELECT + 'WARNING'::text, + ns_type, + g_count, + backup_exists, + 'Backup table exists but schema not converted. Run age_prepare_pg_upgrade() again.'::text; + ELSIF ns_type = 'oid' AND backup_exists THEN + -- Prepared and ready for pg_upgrade, or awaiting finish after pg_upgrade + RETURN QUERY SELECT + 'PREPARED - AWAITING FINISH'::text, + ns_type, + g_count, + backup_exists, + 'After pg_upgrade, run SELECT age_finish_pg_upgrade();'::text; + ELSE + -- oid type without backup - manually converted or partial state + RETURN QUERY SELECT + 'CONVERTED'::text, + ns_type, + g_count, + backup_exists, + 'Namespace is oid type. If upgrading, ensure backup table exists.'::text; + END IF; +END; +$function$; + +COMMENT ON FUNCTION ag_catalog.age_pg_upgrade_status() IS +'Returns the current pg_upgrade readiness status of the AGE installation.'; diff --git a/sql/sql_files b/sql/sql_files index b10f1bcc6..32f9a7099 100644 --- a/sql/sql_files +++ b/sql/sql_files @@ -14,3 +14,4 @@ age_string age_trig age_aggregate agtype_typecast +age_pg_upgrade From 887564d9ce0f153ad7dc9f30223aa303c3fb60d0 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Fri, 13 Feb 2026 01:54:06 -0800 Subject: [PATCH 026/100] Fix JDBC driver CI test failures (#2333) Fix JDBC driver CI test failures: encoding, Testcontainers, and Docker compatibility. Note: This PR was created with the help of AI tools and a human. - Set JavaCompile encoding to UTF-8 to fix unicode test failures in CI environments that default to US-ASCII. - Add Testcontainers wait strategy (Wait.forLogMessage) to wait for PostgreSQL to be fully ready before connecting. - Use agensGraphContainer.getHost() instead of hardcoded localhost for Docker-in-Docker compatibility. - Add sslmode=disable to JDBC URL since PostgreSQL driver 42.6.0+ attempts SSL by default. - Remove silent exception swallowing around connection setup to fail fast with meaningful errors instead of NullPointerException. - Upgrade Testcontainers from 1.18.0 to 1.21.4 to fix Docker 29.x detection failure on ubuntu-24.04 runners (docker-java 3.3.x in 1.18.0 cannot negotiate with Docker Engine 29.1.5 API). modified: drivers/jdbc/lib/build.gradle.kts modified: drivers/jdbc/lib/src/test/java/org/apache/age/jdbc/BaseDockerizedTest.java --- drivers/jdbc/lib/build.gradle.kts | 6 +++++- .../apache/age/jdbc/BaseDockerizedTest.java | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/drivers/jdbc/lib/build.gradle.kts b/drivers/jdbc/lib/build.gradle.kts index 0b63bc5a6..4965c2a59 100644 --- a/drivers/jdbc/lib/build.gradle.kts +++ b/drivers/jdbc/lib/build.gradle.kts @@ -38,13 +38,17 @@ dependencies { testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") testRuntimeOnly("org.junit.platform:junit-platform-launcher") - testImplementation("org.testcontainers:testcontainers:1.18.0") + testImplementation("org.testcontainers:testcontainers:1.21.4") testImplementation("org.postgresql:postgresql:42.6.0") testImplementation("org.slf4j:slf4j-api:2.0.7") testImplementation("org.slf4j:slf4j-simple:2.0.7") } +tasks.withType { + options.encoding = "UTF-8" +} + tasks.generateGrammarSource { maxHeapSize = "64m" source = project.objects diff --git a/drivers/jdbc/lib/src/test/java/org/apache/age/jdbc/BaseDockerizedTest.java b/drivers/jdbc/lib/src/test/java/org/apache/age/jdbc/BaseDockerizedTest.java index 393175c3d..f46c5b5d3 100644 --- a/drivers/jdbc/lib/src/test/java/org/apache/age/jdbc/BaseDockerizedTest.java +++ b/drivers/jdbc/lib/src/test/java/org/apache/age/jdbc/BaseDockerizedTest.java @@ -21,6 +21,7 @@ import java.sql.DriverManager; import java.sql.Statement; +import java.time.Duration; import org.apache.age.jdbc.base.Agtype; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -28,6 +29,7 @@ import org.junit.jupiter.api.TestInstance.Lifecycle; import org.postgresql.jdbc.PgConnection; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerImageName; @TestInstance(Lifecycle.PER_CLASS) @@ -54,20 +56,19 @@ public void beforeAll() throws Exception { agensGraphContainer = new GenericContainer<>(DockerImageName .parse("apache/age:dev_snapshot_master")) .withEnv("POSTGRES_PASSWORD", CORRECT_DB_PASSWORDS) - .withExposedPorts(5432); + .withExposedPorts(5432) + .waitingFor(Wait.forLogMessage(".*database system is ready to accept connections.*\\n", 2) + .withStartupTimeout(Duration.ofSeconds(60))); agensGraphContainer.start(); + String host = agensGraphContainer.getHost(); int mappedPort = agensGraphContainer.getMappedPort(5432); String jdbcUrl = String - .format("jdbc:postgresql://%s:%d/%s", "localhost", mappedPort, "postgres"); + .format("jdbc:postgresql://%s:%d/%s?sslmode=disable", host, mappedPort, "postgres"); - try { - this.connection = DriverManager.getConnection(jdbcUrl, "postgres", CORRECT_DB_PASSWORDS) - .unwrap(PgConnection.class); - this.connection.addDataType("agtype", Agtype.class); - } catch (Exception e) { - System.out.println(e); - } + this.connection = DriverManager.getConnection(jdbcUrl, "postgres", CORRECT_DB_PASSWORDS) + .unwrap(PgConnection.class); + this.connection.addDataType("agtype", Agtype.class); try (Statement statement = connection.createStatement()) { statement.execute("CREATE EXTENSION IF NOT EXISTS age;"); statement.execute("LOAD 'age'"); From 5f5b744a08641225652de83332d73bc7acfc889d Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Fri, 13 Feb 2026 09:30:05 -0800 Subject: [PATCH 027/100] Update python-driver security and formatting (#2330) Note: This PR was created with AI tools and a human. - Add parameterized query construction using psycopg.sql to prevent SQL injection in all Cypher execution paths (age.py, networkx/lib.py) - Replace all %-format and f-string SQL in networkx/lib.py with sql.Identifier() for schema/table names and sql.Literal() for values - Add validate_graph_name() with AGE-aligned VALID_GRAPH_NAME regex: start with letter/underscore, allow dots and hyphens in middle positions, end with letter/digit/underscore, min 3 chars, max 63 chars - Add validate_identifier() with strict VALID_IDENTIFIER regex for labels, column names, and SQL types (no dots or hyphens) - Add validation calls to all networkx/lib.py entry points: graph names validated on entry, labels validated before SQL construction - Add _validate_column() to sanitize column specifications in buildCypher() - Fix exception constructors (AgeNotSet, GraphNotFound, GraphAlreadyExists) to always call super().__init__() with a meaningful default message so that str(exception) never returns an empty string - Add InvalidGraphName and InvalidIdentifier exception classes with structured name/reason/context fields - Fix builder.py: change erroneous 'return Exception(...)' to 'raise ValueError(...)' for unknown float expressions - Fix copy-paste docstring in create_elabel() ('create_vlabels' -> 'create_elabels') - Remove unused 'from psycopg.adapt import Loader' import in age.py - Add design documentation in source explaining: - VALID_GRAPH_NAME regex uses '*' (not '+') intentionally so that the min-length check fires first with a clear error message - buildCypher uses string concatenation (not sql.Identifier) because column specs are pre-validated 'name type' pairs that don't map to sql.Identifier(); graphName and cypherStmt are NOT embedded - Update test_networkx.py GraphNotFound assertion to use assertIn() instead of assertEqual() to match the improved exception messages - Strip Windows carriage returns (^M) from 7 source files - Fix requirements.txt: convert from UTF-16LE+BOM+CRLF to clean UTF-8+LF, move --no-binary flag from requirements.txt to CI workflow pip command - Upgrade actions/setup-python from v4 (deprecated) to v5 in CI workflow - Add 46 security unit tests in test_security.py covering: - Graph name validation (AGE naming rules, injection, edge cases) - SQL identifier validation (labels, columns, types) - Column spec sanitization - buildCypher injection prevention - Exception constructor correctness (str() never empty) - Add test_security.py to CI pipeline (python-driver.yaml) - pip-audit: 0 known vulnerabilities in all dependencies modified: .github/workflows/python-driver.yaml modified: drivers/python/age/VERSION.py modified: drivers/python/age/__init__.py modified: drivers/python/age/age.py modified: drivers/python/age/builder.py modified: drivers/python/age/exceptions.py modified: drivers/python/age/models.py modified: drivers/python/age/networkx/lib.py modified: drivers/python/requirements.txt modified: drivers/python/setup.py modified: drivers/python/test_agtypes.py modified: drivers/python/test_networkx.py new file: drivers/python/test_security.py --- .github/workflows/python-driver.yaml | 5 +- drivers/python/age/VERSION.py | 44 +- drivers/python/age/__init__.py | 80 ++-- drivers/python/age/age.py | 611 ++++++++++++++++----------- drivers/python/age/builder.py | 420 +++++++++--------- drivers/python/age/exceptions.py | 59 ++- drivers/python/age/models.py | 586 ++++++++++++------------- drivers/python/age/networkx/lib.py | 97 +++-- drivers/python/requirements.txt | Bin 176 -> 59 bytes drivers/python/setup.py | 44 +- drivers/python/test_agtypes.py | 264 ++++++------ drivers/python/test_networkx.py | 2 +- drivers/python/test_security.py | 274 ++++++++++++ 13 files changed, 1481 insertions(+), 1005 deletions(-) create mode 100644 drivers/python/test_security.py diff --git a/.github/workflows/python-driver.yaml b/.github/workflows/python-driver.yaml index 4dad14638..16ccface4 100644 --- a/.github/workflows/python-driver.yaml +++ b/.github/workflows/python-driver.yaml @@ -22,14 +22,14 @@ jobs: run: docker compose up -d - name: Set up python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install pre-requisites run: | sudo apt-get install python3-dev libpq-dev - pip install -r requirements.txt + pip install --no-binary psycopg -r requirements.txt - name: Build run: | @@ -40,3 +40,4 @@ jobs: python test_age_py.py -db "postgres" -u "postgres" -pass "agens" python test_networkx.py -db "postgres" -u "postgres" -pass "agens" python -m unittest -v test_agtypes.py + python -m unittest -v test_security.py diff --git a/drivers/python/age/VERSION.py b/drivers/python/age/VERSION.py index 3b014ea5b..5136181ae 100644 --- a/drivers/python/age/VERSION.py +++ b/drivers/python/age/VERSION.py @@ -1,22 +1,22 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - - -VER_MAJOR = 1 -VER_MINOR = 0 -VER_MICRO = 0 - -VERSION = '.'.join([str(VER_MAJOR),str(VER_MINOR),str(VER_MICRO)]) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + + +VER_MAJOR = 1 +VER_MINOR = 0 +VER_MICRO = 0 + +VERSION = '.'.join([str(VER_MAJOR),str(VER_MINOR),str(VER_MICRO)]) diff --git a/drivers/python/age/__init__.py b/drivers/python/age/__init__.py index fd50135af..685f0fe74 100644 --- a/drivers/python/age/__init__.py +++ b/drivers/python/age/__init__.py @@ -1,40 +1,40 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import psycopg.conninfo as conninfo -from . import age -from .age import * -from .models import * -from .builder import ResultHandler, DummyResultHandler, parseAgeValue, newResultHandler -from . import VERSION - -def version(): - return VERSION.VERSION - - -def connect(dsn=None, graph=None, connection_factory=None, cursor_factory=ClientCursor, load_from_plugins=False, - **kwargs): - - dsn = conninfo.make_conninfo('' if dsn is None else dsn, **kwargs) - - ag = Age() - ag.connect(dsn=dsn, graph=graph, connection_factory=connection_factory, cursor_factory=cursor_factory, - load_from_plugins=load_from_plugins, **kwargs) - return ag - -# Dummy ResultHandler -rawPrinter = DummyResultHandler() - -__name__="age" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import psycopg.conninfo as conninfo +from . import age +from .age import * +from .models import * +from .builder import ResultHandler, DummyResultHandler, parseAgeValue, newResultHandler +from . import VERSION + +def version(): + return VERSION.VERSION + + +def connect(dsn=None, graph=None, connection_factory=None, cursor_factory=ClientCursor, load_from_plugins=False, + **kwargs): + + dsn = conninfo.make_conninfo('' if dsn is None else dsn, **kwargs) + + ag = Age() + ag.connect(dsn=dsn, graph=graph, connection_factory=connection_factory, cursor_factory=cursor_factory, + load_from_plugins=load_from_plugins, **kwargs) + return ag + +# Dummy ResultHandler +rawPrinter = DummyResultHandler() + +__name__="age" diff --git a/drivers/python/age/age.py b/drivers/python/age/age.py index b1aa82158..fad1f27b1 100644 --- a/drivers/python/age/age.py +++ b/drivers/python/age/age.py @@ -1,236 +1,375 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import re -import psycopg -from psycopg.types import TypeInfo -from psycopg.adapt import Loader -from psycopg import sql -from psycopg.client_cursor import ClientCursor -from .exceptions import * -from .builder import parseAgeValue - - -_EXCEPTION_NoConnection = NoConnection() -_EXCEPTION_GraphNotSet = GraphNotSet() - -WHITESPACE = re.compile(r'\s') - - -class AgeDumper(psycopg.adapt.Dumper): - def dump(self, obj: Any) -> bytes | bytearray | memoryview: - pass - - -class AgeLoader(psycopg.adapt.Loader): - def load(self, data: bytes | bytearray | memoryview) -> Any | None: - if isinstance(data, memoryview): - data_bytes = data.tobytes() - else: - data_bytes = data - - return parseAgeValue(data_bytes.decode('utf-8')) - - -def setUpAge(conn:psycopg.connection, graphName:str, load_from_plugins:bool=False): - with conn.cursor() as cursor: - if load_from_plugins: - cursor.execute("LOAD '$libdir/plugins/age';") - else: - cursor.execute("LOAD 'age';") - - cursor.execute("SET search_path = ag_catalog, '$user', public;") - - ag_info = TypeInfo.fetch(conn, 'agtype') - - if not ag_info: - raise AgeNotSet() - - conn.adapters.register_loader(ag_info.oid, AgeLoader) - conn.adapters.register_loader(ag_info.array_oid, AgeLoader) - - # Check graph exists - if graphName != None: - checkGraphCreated(conn, graphName) - -# Create the graph, if it does not exist -def checkGraphCreated(conn:psycopg.connection, graphName:str): - with conn.cursor() as cursor: - cursor.execute(sql.SQL("SELECT count(*) FROM ag_graph WHERE name={graphName}").format(graphName=sql.Literal(graphName))) - if cursor.fetchone()[0] == 0: - cursor.execute(sql.SQL("SELECT create_graph({graphName});").format(graphName=sql.Literal(graphName))) - conn.commit() - - -def deleteGraph(conn:psycopg.connection, graphName:str): - with conn.cursor() as cursor: - cursor.execute(sql.SQL("SELECT drop_graph({graphName}, true);").format(graphName=sql.Literal(graphName))) - conn.commit() - - -def buildCypher(graphName:str, cypherStmt:str, columns:list) ->str: - if graphName == None: - raise _EXCEPTION_GraphNotSet - - columnExp=[] - if columns != None and len(columns) > 0: - for col in columns: - if col.strip() == '': - continue - elif WHITESPACE.search(col) != None: - columnExp.append(col) - else: - columnExp.append(col + " agtype") - else: - columnExp.append('v agtype') - - stmtArr = [] - stmtArr.append("SELECT * from cypher(NULL,NULL) as (") - stmtArr.append(','.join(columnExp)) - stmtArr.append(");") - return "".join(stmtArr) - -def execSql(conn:psycopg.connection, stmt:str, commit:bool=False, params:tuple=None) -> psycopg.cursor : - if conn == None or conn.closed: - raise _EXCEPTION_NoConnection - - cursor = conn.cursor() - try: - cursor.execute(stmt, params) - if commit: - conn.commit() - - return cursor - except SyntaxError as cause: - conn.rollback() - raise cause - except Exception as cause: - conn.rollback() - raise SqlExecutionError("Execution ERR[" + str(cause) +"](" + stmt +")", cause) - - -def querySql(conn:psycopg.connection, stmt:str, params:tuple=None) -> psycopg.cursor : - return execSql(conn, stmt, False, params) - -# Execute cypher statement and return cursor. -# If cypher statement changes data (create, set, remove), -# You must commit session(ag.commit()) -# (Otherwise the execution cannot make any effect.) -def execCypher(conn:psycopg.connection, graphName:str, cypherStmt:str, cols:list=None, params:tuple=None) -> psycopg.cursor : - if conn == None or conn.closed: - raise _EXCEPTION_NoConnection - - cursor = conn.cursor() - #clean up the string for mogrification - cypherStmt = cypherStmt.replace("\n", "") - cypherStmt = cypherStmt.replace("\t", "") - cypher = str(cursor.mogrify(cypherStmt, params)) - cypher = cypher.strip() - - preparedStmt = "SELECT * FROM age_prepare_cypher({graphName},{cypherStmt})" - - cursor = conn.cursor() - try: - cursor.execute(sql.SQL(preparedStmt).format(graphName=sql.Literal(graphName),cypherStmt=sql.Literal(cypher))) - except SyntaxError as cause: - conn.rollback() - raise cause - except Exception as cause: - conn.rollback() - raise SqlExecutionError("Execution ERR[" + str(cause) +"](" + preparedStmt +")", cause) - - stmt = buildCypher(graphName, cypher, cols) - - cursor = conn.cursor() - try: - cursor.execute(stmt) - return cursor - except SyntaxError as cause: - conn.rollback() - raise cause - except Exception as cause: - conn.rollback() - raise SqlExecutionError("Execution ERR[" + str(cause) +"](" + stmt +")", cause) - - -def cypher(cursor:psycopg.cursor, graphName:str, cypherStmt:str, cols:list=None, params:tuple=None) -> psycopg.cursor : - #clean up the string for mogrification - cypherStmt = cypherStmt.replace("\n", "") - cypherStmt = cypherStmt.replace("\t", "") - cypher = str(cursor.mogrify(cypherStmt, params)) - cypher = cypher.strip() - - preparedStmt = "SELECT * FROM age_prepare_cypher({graphName},{cypherStmt})" - cursor.execute(sql.SQL(preparedStmt).format(graphName=sql.Literal(graphName),cypherStmt=sql.Literal(cypher))) - - stmt = buildCypher(graphName, cypher, cols) - cursor.execute(stmt) - - -# def execCypherWithReturn(conn:psycopg.connection, graphName:str, cypherStmt:str, columns:list=None , params:tuple=None) -> psycopg.cursor : -# stmt = buildCypher(graphName, cypherStmt, columns) -# return execSql(conn, stmt, False, params) - -# def queryCypher(conn:psycopg.connection, graphName:str, cypherStmt:str, columns:list=None , params:tuple=None) -> psycopg.cursor : -# return execCypherWithReturn(conn, graphName, cypherStmt, columns, params) - - -class Age: - def __init__(self): - self.connection = None # psycopg connection] - self.graphName = None - - # Connect to PostgreSQL Server and establish session and type extension environment. - def connect(self, graph:str=None, dsn:str=None, connection_factory=None, cursor_factory=ClientCursor, - load_from_plugins:bool=False, **kwargs): - conn = psycopg.connect(dsn, cursor_factory=cursor_factory, **kwargs) - setUpAge(conn, graph, load_from_plugins) - self.connection = conn - self.graphName = graph - return self - - def close(self): - self.connection.close() - - def setGraph(self, graph:str): - checkGraphCreated(self.connection, graph) - self.graphName = graph - return self - - def commit(self): - self.connection.commit() - - def rollback(self): - self.connection.rollback() - - def execCypher(self, cypherStmt:str, cols:list=None, params:tuple=None) -> psycopg.cursor : - return execCypher(self.connection, self.graphName, cypherStmt, cols=cols, params=params) - - def cypher(self, cursor:psycopg.cursor, cypherStmt:str, cols:list=None, params:tuple=None) -> psycopg.cursor : - return cypher(cursor, self.graphName, cypherStmt, cols=cols, params=params) - - # def execSql(self, stmt:str, commit:bool=False, params:tuple=None) -> psycopg.cursor : - # return execSql(self.connection, stmt, commit, params) - - - # def execCypher(self, cypherStmt:str, commit:bool=False, params:tuple=None) -> psycopg.cursor : - # return execCypher(self.connection, self.graphName, cypherStmt, commit, params) - - # def execCypherWithReturn(self, cypherStmt:str, columns:list=None , params:tuple=None) -> psycopg.cursor : - # return execCypherWithReturn(self.connection, self.graphName, cypherStmt, columns, params) - - # def queryCypher(self, cypherStmt:str, columns:list=None , params:tuple=None) -> psycopg.cursor : - # return queryCypher(self.connection, self.graphName, cypherStmt, columns, params) - +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re +import psycopg +from psycopg.types import TypeInfo +from psycopg import sql +from psycopg.client_cursor import ClientCursor +from .exceptions import * +from .builder import parseAgeValue + + +_EXCEPTION_NoConnection = NoConnection() +_EXCEPTION_GraphNotSet = GraphNotSet() + +WHITESPACE = re.compile(r'\s') + +# Valid AGE graph name pattern aligned with Apache AGE's internal validation +# and Neo4j/openCypher naming conventions. +# Start: letter or underscore +# Middle: letter, digit, underscore, dot, or hyphen +# End: letter, digit, or underscore +# +# Design note: The middle segment uses `*` (not `+`) intentionally. +# This makes the regex match names as short as 2 characters at the +# regex level. However, validate_graph_name() checks MIN_GRAPH_NAME_LENGTH +# *before* applying this regex, so 2-character names are rejected with a +# clear "must be at least 3 characters" error rather than a confusing +# regex-mismatch error. This ordering gives users actionable feedback. +VALID_GRAPH_NAME = re.compile(r'^[A-Za-z_][A-Za-z0-9_.\-]*[A-Za-z0-9_]$') +MIN_GRAPH_NAME_LENGTH = 3 + +# Valid SQL identifier for labels, column names, and types. +# Stricter than graph names — no dots or hyphens. +VALID_IDENTIFIER = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') +MAX_IDENTIFIER_LENGTH = 63 + + +def validate_graph_name(graph_name: str) -> None: + """Validate that a graph name conforms to Apache AGE's naming rules. + + Graph names must: + - Be at least 3 characters and at most 63 characters + - Start with a letter or underscore + - Contain only letters, digits, underscores, dots, and hyphens + - End with a letter, digit, or underscore + + This aligns with AGE's internal validation and Neo4j/openCypher + naming conventions. + + Args: + graph_name: The graph name to validate. + + Raises: + InvalidGraphName: If the graph name is invalid. + """ + if not graph_name or not isinstance(graph_name, str): + raise InvalidGraphName( + str(graph_name), + "Graph name must be a non-empty string." + ) + if len(graph_name) < MIN_GRAPH_NAME_LENGTH: + raise InvalidGraphName( + graph_name, + f"Graph names must be at least {MIN_GRAPH_NAME_LENGTH} characters." + ) + if len(graph_name) > MAX_IDENTIFIER_LENGTH: + raise InvalidGraphName( + graph_name, + f"Must not exceed {MAX_IDENTIFIER_LENGTH} characters " + "(PostgreSQL name limit)." + ) + if not VALID_GRAPH_NAME.match(graph_name): + raise InvalidGraphName( + graph_name, + "Graph names must start with a letter or underscore, " + "may contain letters, digits, underscores, dots, and hyphens, " + "and must end with a letter, digit, or underscore." + ) + + +def validate_identifier(name: str, context: str = "identifier") -> None: + """Validate that a name is a safe SQL identifier for labels, columns, or types. + + This follows stricter rules than graph names — only letters, digits, + and underscores are permitted (no dots or hyphens). + + Args: + name: The identifier to validate. + context: What the identifier represents (for error messages). + + Raises: + InvalidIdentifier: If the identifier is invalid. + """ + if not name or not isinstance(name, str): + raise InvalidIdentifier( + str(name), + f"{context} must be a non-empty string." + ) + if len(name) > MAX_IDENTIFIER_LENGTH: + raise InvalidIdentifier( + name, + f"{context} must not exceed {MAX_IDENTIFIER_LENGTH} characters." + ) + if not VALID_IDENTIFIER.match(name): + raise InvalidIdentifier( + name, + f"{context} must start with a letter or underscore " + "and contain only letters, digits, and underscores." + ) + + +class AgeDumper(psycopg.adapt.Dumper): + def dump(self, obj: Any) -> bytes | bytearray | memoryview: + pass + + +class AgeLoader(psycopg.adapt.Loader): + def load(self, data: bytes | bytearray | memoryview) -> Any | None: + if isinstance(data, memoryview): + data_bytes = data.tobytes() + else: + data_bytes = data + + return parseAgeValue(data_bytes.decode('utf-8')) + + +def setUpAge(conn:psycopg.connection, graphName:str, load_from_plugins:bool=False): + with conn.cursor() as cursor: + if load_from_plugins: + cursor.execute("LOAD '$libdir/plugins/age';") + else: + cursor.execute("LOAD 'age';") + + cursor.execute("SET search_path = ag_catalog, '$user', public;") + + ag_info = TypeInfo.fetch(conn, 'agtype') + + if not ag_info: + raise AgeNotSet( + "AGE agtype type not found. Ensure the AGE extension is " + "installed and loaded in the current database. " + "Run CREATE EXTENSION age; first." + ) + + conn.adapters.register_loader(ag_info.oid, AgeLoader) + conn.adapters.register_loader(ag_info.array_oid, AgeLoader) + + # Check graph exists + if graphName != None: + checkGraphCreated(conn, graphName) + +# Create the graph, if it does not exist +def checkGraphCreated(conn:psycopg.connection, graphName:str): + validate_graph_name(graphName) + with conn.cursor() as cursor: + cursor.execute(sql.SQL("SELECT count(*) FROM ag_graph WHERE name={graphName}").format(graphName=sql.Literal(graphName))) + if cursor.fetchone()[0] == 0: + cursor.execute(sql.SQL("SELECT create_graph({graphName});").format(graphName=sql.Literal(graphName))) + conn.commit() + + +def deleteGraph(conn:psycopg.connection, graphName:str): + validate_graph_name(graphName) + with conn.cursor() as cursor: + cursor.execute(sql.SQL("SELECT drop_graph({graphName}, true);").format(graphName=sql.Literal(graphName))) + conn.commit() + + +def _validate_column(col: str) -> str: + """Validate and normalize a column specification for use in SQL. + + Accepts either a plain column name (e.g. 'v') or a name with type + (e.g. 'v agtype'). Validates each component to prevent SQL injection. + + Args: + col: Column specification string. + + Returns: + Normalized column specification, or empty string if blank. + + Raises: + InvalidIdentifier: If any component is invalid. + """ + col = col.strip() + if not col: + return '' + + if WHITESPACE.search(col): + parts = col.split() + if len(parts) != 2: + raise InvalidIdentifier( + col, + "Column specification must be 'name' or 'name type'." + ) + name, type_name = parts + validate_identifier(name, "Column name") + validate_identifier(type_name, "Column type") + return f"{name} {type_name}" + else: + validate_identifier(col, "Column name") + return f"{col} agtype" + + +def buildCypher(graphName:str, cypherStmt:str, columns:list) ->str: + if graphName == None: + raise _EXCEPTION_GraphNotSet + + columnExp=[] + if columns != None and len(columns) > 0: + for col in columns: + validated = _validate_column(col) + if validated: + columnExp.append(validated) + else: + columnExp.append('v agtype') + + # Design note: String concatenation is used here instead of + # psycopg.sql.Identifier() because column specifications are + # "name type" pairs (e.g. "v agtype") that don't map directly to + # sql.Identifier(). Each component has already been validated by + # _validate_column() → validate_identifier(), which restricts + # names to ^[A-Za-z_][A-Za-z0-9_]*$ and max 63 chars. The + # graphName and cypherStmt are NOT embedded here — this template + # only contains the validated column list and static SQL keywords. + stmtArr = [] + stmtArr.append("SELECT * from cypher(NULL,NULL) as (") + stmtArr.append(','.join(columnExp)) + stmtArr.append(");") + return "".join(stmtArr) + +def execSql(conn:psycopg.connection, stmt:str, commit:bool=False, params:tuple=None) -> psycopg.cursor : + if conn == None or conn.closed: + raise _EXCEPTION_NoConnection + + cursor = conn.cursor() + try: + cursor.execute(stmt, params) + if commit: + conn.commit() + + return cursor + except SyntaxError as cause: + conn.rollback() + raise cause + except Exception as cause: + conn.rollback() + raise SqlExecutionError("Execution ERR[" + str(cause) +"](" + stmt +")", cause) + + +def querySql(conn:psycopg.connection, stmt:str, params:tuple=None) -> psycopg.cursor : + return execSql(conn, stmt, False, params) + +# Execute cypher statement and return cursor. +# If cypher statement changes data (create, set, remove), +# You must commit session(ag.commit()) +# (Otherwise the execution cannot make any effect.) +def execCypher(conn:psycopg.connection, graphName:str, cypherStmt:str, cols:list=None, params:tuple=None) -> psycopg.cursor : + if conn == None or conn.closed: + raise _EXCEPTION_NoConnection + + cursor = conn.cursor() + #clean up the string for mogrification + cypherStmt = cypherStmt.replace("\n", "") + cypherStmt = cypherStmt.replace("\t", "") + cypher = str(cursor.mogrify(cypherStmt, params)) + cypher = cypher.strip() + + preparedStmt = "SELECT * FROM age_prepare_cypher({graphName},{cypherStmt})" + + cursor = conn.cursor() + try: + cursor.execute(sql.SQL(preparedStmt).format(graphName=sql.Literal(graphName),cypherStmt=sql.Literal(cypher))) + except SyntaxError as cause: + conn.rollback() + raise cause + except Exception as cause: + conn.rollback() + raise SqlExecutionError("Execution ERR[" + str(cause) +"](" + preparedStmt +")", cause) + + stmt = buildCypher(graphName, cypher, cols) + + cursor = conn.cursor() + try: + cursor.execute(stmt) + return cursor + except SyntaxError as cause: + conn.rollback() + raise cause + except Exception as cause: + conn.rollback() + raise SqlExecutionError("Execution ERR[" + str(cause) +"](" + stmt +")", cause) + + +def cypher(cursor:psycopg.cursor, graphName:str, cypherStmt:str, cols:list=None, params:tuple=None) -> psycopg.cursor : + #clean up the string for mogrification + cypherStmt = cypherStmt.replace("\n", "") + cypherStmt = cypherStmt.replace("\t", "") + cypher = str(cursor.mogrify(cypherStmt, params)) + cypher = cypher.strip() + + preparedStmt = "SELECT * FROM age_prepare_cypher({graphName},{cypherStmt})" + cursor.execute(sql.SQL(preparedStmt).format(graphName=sql.Literal(graphName),cypherStmt=sql.Literal(cypher))) + + stmt = buildCypher(graphName, cypher, cols) + cursor.execute(stmt) + + +# def execCypherWithReturn(conn:psycopg.connection, graphName:str, cypherStmt:str, columns:list=None , params:tuple=None) -> psycopg.cursor : +# stmt = buildCypher(graphName, cypherStmt, columns) +# return execSql(conn, stmt, False, params) + +# def queryCypher(conn:psycopg.connection, graphName:str, cypherStmt:str, columns:list=None , params:tuple=None) -> psycopg.cursor : +# return execCypherWithReturn(conn, graphName, cypherStmt, columns, params) + + +class Age: + def __init__(self): + self.connection = None # psycopg connection] + self.graphName = None + + # Connect to PostgreSQL Server and establish session and type extension environment. + def connect(self, graph:str=None, dsn:str=None, connection_factory=None, cursor_factory=ClientCursor, + load_from_plugins:bool=False, **kwargs): + conn = psycopg.connect(dsn, cursor_factory=cursor_factory, **kwargs) + setUpAge(conn, graph, load_from_plugins) + self.connection = conn + self.graphName = graph + return self + + def close(self): + self.connection.close() + + def setGraph(self, graph:str): + checkGraphCreated(self.connection, graph) + self.graphName = graph + return self + + def commit(self): + self.connection.commit() + + def rollback(self): + self.connection.rollback() + + def execCypher(self, cypherStmt:str, cols:list=None, params:tuple=None) -> psycopg.cursor : + return execCypher(self.connection, self.graphName, cypherStmt, cols=cols, params=params) + + def cypher(self, cursor:psycopg.cursor, cypherStmt:str, cols:list=None, params:tuple=None) -> psycopg.cursor : + return cypher(cursor, self.graphName, cypherStmt, cols=cols, params=params) + + # def execSql(self, stmt:str, commit:bool=False, params:tuple=None) -> psycopg.cursor : + # return execSql(self.connection, stmt, commit, params) + + + # def execCypher(self, cypherStmt:str, commit:bool=False, params:tuple=None) -> psycopg.cursor : + # return execCypher(self.connection, self.graphName, cypherStmt, commit, params) + + # def execCypherWithReturn(self, cypherStmt:str, columns:list=None , params:tuple=None) -> psycopg.cursor : + # return execCypherWithReturn(self.connection, self.graphName, cypherStmt, columns, params) + + # def queryCypher(self, cypherStmt:str, columns:list=None , params:tuple=None) -> psycopg.cursor : + # return queryCypher(self.connection, self.graphName, cypherStmt, columns, params) + diff --git a/drivers/python/age/builder.py b/drivers/python/age/builder.py index a3815b829..f1e7a2ce8 100644 --- a/drivers/python/age/builder.py +++ b/drivers/python/age/builder.py @@ -1,210 +1,210 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from . import gen -from .gen.AgtypeLexer import AgtypeLexer -from .gen.AgtypeParser import AgtypeParser -from .gen.AgtypeVisitor import AgtypeVisitor -from .models import * -from .exceptions import * -from antlr4 import InputStream, CommonTokenStream, ParserRuleContext -from antlr4.tree.Tree import TerminalNode -from decimal import Decimal - -resultHandler = None - -class ResultHandler: - def parse(ageData): - pass - -def newResultHandler(query=""): - resultHandler = Antlr4ResultHandler(None, query) - return resultHandler - -def parseAgeValue(value, cursor=None): - if value is None: - return None - - global resultHandler - if (resultHandler == None): - resultHandler = Antlr4ResultHandler(None) - try: - return resultHandler.parse(value) - except Exception as ex: - raise AGTypeError(value, ex) - - -class Antlr4ResultHandler(ResultHandler): - def __init__(self, vertexCache, query=None): - self.lexer = AgtypeLexer() - self.parser = AgtypeParser(None) - self.visitor = ResultVisitor(vertexCache) - - def parse(self, ageData): - if not ageData: - return None - # print("Parse::", ageData) - - self.lexer.inputStream = InputStream(ageData) - self.parser.setTokenStream(CommonTokenStream(self.lexer)) - self.parser.reset() - tree = self.parser.agType() - parsed = tree.accept(self.visitor) - return parsed - - -# print raw result String -class DummyResultHandler(ResultHandler): - def parse(self, ageData): - print(ageData) - -# default agType visitor -class ResultVisitor(AgtypeVisitor): - vertexCache = None - - def __init__(self, cache) -> None: - super().__init__() - self.vertexCache = cache - - - def visitAgType(self, ctx:AgtypeParser.AgTypeContext): - agVal = ctx.agValue() - if agVal != None: - obj = ctx.agValue().accept(self) - return obj - - return None - - def visitAgValue(self, ctx:AgtypeParser.AgValueContext): - annoCtx = ctx.typeAnnotation() - valueCtx = ctx.value() - - if annoCtx is not None: - annoCtx.accept(self) - anno = annoCtx.IDENT().getText() - return self.handleAnnotatedValue(anno, valueCtx) - else: - return valueCtx.accept(self) - - - # Visit a parse tree produced by AgtypeParser#StringValue. - def visitStringValue(self, ctx:AgtypeParser.StringValueContext): - return ctx.STRING().getText().strip('"') - - - # Visit a parse tree produced by AgtypeParser#IntegerValue. - def visitIntegerValue(self, ctx:AgtypeParser.IntegerValueContext): - return int(ctx.INTEGER().getText()) - - # Visit a parse tree produced by AgtypeParser#floatLiteral. - def visitFloatLiteral(self, ctx:AgtypeParser.FloatLiteralContext): - c = ctx.getChild(0) - tp = c.symbol.type - text = ctx.getText() - if tp == AgtypeParser.RegularFloat: - return float(text) - elif tp == AgtypeParser.ExponentFloat: - return float(text) - else: - if text == 'NaN': - return float('nan') - elif text == '-Infinity': - return float('-inf') - elif text == 'Infinity': - return float('inf') - else: - return Exception("Unknown float expression:"+text) - - - # Visit a parse tree produced by AgtypeParser#TrueBoolean. - def visitTrueBoolean(self, ctx:AgtypeParser.TrueBooleanContext): - return True - - - # Visit a parse tree produced by AgtypeParser#FalseBoolean. - def visitFalseBoolean(self, ctx:AgtypeParser.FalseBooleanContext): - return False - - - # Visit a parse tree produced by AgtypeParser#NullValue. - def visitNullValue(self, ctx:AgtypeParser.NullValueContext): - return None - - - # Visit a parse tree produced by AgtypeParser#obj. - def visitObj(self, ctx:AgtypeParser.ObjContext): - obj = dict() - for c in ctx.getChildren(): - if isinstance(c, AgtypeParser.PairContext): - namVal = self.visitPair(c) - name = namVal[0] - valCtx = namVal[1] - val = valCtx.accept(self) - obj[name] = val - return obj - - - # Visit a parse tree produced by AgtypeParser#pair. - def visitPair(self, ctx:AgtypeParser.PairContext): - self.visitChildren(ctx) - return (ctx.STRING().getText().strip('"') , ctx.agValue()) - - - # Visit a parse tree produced by AgtypeParser#array. - def visitArray(self, ctx:AgtypeParser.ArrayContext): - li = list() - for c in ctx.getChildren(): - if not isinstance(c, TerminalNode): - val = c.accept(self) - li.append(val) - return li - - def handleAnnotatedValue(self, anno:str, ctx:ParserRuleContext): - if anno == "numeric": - return Decimal(ctx.getText()) - elif anno == "vertex": - dict = ctx.accept(self) - vid = dict["id"] - vertex = None - if self.vertexCache != None and vid in self.vertexCache : - vertex = self.vertexCache[vid] - else: - vertex = Vertex() - vertex.id = dict["id"] - vertex.label = dict["label"] - vertex.properties = dict["properties"] - - if self.vertexCache != None: - self.vertexCache[vid] = vertex - - return vertex - - elif anno == "edge": - edge = Edge() - dict = ctx.accept(self) - edge.id = dict["id"] - edge.label = dict["label"] - edge.end_id = dict["end_id"] - edge.start_id = dict["start_id"] - edge.properties = dict["properties"] - - return edge - - elif anno == "path": - arr = ctx.accept(self) - path = Path(arr) - - return path - - return ctx.accept(self) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from . import gen +from .gen.AgtypeLexer import AgtypeLexer +from .gen.AgtypeParser import AgtypeParser +from .gen.AgtypeVisitor import AgtypeVisitor +from .models import * +from .exceptions import * +from antlr4 import InputStream, CommonTokenStream, ParserRuleContext +from antlr4.tree.Tree import TerminalNode +from decimal import Decimal + +resultHandler = None + +class ResultHandler: + def parse(ageData): + pass + +def newResultHandler(query=""): + resultHandler = Antlr4ResultHandler(None, query) + return resultHandler + +def parseAgeValue(value, cursor=None): + if value is None: + return None + + global resultHandler + if (resultHandler == None): + resultHandler = Antlr4ResultHandler(None) + try: + return resultHandler.parse(value) + except Exception as ex: + raise AGTypeError(value, ex) + + +class Antlr4ResultHandler(ResultHandler): + def __init__(self, vertexCache, query=None): + self.lexer = AgtypeLexer() + self.parser = AgtypeParser(None) + self.visitor = ResultVisitor(vertexCache) + + def parse(self, ageData): + if not ageData: + return None + # print("Parse::", ageData) + + self.lexer.inputStream = InputStream(ageData) + self.parser.setTokenStream(CommonTokenStream(self.lexer)) + self.parser.reset() + tree = self.parser.agType() + parsed = tree.accept(self.visitor) + return parsed + + +# print raw result String +class DummyResultHandler(ResultHandler): + def parse(self, ageData): + print(ageData) + +# default agType visitor +class ResultVisitor(AgtypeVisitor): + vertexCache = None + + def __init__(self, cache) -> None: + super().__init__() + self.vertexCache = cache + + + def visitAgType(self, ctx:AgtypeParser.AgTypeContext): + agVal = ctx.agValue() + if agVal != None: + obj = ctx.agValue().accept(self) + return obj + + return None + + def visitAgValue(self, ctx:AgtypeParser.AgValueContext): + annoCtx = ctx.typeAnnotation() + valueCtx = ctx.value() + + if annoCtx is not None: + annoCtx.accept(self) + anno = annoCtx.IDENT().getText() + return self.handleAnnotatedValue(anno, valueCtx) + else: + return valueCtx.accept(self) + + + # Visit a parse tree produced by AgtypeParser#StringValue. + def visitStringValue(self, ctx:AgtypeParser.StringValueContext): + return ctx.STRING().getText().strip('"') + + + # Visit a parse tree produced by AgtypeParser#IntegerValue. + def visitIntegerValue(self, ctx:AgtypeParser.IntegerValueContext): + return int(ctx.INTEGER().getText()) + + # Visit a parse tree produced by AgtypeParser#floatLiteral. + def visitFloatLiteral(self, ctx:AgtypeParser.FloatLiteralContext): + c = ctx.getChild(0) + tp = c.symbol.type + text = ctx.getText() + if tp == AgtypeParser.RegularFloat: + return float(text) + elif tp == AgtypeParser.ExponentFloat: + return float(text) + else: + if text == 'NaN': + return float('nan') + elif text == '-Infinity': + return float('-inf') + elif text == 'Infinity': + return float('inf') + else: + raise ValueError("Unknown float expression: " + text) + + + # Visit a parse tree produced by AgtypeParser#TrueBoolean. + def visitTrueBoolean(self, ctx:AgtypeParser.TrueBooleanContext): + return True + + + # Visit a parse tree produced by AgtypeParser#FalseBoolean. + def visitFalseBoolean(self, ctx:AgtypeParser.FalseBooleanContext): + return False + + + # Visit a parse tree produced by AgtypeParser#NullValue. + def visitNullValue(self, ctx:AgtypeParser.NullValueContext): + return None + + + # Visit a parse tree produced by AgtypeParser#obj. + def visitObj(self, ctx:AgtypeParser.ObjContext): + obj = dict() + for c in ctx.getChildren(): + if isinstance(c, AgtypeParser.PairContext): + namVal = self.visitPair(c) + name = namVal[0] + valCtx = namVal[1] + val = valCtx.accept(self) + obj[name] = val + return obj + + + # Visit a parse tree produced by AgtypeParser#pair. + def visitPair(self, ctx:AgtypeParser.PairContext): + self.visitChildren(ctx) + return (ctx.STRING().getText().strip('"') , ctx.agValue()) + + + # Visit a parse tree produced by AgtypeParser#array. + def visitArray(self, ctx:AgtypeParser.ArrayContext): + li = list() + for c in ctx.getChildren(): + if not isinstance(c, TerminalNode): + val = c.accept(self) + li.append(val) + return li + + def handleAnnotatedValue(self, anno:str, ctx:ParserRuleContext): + if anno == "numeric": + return Decimal(ctx.getText()) + elif anno == "vertex": + dict = ctx.accept(self) + vid = dict["id"] + vertex = None + if self.vertexCache != None and vid in self.vertexCache : + vertex = self.vertexCache[vid] + else: + vertex = Vertex() + vertex.id = dict["id"] + vertex.label = dict["label"] + vertex.properties = dict["properties"] + + if self.vertexCache != None: + self.vertexCache[vid] = vertex + + return vertex + + elif anno == "edge": + edge = Edge() + dict = ctx.accept(self) + edge.id = dict["id"] + edge.label = dict["label"] + edge.end_id = dict["end_id"] + edge.start_id = dict["start_id"] + edge.properties = dict["properties"] + + return edge + + elif anno == "path": + arr = ctx.accept(self) + path = Path(arr) + + return path + + return ctx.accept(self) diff --git a/drivers/python/age/exceptions.py b/drivers/python/age/exceptions.py index 3aa94f4b8..18292cc08 100644 --- a/drivers/python/age/exceptions.py +++ b/drivers/python/age/exceptions.py @@ -16,39 +16,74 @@ from psycopg.errors import * class AgeNotSet(Exception): - def __init__(self, name): + def __init__(self, name=None): self.name = name + super().__init__(name or 'AGE extension is not set.') - def __repr__(self) : + def __repr__(self): return 'AGE extension is not set.' class GraphNotFound(Exception): - def __init__(self, name): + def __init__(self, name=None): self.name = name + super().__init__(f'Graph[{name}] does not exist.' if name else 'Graph does not exist.') - def __repr__(self) : - return 'Graph[' + self.name + '] does not exist.' + def __repr__(self): + if self.name: + return 'Graph[' + self.name + '] does not exist.' + return 'Graph does not exist.' class GraphAlreadyExists(Exception): - def __init__(self, name): + def __init__(self, name=None): self.name = name + super().__init__(f'Graph[{name}] already exists.' if name else 'Graph already exists.') - def __repr__(self) : - return 'Graph[' + self.name + '] already exists.' + def __repr__(self): + if self.name: + return 'Graph[' + self.name + '] already exists.' + return 'Graph already exists.' + + +class InvalidGraphName(Exception): + """Raised when a graph name contains invalid characters.""" + def __init__(self, name, reason=None): + self.name = name + self.reason = reason + msg = f"Invalid graph name: '{name}'." + if reason: + msg += f" {reason}" + super().__init__(msg) + + def __repr__(self): + return f"InvalidGraphName('{self.name}')" + + +class InvalidIdentifier(Exception): + """Raised when an identifier (column, label, etc.) is invalid.""" + def __init__(self, name, context=None): + self.name = name + self.context = context + msg = f"Invalid identifier: '{name}'." + if context: + msg += f" {context}" + super().__init__(msg) + + def __repr__(self): + return f"InvalidIdentifier('{self.name}')" class GraphNotSet(Exception): - def __repr__(self) : + def __repr__(self): return 'Graph name is not set.' class NoConnection(Exception): - def __repr__(self) : + def __repr__(self): return 'No Connection' class NoCursor(Exception): - def __repr__(self) : + def __repr__(self): return 'No Cursor' class SqlExecutionError(Exception): @@ -57,7 +92,7 @@ def __init__(self, msg, cause): self.cause = cause super().__init__(msg, cause) - def __repr__(self) : + def __repr__(self): return 'SqlExecution [' + self.msg + ']' class AGTypeError(Exception): diff --git a/drivers/python/age/models.py b/drivers/python/age/models.py index aee1b7599..6d9095485 100644 --- a/drivers/python/age/models.py +++ b/drivers/python/age/models.py @@ -1,294 +1,294 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import json -from io import StringIO - - -TP_NONE = 0 -TP_VERTEX = 1 -TP_EDGE = 2 -TP_PATH = 3 - - -class Graph(): - def __init__(self, stmt=None) -> None: - self.statement = stmt - self.rows = list() - self.vertices = dict() - - def __iter__(self): - return self.rows.__iter__() - - def __len__(self): - return self.rows.__len__() - - def __getitem__(self,index): - return self.rows[index] - - def size(self): - return self.rows.__len__() - - def append(self, agObj): - self.rows.append(agObj) - - def getVertices(self): - return self.vertices - - def getVertex(self, id): - if id in self.vertices: - return self.vertices[id] - else: - return None - -class AGObj: - @property - def gtype(self): - return TP_NONE - - -class Path(AGObj): - entities = [] - def __init__(self, entities=None) -> None: - self.entities = entities - - @property - def gtype(self): - return TP_PATH - - def __iter__(self): - return self.entities.__iter__() - - def __len__(self): - return self.entities.__len__() - - def __getitem__(self,index): - return self.entities[index] - - def size(self): - return self.entities.__len__() - - def append(self, agObj:AGObj ): - self.entities.append(agObj) - - def __str__(self) -> str: - return self.toString() - - def __repr__(self) -> str: - return self.toString() - - def toString(self) -> str: - buf = StringIO() - buf.write("[") - max = len(self.entities) - idx = 0 - while idx < max: - if idx > 0: - buf.write(",") - self.entities[idx]._toString(buf) - idx += 1 - buf.write("]::PATH") - - return buf.getvalue() - - def toJson(self) -> str: - buf = StringIO() - buf.write("{\"gtype\": \"path\", \"elements\": [") - - max = len(self.entities) - idx = 0 - while idx < max: - if idx > 0: - buf.write(",") - self.entities[idx]._toJson(buf) - idx += 1 - buf.write("]}") - - return buf.getvalue() - - - - -class Vertex(AGObj): - def __init__(self, id=None, label=None, properties=None) -> None: - self.id = id - self.label = label - self.properties = properties - - @property - def gtype(self): - return TP_VERTEX - - def __setitem__(self,name, value): - self.properties[name]=value - - def __getitem__(self,name): - if name in self.properties: - return self.properties[name] - else: - return None - - def __str__(self) -> str: - return self.toString() - - def __repr__(self) -> str: - return self.toString() - - def toString(self) -> str: - return nodeToString(self) - - def _toString(self, buf): - _nodeToString(self, buf) - - def toJson(self) -> str: - return nodeToJson(self) - - def _toJson(self, buf): - _nodeToJson(self, buf) - - -class Edge(AGObj): - def __init__(self, id=None, label=None, properties=None) -> None: - self.id = id - self.label = label - self.start_id = None - self.end_id = None - self.properties = properties - - @property - def gtype(self): - return TP_EDGE - - def __setitem__(self,name, value): - self.properties[name]=value - - def __getitem__(self,name): - if name in self.properties: - return self.properties[name] - else: - return None - - def __str__(self) -> str: - return self.toString() - - def __repr__(self) -> str: - return self.toString() - - def extraStrFormat(node, buf): - if node.start_id != None: - buf.write(", start_id:") - buf.write(str(node.start_id)) - - if node.end_id != None: - buf.write(", end_id:") - buf.write(str(node.end_id)) - - - def toString(self) -> str: - return nodeToString(self, Edge.extraStrFormat) - - def _toString(self, buf): - _nodeToString(self, buf, Edge.extraStrFormat) - - def extraJsonFormat(node, buf): - if node.start_id != None: - buf.write(", \"start_id\": \"") - buf.write(str(node.start_id)) - buf.write("\"") - - if node.end_id != None: - buf.write(", \"end_id\": \"") - buf.write(str(node.end_id)) - buf.write("\"") - - def toJson(self) -> str: - return nodeToJson(self, Edge.extraJsonFormat) - - def _toJson(self, buf): - _nodeToJson(self, buf, Edge.extraJsonFormat) - - -def nodeToString(node, extraFormatter=None): - buf = StringIO() - _nodeToString(node,buf,extraFormatter=extraFormatter) - return buf.getvalue() - - -def _nodeToString(node, buf, extraFormatter=None): - buf.write("{") - if node.label != None: - buf.write("label:") - buf.write(node.label) - - if node.id != None: - buf.write(", id:") - buf.write(str(node.id)) - - if node.properties != None: - buf.write(", properties:{") - prop_list = [] - for k, v in node.properties.items(): - prop_list.append(f"{k}: {str(v)}") - - # Join properties with comma and write to buffer - buf.write(", ".join(prop_list)) - buf.write("}") - - if extraFormatter != None: - extraFormatter(node, buf) - - if node.gtype == TP_VERTEX: - buf.write("}::VERTEX") - if node.gtype == TP_EDGE: - buf.write("}::EDGE") - - -def nodeToJson(node, extraFormatter=None): - buf = StringIO() - _nodeToJson(node, buf, extraFormatter=extraFormatter) - return buf.getvalue() - - -def _nodeToJson(node, buf, extraFormatter=None): - buf.write("{\"gtype\": ") - if node.gtype == TP_VERTEX: - buf.write("\"vertex\", ") - if node.gtype == TP_EDGE: - buf.write("\"edge\", ") - - if node.label != None: - buf.write("\"label\":\"") - buf.write(node.label) - buf.write("\"") - - if node.id != None: - buf.write(", \"id\":") - buf.write(str(node.id)) - - if extraFormatter != None: - extraFormatter(node, buf) - - if node.properties != None: - buf.write(", \"properties\":{") - - prop_list = [] - for k, v in node.properties.items(): - prop_list.append(f"\"{k}\": \"{str(v)}\"") - - # Join properties with comma and write to buffer - buf.write(", ".join(prop_list)) - buf.write("}") - buf.write("}") +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from io import StringIO + + +TP_NONE = 0 +TP_VERTEX = 1 +TP_EDGE = 2 +TP_PATH = 3 + + +class Graph(): + def __init__(self, stmt=None) -> None: + self.statement = stmt + self.rows = list() + self.vertices = dict() + + def __iter__(self): + return self.rows.__iter__() + + def __len__(self): + return self.rows.__len__() + + def __getitem__(self,index): + return self.rows[index] + + def size(self): + return self.rows.__len__() + + def append(self, agObj): + self.rows.append(agObj) + + def getVertices(self): + return self.vertices + + def getVertex(self, id): + if id in self.vertices: + return self.vertices[id] + else: + return None + +class AGObj: + @property + def gtype(self): + return TP_NONE + + +class Path(AGObj): + entities = [] + def __init__(self, entities=None) -> None: + self.entities = entities + + @property + def gtype(self): + return TP_PATH + + def __iter__(self): + return self.entities.__iter__() + + def __len__(self): + return self.entities.__len__() + + def __getitem__(self,index): + return self.entities[index] + + def size(self): + return self.entities.__len__() + + def append(self, agObj:AGObj ): + self.entities.append(agObj) + + def __str__(self) -> str: + return self.toString() + + def __repr__(self) -> str: + return self.toString() + + def toString(self) -> str: + buf = StringIO() + buf.write("[") + max = len(self.entities) + idx = 0 + while idx < max: + if idx > 0: + buf.write(",") + self.entities[idx]._toString(buf) + idx += 1 + buf.write("]::PATH") + + return buf.getvalue() + + def toJson(self) -> str: + buf = StringIO() + buf.write("{\"gtype\": \"path\", \"elements\": [") + + max = len(self.entities) + idx = 0 + while idx < max: + if idx > 0: + buf.write(",") + self.entities[idx]._toJson(buf) + idx += 1 + buf.write("]}") + + return buf.getvalue() + + + + +class Vertex(AGObj): + def __init__(self, id=None, label=None, properties=None) -> None: + self.id = id + self.label = label + self.properties = properties + + @property + def gtype(self): + return TP_VERTEX + + def __setitem__(self,name, value): + self.properties[name]=value + + def __getitem__(self,name): + if name in self.properties: + return self.properties[name] + else: + return None + + def __str__(self) -> str: + return self.toString() + + def __repr__(self) -> str: + return self.toString() + + def toString(self) -> str: + return nodeToString(self) + + def _toString(self, buf): + _nodeToString(self, buf) + + def toJson(self) -> str: + return nodeToJson(self) + + def _toJson(self, buf): + _nodeToJson(self, buf) + + +class Edge(AGObj): + def __init__(self, id=None, label=None, properties=None) -> None: + self.id = id + self.label = label + self.start_id = None + self.end_id = None + self.properties = properties + + @property + def gtype(self): + return TP_EDGE + + def __setitem__(self,name, value): + self.properties[name]=value + + def __getitem__(self,name): + if name in self.properties: + return self.properties[name] + else: + return None + + def __str__(self) -> str: + return self.toString() + + def __repr__(self) -> str: + return self.toString() + + def extraStrFormat(node, buf): + if node.start_id != None: + buf.write(", start_id:") + buf.write(str(node.start_id)) + + if node.end_id != None: + buf.write(", end_id:") + buf.write(str(node.end_id)) + + + def toString(self) -> str: + return nodeToString(self, Edge.extraStrFormat) + + def _toString(self, buf): + _nodeToString(self, buf, Edge.extraStrFormat) + + def extraJsonFormat(node, buf): + if node.start_id != None: + buf.write(", \"start_id\": \"") + buf.write(str(node.start_id)) + buf.write("\"") + + if node.end_id != None: + buf.write(", \"end_id\": \"") + buf.write(str(node.end_id)) + buf.write("\"") + + def toJson(self) -> str: + return nodeToJson(self, Edge.extraJsonFormat) + + def _toJson(self, buf): + _nodeToJson(self, buf, Edge.extraJsonFormat) + + +def nodeToString(node, extraFormatter=None): + buf = StringIO() + _nodeToString(node,buf,extraFormatter=extraFormatter) + return buf.getvalue() + + +def _nodeToString(node, buf, extraFormatter=None): + buf.write("{") + if node.label != None: + buf.write("label:") + buf.write(node.label) + + if node.id != None: + buf.write(", id:") + buf.write(str(node.id)) + + if node.properties != None: + buf.write(", properties:{") + prop_list = [] + for k, v in node.properties.items(): + prop_list.append(f"{k}: {str(v)}") + + # Join properties with comma and write to buffer + buf.write(", ".join(prop_list)) + buf.write("}") + + if extraFormatter != None: + extraFormatter(node, buf) + + if node.gtype == TP_VERTEX: + buf.write("}::VERTEX") + if node.gtype == TP_EDGE: + buf.write("}::EDGE") + + +def nodeToJson(node, extraFormatter=None): + buf = StringIO() + _nodeToJson(node, buf, extraFormatter=extraFormatter) + return buf.getvalue() + + +def _nodeToJson(node, buf, extraFormatter=None): + buf.write("{\"gtype\": ") + if node.gtype == TP_VERTEX: + buf.write("\"vertex\", ") + if node.gtype == TP_EDGE: + buf.write("\"edge\", ") + + if node.label != None: + buf.write("\"label\":\"") + buf.write(node.label) + buf.write("\"") + + if node.id != None: + buf.write(", \"id\":") + buf.write(str(node.id)) + + if extraFormatter != None: + extraFormatter(node, buf) + + if node.properties != None: + buf.write(", \"properties\":{") + + prop_list = [] + for k, v in node.properties.items(): + prop_list.append(f"\"{k}\": \"{str(v)}\"") + + # Join properties with comma and write to buffer + buf.write(", ".join(prop_list)) + buf.write("}") + buf.write("}") \ No newline at end of file diff --git a/drivers/python/age/networkx/lib.py b/drivers/python/age/networkx/lib.py index 308658620..5df761eae 100644 --- a/drivers/python/age/networkx/lib.py +++ b/drivers/python/age/networkx/lib.py @@ -20,17 +20,18 @@ from psycopg import sql from typing import Dict, Any, List, Set from age.models import Vertex, Edge, Path +from age.age import validate_graph_name, validate_identifier def checkIfGraphNameExistInAGE(connection: psycopg.connect, graphName: str): """Check if the age graph exists""" + validate_graph_name(graphName) with connection.cursor() as cursor: - cursor.execute(sql.SQL(""" - SELECT count(*) - FROM ag_catalog.ag_graph - WHERE name='%s' - """ % (graphName))) + cursor.execute( + sql.SQL("SELECT count(*) FROM ag_catalog.ag_graph WHERE name={gn}") + .format(gn=sql.Literal(graphName)) + ) if cursor.fetchone()[0] == 0: raise GraphNotFound(graphName) @@ -38,11 +39,13 @@ def checkIfGraphNameExistInAGE(connection: psycopg.connect, def getOidOfGraph(connection: psycopg.connect, graphName: str) -> int: """Returns oid of a graph""" + validate_graph_name(graphName) try: with connection.cursor() as cursor: - cursor.execute(sql.SQL(""" - SELECT graphid FROM ag_catalog.ag_graph WHERE name='%s' ; - """ % (graphName))) + cursor.execute( + sql.SQL("SELECT graphid FROM ag_catalog.ag_graph WHERE name={gn}") + .format(gn=sql.Literal(graphName)) + ) oid = cursor.fetchone()[0] return oid except Exception as e: @@ -56,7 +59,9 @@ def get_vlabel(connection: psycopg.connect, try: with connection.cursor() as cursor: cursor.execute( - """SELECT name FROM ag_catalog.ag_label WHERE kind='v' AND graph=%s;""" % oid) + sql.SQL("SELECT name FROM ag_catalog.ag_label WHERE kind='v' AND graph={oid}") + .format(oid=sql.Literal(oid)) + ) for row in cursor: node_label_list.append(row[0]) @@ -69,18 +74,19 @@ def create_vlabel(connection: psycopg.connect, graphName: str, node_label_list: List): """create_vlabels from list if not exist""" + validate_graph_name(graphName) try: node_label_set = set(get_vlabel(connection, graphName)) - crete_label_statement = '' for label in node_label_list: if label in node_label_set: continue - crete_label_statement += """SELECT create_vlabel('%s','%s');\n""" % ( - graphName, label) - if crete_label_statement != '': + validate_identifier(label, "Vertex label") with connection.cursor() as cursor: - cursor.execute(crete_label_statement) - connection.commit() + cursor.execute( + sql.SQL("SELECT create_vlabel({gn},{lbl})") + .format(gn=sql.Literal(graphName), lbl=sql.Literal(label)) + ) + connection.commit() except Exception as e: raise Exception(e) @@ -92,7 +98,9 @@ def get_elabel(connection: psycopg.connect, try: with connection.cursor() as cursor: cursor.execute( - """SELECT name FROM ag_catalog.ag_label WHERE kind='e' AND graph=%s;""" % oid) + sql.SQL("SELECT name FROM ag_catalog.ag_label WHERE kind='e' AND graph={oid}") + .format(oid=sql.Literal(oid)) + ) for row in cursor: edge_label_list.append(row[0]) except Exception as ex: @@ -103,19 +111,20 @@ def get_elabel(connection: psycopg.connect, def create_elabel(connection: psycopg.connect, graphName: str, edge_label_list: List): - """create_vlabels from list if not exist""" + """create_elabels from list if not exist""" + validate_graph_name(graphName) try: edge_label_set = set(get_elabel(connection, graphName)) - crete_label_statement = '' for label in edge_label_list: if label in edge_label_set: continue - crete_label_statement += """SELECT create_elabel('%s','%s');\n""" % ( - graphName, label) - if crete_label_statement != '': + validate_identifier(label, "Edge label") with connection.cursor() as cursor: - cursor.execute(crete_label_statement) - connection.commit() + cursor.execute( + sql.SQL("SELECT create_elabel({gn},{lbl})") + .format(gn=sql.Literal(graphName), lbl=sql.Literal(label)) + ) + connection.commit() except Exception as e: raise Exception(e) @@ -171,6 +180,7 @@ def getEdgeLabelListAfterPreprocessing(G: nx.DiGraph): def addAllNodesIntoAGE(connection: psycopg.connect, graphName: str, G: nx.DiGraph, node_label_list: Set): """Add all node to AGE""" + validate_graph_name(graphName) try: queue_data = {label: [] for label in node_label_list} id_data = {} @@ -180,8 +190,11 @@ def addAllNodesIntoAGE(connection: psycopg.connect, graphName: str, G: nx.DiGrap queue_data[data['label']].append((json_string,)) for label, rows in queue_data.items(): - table_name = """%s."%s" """ % (graphName, label) - insert_query = f"INSERT INTO {table_name} (properties) VALUES (%s) RETURNING id" + validate_identifier(label, "Node label") + insert_query = sql.SQL("INSERT INTO {schema}.{table} (properties) VALUES (%s) RETURNING id").format( + schema=sql.Identifier(graphName), + table=sql.Identifier(label) + ) cursor = connection.cursor() cursor.executemany(insert_query, rows, returning=True) ids = [] @@ -205,6 +218,7 @@ def addAllNodesIntoAGE(connection: psycopg.connect, graphName: str, G: nx.DiGrap def addAllEdgesIntoAGE(connection: psycopg.connect, graphName: str, G: nx.DiGraph, edge_label_list: Set): """Add all edge to AGE""" + validate_graph_name(graphName) try: queue_data = {label: [] for label in edge_label_list} for u, v, data in G.edges(data=True): @@ -213,8 +227,11 @@ def addAllEdgesIntoAGE(connection: psycopg.connect, graphName: str, G: nx.DiGrap (G.nodes[u]['properties']['__gid__'], G.nodes[v]['properties']['__gid__'], json_string,)) for label, rows in queue_data.items(): - table_name = """%s."%s" """ % (graphName, label) - insert_query = f"INSERT INTO {table_name} (start_id,end_id,properties) VALUES (%s, %s, %s)" + validate_identifier(label, "Edge label") + insert_query = sql.SQL("INSERT INTO {schema}.{table} (start_id,end_id,properties) VALUES (%s, %s, %s)").format( + schema=sql.Identifier(graphName), + table=sql.Identifier(label) + ) cursor = connection.cursor() cursor.executemany(insert_query, rows) connection.commit() @@ -225,14 +242,19 @@ def addAllEdgesIntoAGE(connection: psycopg.connect, graphName: str, G: nx.DiGrap def addAllNodesIntoNetworkx(connection: psycopg.connect, graphName: str, G: nx.DiGraph): """Add all nodes to Networkx""" + validate_graph_name(graphName) node_label_list = get_vlabel(connection, graphName) try: for label in node_label_list: + validate_identifier(label, "Node label") with connection.cursor() as cursor: - cursor.execute(""" - SELECT id, CAST(properties AS VARCHAR) - FROM %s."%s"; - """ % (graphName, label)) + cursor.execute( + sql.SQL("SELECT id, CAST(properties AS VARCHAR) FROM {schema}.{table}") + .format( + schema=sql.Identifier(graphName), + table=sql.Identifier(label) + ) + ) rows = cursor.fetchall() for row in rows: G.add_node(int(row[0]), label=label, @@ -243,14 +265,19 @@ def addAllNodesIntoNetworkx(connection: psycopg.connect, graphName: str, G: nx.D def addAllEdgesIntoNetworkx(connection: psycopg.connect, graphName: str, G: nx.DiGraph): """Add All edges to Networkx""" + validate_graph_name(graphName) try: edge_label_list = get_elabel(connection, graphName) for label in edge_label_list: + validate_identifier(label, "Edge label") with connection.cursor() as cursor: - cursor.execute(""" - SELECT start_id, end_id, CAST(properties AS VARCHAR) - FROM %s."%s"; - """ % (graphName, label)) + cursor.execute( + sql.SQL("SELECT start_id, end_id, CAST(properties AS VARCHAR) FROM {schema}.{table}") + .format( + schema=sql.Identifier(graphName), + table=sql.Identifier(label) + ) + ) rows = cursor.fetchall() for row in rows: G.add_edge(int(row[0]), int( diff --git a/drivers/python/requirements.txt b/drivers/python/requirements.txt index b0593b79218c7fc2ac7cff48c8fcabb87b24fd10..449d38c673c0d668086c21e2481caa9693dca840 100644 GIT binary patch literal 59 zcmXRYu1wA^Nasq-E6FJ`(JiPf$;i($)-5W{E6L1FwY4?TGc?pQ|;&(A65 O%1bRN&o9cZ-~s@K1{F5| literal 176 zcmY+6K?=e^5CrQi_=kKTf|ygl$3zJljGJY%qWOHZHaBS)da9@AyGCXfu1rL3RMaZC z)m#{K9m%|+)s3pv|9AH6%mUdo(b$YOGIzfOPVR}PM^$F&&+_b5bWUoN N6dpGImLwj0_yN-MAWQ%N diff --git a/drivers/python/setup.py b/drivers/python/setup.py index d0eed26be..853f1006a 100644 --- a/drivers/python/setup.py +++ b/drivers/python/setup.py @@ -1,22 +1,22 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This setup.py is maintained for backward compatibility. -# All package configuration is in pyproject.toml. For installation, -# use: pip install . - -from setuptools import setup - -setup() +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This setup.py is maintained for backward compatibility. +# All package configuration is in pyproject.toml. For installation, +# use: pip install . + +from setuptools import setup + +setup() diff --git a/drivers/python/test_agtypes.py b/drivers/python/test_agtypes.py index 69bbbc298..4e9752e61 100644 --- a/drivers/python/test_agtypes.py +++ b/drivers/python/test_agtypes.py @@ -1,132 +1,132 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -from decimal import Decimal -import math -import age - - -class TestAgtype(unittest.TestCase): - resultHandler = None - - def __init__(self, methodName: str) -> None: - super().__init__(methodName=methodName) - self.resultHandler = age.newResultHandler() - - def parse(self, exp): - return self.resultHandler.parse(exp) - - def test_scalar(self): - print("\nTesting Scalar Value Parsing. Result : ", end='') - - mapStr = '{"name": "Smith", "num":123, "yn":true, "bigInt":123456789123456789123456789123456789::numeric}' - arrStr = '["name", "Smith", "num", 123, "yn", true, 123456789123456789123456789123456789.8888::numeric]' - strStr = '"abcd"' - intStr = '1234' - floatStr = '1234.56789' - floatStr2 = '6.45161290322581e+46' - numericStr1 = '12345678901234567890123456789123456789.789::numeric' - numericStr2 = '12345678901234567890123456789123456789::numeric' - boolStr = 'true' - nullStr = '' - nanStr = "NaN" - infpStr = "Infinity" - infnStr = "-Infinity" - - mapVal = self.parse(mapStr) - arrVal = self.parse(arrStr) - str = self.parse(strStr) - intVal = self.parse(intStr) - floatVal = self.parse(floatStr) - floatVal2 = self.parse(floatStr2) - bigFloat = self.parse(numericStr1) - bigInt = self.parse(numericStr2) - boolVal = self.parse(boolStr) - nullVal = self.parse(nullStr) - nanVal = self.parse(nanStr) - infpVal = self.parse(infpStr) - infnVal = self.parse(infnStr) - - self.assertEqual(mapVal, {'name': 'Smith', 'num': 123, 'yn': True, 'bigInt': Decimal( - '123456789123456789123456789123456789')}) - self.assertEqual(arrVal, ["name", "Smith", "num", 123, "yn", True, Decimal( - "123456789123456789123456789123456789.8888")]) - self.assertEqual(str, "abcd") - self.assertEqual(intVal, 1234) - self.assertEqual(floatVal, 1234.56789) - self.assertEqual(floatVal2, 6.45161290322581e+46) - self.assertEqual(bigFloat, Decimal( - "12345678901234567890123456789123456789.789")) - self.assertEqual(bigInt, Decimal( - "12345678901234567890123456789123456789")) - self.assertEqual(boolVal, True) - self.assertTrue(math.isnan(nanVal)) - self.assertTrue(math.isinf(infpVal)) - self.assertTrue(math.isinf(infnVal)) - - def test_vertex(self): - - print("\nTesting vertex Parsing. Result : ", end='') - - vertexExp = '''{"id": 2251799813685425, "label": "Person", - "properties": {"name": "Smith", "numInt":123, "numFloat": 384.23424, - "bigInt":123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789::numeric, - "bigFloat":123456789123456789123456789123456789.12345::numeric, - "yn":true, "nullVal": null}}::vertex''' - - vertex = self.parse(vertexExp) - self.assertEqual(vertex.id, 2251799813685425) - self.assertEqual(vertex.label, "Person") - self.assertEqual(vertex["name"], "Smith") - self.assertEqual(vertex["numInt"], 123) - self.assertEqual(vertex["numFloat"], 384.23424) - self.assertEqual(vertex["bigInt"], Decimal( - "123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789")) - self.assertEqual(vertex["bigFloat"], Decimal( - "123456789123456789123456789123456789.12345")) - self.assertEqual(vertex["yn"], True) - self.assertEqual(vertex["nullVal"], None) - - def test_path(self): - - print("\nTesting Path Parsing. Result : ", end='') - - pathExp = '''[{"id": 2251799813685425, "label": "Person", "properties": {"name": "Smith"}}::vertex, - {"id": 2533274790396576, "label": "workWith", "end_id": 2251799813685425, "start_id": 2251799813685424, - "properties": {"weight": 3, "bigFloat":123456789123456789123456789.12345::numeric}}::edge, - {"id": 2251799813685424, "label": "Person", "properties": {"name": "Joe"}}::vertex]::path''' - - path = self.parse(pathExp) - vertexStart = path[0] - edge = path[1] - vertexEnd = path[2] - self.assertEqual(vertexStart.id, 2251799813685425) - self.assertEqual(vertexStart.label, "Person") - self.assertEqual(vertexStart["name"], "Smith") - - self.assertEqual(edge.id, 2533274790396576) - self.assertEqual(edge.label, "workWith") - self.assertEqual(edge["weight"], 3) - self.assertEqual(edge["bigFloat"], Decimal( - "123456789123456789123456789.12345")) - - self.assertEqual(vertexEnd.id, 2251799813685424) - self.assertEqual(vertexEnd.label, "Person") - self.assertEqual(vertexEnd["name"], "Joe") - - -if __name__ == '__main__': - unittest.main() +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +from decimal import Decimal +import math +import age + + +class TestAgtype(unittest.TestCase): + resultHandler = None + + def __init__(self, methodName: str) -> None: + super().__init__(methodName=methodName) + self.resultHandler = age.newResultHandler() + + def parse(self, exp): + return self.resultHandler.parse(exp) + + def test_scalar(self): + print("\nTesting Scalar Value Parsing. Result : ", end='') + + mapStr = '{"name": "Smith", "num":123, "yn":true, "bigInt":123456789123456789123456789123456789::numeric}' + arrStr = '["name", "Smith", "num", 123, "yn", true, 123456789123456789123456789123456789.8888::numeric]' + strStr = '"abcd"' + intStr = '1234' + floatStr = '1234.56789' + floatStr2 = '6.45161290322581e+46' + numericStr1 = '12345678901234567890123456789123456789.789::numeric' + numericStr2 = '12345678901234567890123456789123456789::numeric' + boolStr = 'true' + nullStr = '' + nanStr = "NaN" + infpStr = "Infinity" + infnStr = "-Infinity" + + mapVal = self.parse(mapStr) + arrVal = self.parse(arrStr) + str = self.parse(strStr) + intVal = self.parse(intStr) + floatVal = self.parse(floatStr) + floatVal2 = self.parse(floatStr2) + bigFloat = self.parse(numericStr1) + bigInt = self.parse(numericStr2) + boolVal = self.parse(boolStr) + nullVal = self.parse(nullStr) + nanVal = self.parse(nanStr) + infpVal = self.parse(infpStr) + infnVal = self.parse(infnStr) + + self.assertEqual(mapVal, {'name': 'Smith', 'num': 123, 'yn': True, 'bigInt': Decimal( + '123456789123456789123456789123456789')}) + self.assertEqual(arrVal, ["name", "Smith", "num", 123, "yn", True, Decimal( + "123456789123456789123456789123456789.8888")]) + self.assertEqual(str, "abcd") + self.assertEqual(intVal, 1234) + self.assertEqual(floatVal, 1234.56789) + self.assertEqual(floatVal2, 6.45161290322581e+46) + self.assertEqual(bigFloat, Decimal( + "12345678901234567890123456789123456789.789")) + self.assertEqual(bigInt, Decimal( + "12345678901234567890123456789123456789")) + self.assertEqual(boolVal, True) + self.assertTrue(math.isnan(nanVal)) + self.assertTrue(math.isinf(infpVal)) + self.assertTrue(math.isinf(infnVal)) + + def test_vertex(self): + + print("\nTesting vertex Parsing. Result : ", end='') + + vertexExp = '''{"id": 2251799813685425, "label": "Person", + "properties": {"name": "Smith", "numInt":123, "numFloat": 384.23424, + "bigInt":123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789::numeric, + "bigFloat":123456789123456789123456789123456789.12345::numeric, + "yn":true, "nullVal": null}}::vertex''' + + vertex = self.parse(vertexExp) + self.assertEqual(vertex.id, 2251799813685425) + self.assertEqual(vertex.label, "Person") + self.assertEqual(vertex["name"], "Smith") + self.assertEqual(vertex["numInt"], 123) + self.assertEqual(vertex["numFloat"], 384.23424) + self.assertEqual(vertex["bigInt"], Decimal( + "123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789123456789")) + self.assertEqual(vertex["bigFloat"], Decimal( + "123456789123456789123456789123456789.12345")) + self.assertEqual(vertex["yn"], True) + self.assertEqual(vertex["nullVal"], None) + + def test_path(self): + + print("\nTesting Path Parsing. Result : ", end='') + + pathExp = '''[{"id": 2251799813685425, "label": "Person", "properties": {"name": "Smith"}}::vertex, + {"id": 2533274790396576, "label": "workWith", "end_id": 2251799813685425, "start_id": 2251799813685424, + "properties": {"weight": 3, "bigFloat":123456789123456789123456789.12345::numeric}}::edge, + {"id": 2251799813685424, "label": "Person", "properties": {"name": "Joe"}}::vertex]::path''' + + path = self.parse(pathExp) + vertexStart = path[0] + edge = path[1] + vertexEnd = path[2] + self.assertEqual(vertexStart.id, 2251799813685425) + self.assertEqual(vertexStart.label, "Person") + self.assertEqual(vertexStart["name"], "Smith") + + self.assertEqual(edge.id, 2533274790396576) + self.assertEqual(edge.label, "workWith") + self.assertEqual(edge["weight"], 3) + self.assertEqual(edge["bigFloat"], Decimal( + "123456789123456789123456789.12345")) + + self.assertEqual(vertexEnd.id, 2251799813685424) + self.assertEqual(vertexEnd.label, "Person") + self.assertEqual(vertexEnd["name"], "Joe") + + +if __name__ == '__main__': + unittest.main() diff --git a/drivers/python/test_networkx.py b/drivers/python/test_networkx.py index 310d2cf5e..dbaaf8664 100644 --- a/drivers/python/test_networkx.py +++ b/drivers/python/test_networkx.py @@ -224,7 +224,7 @@ def test_existing_graph(self): with self.assertRaises(GraphNotFound) as context: age_to_networkx(ag.connection, graphName=non_existing_graph) # Check the raised exception has the expected error message - self.assertEqual(str(context.exception), non_existing_graph) + self.assertIn(non_existing_graph, str(context.exception)) class TestNetworkxToAGE(unittest.TestCase): diff --git a/drivers/python/test_security.py b/drivers/python/test_security.py new file mode 100644 index 000000000..55347868e --- /dev/null +++ b/drivers/python/test_security.py @@ -0,0 +1,274 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Security tests for the Apache AGE Python driver. + +Tests input validation, SQL injection prevention, and exception handling. +""" + +import unittest +from age.age import ( + validate_graph_name, + validate_identifier, + buildCypher, + _validate_column, +) +from age.exceptions import ( + AgeNotSet, + GraphNotFound, + GraphAlreadyExists, + GraphNotSet, + InvalidGraphName, + InvalidIdentifier, +) + + +class TestGraphNameValidation(unittest.TestCase): + """Test validate_graph_name rejects dangerous inputs.""" + + def test_rejects_empty_string(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name('') + + def test_rejects_none(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name(None) + + def test_rejects_non_string(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name(123) + + def test_rejects_digit_start(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name('123graph') + + def test_rejects_sql_injection_drop_table(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name("'; DROP TABLE ag_graph; --") + + def test_rejects_sql_injection_semicolon(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name("test'); DROP TABLE users; --") + + def test_rejects_sql_injection_select(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name("graph; SELECT * FROM pg_shadow") + + def test_accepts_hyphenated_graph_name(self): + # AGE allows hyphens in middle positions of graph names. + validate_graph_name('my-graph') + + def test_rejects_space(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name('my graph') + + def test_accepts_dotted_graph_name(self): + # AGE allows dots in middle positions of graph names. + validate_graph_name('my.graph') + + def test_rejects_dollar(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name('my$graph') + + def test_rejects_exceeding_63_chars(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name('a' * 64) + + def test_accepts_valid_names(self): + # These should NOT raise + validate_graph_name('my_graph') + validate_graph_name('MyGraph') + validate_graph_name('_pr_ivate') + validate_graph_name('graph123') + validate_graph_name('my-graph') + validate_graph_name('my.graph') + validate_graph_name('a-b.c_d') + validate_graph_name('abc') + validate_graph_name('a' * 63) + + def test_rejects_shorter_than_3_chars(self): + # AGE requires minimum 3 character graph names. + with self.assertRaises(InvalidGraphName): + validate_graph_name('a') + with self.assertRaises(InvalidGraphName): + validate_graph_name('ab') + + def test_rejects_name_ending_with_hyphen(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name('graph-') + + def test_rejects_name_ending_with_dot(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name('graph.') + + def test_rejects_name_starting_with_hyphen(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name('-graph') + + def test_rejects_name_starting_with_dot(self): + with self.assertRaises(InvalidGraphName): + validate_graph_name('.graph') + + def test_error_message_contains_name(self): + try: + validate_graph_name("bad;name") + self.fail("Expected InvalidGraphName") + except InvalidGraphName as e: + self.assertIn("bad;name", str(e)) + self.assertIn("Invalid graph name", str(e)) + + +class TestIdentifierValidation(unittest.TestCase): + """Test validate_identifier rejects dangerous inputs.""" + + def test_rejects_empty_string(self): + with self.assertRaises(InvalidIdentifier): + validate_identifier('') + + def test_rejects_none(self): + with self.assertRaises(InvalidIdentifier): + validate_identifier(None) + + def test_rejects_sql_injection(self): + with self.assertRaises(InvalidIdentifier): + validate_identifier("Person'; DROP TABLE--") + + def test_rejects_special_chars(self): + with self.assertRaises(InvalidIdentifier): + validate_identifier("col; DROP TABLE") + + def test_accepts_valid_identifiers(self): + validate_identifier('Person') + validate_identifier('KNOWS') + validate_identifier('_internal') + validate_identifier('col1') + + def test_error_includes_context(self): + try: + validate_identifier("bad;name", "Column name") + self.fail("Expected InvalidIdentifier") + except InvalidIdentifier as e: + self.assertIn("Column name", str(e)) + + +class TestColumnValidation(unittest.TestCase): + """Test _validate_column prevents injection through column specs.""" + + def test_plain_column_name(self): + self.assertEqual(_validate_column('v'), 'v agtype') + + def test_column_with_type(self): + self.assertEqual(_validate_column('n agtype'), 'n agtype') + + def test_empty_column(self): + self.assertEqual(_validate_column(''), '') + self.assertEqual(_validate_column(' '), '') + + def test_rejects_injection_in_column_name(self): + with self.assertRaises(InvalidIdentifier): + _validate_column("v); DROP TABLE ag_graph; --") + + def test_rejects_injection_in_column_type(self): + with self.assertRaises(InvalidIdentifier): + _validate_column("v agtype); DROP TABLE") + + def test_rejects_three_part_column(self): + with self.assertRaises(InvalidIdentifier): + _validate_column("a b c") + + def test_rejects_semicolon_in_name(self): + with self.assertRaises(InvalidIdentifier): + _validate_column("col;") + + +class TestBuildCypher(unittest.TestCase): + """Test buildCypher validates columns and rejects injection.""" + + def test_default_column(self): + result = buildCypher('test_graph', 'MATCH (n) RETURN n', None) + self.assertIn('v agtype', result) + + def test_single_column(self): + result = buildCypher('test_graph', 'MATCH (n) RETURN n', ['n']) + self.assertIn('n agtype', result) + + def test_typed_column(self): + result = buildCypher('test_graph', 'MATCH (n) RETURN n', ['n agtype']) + self.assertIn('n agtype', result) + + def test_multiple_columns(self): + result = buildCypher('test_graph', 'MATCH (n) RETURN n', ['a', 'b']) + self.assertIn('a agtype', result) + self.assertIn('b agtype', result) + + def test_rejects_injection_in_column(self): + with self.assertRaises(InvalidIdentifier): + buildCypher('test_graph', 'MATCH (n) RETURN n', + ["v); DROP TABLE ag_graph;--"]) + + def test_rejects_none_graph_name(self): + with self.assertRaises(GraphNotSet): + buildCypher(None, 'MATCH (n) RETURN n', None) + + +class TestExceptionConstructors(unittest.TestCase): + """Test that exception constructors work correctly.""" + + def test_age_not_set_no_args(self): + """AgeNotSet() must work without arguments (previously crashed).""" + e = AgeNotSet() + self.assertIsNone(e.name) + self.assertIn('not set', repr(e)) + + def test_age_not_set_with_message(self): + e = AgeNotSet("custom message") + self.assertEqual(e.name, "custom message") + + def test_graph_not_found_no_args(self): + e = GraphNotFound() + self.assertIsNone(e.name) + self.assertIn('does not exist', repr(e)) + + def test_graph_not_found_with_name(self): + e = GraphNotFound("test_graph") + self.assertEqual(e.name, "test_graph") + self.assertIn('test_graph', repr(e)) + + def test_graph_already_exists_no_args(self): + e = GraphAlreadyExists() + self.assertIsNone(e.name) + self.assertIn('already exists', repr(e)) + + def test_graph_already_exists_with_name(self): + e = GraphAlreadyExists("test_graph") + self.assertEqual(e.name, "test_graph") + self.assertIn('test_graph', repr(e)) + + def test_invalid_graph_name_fields(self): + e = InvalidGraphName("bad;name", "must be valid") + self.assertEqual(e.name, "bad;name") + self.assertEqual(e.reason, "must be valid") + self.assertIn("bad;name", str(e)) + self.assertIn("must be valid", str(e)) + + def test_invalid_identifier_fields(self): + e = InvalidIdentifier("col;drop", "Column name") + self.assertEqual(e.name, "col;drop") + self.assertEqual(e.context, "Column name") + self.assertIn("col;drop", str(e)) + + +if __name__ == '__main__': + unittest.main() From 77a16ece0bbd8f137ad40b4e203c9622c249352d Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Fri, 13 Feb 2026 09:31:35 -0800 Subject: [PATCH 028/100] Fix security vulnerabilities in Node.js driver (#2329) Fix security vulnerabilities in Node.js driver and harden input validation and query construction. Note: This PR was created with AI tools and a human. - Add input validation for graph names, label names, and column names to prevent SQL injection via string interpolation. Graph name rules are based on Apache AGE's naming conventions and Neo4j/openCypher compatibility (hyphens and dots permitted, min 3 chars, max 63 chars). Label names follow AGE's stricter rules (no hyphens or dots). - Add design documentation noting intentional ASCII-only restriction in driver-side regex validation as a security hardening measure (homoglyph/encoding attack surface reduction). AGE's server-side validation (name_validation.h) uses full Unicode ID_Start/ID_Continue and remains the authoritative check for Unicode names. - Add safe query helpers: queryCypher(), createGraph(), dropGraph() with graph name validation and dollar-quoting for Cypher strings - Add runtime typeof check on dropGraph cascade parameter to prevent injection from plain JavaScript consumers (TypeScript types are erased at runtime) - Use BigInt for integer values exceeding Number.MAX_SAFE_INTEGER to prevent silent precision loss with 64-bit AGE graph IDs - Make CREATE EXTENSION opt-in via SetAGETypesOptions.createExtension instead of running DDL automatically without user consent - Wrap LOAD/search_path setup in try/catch with actionable error message mentioning CREATE EXTENSION and { createExtension: true } - Improve agtype-not-found error message with installation guidance - Tighten pg dependency from >=6.0.0 to >=8.0.0 - Add comprehensive test suites covering validation, SQL injection prevention, cascade type safety, hyphenated graph name integration, BigInt parsing, and setAGETypes error handling - Add design note in tests documenting why createExtension: false is the correct default (CI image has AGE pre-installed, auto-creating extensions requires SUPERUSER and conflates concerns) modified: drivers/nodejs/package.json modified: drivers/nodejs/src/antlr4/CustomAgTypeListener.ts modified: drivers/nodejs/src/index.ts modified: drivers/nodejs/test/Agtype.test.ts modified: drivers/nodejs/test/index.test.ts --- drivers/nodejs/package.json | 2 +- .../nodejs/src/antlr4/CustomAgTypeListener.ts | 6 +- drivers/nodejs/src/index.ts | 301 +++++++++++++++++- drivers/nodejs/test/Agtype.test.ts | 27 ++ drivers/nodejs/test/index.test.ts | 285 +++++++++++++---- 5 files changed, 539 insertions(+), 82 deletions(-) diff --git a/drivers/nodejs/package.json b/drivers/nodejs/package.json index 6be11c780..15c2371f4 100644 --- a/drivers/nodejs/package.json +++ b/drivers/nodejs/package.json @@ -30,7 +30,7 @@ "author": "Alex Kwak ", "dependencies": { "antlr4ts": "^0.5.0-alpha.4", - "pg": ">=6.0.0" + "pg": ">=8.0.0" }, "devDependencies": { "@types/jest": "^29.5.14", diff --git a/drivers/nodejs/src/antlr4/CustomAgTypeListener.ts b/drivers/nodejs/src/antlr4/CustomAgTypeListener.ts index fceabee2b..6089ee994 100644 --- a/drivers/nodejs/src/antlr4/CustomAgTypeListener.ts +++ b/drivers/nodejs/src/antlr4/CustomAgTypeListener.ts @@ -92,7 +92,11 @@ class CustomAgTypeListener implements AgtypeListener, ParseTreeListener { } exitIntegerValue (ctx: IntegerValueContext): void { - const value = parseInt(ctx.text) + // Use BigInt for values that exceed Number.MAX_SAFE_INTEGER to + // prevent silent precision loss with large AGE graph IDs (64-bit). + const text = ctx.text + const num = Number(text) + const value = Number.isSafeInteger(num) ? num : BigInt(text) if (!this.pushIfArray(value)) { this.lastValue = value } diff --git a/drivers/nodejs/src/index.ts b/drivers/nodejs/src/index.ts index 416cc3da5..dad004a81 100644 --- a/drivers/nodejs/src/index.ts +++ b/drivers/nodejs/src/index.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Client } from 'pg' +import { Client, QueryResult, QueryResultRow } from 'pg' import pgTypes from 'pg-types' import { CharStreams, CommonTokenStream } from 'antlr4ts' import { AgtypeLexer } from './antlr4/AgtypeLexer' @@ -25,6 +25,122 @@ import { AgtypeParser } from './antlr4/AgtypeParser' import CustomAgTypeListener from './antlr4/CustomAgTypeListener' import { ParseTreeWalker } from 'antlr4ts/tree' +/** + * Valid graph name pattern based on Apache AGE's naming conventions + * (Neo4j/openCypher compatible). Allows ASCII letters, digits, + * underscores, plus dots and hyphens in middle positions. + * + * DESIGN NOTE: AGE's server-side validation (name_validation.h) uses + * full Unicode ID_Start/ID_Continue character classes, accepting names + * like 'mydätabase' or 'mydঅtabase'. This driver intentionally restricts + * to ASCII-only as a security hardening measure — it reduces the attack + * surface for homoglyph and encoding-based injection vectors. Server-side + * validation remains the authoritative check for Unicode names. + * + * Start: ASCII letter or underscore + * Middle: ASCII letter, digit, underscore, dot, or hyphen + * End: ASCII letter, digit, or underscore + */ +const VALID_GRAPH_NAME = /^[A-Za-z_][A-Za-z0-9_.\-]*[A-Za-z0-9_]$/ + +/** + * Valid label name pattern based on Apache AGE's naming rules. + * Labels follow stricter identifier rules than graph names — dots and + * hyphens are NOT permitted in label names. + * + * Note: ASCII-only restriction (see VALID_GRAPH_NAME design note above). + */ +const VALID_LABEL_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/ + +/** + * Valid SQL identifier pattern for column names and types in the + * query result AS clause. Standard PostgreSQL unquoted identifier rules. + * + * Note: ASCII-only restriction (see VALID_GRAPH_NAME design note above). + */ +const VALID_SQL_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/ + +/** + * Validates that a graph name conforms to Apache AGE's naming conventions + * and is safe for use in SQL queries. + * + * Graph names must: + * - Be at least 3 characters and at most 63 characters + * - Start with an ASCII letter or underscore + * - Contain only ASCII letters, digits, underscores, dots, and hyphens + * - End with an ASCII letter, digit, or underscore + * + * Note: This is intentionally stricter than AGE's server-side validation + * (which accepts Unicode letters). See VALID_GRAPH_NAME design note. + * + * @param graphName - The graph name to validate + * @throws Error if the graph name is invalid + */ +function validateGraphName (graphName: string): void { + if (!graphName || typeof graphName !== 'string') { + throw new Error('Graph name must be a non-empty string') + } + if (graphName.length < 3) { + throw new Error( + `Invalid graph name: '${graphName}'. Graph names must be at least 3 characters.` + ) + } + if (graphName.length > 63) { + throw new Error('Graph name must not exceed 63 characters (PostgreSQL name limit)') + } + if (!VALID_GRAPH_NAME.test(graphName)) { + throw new Error( + `Invalid graph name: '${graphName}'. Graph names must start with a letter ` + + 'or underscore, may contain letters, digits, underscores, dots, and hyphens, ' + + 'and must end with a letter, digit, or underscore.' + ) + } +} + +/** + * Validates that a label name conforms to Apache AGE's naming rules. + * Label names are stricter than graph names — only letters, digits, + * and underscores are permitted (no dots or hyphens). + * + * @param labelName - The label name to validate + * @throws Error if the label name is invalid + */ +function validateLabelName (labelName: string): void { + if (!labelName || typeof labelName !== 'string') { + throw new Error('Label name must be a non-empty string') + } + if (labelName.length > 63) { + throw new Error('Label name must not exceed 63 characters (PostgreSQL name limit)') + } + if (!VALID_LABEL_NAME.test(labelName)) { + throw new Error( + `Invalid label name: '${labelName}'. Label names must start with a letter ` + + 'or underscore and contain only letters, digits, and underscores.' + ) + } +} + +/** + * Escapes a PostgreSQL dollar-quoted string literal by ensuring the + * cypher query does not contain the dollar-quote delimiter. If the + * default $$ delimiter conflicts, a unique tagged delimiter is used. + * + * @param cypher - The Cypher query string + * @returns An object with the delimiter and safely quoted string + */ +function cypherDollarQuote (cypher: string): { delimiter: string; quoted: string } { + if (!cypher.includes('$$')) { + return { delimiter: '$$', quoted: `$$${cypher}$$` } + } + // Generate a unique tag that doesn't appear in the cypher query + let tag = 'cypher' + let counter = 0 + while (cypher.includes(`$${tag}$`)) { + tag = `cypher${counter++}` + } + return { delimiter: `$${tag}$`, quoted: `$${tag}$${cypher}$${tag}$` } +} + function AGTypeParse (input: string) { const chars = CharStreams.fromString(input) const lexer = new AgtypeLexer(chars) @@ -36,21 +152,180 @@ function AGTypeParse (input: string) { return printer.getResult() } -async function setAGETypes (client: Client, types: typeof pgTypes) { - await client.query(` - CREATE EXTENSION IF NOT EXISTS age; - LOAD 'age'; - SET search_path = ag_catalog, "$user", public; - `) +/** + * Options for setAGETypes configuration. + */ +interface SetAGETypesOptions { + /** + * If true, will attempt to CREATE EXTENSION IF NOT EXISTS age. + * Defaults to false. Set to true only if the connected user has + * sufficient privileges. + */ + createExtension?: boolean +} + +/** + * Configures the pg client to properly parse AGE agtype results. + * + * This function: + * 1. Loads the AGE extension into the session + * 2. Sets the search_path to include ag_catalog + * 3. Registers the agtype type parser + * + * @param client - A connected pg Client instance + * @param types - The pg types module for registering type parsers + * @param options - Optional configuration settings + * @throws Error if AGE extension is not installed or agtype is not found + */ +async function setAGETypes (client: Client, types: typeof pgTypes, options?: SetAGETypesOptions) { + const createExtension = options?.createExtension ?? false + + if (createExtension) { + await client.query('CREATE EXTENSION IF NOT EXISTS age;') + } + + try { + await client.query("LOAD 'age';") + await client.query('SET search_path = ag_catalog, "$user", public;') + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err) + throw new Error( + `Failed to load AGE extension: ${msg}. ` + + 'Ensure the AGE extension is installed in the database. ' + + 'You may need to run CREATE EXTENSION age; first, or pass ' + + '{ createExtension: true } to setAGETypes().' + ) + } - const oidResults = await client.query(` - select typelem - from pg_type - where typname = '_agtype';`) + const oidResults = await client.query( + "SELECT typelem FROM pg_type WHERE typname = '_agtype';" + ) - if (oidResults.rows.length < 1) { throw new Error() } + if (oidResults.rows.length < 1) { + throw new Error( + 'AGE agtype type not found. Ensure the AGE extension is installed ' + + 'and loaded in the current database. Run CREATE EXTENSION age; first, ' + + 'or pass { createExtension: true } to setAGETypes().' + ) + } types.setTypeParser(oidResults.rows[0].typelem, AGTypeParse) } -export { setAGETypes, AGTypeParse } +/** + * Column definition for Cypher query results. + */ +interface CypherColumn { + /** Column alias name */ + name: string + /** Column type (defaults to 'agtype') */ + type?: string +} + +/** + * Executes a Cypher query safely against an AGE graph. + * + * This function validates the graph name to prevent SQL injection, + * properly quotes the Cypher query using dollar-quoting, and builds + * the required SQL wrapper. + * + * @param client - A connected pg Client instance (with AGE types set) + * @param graphName - The target graph name (must be a valid AGE graph name) + * @param cypher - The Cypher query string + * @param columns - Column definitions for the result set + * @returns The query result + * @throws Error if graphName is invalid or query fails + * + * @example + * ```typescript + * const result = await queryCypher( + * client, + * 'my_graph', + * 'MATCH (n:Person) WHERE n.name = $name RETURN n', + * [{ name: 'n' }] + * ); + * ``` + */ +async function queryCypher ( + client: Client, + graphName: string, + cypher: string, + columns: CypherColumn[] = [{ name: 'result' }] +): Promise> { + // Validate graph name against injection + validateGraphName(graphName) + + // Validate column definitions + if (!columns || columns.length === 0) { + throw new Error('At least one column definition is required') + } + + for (const col of columns) { + if (!col.name || typeof col.name !== 'string') { + throw new Error('Column name must be a non-empty string') + } + // Column names must be valid SQL identifiers + if (!VALID_SQL_IDENTIFIER.test(col.name)) { + throw new Error( + `Invalid column name: '${col.name}'. Column names must be valid SQL identifiers.` + ) + } + if (col.type && !VALID_SQL_IDENTIFIER.test(col.type)) { + throw new Error( + `Invalid column type: '${col.type}'. Column types must be valid SQL type identifiers.` + ) + } + } + + // Build column list safely + const columnList = columns + .map(col => `${col.name} ${col.type ?? 'agtype'}`) + .join(', ') + + // Safely dollar-quote the cypher query + const { quoted } = cypherDollarQuote(cypher) + + const sql = `SELECT * FROM cypher('${graphName}', ${quoted}) AS (${columnList});` + + return client.query(sql) +} + +/** + * Creates a new graph safely. + * + * @param client - A connected pg Client instance + * @param graphName - Name for the new graph (must be a valid AGE graph name) + * @throws Error if graphName is invalid + */ +async function createGraph (client: Client, graphName: string): Promise { + validateGraphName(graphName) + await client.query(`SELECT * FROM ag_catalog.create_graph('${graphName}');`) +} + +/** + * Drops an existing graph safely. + * + * @param client - A connected pg Client instance + * @param graphName - Name of the graph to drop (must be a valid AGE graph name) + * @param cascade - If true, drop dependent objects (default: false) + * @throws Error if graphName is invalid + */ +async function dropGraph (client: Client, graphName: string, cascade: boolean = false): Promise { + validateGraphName(graphName) + if (typeof cascade !== 'boolean') { + throw new Error('cascade parameter must be a boolean') + } + await client.query(`SELECT * FROM ag_catalog.drop_graph('${graphName}', ${cascade});`) +} + +export { + setAGETypes, + AGTypeParse, + queryCypher, + createGraph, + dropGraph, + validateGraphName, + validateLabelName, + SetAGETypesOptions, + CypherColumn +} diff --git a/drivers/nodejs/test/Agtype.test.ts b/drivers/nodejs/test/Agtype.test.ts index 5ae26b549..f1b1463a9 100644 --- a/drivers/nodejs/test/Agtype.test.ts +++ b/drivers/nodejs/test/Agtype.test.ts @@ -101,4 +101,31 @@ describe('Parsing', () => { })) }))) }) + + it('Large integer uses BigInt when exceeding MAX_SAFE_INTEGER', () => { + // 2^53 = 9007199254740992 exceeds MAX_SAFE_INTEGER (2^53 - 1) + const result = AGTypeParse('{"id": 9007199254740993, "label": "test", "properties": {}}::vertex') + const id = (result as Map).get('id') + expect(typeof id).toBe('bigint') + expect(id).toBe(BigInt('9007199254740993')) + }) + + it('Safe integers remain as Number type', () => { + const result = AGTypeParse('{"id": 844424930131969, "label": "test", "properties": {}}::vertex') + const id = (result as Map).get('id') + expect(typeof id).toBe('number') + expect(id).toBe(844424930131969) + }) + + it('Small integers remain as Number type', () => { + const result = AGTypeParse('42') + expect(typeof result).toBe('number') + expect(result).toBe(42) + }) + + it('Negative integers parsed correctly', () => { + const result = AGTypeParse('-100') + expect(typeof result).toBe('number') + expect(result).toBe(-100) + }) }) diff --git a/drivers/nodejs/test/index.test.ts b/drivers/nodejs/test/index.test.ts index ea386ddaa..ae2c5c319 100644 --- a/drivers/nodejs/test/index.test.ts +++ b/drivers/nodejs/test/index.test.ts @@ -17,18 +17,33 @@ * under the License. */ -import { types, Client, QueryResultRow } from 'pg' -import { setAGETypes } from '../src' +import { types, Client } from 'pg' +import { + setAGETypes, + queryCypher, + createGraph, + dropGraph, + validateGraphName, + validateLabelName +} from '../src' const config = { - user: 'postgres', - host: '127.0.0.1', - database: 'postgres', - password: 'agens', - port: 5432 + user: process.env.PGUSER || 'postgres', + password: process.env.PGPASSWORD || 'agens', + host: process.env.PGHOST || '127.0.0.1', + database: process.env.PGDATABASE || 'postgres', + port: parseInt(process.env.PGPORT || '5432', 10) } -const testGraphName = 'age-test' +const testGraphName = 'age_test' + +// DESIGN NOTE: All test suites use { createExtension: false } intentionally. +// The CI Docker image (apache/age:dev_snapshot_master) has the AGE extension +// pre-installed, matching the GitHub Actions workflow. Using createExtension: false +// is the correct security default — auto-creating extensions requires SUPERUSER +// privileges and conflates extension lifecycle management with session setup. +// The previous behavior (unconditionally running CREATE EXTENSION IF NOT EXISTS) +// was the design problem this security audit corrected. describe('Pre-connected Connection', () => { let client: Client | null @@ -36,67 +51,203 @@ describe('Pre-connected Connection', () => { beforeAll(async () => { client = new Client(config) await client.connect() - await setAGETypes(client, types) - await client.query(`SELECT create_graph('${testGraphName}');`) + await setAGETypes(client, types, { createExtension: false }) + await createGraph(client, testGraphName) }) afterAll(async () => { - await client?.query(`SELECT drop_graph('${testGraphName}', true);`) - await client?.end() - }) - it('simple CREATE & MATCH', async () => { - await client?.query(` - SELECT * - from cypher('${testGraphName}', $$ CREATE (a:Part {part_num: '123'}), - (b:Part {part_num: '345'}), - (c:Part {part_num: '456'}), - (d:Part {part_num: '789'}) - $$) as (a agtype); - `) - const results: QueryResultRow = await client?.query(` - SELECT * - from cypher('${testGraphName}', $$ - MATCH (a) RETURN a - $$) as (a agtype); - `)! - expect(results.rows).toStrictEqual( - [ - { - a : new Map(Object.entries({ - id: 844424930131969, - label: 'Part', - properties: new Map(Object.entries({ - part_num: '123' - })) - })), - }, - { - a : new Map(Object.entries({ - id: 844424930131970, - label: 'Part', - properties: new Map(Object.entries({ - part_num: '345' - })) - })), - }, - { - a : new Map(Object.entries({ - id: 844424930131971, - label: 'Part', - properties: new Map(Object.entries({ - part_num: '456' - })) - })), - }, - { - a : new Map(Object.entries({ - id: 844424930131972, - label: 'Part', - properties: new Map(Object.entries({ - part_num: '789' - })) - })), - } - ] + if (client) { + await dropGraph(client, testGraphName, true) + await client.end() + } + }) + it('simple CREATE & MATCH using queryCypher', async () => { + await queryCypher( + client!, + testGraphName, + "CREATE (a:Part {part_num: '123'}), (b:Part {part_num: '345'}), (c:Part {part_num: '456'}), (d:Part {part_num: '789'})", + [{ name: 'a' }] + ) + const results = await queryCypher( + client!, + testGraphName, + 'MATCH (a) RETURN a', + [{ name: 'a' }] ) + expect(results.rows.length).toBe(4) + // Verify node properties are preserved + const partNums = results.rows.map((row: any) => row.a.get('properties').get('part_num')) + expect(partNums).toContain('123') + expect(partNums).toContain('345') + expect(partNums).toContain('456') + expect(partNums).toContain('789') + }) +}) + +describe('Graph Name Validation', () => { + it('rejects empty graph name', () => { + expect(() => validateGraphName('')).toThrow('non-empty string') + }) + + it('rejects null/undefined graph name', () => { + expect(() => validateGraphName(null as any)).toThrow('non-empty string') + expect(() => validateGraphName(undefined as any)).toThrow('non-empty string') + }) + + it('rejects graph names shorter than 3 characters', () => { + expect(() => validateGraphName('ab')).toThrow('at least 3 characters') + expect(() => validateGraphName('a')).toThrow('at least 3 characters') + }) + + it('rejects graph names exceeding 63 characters', () => { + const longName = 'a'.repeat(64) + expect(() => validateGraphName(longName)).toThrow('63 characters') + }) + + it('rejects graph names starting with digits', () => { + expect(() => validateGraphName('123graph')).toThrow('Invalid graph name') + }) + + it('rejects graph names with SQL injection attempts', () => { + expect(() => validateGraphName("'; DROP TABLE ag_graph; --")).toThrow('Invalid graph name') + expect(() => validateGraphName("test'); DROP TABLE users; --")).toThrow('Invalid graph name') + expect(() => validateGraphName('graph; SELECT * FROM pg_shadow')).toThrow('Invalid graph name') + }) + + it('rejects graph names with disallowed characters', () => { + expect(() => validateGraphName('my graph')).toThrow('Invalid graph name') + expect(() => validateGraphName('my$graph')).toThrow('Invalid graph name') + expect(() => validateGraphName("my'graph")).toThrow('Invalid graph name') + expect(() => validateGraphName('my@graph')).toThrow('Invalid graph name') + }) + + it('rejects graph names ending with dot or hyphen', () => { + expect(() => validateGraphName('graph-')).toThrow('Invalid graph name') + expect(() => validateGraphName('graph.')).toThrow('Invalid graph name') + }) + + it('accepts graph names with hyphens (Neo4j/openCypher compatible)', () => { + expect(() => validateGraphName('my-graph')).not.toThrow() + expect(() => validateGraphName('age-test')).not.toThrow() + expect(() => validateGraphName('my-multi-part-name')).not.toThrow() + }) + + it('accepts graph names with dots', () => { + expect(() => validateGraphName('my.graph')).not.toThrow() + expect(() => validateGraphName('tenant.db')).not.toThrow() + }) + + it('accepts standard identifier graph names', () => { + expect(() => validateGraphName('my_graph')).not.toThrow() + expect(() => validateGraphName('MyGraph')).not.toThrow() + expect(() => validateGraphName('_private')).not.toThrow() + expect(() => validateGraphName('graph123')).not.toThrow() + expect(() => validateGraphName('abc')).not.toThrow() + }) +}) + +describe('Label Name Validation', () => { + it('rejects SQL injection in label names', () => { + expect(() => validateLabelName("Person'; DROP TABLE--")).toThrow('Invalid label name') + }) + + it('rejects label names with hyphens and dots (per AGE rules)', () => { + expect(() => validateLabelName('Has-Relationship')).toThrow('Invalid label name') + expect(() => validateLabelName('label.name')).toThrow('Invalid label name') + }) + + it('accepts valid label names', () => { + expect(() => validateLabelName('Person')).not.toThrow() + expect(() => validateLabelName('KNOWS')).not.toThrow() + expect(() => validateLabelName('_internal')).not.toThrow() + expect(() => validateLabelName('Label123')).not.toThrow() + }) +}) + +describe('SQL Injection Prevention', () => { + let client: Client + + beforeAll(async () => { + client = new Client(config) + await client.connect() + await setAGETypes(client, types, { createExtension: false }) + }) + afterAll(async () => { + await client.end() + }) + + it('queryCypher rejects injected graph name', async () => { + await expect( + queryCypher(client, "test'); DROP TABLE ag_graph;--", 'MATCH (n) RETURN n', [{ name: 'n' }]) + ).rejects.toThrow('Invalid graph name') + }) + + it('queryCypher rejects injected column name', async () => { + await expect( + queryCypher(client, 'age_test', 'MATCH (n) RETURN n', [{ name: "n); DROP TABLE ag_graph;--" }]) + ).rejects.toThrow('Invalid column name') + }) + + it('createGraph rejects injected graph name', async () => { + await expect( + createGraph(client, "test'); DROP TABLE ag_graph;--") + ).rejects.toThrow('Invalid graph name') + }) + + it('dropGraph rejects injected graph name', async () => { + await expect( + dropGraph(client, "test'); DROP TABLE ag_graph;--") + ).rejects.toThrow('Invalid graph name') + }) + + it('dropGraph rejects non-boolean cascade from JS', async () => { + await expect( + dropGraph(client, 'age_test', 'true; DROP TABLE ag_graph;--' as any) + ).rejects.toThrow('cascade parameter must be a boolean') + }) +}) + +describe('Hyphenated Graph Name (Neo4j/openCypher compatible)', () => { + const hyphenatedGraphName = 'age-test' + let client: Client | null + + beforeAll(async () => { + client = new Client(config) + await client.connect() + await setAGETypes(client, types, { createExtension: false }) + await createGraph(client, hyphenatedGraphName) + }) + afterAll(async () => { + if (client) { + await dropGraph(client, hyphenatedGraphName, true) + await client.end() + } + }) + + it('creates and queries a graph with hyphens in the name', async () => { + await queryCypher( + client!, + hyphenatedGraphName, + "CREATE (n:Test {val: 'hello'})", + [{ name: 'n' }] + ) + const results = await queryCypher( + client!, + hyphenatedGraphName, + 'MATCH (n:Test) RETURN n', + [{ name: 'n' }] + ) + expect(results.rows.length).toBe(1) + }) +}) + +describe('setAGETypes error handling', () => { + it('throws when client connection has been closed', async () => { + const tempClient = new Client(config) + await tempClient.connect() + await tempClient.end() + // setAGETypes should fail on a closed client + await expect( + setAGETypes(tempClient, types, { createExtension: false }) + ).rejects.toThrow() }) }) From 0c9a527860cde710175fc9c171b6bbe9aaed45d6 Mon Sep 17 00:00:00 2001 From: Maxim Korotkov Date: Fri, 13 Feb 2026 21:57:51 +0300 Subject: [PATCH 029/100] Fix null pointer handling in array iteration (#2313) Previously, when iterating through an agtype container, the code would access `elem->val` even when `elem` was null. This adds a null check to set the result type to AGTV_NULL when the element is null, preventing a potential segmentation fault. Fixes: 4274f10 ("Added the toStringList() function (#1084)") Found by PostgresPro. Signed-off-by: Maksim Korotkov --- src/backend/utils/adt/agtype.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index c552727d8..2526f41f7 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -7500,19 +7500,12 @@ Datum age_tostringlist(PG_FUNCTION_ARGS) /* TODO: check element's type, it's value, and convert it to string if possible. */ elem = get_ith_agtype_value_from_container(&agt_arg->root, i); string_elem.type = AGTV_STRING; + enum agtype_value_type elem_type = elem ? elem->type : AGTV_NULL; - switch (elem->type) + switch (elem_type) { case AGTV_STRING: - if(!elem) - { - string_elem.type = AGTV_NULL; - - agis_result.res = push_agtype_value(&agis_result.parse_state, - WAGT_ELEM, &string_elem); - } - string_elem.val.string.val = elem->val.string.val; string_elem.val.string.len = elem->val.string.len; From fa91350b0a524e596e4fc85c9086565c60c00548 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Fri, 13 Feb 2026 23:24:54 -0800 Subject: [PATCH 030/100] Fix ISO C90 forbids mixed declarations and code warning (#2334) Fixed the following ISO C90 warning - src/backend/utils/adt/agtype.c:7503:9: warning: ISO C90 forbids mixed declarations and code [-Wdeclaration-after-statement] 7503 | enum agtype_value_type elem_type = elem ? elem->type : AGTV_NULL; | ^~~~ No regression tests impacted. modified: src/backend/utils/adt/agtype.c --- src/backend/utils/adt/agtype.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index 2526f41f7..c0c54e5a4 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -7498,9 +7498,11 @@ Datum age_tostringlist(PG_FUNCTION_ARGS) for (i = 0; i < count; i++) { /* TODO: check element's type, it's value, and convert it to string if possible. */ + enum agtype_value_type elem_type; + elem = get_ith_agtype_value_from_container(&agt_arg->root, i); string_elem.type = AGTV_STRING; - enum agtype_value_type elem_type = elem ? elem->type : AGTV_NULL; + elem_type = elem ? elem->type : AGTV_NULL; switch (elem_type) { From 3c4d9cc7456703f37d3f2d02e4cc79731b06542a Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Mon, 16 Feb 2026 22:09:01 +0500 Subject: [PATCH 031/100] Remove labeler github action (#2335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - dropped the labeler workflow because `pull_request_target` trigger can be dangerous and the workflow wasn’t particularly useful. --- .github/labeler.yml | 26 -------------------------- .github/workflows/labeler.yml | 31 ------------------------------- 2 files changed, 57 deletions(-) delete mode 100644 .github/labeler.yml delete mode 100644 .github/workflows/labeler.yml diff --git a/.github/labeler.yml b/.github/labeler.yml deleted file mode 100644 index f860c2b19..000000000 --- a/.github/labeler.yml +++ /dev/null @@ -1,26 +0,0 @@ -PG11: -- base-branch: 'PG11' - -PG12: -- base-branch: 'PG12' - -PG13: -- base-branch: 'PG13' - -PG14: -- base-branch: 'PG14' - -PG15: -- base-branch: 'PG15' - -PG16: -- base-branch: 'PG16' - -PG17: -- base-branch: 'PG17' - -PG18: -- base-branch: 'PG18' - -master: -- base-branch: 'master' diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml deleted file mode 100644 index d5fc8c835..000000000 --- a/.github/workflows/labeler.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: "Pull Request Labeler" -on: -- pull_request_target - -jobs: - triage: - permissions: - contents: write - pull-requests: write - issues: write - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Apply branch labels - uses: actions/labeler@v5.0.0 - - - name: Apply label based on author - if: | - contains('["jrgemignani", "dehowef", "eyab" "rafsun42", "Zainab-Saad", "MuhammadTahaNaveed"]', github.event.pull_request.user.login) - uses: actions/github-script@v7 - with: - script: | - const labelsToAdd = ['override-stale']; - github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: labelsToAdd - }); From 55476ad2875f37f5b65919b6eeab7d81fd65aa7c Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Wed, 25 Feb 2026 09:41:41 -0800 Subject: [PATCH 032/100] fix incorrect variable assignment (#2336) Fixed an incorrect variable assignment, that was causing a warning message during compilation, on some compilers. The create_index_on_column function assigned InvalidOid, instead of NIL or NULL. - index_col->collation = InvalidOid; + index_col->collation = NIL; No regression tests were impacted. modified: src/backend/commands/label_commands.c --- src/backend/commands/label_commands.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/commands/label_commands.c b/src/backend/commands/label_commands.c index c9fdfad89..051bbc8a0 100644 --- a/src/backend/commands/label_commands.c +++ b/src/backend/commands/label_commands.c @@ -448,7 +448,7 @@ static void create_index_on_column(char *schema_name, index_col->name = colname; index_col->expr = NULL; index_col->indexcolname = NULL; - index_col->collation = InvalidOid; + index_col->collation = NIL; index_col->opclass = list_make1(makeString("graphid_ops")); index_col->opclassopts = NIL; index_col->ordering = SORTBY_DEFAULT; From 5005c21e5c2aa5daaca909fee7c4f9ed8ccdf984 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Thu, 26 Feb 2026 18:30:56 -0500 Subject: [PATCH 033/100] Fix VLE NULL handling for chained OPTIONAL MATCH (#2337) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix VLE NULL handling for chained OPTIONAL MATCH (#2092) VLE functions (age_match_vle_terminal_edge, age_match_two_vle_edges, age_match_vle_edge_to_id_qual) threw errors when receiving NULL arguments from OPTIONAL MATCH (LEFT JOIN) contexts. Additionally, build_local_vle_context crashed with a segfault when dereferencing a NULL next_vertex pointer in the cached VLE context path. These functions are used as join quals. In a LEFT JOIN, NULL arguments mean the inner side produced no match. The correct response is FALSE (no match), which lets PostgreSQL emit NULL-extended rows — the expected OPTIONAL MATCH behavior. Errors or crashes are incorrect. Changes: - build_local_vle_context: guard against NULL next_vertex in cached path; return NULL when vertex list is exhausted - age_vle: handle NULL return from build_local_vle_context with SRF_RETURN_DONE - age_match_vle_terminal_edge: return FALSE on NULL arguments instead of ereport(ERROR) - age_match_two_vle_edges: return FALSE on NULL arguments - age_match_vle_edge_to_id_qual: return FALSE on NULL arguments All 32 regression tests pass including new tests for this fix. * Address review feedback: fix error message and add ORDER BY to tests - Fix errmsg in age_match_vle_terminal_edge() to use the correct function name (was age_match_terminal_edge) - Add ORDER BY p.name to regression test queries to avoid nondeterministic row ordering in expected output AI-assisted: Claude (Anthropic) was used in developing this fix. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- regress/expected/cypher_vle.out | 89 +++++++++++++++++++++++++++++++++ regress/sql/cypher_vle.sql | 53 ++++++++++++++++++++ src/backend/utils/adt/age_vle.c | 62 ++++++++++++++++------- 3 files changed, 185 insertions(+), 19 deletions(-) diff --git a/regress/expected/cypher_vle.out b/regress/expected/cypher_vle.out index 57f930d98..6574e0608 100644 --- a/regress/expected/cypher_vle.out +++ b/regress/expected/cypher_vle.out @@ -1109,6 +1109,95 @@ NOTICE: graph "issue_1910" has been dropped (1 row) +-- issue 2092: VLE with chained OPTIONAL MATCH and NULL handling +-- Previously, chained OPTIONAL MATCH with VLE would either segfault +-- (with WHERE IS NOT NULL) or error with "arguments cannot be NULL" +-- (without WHERE) instead of producing correct NULL-extended rows. +SELECT create_graph('issue_2092'); +NOTICE: graph "issue_2092" has been created + create_graph +-------------- + +(1 row) + +-- Set up a small graph where some OPTIONAL MATCH paths exist and some don't +SELECT * FROM cypher('issue_2092', $$ + CREATE (a:Person {name: 'Alice'}) + CREATE (b:Person {name: 'Bob'}) + CREATE (c:City {name: 'NYC'}) + CREATE (d:City {name: 'LA'}) + CREATE (e:Place {name: 'Central Park'}) + CREATE (a)-[:LIVES_IN]->(c) + CREATE (c)-[:HAS_PLACE]->(e) + CREATE (b)-[:LIVES_IN]->(d) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- Alice lives in NYC which has Central Park. +-- Bob lives in LA which has no places. +-- VLE + chained OPTIONAL MATCH + WHERE IS NOT NULL: should return rows +-- without crashing (was: segfault) +SELECT * FROM cypher('issue_2092', $$ + MATCH (p:Person)-[:LIVES_IN*]->(c:City) + OPTIONAL MATCH (c)-[:HAS_PLACE*]->(place) + OPTIONAL MATCH (place)-[:NEARBY*]->(other) + WHERE place IS NOT NULL + RETURN p.name, place.name, other + ORDER BY p.name +$$) AS (person agtype, place agtype, other agtype); + person | place | other +---------+----------------+------- + "Alice" | "Central Park" | + "Bob" | | +(2 rows) + +-- VLE + chained OPTIONAL MATCH without WHERE: should return NULL-extended +-- rows without error (was: "match_vle_terminal_edge() arguments cannot be +-- NULL") +SELECT * FROM cypher('issue_2092', $$ + MATCH (p:Person)-[:LIVES_IN*]->(c:City) + OPTIONAL MATCH (c)-[:HAS_PLACE*]->(place) + OPTIONAL MATCH (place)-[:NEARBY*]->(other) + RETURN p.name, place.name, other + ORDER BY p.name +$$) AS (person agtype, place agtype, other agtype); + person | place | other +---------+----------------+------- + "Alice" | "Central Park" | + "Bob" | | +(2 rows) + +-- Verify the happy path still works: Alice's full chain resolves +SELECT * FROM cypher('issue_2092', $$ + MATCH (p:Person)-[:LIVES_IN*]->(c:City) + OPTIONAL MATCH (c)-[:HAS_PLACE*]->(place) + WHERE place IS NOT NULL + RETURN p.name, c.name, place.name + ORDER BY p.name +$$) AS (person agtype, city agtype, place agtype); + person | city | place +---------+-------+---------------- + "Alice" | "NYC" | "Central Park" + "Bob" | "LA" | +(2 rows) + +SELECT drop_graph('issue_2092', true); +NOTICE: drop cascades to 7 other objects +DETAIL: drop cascades to table issue_2092._ag_label_vertex +drop cascades to table issue_2092._ag_label_edge +drop cascades to table issue_2092."Person" +drop cascades to table issue_2092."City" +drop cascades to table issue_2092."Place" +drop cascades to table issue_2092."LIVES_IN" +drop cascades to table issue_2092."HAS_PLACE" +NOTICE: graph "issue_2092" has been dropped + drop_graph +------------ + +(1 row) + -- -- Clean up -- diff --git a/regress/sql/cypher_vle.sql b/regress/sql/cypher_vle.sql index 5835627cc..c960aa7a4 100644 --- a/regress/sql/cypher_vle.sql +++ b/regress/sql/cypher_vle.sql @@ -357,6 +357,59 @@ SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*2..2]-({name: SELECT drop_graph('issue_1910', true); +-- issue 2092: VLE with chained OPTIONAL MATCH and NULL handling +-- Previously, chained OPTIONAL MATCH with VLE would either segfault +-- (with WHERE IS NOT NULL) or error with "arguments cannot be NULL" +-- (without WHERE) instead of producing correct NULL-extended rows. +SELECT create_graph('issue_2092'); + +-- Set up a small graph where some OPTIONAL MATCH paths exist and some don't +SELECT * FROM cypher('issue_2092', $$ + CREATE (a:Person {name: 'Alice'}) + CREATE (b:Person {name: 'Bob'}) + CREATE (c:City {name: 'NYC'}) + CREATE (d:City {name: 'LA'}) + CREATE (e:Place {name: 'Central Park'}) + CREATE (a)-[:LIVES_IN]->(c) + CREATE (c)-[:HAS_PLACE]->(e) + CREATE (b)-[:LIVES_IN]->(d) +$$) AS (result agtype); + +-- Alice lives in NYC which has Central Park. +-- Bob lives in LA which has no places. +-- VLE + chained OPTIONAL MATCH + WHERE IS NOT NULL: should return rows +-- without crashing (was: segfault) +SELECT * FROM cypher('issue_2092', $$ + MATCH (p:Person)-[:LIVES_IN*]->(c:City) + OPTIONAL MATCH (c)-[:HAS_PLACE*]->(place) + OPTIONAL MATCH (place)-[:NEARBY*]->(other) + WHERE place IS NOT NULL + RETURN p.name, place.name, other + ORDER BY p.name +$$) AS (person agtype, place agtype, other agtype); + +-- VLE + chained OPTIONAL MATCH without WHERE: should return NULL-extended +-- rows without error (was: "match_vle_terminal_edge() arguments cannot be +-- NULL") +SELECT * FROM cypher('issue_2092', $$ + MATCH (p:Person)-[:LIVES_IN*]->(c:City) + OPTIONAL MATCH (c)-[:HAS_PLACE*]->(place) + OPTIONAL MATCH (place)-[:NEARBY*]->(other) + RETURN p.name, place.name, other + ORDER BY p.name +$$) AS (person agtype, place agtype, other agtype); + +-- Verify the happy path still works: Alice's full chain resolves +SELECT * FROM cypher('issue_2092', $$ + MATCH (p:Person)-[:LIVES_IN*]->(c:City) + OPTIONAL MATCH (c)-[:HAS_PLACE*]->(place) + WHERE place IS NOT NULL + RETURN p.name, c.name, place.name + ORDER BY p.name +$$) AS (person agtype, city agtype, place agtype); + +SELECT drop_graph('issue_2092', true); + -- -- Clean up -- diff --git a/src/backend/utils/adt/age_vle.c b/src/backend/utils/adt/age_vle.c index f9e4c70b8..9224ed612 100644 --- a/src/backend/utils/adt/age_vle.c +++ b/src/backend/utils/adt/age_vle.c @@ -561,6 +561,11 @@ static VLE_local_context *build_local_vle_context(FunctionCallInfo fcinfo, /* get and update the start vertex id */ if (PG_ARGISNULL(1) || is_agtype_null(AG_GET_ARG_AGTYPE_P(1))) { + /* if there are no more vertices to process, return NULL */ + if (vlelctx->next_vertex == NULL) + { + return NULL; + } vlelctx->vsid = get_graphid(vlelctx->next_vertex); /* increment to the next vertex */ vlelctx->next_vertex = next_GraphIdNode(vlelctx->next_vertex); @@ -1733,6 +1738,16 @@ Datum age_vle(PG_FUNCTION_ARGS) /* build the local vle context */ vlelctx = build_local_vle_context(fcinfo, funcctx); + /* + * If the context is NULL, there are no paths to find. + * This can happen when a cached VLE context has exhausted + * its vertex list (e.g., from a NULL OPTIONAL MATCH variable). + */ + if (vlelctx == NULL) + { + SRF_RETURN_DONE(funcctx); + } + /* * Point the function call context's user pointer to the local VLE * context just created @@ -1934,6 +1949,16 @@ Datum age_match_two_vle_edges(PG_FUNCTION_ARGS) graphid *left_array, *right_array; int left_array_size; + /* + * If either argument is NULL, return FALSE. This can occur in + * OPTIONAL MATCH (LEFT JOIN) contexts where a preceding clause + * produced no results. + */ + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) + { + PG_RETURN_BOOL(false); + } + /* get the VLE_path_container argument */ agt_arg_vpc = AG_GET_ARG_AGTYPE_P(0); @@ -2008,12 +2033,14 @@ Datum age_match_vle_edge_to_id_qual(PG_FUNCTION_ARGS) errmsg("age_match_vle_edge_to_id_qual() invalid number of arguments"))); } - /* the arguments cannot be NULL */ + /* + * If any argument is NULL, return FALSE. This can occur in + * OPTIONAL MATCH (LEFT JOIN) contexts where a preceding clause + * produced no results. + */ if (nulls[0] || nulls[1] || nulls[2]) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("age_match_vle_edge_to_id_qual() arguments must be non NULL"))); + PG_RETURN_BOOL(false); } /* get the VLE_path_container argument */ @@ -2233,26 +2260,27 @@ Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) if (nargs != 3) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("age_match_terminal_edge() invalid number of arguments"))); + errmsg("age_match_vle_terminal_edge() invalid number of arguments"))); } - /* the arguments cannot be NULL */ + /* + * If any argument is NULL, return FALSE. This can occur when this + * function is used as a join qual in an OPTIONAL MATCH (LEFT JOIN) + * where a preceding OPTIONAL MATCH produced no results. Returning + * FALSE allows PostgreSQL to produce the correct NULL-extended rows. + */ if (nulls[0] || nulls[1] || nulls[2]) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("match_vle_terminal_edge() arguments cannot be NULL"))); + PG_RETURN_BOOL(false); } /* get the vpc */ agt_arg_path = DATUM_GET_AGTYPE_P(args[2]); - /* it cannot be NULL */ + /* if the vpc is an agtype NULL, return FALSE */ if (is_agtype_null(agt_arg_path)) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("match_vle_terminal_edge() argument 3 cannot be NULL"))); + PG_RETURN_BOOL(false); } /* @@ -2290,9 +2318,7 @@ Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) } else { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("match_vle_terminal_edge() argument 1 must be non NULL"))); + PG_RETURN_BOOL(false); } } else if (types[0] == GRAPHIDOID) @@ -2320,9 +2346,7 @@ Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) } else { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("match_vle_terminal_edge() argument 2 must be non NULL"))); + PG_RETURN_BOOL(false); } } else if (types[1] == GRAPHIDOID) From 346f319459e18db89deae30a3af7ea945bba101d Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Thu, 26 Feb 2026 19:24:58 -0500 Subject: [PATCH 034/100] Fix VLE queries failing on read-only replicas (#2160) (#2345) The global graph cache used by VLE acquired ShareLock when scanning vertex and edge label tables to populate in-memory hashtables. On read-only replicas (standby servers in recovery), PostgreSQL only allows RowExclusiveLock or less, so VLE queries would fail with: "cannot acquire lock mode ShareLock on database objects while recovery is in progress" Change all three functions in age_global_graph.c to use AccessShareLock instead, which is sufficient for read-only table scans and is consistent with the existing ag_cache.c code that performs identical operations. Co-authored-by: Claude Opus 4.6 --- src/backend/utils/adt/age_global_graph.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backend/utils/adt/age_global_graph.c b/src/backend/utils/adt/age_global_graph.c index c34e51ee3..4e5b58632 100644 --- a/src/backend/utils/adt/age_global_graph.c +++ b/src/backend/utils/adt/age_global_graph.c @@ -215,7 +215,7 @@ static List *get_ag_labels_names(Snapshot snapshot, Oid graph_oid, F_CHAREQ, CharGetDatum(label_type)); /* setup the table to be scanned, ag_label in this case */ - ag_label = table_open(ag_label_relation_id(), ShareLock); + ag_label = table_open(ag_label_relation_id(), AccessShareLock); scan_desc = table_beginscan(ag_label, snapshot, 2, scan_keys); /* get the tupdesc - we don't need to release this one */ @@ -241,7 +241,7 @@ static List *get_ag_labels_names(Snapshot snapshot, Oid graph_oid, /* close up scan */ table_endscan(scan_desc); - table_close(ag_label, ShareLock); + table_close(ag_label, AccessShareLock); return labels; } @@ -493,7 +493,7 @@ static void load_vertex_hashtable(GRAPH_global_context *ggctx) vertex_label_table_oid = get_relname_relid(vertex_label_name, graph_namespace_oid); /* open the relation (table) and begin the scan */ - graph_vertex_label = table_open(vertex_label_table_oid, ShareLock); + graph_vertex_label = table_open(vertex_label_table_oid, AccessShareLock); scan_desc = table_beginscan(graph_vertex_label, snapshot, 0, NULL); /* get the tupdesc - we don't need to release this one */ tupdesc = RelationGetDescr(graph_vertex_label); @@ -544,7 +544,7 @@ static void load_vertex_hashtable(GRAPH_global_context *ggctx) /* end the scan and close the relation */ table_endscan(scan_desc); - table_close(graph_vertex_label, ShareLock); + table_close(graph_vertex_label, AccessShareLock); } } @@ -601,7 +601,7 @@ static void load_edge_hashtable(GRAPH_global_context *ggctx) edge_label_table_oid = get_relname_relid(edge_label_name, graph_namespace_oid); /* open the relation (table) and begin the scan */ - graph_edge_label = table_open(edge_label_table_oid, ShareLock); + graph_edge_label = table_open(edge_label_table_oid, AccessShareLock); scan_desc = table_beginscan(graph_edge_label, snapshot, 0, NULL); /* get the tupdesc - we don't need to release this one */ tupdesc = RelationGetDescr(graph_edge_label); @@ -678,7 +678,7 @@ static void load_edge_hashtable(GRAPH_global_context *ggctx) /* end the scan and close the relation */ table_endscan(scan_desc); - table_close(graph_edge_label, ShareLock); + table_close(graph_edge_label, AccessShareLock); } } From 20ada845280c4370909d82b8652f6a082e5aa5df Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Fri, 27 Feb 2026 15:05:31 -0500 Subject: [PATCH 035/100] Fix chained MERGE not seeing sibling MERGE's changes (#1446) (#2344) * Fix chained MERGE not seeing sibling MERGE's changes (#1446) When multiple MERGEs are chained (e.g. MATCH ... MERGE ... MERGE ...), the non-terminal (first) MERGE returned rows one at a time to the parent plan node. The parent MERGE's lateral join would materialize its hash table on the first row, before the child MERGE had finished all its iterations. This caused the second MERGE to not see entities created by the first MERGE, leading to duplicate nodes. Fix by making non-terminal MERGE eager: it processes ALL input rows and buffers the projected results before returning any to the parent. This ensures all entity creations are committed before any parent plan node scans the tables. Co-Authored-By: Claude Opus 4.6 * Fix non-terminal MERGE empty-buffer fallthrough and add test When a non-terminal MERGE receives no input rows from its predecessor (e.g., MATCH returns 0 rows), the eager buffer is filled but empty. The condition at line 688 checked `css->eager_tuples != NIL`, which evaluated to false for an empty buffer, causing execution to fall through to the terminal MERGE code path. This could incorrectly create entities when none should be created. Fix by checking `css->eager_buffer_filled` instead, which correctly distinguishes "buffer not yet filled" from "buffer filled but empty". Add regression test for chained MERGE with empty MATCH result. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- regress/expected/cypher_merge.out | 173 +++++++++++++++++++++++++++ regress/sql/cypher_merge.sql | 77 ++++++++++++ src/backend/executor/cypher_merge.c | 177 ++++++++++++++++++++-------- src/include/executor/cypher_utils.h | 3 + 4 files changed, 379 insertions(+), 51 deletions(-) diff --git a/regress/expected/cypher_merge.out b/regress/expected/cypher_merge.out index 56a23f513..8c37dc2de 100644 --- a/regress/expected/cypher_merge.out +++ b/regress/expected/cypher_merge.out @@ -1717,6 +1717,154 @@ SELECT * FROM cypher('issue_1907', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype {"id": 1125899906842626, "label": "RELATED_TO", "end_id": 281474976710660, "start_id": 281474976710659, "properties": {"property1": "something", "property2": "else"}}::edge (1 row) +-- +-- Fix issue 1446: First MERGE does not see the second MERGE's changes +-- +-- When chained MERGEs appear (MATCH ... MERGE ... MERGE ...), the first +-- (non-terminal) MERGE returned rows one at a time, so the second MERGE's +-- lateral join was materialized before the first finished all iterations. +-- This caused duplicate nodes. The fix makes non-terminal MERGE eager: +-- it processes ALL input rows before returning any. +-- +SELECT * FROM create_graph('issue_1446'); +NOTICE: graph "issue_1446" has been created + create_graph +-------------- + +(1 row) + +-- Reporter's exact setup: two initial nodes +SELECT * FROM cypher('issue_1446', $$ CREATE (:A), (:C) $$) AS (a agtype); + a +--- +(0 rows) + +-- Reporter's exact reproduction case: two chained MERGEs +-- Without fix: C is created multiple times (once per MATCH row) because the +-- second MERGE's lateral join materializes before the first MERGE finishes. +-- With fix: returns 2 rows, C is found and reused by the second MERGE. +SELECT * FROM cypher('issue_1446', $$ + MATCH (x) + MERGE (x)-[:r]->(:t) + MERGE (:C)-[:r]->(:t) + RETURN x +$$) AS (a agtype); + a +------------------------------------------------------------------ + {"id": 844424930131969, "label": "A", "properties": {}}::vertex + {"id": 1125899906842625, "label": "C", "properties": {}}::vertex +(2 rows) + +-- Verify: A(1), C(1), t(2) = 4 nodes, 2 edges +SELECT * FROM cypher('issue_1446', $$ + MATCH (n) + RETURN labels(n) AS label, count(*) AS cnt + ORDER BY label +$$) AS (label agtype, cnt agtype); + label | cnt +-------+----- + ["A"] | 1 + ["C"] | 1 + ["t"] | 2 +(3 rows) + +SELECT * FROM cypher('issue_1446', $$ + MATCH ()-[e]->() + RETURN count(*) AS edge_count +$$) AS (edge_count agtype); + edge_count +------------ + 2 +(1 row) + +-- Test with 3 initial nodes: ensures eager buffering works for larger sets +SELECT * FROM cypher('issue_1446', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('issue_1446', $$ CREATE (:X), (:Y), (:Z) $$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('issue_1446', $$ + MATCH (n) + MERGE (n)-[:link]->(:shared) + MERGE (:hub)-[:link]->(:shared) + RETURN n +$$) AS (a agtype); + a +------------------------------------------------------------------ + {"id": 1970324836974593, "label": "X", "properties": {}}::vertex + {"id": 2251799813685249, "label": "Y", "properties": {}}::vertex + {"id": 2533274790395905, "label": "Z", "properties": {}}::vertex +(3 rows) + +-- Without fix: hub is created 3 times (once per MATCH row). +-- With fix: hub(1), shared(4), X(1), Y(1), Z(1) = 8 nodes, 4 edges +-- (3 n->shared edges + 1 hub->shared edge; hub reused for rows 2 & 3) +SELECT * FROM cypher('issue_1446', $$ + MATCH (n) + RETURN labels(n) AS label, count(*) AS cnt + ORDER BY label +$$) AS (label agtype, cnt agtype); + label | cnt +------------+----- + ["X"] | 1 + ["Y"] | 1 + ["Z"] | 1 + ["hub"] | 1 + ["shared"] | 4 +(5 rows) + +SELECT * FROM cypher('issue_1446', $$ + MATCH ()-[e]->() + RETURN count(*) AS edge_count +$$) AS (edge_count agtype); + edge_count +------------ + 4 +(1 row) + +-- Test chained MERGE with empty MATCH result (empty buffer scenario) +-- Without fix: non-terminal MERGE falls through to terminal logic, +-- incorrectly creating entities when MATCH returns 0 rows. +SELECT * FROM cypher('issue_1446', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('issue_1446', $$ + MATCH (x:NonExistent) + MERGE (x)-[:r]->(:t) + MERGE (:C)-[:r]->(:t) + RETURN count(*) AS cnt +$$) AS (cnt agtype); + cnt +----- + 0 +(1 row) + +-- Verify no nodes or edges were created +SELECT * FROM cypher('issue_1446', $$ + MATCH (n) + RETURN count(*) AS node_count +$$) AS (node_count agtype); + node_count +------------ + 0 +(1 row) + +SELECT * FROM cypher('issue_1446', $$ + MATCH ()-[e]->() + RETURN count(*) AS edge_count +$$) AS (edge_count agtype); + edge_count +------------ + 0 +(1 row) + -- -- clean up graphs -- @@ -1735,6 +1883,11 @@ SELECT * FROM cypher('issue_1709', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype --- (0 rows) +SELECT * FROM cypher('issue_1446', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); + a +--- +(0 rows) + -- -- delete graphs -- @@ -1812,6 +1965,26 @@ NOTICE: graph "issue_1709" has been dropped (1 row) +SELECT drop_graph('issue_1446', true); +NOTICE: drop cascades to 12 other objects +DETAIL: drop cascades to table issue_1446._ag_label_vertex +drop cascades to table issue_1446._ag_label_edge +drop cascades to table issue_1446."A" +drop cascades to table issue_1446."C" +drop cascades to table issue_1446.r +drop cascades to table issue_1446.t +drop cascades to table issue_1446."X" +drop cascades to table issue_1446."Y" +drop cascades to table issue_1446."Z" +drop cascades to table issue_1446.link +drop cascades to table issue_1446.shared +drop cascades to table issue_1446.hub +NOTICE: graph "issue_1446" has been dropped + drop_graph +------------ + +(1 row) + -- -- End -- diff --git a/regress/sql/cypher_merge.sql b/regress/sql/cypher_merge.sql index 02c9d21c2..cc900e73d 100644 --- a/regress/sql/cypher_merge.sql +++ b/regress/sql/cypher_merge.sql @@ -785,12 +785,88 @@ SELECT * FROM cypher('issue_1907', $$ MERGE (a {name: 'Test Node A'})-[r:RELATED -- should return properties added SELECT * FROM cypher('issue_1907', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype); +-- +-- Fix issue 1446: First MERGE does not see the second MERGE's changes +-- +-- When chained MERGEs appear (MATCH ... MERGE ... MERGE ...), the first +-- (non-terminal) MERGE returned rows one at a time, so the second MERGE's +-- lateral join was materialized before the first finished all iterations. +-- This caused duplicate nodes. The fix makes non-terminal MERGE eager: +-- it processes ALL input rows before returning any. +-- +SELECT * FROM create_graph('issue_1446'); +-- Reporter's exact setup: two initial nodes +SELECT * FROM cypher('issue_1446', $$ CREATE (:A), (:C) $$) AS (a agtype); +-- Reporter's exact reproduction case: two chained MERGEs +-- Without fix: C is created multiple times (once per MATCH row) because the +-- second MERGE's lateral join materializes before the first MERGE finishes. +-- With fix: returns 2 rows, C is found and reused by the second MERGE. +SELECT * FROM cypher('issue_1446', $$ + MATCH (x) + MERGE (x)-[:r]->(:t) + MERGE (:C)-[:r]->(:t) + RETURN x +$$) AS (a agtype); +-- Verify: A(1), C(1), t(2) = 4 nodes, 2 edges +SELECT * FROM cypher('issue_1446', $$ + MATCH (n) + RETURN labels(n) AS label, count(*) AS cnt + ORDER BY label +$$) AS (label agtype, cnt agtype); +SELECT * FROM cypher('issue_1446', $$ + MATCH ()-[e]->() + RETURN count(*) AS edge_count +$$) AS (edge_count agtype); + +-- Test with 3 initial nodes: ensures eager buffering works for larger sets +SELECT * FROM cypher('issue_1446', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); +SELECT * FROM cypher('issue_1446', $$ CREATE (:X), (:Y), (:Z) $$) AS (a agtype); +SELECT * FROM cypher('issue_1446', $$ + MATCH (n) + MERGE (n)-[:link]->(:shared) + MERGE (:hub)-[:link]->(:shared) + RETURN n +$$) AS (a agtype); +-- Without fix: hub is created 3 times (once per MATCH row). +-- With fix: hub(1), shared(4), X(1), Y(1), Z(1) = 8 nodes, 4 edges +-- (3 n->shared edges + 1 hub->shared edge; hub reused for rows 2 & 3) +SELECT * FROM cypher('issue_1446', $$ + MATCH (n) + RETURN labels(n) AS label, count(*) AS cnt + ORDER BY label +$$) AS (label agtype, cnt agtype); +SELECT * FROM cypher('issue_1446', $$ + MATCH ()-[e]->() + RETURN count(*) AS edge_count +$$) AS (edge_count agtype); + +-- Test chained MERGE with empty MATCH result (empty buffer scenario) +-- Without fix: non-terminal MERGE falls through to terminal logic, +-- incorrectly creating entities when MATCH returns 0 rows. +SELECT * FROM cypher('issue_1446', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); +SELECT * FROM cypher('issue_1446', $$ + MATCH (x:NonExistent) + MERGE (x)-[:r]->(:t) + MERGE (:C)-[:r]->(:t) + RETURN count(*) AS cnt +$$) AS (cnt agtype); +-- Verify no nodes or edges were created +SELECT * FROM cypher('issue_1446', $$ + MATCH (n) + RETURN count(*) AS node_count +$$) AS (node_count agtype); +SELECT * FROM cypher('issue_1446', $$ + MATCH ()-[e]->() + RETURN count(*) AS edge_count +$$) AS (edge_count agtype); + -- -- clean up graphs -- SELECT * FROM cypher('cypher_merge', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); SELECT * FROM cypher('issue_1630', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); SELECT * FROM cypher('issue_1709', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); +SELECT * FROM cypher('issue_1446', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); -- -- delete graphs @@ -800,6 +876,7 @@ SELECT drop_graph('cypher_merge', true); SELECT drop_graph('issue_1630', true); SELECT drop_graph('issue_1691', true); SELECT drop_graph('issue_1709', true); +SELECT drop_graph('issue_1446', true); -- -- End diff --git a/src/backend/executor/cypher_merge.c b/src/backend/executor/cypher_merge.c index a1bb4686c..1edfc812d 100644 --- a/src/backend/executor/cypher_merge.c +++ b/src/backend/executor/cypher_merge.c @@ -596,23 +596,126 @@ static TupleTableSlot *exec_cypher_merge(CustomScanState *node) * it did, we don't need to create the pattern. If the lateral join did * not find the whole path, create the whole path. * - * If this is a terminal clause, process all tuples. If not, pass the - * tuple to up the execution tree. + * For non-terminal MERGE, we eagerly process ALL input rows before + * returning any results. This ensures that all entities created by + * this MERGE are committed to the database before any parent plan + * node (such as another MERGE's lateral join) scans the tables. + * Without eager processing, chained MERGEs cannot see each other's + * changes because the parent's join scan is materialized before the + * child MERGE finishes all its iterations. + * + * For terminal MERGE, all tuples are processed in a single pass + * and NULL is returned. + */ + + /* + * Non-terminal: eagerly process all rows and buffer the results. + */ + if (!terminal && !css->eager_buffer_filled) + { + css->eager_tuples = NIL; + css->eager_tuples_index = 0; + + while (true) + { + TupleTableSlot *projected; + HeapTuple htup; + + /* Process the subtree first */ + Decrement_Estate_CommandId(estate) + slot = ExecProcNode(node->ss.ps.lefttree); + Increment_Estate_CommandId(estate) + + if (TupIsNull(slot)) + { + break; + } + + /* setup the scantuple that the process_path needs */ + econtext->ecxt_scantuple = + node->ss.ps.lefttree->ps_ProjInfo->pi_exprContext->ecxt_scantuple; + + /* + * Check the subtree to see if the lateral join + * representing the MERGE path found results. If not, + * we need to create the path. + */ + if (check_path(css, econtext->ecxt_scantuple)) + { + path_entry **prebuilt_path_array = NULL; + path_entry **found_path_array = NULL; + int path_length = + list_length(css->path->target_nodes); + + prebuilt_path_array = prebuild_path(node); + + found_path_array = + find_duplicate_path(node, prebuilt_path_array); + + if (found_path_array) + { + free_path_entry_array(prebuilt_path_array, + path_length); + process_path(css, found_path_array, false); + } + else + { + created_path *new_path = + palloc0(sizeof(created_path)); + + new_path->next = css->created_paths_list; + new_path->entry = prebuilt_path_array; + css->created_paths_list = new_path; + + process_path(css, prebuilt_path_array, true); + } + } + + /* Project the result and save a copy */ + econtext->ecxt_scantuple = + ExecProject(node->ss.ps.lefttree->ps_ProjInfo); + projected = ExecProject(node->ss.ps.ps_ProjInfo); + + htup = ExecCopySlotHeapTuple(projected); + css->eager_tuples = + lappend(css->eager_tuples, htup); + } + + css->eager_buffer_filled = true; + } + + /* Non-terminal: return the next buffered row (or NULL if empty) */ + if (!terminal && css->eager_buffer_filled) + { + if (css->eager_tuples_index < list_length(css->eager_tuples)) + { + HeapTuple htup; + TupleTableSlot *result_slot = + node->ss.ps.ps_ResultTupleSlot; + + htup = (HeapTuple) + list_nth(css->eager_tuples, css->eager_tuples_index); + css->eager_tuples_index++; + + ExecForceStoreHeapTuple(htup, result_slot, false); + return result_slot; + } + + return NULL; + } + + /* + * Terminal: process all tuples and return NULL. */ do { - /*Process the subtree first */ + /* Process the subtree first */ Decrement_Estate_CommandId(estate) slot = ExecProcNode(node->ss.ps.lefttree); Increment_Estate_CommandId(estate) - /* - * We are done processing the subtree, mark as terminal - * so the function returns NULL. - */ if (TupIsNull(slot)) { - terminal = true; break; } @@ -622,7 +725,7 @@ static TupleTableSlot *exec_cypher_merge(CustomScanState *node) /* * Check the subtree to see if the lateral join representing the - * MERGE path found results. If not, we need to create the path + * MERGE path found results. If not, we need to create the path. */ if (check_path(css, econtext->ecxt_scantuple)) { @@ -630,71 +733,31 @@ static TupleTableSlot *exec_cypher_merge(CustomScanState *node) path_entry **found_path_array = NULL; int path_length = list_length(css->path->target_nodes); - /* - * Prebuild our path and verify that it wasn't already created. - * - * Note: This is currently only needed when there is a previous - * clause. This is due to the fact that MERGE can't see - * what it has just created. This isn't due to transaction - * or command ids, it's due to the join's scan not being - * able to add in the newly inserted tuples and rescan - * with these tuples. - * - * Note: The prebuilt path is purposely generic as it needs to - * only match a path. The more specific items will be - * added by merge_vertex and merge_edge if it is inserted. - * - * Note: The IDs are purposely not created here because we may - * need to throw them away if a path was previously - * created. Remember, the IDs are automatically - * incremented when fetched. - */ prebuilt_path_array = prebuild_path(node); found_path_array = find_duplicate_path(node, prebuilt_path_array); - /* if found we don't need to insert anything, just reuse it */ if (found_path_array) { - /* we don't need our prebuilt path anymore */ free_path_entry_array(prebuilt_path_array, path_length); - - /* as this path exists, we don't need to insert it */ process_path(css, found_path_array, false); } - /* otherwise, we need to insert the new, prebuilt, path */ else { created_path *new_path = palloc0(sizeof(created_path)); - /* build the next linked list entry for our created_paths */ - new_path = palloc0(sizeof(created_path)); new_path->next = css->created_paths_list; new_path->entry = prebuilt_path_array; - - /* we need to push our prebuilt path onto the list */ css->created_paths_list = new_path; - /* - * We need to pass in the prebuilt path so that it can get - * filled in with more specific information - */ process_path(css, prebuilt_path_array, true); } } - } while (terminal); - - /* if this was a terminal MERGE just return NULL */ - if (terminal) - { - return NULL; - } - - econtext->ecxt_scantuple = ExecProject(node->ss.ps.lefttree->ps_ProjInfo); + } while (true); - return ExecProject(node->ss.ps.ps_ProjInfo); + return NULL; } else if (terminal) @@ -910,6 +973,18 @@ static void end_cypher_merge(CustomScanState *node) css->created_paths_list = next; } + /* free the eager buffer if it was used */ + if (css->eager_tuples != NIL) + { + ListCell *lc2; + + foreach(lc2, css->eager_tuples) + { + heap_freetuple((HeapTuple) lfirst(lc2)); + } + list_free(css->eager_tuples); + css->eager_tuples = NIL; + } } /* diff --git a/src/include/executor/cypher_utils.h b/src/include/executor/cypher_utils.h index fc4067455..278094e07 100644 --- a/src/include/executor/cypher_utils.h +++ b/src/include/executor/cypher_utils.h @@ -108,6 +108,9 @@ typedef struct cypher_merge_custom_scan_state bool found_a_path; CommandId base_currentCommandId; struct created_path *created_paths_list; + List *eager_tuples; + int eager_tuples_index; + bool eager_buffer_filled; } cypher_merge_custom_scan_state; TupleTableSlot *populate_vertex_tts(TupleTableSlot *elemTupleSlot, From 217467a36a1c29df4b918faf6adb6e75aec28817 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Fri, 27 Feb 2026 17:10:04 -0500 Subject: [PATCH 036/100] Fix MATCH after CREATE returning 0 rows (issue #2308) (#2340) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a MATCH clause follows CREATE + WITH and re-uses bound variables (e.g. CREATE (a)-[e]->(b) WITH a,e,b MATCH p=(a)-[e]->(b)), the MATCH generates filter quals (age_start_id(e) = age_id(a), etc.) that reference only columns from the predecessor subquery. PostgreSQL's optimizer pushes these quals through the transparent subquery layers into the CREATE's child plan, where they evaluate on NULL values before CREATE has executed — always yielding 0 rows. Fix: mark the predecessor subquery RTE as security_barrier when the clause chain contains a data-modifying operation (CREATE, SET, DELETE, or MERGE). This prevents PostgreSQL from pushing filter quals into the subquery, ensuring they evaluate after the DML produces output values. Co-authored-by: Claude Opus 4.6 --- regress/expected/cypher_match.out | 100 +++++++++++++++++++++++++++++ regress/sql/cypher_match.sql | 55 ++++++++++++++++ src/backend/parser/cypher_clause.c | 38 +++++++++++ 3 files changed, 193 insertions(+) diff --git a/regress/expected/cypher_match.out b/regress/expected/cypher_match.out index ea425e463..ff2825ae0 100644 --- a/regress/expected/cypher_match.out +++ b/regress/expected/cypher_match.out @@ -3533,6 +3533,106 @@ SELECT * FROM cypher('test_enable_containment', $$ EXPLAIN (costs off) MATCH (x: Filter: ((agtype_access_operator(VARIADIC ARRAY[properties, '"school"'::agtype]) = '{"name": "XYZ College", "program": {"major": "Psyc", "degree": "BSc"}}'::agtype) AND (agtype_access_operator(VARIADIC ARRAY[properties, '"phone"'::agtype]) = '[123456789, 987654321, 456987123]'::agtype)) (2 rows) +-- +-- issue 2308: MATCH after CREATE returns 0 rows +-- +-- When all MATCH variables are already bound from a preceding CREATE + WITH, +-- the MATCH filter quals must evaluate after CREATE, not before. +-- +SELECT create_graph('issue_2308'); +NOTICE: graph "issue_2308" has been created + create_graph +-------------- + +(1 row) + +-- Reporter's exact case: CREATE + WITH + MATCH + SET + RETURN +SELECT * FROM cypher('issue_2308', $$ + CREATE (a:TestB3)-[e:B3REL]->(b:TestB3) + WITH a, e, b + MATCH p = (a)-[e]->(b) + SET a.something = 'something' + RETURN a +$$) AS (a agtype); + a +---------------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "TestB3", "properties": {"something": "something"}}::vertex +(1 row) + +-- Bound variables, no SET +SELECT * FROM cypher('issue_2308', $$ + CREATE (a:T2)-[e:R2]->(b:T2) + WITH a, e, b + MATCH (a)-[e]->(b) + RETURN a, e, b +$$) AS (a agtype, e agtype, b agtype); + a | e | b +-------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------- + {"id": 1407374883553281, "label": "T2", "properties": {}}::vertex | {"id": 1688849860263937, "label": "R2", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {}}::edge | {"id": 1407374883553282, "label": "T2", "properties": {}}::vertex +(1 row) + +-- Reversed direction: filter should reject (0 rows expected) +SELECT * FROM cypher('issue_2308', $$ + CREATE (a:T3)-[e:R3]->(b:T3) + WITH a, e, b + MATCH (b)-[e]->(a) + RETURN a +$$) AS (a agtype); + a +--- +(0 rows) + +-- Node-only MATCH with bound variable +SELECT * FROM cypher('issue_2308', $$ + CREATE (a:T4 {name: 'test'}) + WITH a + MATCH (a) + RETURN a +$$) AS (a agtype); + a +--------------------------------------------------------------------------------- + {"id": 2533274790395905, "label": "T4", "properties": {"name": "test"}}::vertex +(1 row) + +-- MATCH after SET (SET is also DML, chain must be protected) +SELECT * FROM cypher('issue_2308', $$ + CREATE (a:T5 {val: 1})-[e:R5]->(b:T5 {val: 2}) +$$) AS (r agtype); + r +--- +(0 rows) + +SELECT * FROM cypher('issue_2308', $$ + MATCH (a:T5)-[e:R5]->(b:T5) + SET a.val = 10 + WITH a, e, b + MATCH (a)-[e]->(b) + RETURN a.val +$$) AS (val agtype); + val +----- + 10 +(1 row) + +SELECT drop_graph('issue_2308', true); +NOTICE: drop cascades to 11 other objects +DETAIL: drop cascades to table issue_2308._ag_label_vertex +drop cascades to table issue_2308._ag_label_edge +drop cascades to table issue_2308."TestB3" +drop cascades to table issue_2308."B3REL" +drop cascades to table issue_2308."T2" +drop cascades to table issue_2308."R2" +drop cascades to table issue_2308."T3" +drop cascades to table issue_2308."R3" +drop cascades to table issue_2308."T4" +drop cascades to table issue_2308."T5" +drop cascades to table issue_2308."R5" +NOTICE: graph "issue_2308" has been dropped + drop_graph +------------ + +(1 row) + -- -- Clean up -- diff --git a/regress/sql/cypher_match.sql b/regress/sql/cypher_match.sql index 2817f36f6..ebcd67b84 100644 --- a/regress/sql/cypher_match.sql +++ b/regress/sql/cypher_match.sql @@ -1437,6 +1437,61 @@ SELECT count(*) FROM cypher('test_enable_containment', $$ MATCH p=(x:Customer)-[ SELECT * FROM cypher('test_enable_containment', $$ EXPLAIN (costs off) MATCH (x:Customer)-[:bought ={store: 'Amazon', addr:{city: 'Vancouver', street: 30}}]->(y:Product) RETURN 0 $$) as (a agtype); SELECT * FROM cypher('test_enable_containment', $$ EXPLAIN (costs off) MATCH (x:Customer ={school: { name: 'XYZ College',program: { major: 'Psyc', degree: 'BSc'} },phone: [ 123456789, 987654321, 456987123 ]}) RETURN 0 $$) as (a agtype); +-- +-- issue 2308: MATCH after CREATE returns 0 rows +-- +-- When all MATCH variables are already bound from a preceding CREATE + WITH, +-- the MATCH filter quals must evaluate after CREATE, not before. +-- +SELECT create_graph('issue_2308'); + +-- Reporter's exact case: CREATE + WITH + MATCH + SET + RETURN +SELECT * FROM cypher('issue_2308', $$ + CREATE (a:TestB3)-[e:B3REL]->(b:TestB3) + WITH a, e, b + MATCH p = (a)-[e]->(b) + SET a.something = 'something' + RETURN a +$$) AS (a agtype); + +-- Bound variables, no SET +SELECT * FROM cypher('issue_2308', $$ + CREATE (a:T2)-[e:R2]->(b:T2) + WITH a, e, b + MATCH (a)-[e]->(b) + RETURN a, e, b +$$) AS (a agtype, e agtype, b agtype); + +-- Reversed direction: filter should reject (0 rows expected) +SELECT * FROM cypher('issue_2308', $$ + CREATE (a:T3)-[e:R3]->(b:T3) + WITH a, e, b + MATCH (b)-[e]->(a) + RETURN a +$$) AS (a agtype); + +-- Node-only MATCH with bound variable +SELECT * FROM cypher('issue_2308', $$ + CREATE (a:T4 {name: 'test'}) + WITH a + MATCH (a) + RETURN a +$$) AS (a agtype); + +-- MATCH after SET (SET is also DML, chain must be protected) +SELECT * FROM cypher('issue_2308', $$ + CREATE (a:T5 {val: 1})-[e:R5]->(b:T5 {val: 2}) +$$) AS (r agtype); +SELECT * FROM cypher('issue_2308', $$ + MATCH (a:T5)-[e:R5]->(b:T5) + SET a.val = 10 + WITH a, e, b + MATCH (a)-[e]->(b) + RETURN a.val +$$) AS (val agtype); + +SELECT drop_graph('issue_2308', true); + -- -- Clean up -- diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 991e3f785..446e97b3f 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -345,6 +345,7 @@ static bool isa_special_VLE_case(cypher_path *path); static ParseNamespaceItem *find_pnsi(cypher_parsestate *cpstate, char *varname); static bool has_list_comp_or_subquery(Node *expr, void *context); +static bool clause_chain_has_dml(cypher_clause *clause); /* * Add required permissions to the RTEPermissionInfo for a relation. @@ -2917,6 +2918,21 @@ static Query *transform_cypher_match_pattern(cypher_parsestate *cpstate, pnsi = transform_prev_cypher_clause(cpstate, clause->prev, true); rte = pnsi->p_rte; + + /* + * If the predecessor clause chain contains a data-modifying + * operation (CREATE, SET, DELETE, MERGE), mark the subquery + * RTE as a security barrier. This prevents PostgreSQL's + * optimizer from pushing MATCH filter quals down into the + * subquery, which would cause them to evaluate before the + * DML executes -- resulting in quals checking NULL values + * and filtering out all rows. + */ + if (clause_chain_has_dml(clause->prev)) + { + rte->security_barrier = true; + } + rtindex = list_length(pstate->p_rtable); /* rte is the first RangeTblEntry in pstate */ if (rtindex != 1) @@ -6545,6 +6561,28 @@ static void advance_transform_entities_to_next_clause(List *entities) } } +/* + * Walk the clause chain and return true if any clause is a + * data-modifying operation (CREATE, SET, DELETE, or MERGE). + */ +static bool clause_chain_has_dml(cypher_clause *clause) +{ + while (clause != NULL) + { + if (is_ag_node(clause->self, cypher_create) || + is_ag_node(clause->self, cypher_set) || + is_ag_node(clause->self, cypher_delete) || + is_ag_node(clause->self, cypher_merge)) + { + return true; + } + + clause = clause->prev; + } + + return false; +} + static Query *analyze_cypher_clause(transform_method transform, cypher_clause *clause, cypher_parsestate *parent_cpstate) From a21120bf139b156661ef10374630e688c0cc46fb Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Mon, 2 Mar 2026 11:27:04 -0500 Subject: [PATCH 037/100] Support doubled-quote escaping in Cypher string literals (issue #2222) (#2342) SQL drivers (psycopg2, JDBC, etc.) escape single quotes by doubling them ('isn''t') when substituting parameters into queries. When these substitutions land inside Cypher's $$ block, the Cypher scanner rejects them because it only recognizes backslash escaping (\'). This makes it difficult to pass strings containing apostrophes through SQL drivers. Add '' and "" as escape sequences in the Cypher scanner, following the same pattern already used for backtick-quoted identifiers (``). Flex picks the longer two-character match over the one-character closing quote, so the change is backwards-compatible -- '' was previously a syntax error. Co-authored-by: Claude Opus 4.6 --- regress/expected/scan.out | 41 +++++++++++++++++++++++++++++++++ regress/sql/scan.sql | 21 +++++++++++++++++ src/backend/parser/ag_scanner.l | 10 ++++++++ 3 files changed, 72 insertions(+) diff --git a/regress/expected/scan.out b/regress/expected/scan.out index 46d5676d0..fb7c626a3 100644 --- a/regress/expected/scan.out +++ b/regress/expected/scan.out @@ -294,6 +294,47 @@ $$) AS t(a agtype, b agtype, c agtype); " \" \" ' ' " | " ' ' \" \" " | " / / \\ \b \f \n \r \t " (1 row) +-- doubled-quote escape sequences for SQL driver compatibility (issue #2222) +SELECT * FROM cypher('scan', $$ +RETURN 'isn''t' +$$) AS t(a agtype); + a +--------- + "isn't" +(1 row) + +SELECT * FROM cypher('scan', $$ +RETURN '''hello' +$$) AS t(a agtype); + a +---------- + "'hello" +(1 row) + +SELECT * FROM cypher('scan', $$ +RETURN 'hello''' +$$) AS t(a agtype); + a +---------- + "hello'" +(1 row) + +SELECT * FROM cypher('scan', $$ +RETURN 'it''s a ''test''' +$$) AS t(a agtype); + a +----------------- + "it's a 'test'" +(1 row) + +SELECT * FROM cypher('scan', $$ +RETURN "she said ""hello""" +$$) AS t(a agtype); + a +---------------------- + "she said \"hello\"" +(1 row) + -- invalid escape sequence SELECT * FROM cypher('scan', $$ RETURN "\a" diff --git a/regress/sql/scan.sql b/regress/sql/scan.sql index 4d35fe0fe..614447e33 100644 --- a/regress/sql/scan.sql +++ b/regress/sql/scan.sql @@ -202,6 +202,27 @@ SELECT * FROM cypher('scan', $$ RETURN " \" \" ' \' ", ' \' \' " \" ', " / \/ \\ \b \f \n \r \t " $$) AS t(a agtype, b agtype, c agtype); +-- doubled-quote escape sequences for SQL driver compatibility (issue #2222) +SELECT * FROM cypher('scan', $$ +RETURN 'isn''t' +$$) AS t(a agtype); + +SELECT * FROM cypher('scan', $$ +RETURN '''hello' +$$) AS t(a agtype); + +SELECT * FROM cypher('scan', $$ +RETURN 'hello''' +$$) AS t(a agtype); + +SELECT * FROM cypher('scan', $$ +RETURN 'it''s a ''test''' +$$) AS t(a agtype); + +SELECT * FROM cypher('scan', $$ +RETURN "she said ""hello""" +$$) AS t(a agtype); + -- invalid escape sequence SELECT * FROM cypher('scan', $$ RETURN "\a" diff --git a/src/backend/parser/ag_scanner.l b/src/backend/parser/ag_scanner.l index d5d72b926..c1e156c39 100644 --- a/src/backend/parser/ag_scanner.l +++ b/src/backend/parser/ag_scanner.l @@ -195,6 +195,8 @@ dquote \" dqchars [^"\\]+ squote ' sqchars [^'\\]+ +esdquote {dquote}{dquote} +essquote {squote}{squote} esascii \\["'/\\bfnrt] esasciifail \\[^Uu]? esunicode \\(U{hexdigit}{8}|u{hexdigit}{4}) @@ -420,6 +422,14 @@ ag_token token; strbuf_append_buf(&yyextra.literal_buf, yytext, yyleng); } +{esdquote} { + strbuf_append_char(&yyextra.literal_buf, '"'); +} + +{essquote} { + strbuf_append_char(&yyextra.literal_buf, '\''); +} + {esascii} { char c; From d0741d8dc90f0c2f636f049b48b9d41eb54516cf Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Mon, 2 Mar 2026 13:09:19 -0500 Subject: [PATCH 038/100] Fix entity_exists() CID visibility for CREATE + WITH + MERGE (#2343) * Fix entity_exists() CID visibility for CREATE + WITH + MERGE (#1954) When a Cypher query chains CREATE ... WITH ... MERGE, vertices created by CREATE become invisible to entity_exists() after a threshold number of input rows. This causes MERGE to throw "vertex assigned to variable was deleted". Root cause: CREATE calls CommandCounterIncrement() which advances the global command ID, but does not update es_snapshot->curcid. The Decrement/Increment CID macros used by the executors bring curcid back to the same value on each iteration. After enough rows, newly inserted vertices have a Cmin >= curcid and HeapTupleSatisfiesMVCC rejects them (requires Cmin < curcid). Fix: In entity_exists(), temporarily set es_snapshot->curcid to the current global command ID (via GetCurrentCommandId) for the duration of the scan, then restore it. This makes all entities inserted by preceding clauses in the same query visible to the existence check. Co-Authored-By: Claude Opus 4.6 * Use Max() to prevent curcid regression in entity_exists() Address review feedback: es_snapshot->curcid can be ahead of the global CID due to Increment_Estate_CommandId macros. Unconditionally assigning GetCurrentCommandId(false) could decrease curcid, making previously visible tuples invisible. Use Max(saved_curcid, GetCurrentCommandId(false)) to ensure we only ever increase visibility. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- regress/expected/cypher_merge.out | 130 ++++++++++++++++++++++++++++ regress/sql/cypher_merge.sql | 65 ++++++++++++++ src/backend/executor/cypher_utils.c | 21 +++++ 3 files changed, 216 insertions(+) diff --git a/regress/expected/cypher_merge.out b/regress/expected/cypher_merge.out index 8c37dc2de..4242f2f59 100644 --- a/regress/expected/cypher_merge.out +++ b/regress/expected/cypher_merge.out @@ -1865,6 +1865,114 @@ $$) AS (edge_count agtype); 0 (1 row) +-- Issue 1954: CREATE + WITH + MERGE causes "vertex was deleted" error +-- when the number of input rows exceeds the snapshot's command ID window. +-- entity_exists() used a stale curcid, making recently-created vertices +-- invisible on later iterations. +-- +SELECT * FROM create_graph('issue_1954'); +NOTICE: graph "issue_1954" has been created + create_graph +-------------- + +(1 row) + +-- Setup: create source nodes and relationships (3 rows to trigger the bug) +SELECT * FROM cypher('issue_1954', $$ + CREATE (:A {name: 'a1'})-[:R]->(:B {name: 'b1'}), + (:A {name: 'a2'})-[:R]->(:B {name: 'b2'}), + (:A {name: 'a3'})-[:R]->(:B {name: 'b3'}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- This query would fail with "vertex assigned to variable c was deleted" +-- on the 3rd row before the fix. +SELECT * FROM cypher('issue_1954', $$ + MATCH (a:A)-[:R]->(b:B) + CREATE (c:C {name: a.name + '|' + b.name}) + WITH a, b, c + MERGE (a)-[:LINK]->(c) + RETURN a.name, b.name, c.name + ORDER BY a.name +$$) AS (a agtype, b agtype, c agtype); + a | b | c +------+------+--------- + "a1" | "b1" | "a1|b1" + "a2" | "b2" | "a2|b2" + "a3" | "b3" | "a3|b3" +(3 rows) + +-- Verify edges were created +SELECT * FROM cypher('issue_1954', $$ + MATCH (a:A)-[:LINK]->(c:C) + RETURN a.name, c.name + ORDER BY a.name +$$) AS (a agtype, c agtype); + a | c +------+--------- + "a1" | "a1|b1" + "a2" | "a2|b2" + "a3" | "a3|b3" +(3 rows) + +-- Test with two MERGEs (more complex case from the original report) +SELECT * FROM cypher('issue_1954', $$ + MATCH ()-[e:LINK]->() DELETE e +$$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('issue_1954', $$ + MATCH (c:C) DELETE c +$$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('issue_1954', $$ + MATCH (a:A)-[:R]->(b:B) + CREATE (c:C {name: a.name + '|' + b.name}) + WITH a, b, c + MERGE (a)-[:LINK1]->(c) + MERGE (b)-[:LINK2]->(c) + RETURN a.name, b.name, c.name + ORDER BY a.name +$$) AS (a agtype, b agtype, c agtype); + a | b | c +------+------+--------- + "a1" | "b1" | "a1|b1" + "a2" | "b2" | "a2|b2" + "a3" | "b3" | "a3|b3" +(3 rows) + +-- Verify both sets of edges +SELECT * FROM cypher('issue_1954', $$ + MATCH (a:A)-[:LINK1]->(c:C) + RETURN a.name, c.name + ORDER BY a.name +$$) AS (a agtype, c agtype); + a | c +------+--------- + "a1" | "a1|b1" + "a2" | "a2|b2" + "a3" | "a3|b3" +(3 rows) + +SELECT * FROM cypher('issue_1954', $$ + MATCH (b:B)-[:LINK2]->(c:C) + RETURN b.name, c.name + ORDER BY b.name +$$) AS (b agtype, c agtype); + b | c +------+--------- + "b1" | "a1|b1" + "b2" | "a2|b2" + "b3" | "a3|b3" +(3 rows) + -- -- clean up graphs -- @@ -1888,6 +1996,11 @@ SELECT * FROM cypher('issue_1446', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype --- (0 rows) +SELECT * FROM cypher('issue_1954', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); + a +--- +(0 rows) + -- -- delete graphs -- @@ -1985,6 +2098,23 @@ NOTICE: graph "issue_1446" has been dropped (1 row) +SELECT drop_graph('issue_1954', true); +NOTICE: drop cascades to 9 other objects +DETAIL: drop cascades to table issue_1954._ag_label_vertex +drop cascades to table issue_1954._ag_label_edge +drop cascades to table issue_1954."A" +drop cascades to table issue_1954."R" +drop cascades to table issue_1954."B" +drop cascades to table issue_1954."C" +drop cascades to table issue_1954."LINK" +drop cascades to table issue_1954."LINK1" +drop cascades to table issue_1954."LINK2" +NOTICE: graph "issue_1954" has been dropped + drop_graph +------------ + +(1 row) + -- -- End -- diff --git a/regress/sql/cypher_merge.sql b/regress/sql/cypher_merge.sql index cc900e73d..5939c42a8 100644 --- a/regress/sql/cypher_merge.sql +++ b/regress/sql/cypher_merge.sql @@ -860,6 +860,69 @@ SELECT * FROM cypher('issue_1446', $$ RETURN count(*) AS edge_count $$) AS (edge_count agtype); +-- Issue 1954: CREATE + WITH + MERGE causes "vertex was deleted" error +-- when the number of input rows exceeds the snapshot's command ID window. +-- entity_exists() used a stale curcid, making recently-created vertices +-- invisible on later iterations. +-- +SELECT * FROM create_graph('issue_1954'); + +-- Setup: create source nodes and relationships (3 rows to trigger the bug) +SELECT * FROM cypher('issue_1954', $$ + CREATE (:A {name: 'a1'})-[:R]->(:B {name: 'b1'}), + (:A {name: 'a2'})-[:R]->(:B {name: 'b2'}), + (:A {name: 'a3'})-[:R]->(:B {name: 'b3'}) +$$) AS (result agtype); + +-- This query would fail with "vertex assigned to variable c was deleted" +-- on the 3rd row before the fix. +SELECT * FROM cypher('issue_1954', $$ + MATCH (a:A)-[:R]->(b:B) + CREATE (c:C {name: a.name + '|' + b.name}) + WITH a, b, c + MERGE (a)-[:LINK]->(c) + RETURN a.name, b.name, c.name + ORDER BY a.name +$$) AS (a agtype, b agtype, c agtype); + +-- Verify edges were created +SELECT * FROM cypher('issue_1954', $$ + MATCH (a:A)-[:LINK]->(c:C) + RETURN a.name, c.name + ORDER BY a.name +$$) AS (a agtype, c agtype); + +-- Test with two MERGEs (more complex case from the original report) +SELECT * FROM cypher('issue_1954', $$ + MATCH ()-[e:LINK]->() DELETE e +$$) AS (result agtype); +SELECT * FROM cypher('issue_1954', $$ + MATCH (c:C) DELETE c +$$) AS (result agtype); + +SELECT * FROM cypher('issue_1954', $$ + MATCH (a:A)-[:R]->(b:B) + CREATE (c:C {name: a.name + '|' + b.name}) + WITH a, b, c + MERGE (a)-[:LINK1]->(c) + MERGE (b)-[:LINK2]->(c) + RETURN a.name, b.name, c.name + ORDER BY a.name +$$) AS (a agtype, b agtype, c agtype); + +-- Verify both sets of edges +SELECT * FROM cypher('issue_1954', $$ + MATCH (a:A)-[:LINK1]->(c:C) + RETURN a.name, c.name + ORDER BY a.name +$$) AS (a agtype, c agtype); + +SELECT * FROM cypher('issue_1954', $$ + MATCH (b:B)-[:LINK2]->(c:C) + RETURN b.name, c.name + ORDER BY b.name +$$) AS (b agtype, c agtype); + -- -- clean up graphs -- @@ -867,6 +930,7 @@ SELECT * FROM cypher('cypher_merge', $$ MATCH (n) DETACH DELETE n $$) AS (a agty SELECT * FROM cypher('issue_1630', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); SELECT * FROM cypher('issue_1709', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); SELECT * FROM cypher('issue_1446', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); +SELECT * FROM cypher('issue_1954', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); -- -- delete graphs @@ -877,6 +941,7 @@ SELECT drop_graph('issue_1630', true); SELECT drop_graph('issue_1691', true); SELECT drop_graph('issue_1709', true); SELECT drop_graph('issue_1446', true); +SELECT drop_graph('issue_1954', true); -- -- End diff --git a/src/backend/executor/cypher_utils.c b/src/backend/executor/cypher_utils.c index eff829925..940284234 100644 --- a/src/backend/executor/cypher_utils.c +++ b/src/backend/executor/cypher_utils.c @@ -208,6 +208,7 @@ bool entity_exists(EState *estate, Oid graph_oid, graphid id) HeapTuple tuple; Relation rel; bool result = true; + CommandId saved_curcid; /* * Extract the label id from the graph id and get the table name @@ -219,6 +220,23 @@ bool entity_exists(EState *estate, Oid graph_oid, graphid id) ScanKeyInit(&scan_keys[0], 1, BTEqualStrategyNumber, F_GRAPHIDEQ, GRAPHID_GET_DATUM(id)); + /* + * Temporarily advance the snapshot's curcid so that entities inserted + * by preceding clauses (e.g., CREATE) in the same query are visible. + * CREATE calls CommandCounterIncrement() which advances the global + * CID, but does not update es_snapshot->curcid. The Decrement/Increment + * CID macros used by the executors can leave curcid behind the global + * CID, making recently created entities invisible to this scan. + * + * Use Max to ensure we never decrease curcid. The executor macros + * (Increment_Estate_CommandId) can push curcid above the global CID, + * and blindly assigning GetCurrentCommandId could make tuples that + * are visible at the current curcid become invisible. + */ + saved_curcid = estate->es_snapshot->curcid; + estate->es_snapshot->curcid = Max(saved_curcid, + GetCurrentCommandId(false)); + rel = table_open(label->relation, RowExclusiveLock); scan_desc = table_beginscan(rel, estate->es_snapshot, 1, scan_keys); @@ -236,6 +254,9 @@ bool entity_exists(EState *estate, Oid graph_oid, graphid id) table_endscan(scan_desc); table_close(rel, RowExclusiveLock); + /* Restore the original curcid */ + estate->es_snapshot->curcid = saved_curcid; + return result; } From 23146a44c36c0ece003b7b91f7942385ae8ac5e7 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Tue, 3 Mar 2026 12:52:48 -0500 Subject: [PATCH 039/100] Fix crash in PREPARE with property parameter when enable_containment is off (#2339) * Fix crash in PREPARE with property parameter when enable_containment is off When age.enable_containment is set to off, executing a PREPARE statement with a property parameter (e.g., MATCH (n $props) RETURN n) causes a segfault. The crash occurs in transform_map_to_ind_recursive because the property_constraints node is a cypher_param, not a cypher_map, but is blindly cast to cypher_map and its keyvals field is dereferenced. Three fixes: - In create_property_constraints, when enable_containment is off and the constraint is a cypher_param, fall back to the containment operator (@>) since map decomposition requires known keys at parse time. - In transform_match_entities, guard the keep_null assignment for both vertex and edge property constraints with is_ag_node checks to avoid writing to the wrong struct layout. Fixes #1964 Co-Authored-By: Claude Opus 4.6 * Fix @> vs @>> for =properties form with PREPARE and add tests When MATCH uses the =properties form (e.g., MATCH (n = $props)), the enable_containment=on path correctly uses @>> (top-level containment). The parameter fallback path unconditionally used @> (deep containment), ignoring the use_equals flag. Fix the fallback to mirror the enable_containment path by selecting @>> when use_equals is set. Add regression tests for =properties form with PREPARE for both vertices and edges, with enable_containment on and off. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- regress/expected/cypher_match.out | 116 +++++++++++++++++++++++++++++ regress/sql/cypher_match.sql | 67 +++++++++++++++++ src/backend/parser/cypher_clause.c | 42 ++++++++++- 3 files changed, 223 insertions(+), 2 deletions(-) diff --git a/regress/expected/cypher_match.out b/regress/expected/cypher_match.out index ff2825ae0..94315f134 100644 --- a/regress/expected/cypher_match.out +++ b/regress/expected/cypher_match.out @@ -3633,6 +3633,110 @@ NOTICE: graph "issue_2308" has been dropped (1 row) +-- Issue 1964 +-- +-- PREPARE with property parameter ($props) crashed the server when +-- age.enable_containment was set to off. The crash was in +-- transform_map_to_ind_recursive which blindly cast cypher_param +-- nodes to cypher_map, accessing invalid memory. +-- +SELECT create_graph('issue_1964'); +NOTICE: graph "issue_1964" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('issue_1964', $$ + CREATE (:Person {name: 'Alice', age: 30}), + (:Person {name: 'Bob', age: 25}) +$$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('issue_1964', $$ + CREATE (:Person {name: 'Alice'})-[:KNOWS {since: 2020}]->(:Person {name: 'Bob'}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- Test PREPARE with enable_containment off (was crashing) +SET age.enable_containment = off; +PREPARE issue_1964_vertex(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH (n $props) RETURN n $$, $1) AS (p agtype); +EXECUTE issue_1964_vertex('{"props": {"name": "Alice"}}'); + p +------------------------------------------------------------------------------------------------ + {"id": 844424930131969, "label": "Person", "properties": {"age": 30, "name": "Alice"}}::vertex + {"id": 844424930131971, "label": "Person", "properties": {"name": "Alice"}}::vertex +(2 rows) + +EXECUTE issue_1964_vertex('{"props": {"age": 25}}'); + p +---------------------------------------------------------------------------------------------- + {"id": 844424930131970, "label": "Person", "properties": {"age": 25, "name": "Bob"}}::vertex +(1 row) + +DEALLOCATE issue_1964_vertex; +-- Test edge property parameter with enable_containment off +PREPARE issue_1964_edge(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH ()-[r $props]->() RETURN r $$, $1) AS (p agtype); +EXECUTE issue_1964_edge('{"props": {"since": 2020}}'); + p +----------------------------------------------------------------------------------------------------------------------------------------- + {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131972, "start_id": 844424930131971, "properties": {"since": 2020}}::edge +(1 row) + +DEALLOCATE issue_1964_edge; +-- Verify enable_containment on still works with PREPARE +SET age.enable_containment = on; +PREPARE issue_1964_vertex_on(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH (n $props) RETURN n $$, $1) AS (p agtype); +EXECUTE issue_1964_vertex_on('{"props": {"name": "Alice"}}'); + p +------------------------------------------------------------------------------------------------ + {"id": 844424930131969, "label": "Person", "properties": {"age": 30, "name": "Alice"}}::vertex + {"id": 844424930131971, "label": "Person", "properties": {"name": "Alice"}}::vertex +(2 rows) + +DEALLOCATE issue_1964_vertex_on; +-- Test =properties form with PREPARE (uses @>> top-level containment) +SET age.enable_containment = off; +PREPARE issue_1964_vertex_eq(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH (n = $props) RETURN n $$, $1) AS (p agtype); +EXECUTE issue_1964_vertex_eq('{"props": {"name": "Alice", "age": 25}}'); + p +--- +(0 rows) + +DEALLOCATE issue_1964_vertex_eq; +PREPARE issue_1964_edge_eq(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH ()-[r = $props]->() RETURN r $$, $1) AS (p agtype); +EXECUTE issue_1964_edge_eq('{"props": {"since": 2020}}'); + p +----------------------------------------------------------------------------------------------------------------------------------------- + {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131972, "start_id": 844424930131971, "properties": {"since": 2020}}::edge +(1 row) + +DEALLOCATE issue_1964_edge_eq; +-- Same with enable_containment on +SET age.enable_containment = on; +PREPARE issue_1964_vertex_eq_on(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH (n = $props) RETURN n $$, $1) AS (p agtype); +EXECUTE issue_1964_vertex_eq_on('{"props": {"name": "Alice", "age": 25}}'); + p +--- +(0 rows) + +DEALLOCATE issue_1964_vertex_eq_on; -- -- Clean up -- @@ -3721,6 +3825,18 @@ NOTICE: graph "issue_1393" has been dropped (1 row) +SELECT drop_graph('issue_1964', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table issue_1964._ag_label_vertex +drop cascades to table issue_1964._ag_label_edge +drop cascades to table issue_1964."Person" +drop cascades to table issue_1964."KNOWS" +NOTICE: graph "issue_1964" has been dropped + drop_graph +------------ + +(1 row) + -- -- End -- diff --git a/regress/sql/cypher_match.sql b/regress/sql/cypher_match.sql index ebcd67b84..d14f45f10 100644 --- a/regress/sql/cypher_match.sql +++ b/regress/sql/cypher_match.sql @@ -1491,6 +1491,72 @@ SELECT * FROM cypher('issue_2308', $$ $$) AS (val agtype); SELECT drop_graph('issue_2308', true); +-- Issue 1964 +-- +-- PREPARE with property parameter ($props) crashed the server when +-- age.enable_containment was set to off. The crash was in +-- transform_map_to_ind_recursive which blindly cast cypher_param +-- nodes to cypher_map, accessing invalid memory. +-- + +SELECT create_graph('issue_1964'); +SELECT * FROM cypher('issue_1964', $$ + CREATE (:Person {name: 'Alice', age: 30}), + (:Person {name: 'Bob', age: 25}) +$$) AS (result agtype); +SELECT * FROM cypher('issue_1964', $$ + CREATE (:Person {name: 'Alice'})-[:KNOWS {since: 2020}]->(:Person {name: 'Bob'}) +$$) AS (result agtype); + +-- Test PREPARE with enable_containment off (was crashing) +SET age.enable_containment = off; + +PREPARE issue_1964_vertex(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH (n $props) RETURN n $$, $1) AS (p agtype); +EXECUTE issue_1964_vertex('{"props": {"name": "Alice"}}'); +EXECUTE issue_1964_vertex('{"props": {"age": 25}}'); +DEALLOCATE issue_1964_vertex; + +-- Test edge property parameter with enable_containment off +PREPARE issue_1964_edge(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH ()-[r $props]->() RETURN r $$, $1) AS (p agtype); +EXECUTE issue_1964_edge('{"props": {"since": 2020}}'); +DEALLOCATE issue_1964_edge; + +-- Verify enable_containment on still works with PREPARE +SET age.enable_containment = on; + +PREPARE issue_1964_vertex_on(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH (n $props) RETURN n $$, $1) AS (p agtype); +EXECUTE issue_1964_vertex_on('{"props": {"name": "Alice"}}'); +DEALLOCATE issue_1964_vertex_on; + +-- Test =properties form with PREPARE (uses @>> top-level containment) +SET age.enable_containment = off; + +PREPARE issue_1964_vertex_eq(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH (n = $props) RETURN n $$, $1) AS (p agtype); +EXECUTE issue_1964_vertex_eq('{"props": {"name": "Alice", "age": 25}}'); +DEALLOCATE issue_1964_vertex_eq; + +PREPARE issue_1964_edge_eq(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH ()-[r = $props]->() RETURN r $$, $1) AS (p agtype); +EXECUTE issue_1964_edge_eq('{"props": {"since": 2020}}'); +DEALLOCATE issue_1964_edge_eq; + +-- Same with enable_containment on +SET age.enable_containment = on; + +PREPARE issue_1964_vertex_eq_on(agtype) AS + SELECT * FROM cypher('issue_1964', + $$MATCH (n = $props) RETURN n $$, $1) AS (p agtype); +EXECUTE issue_1964_vertex_eq_on('{"props": {"name": "Alice", "age": 25}}'); +DEALLOCATE issue_1964_vertex_eq_on; -- -- Clean up @@ -1501,6 +1567,7 @@ SELECT drop_graph('test_enable_containment', true); SELECT drop_graph('issue_945', true); SELECT drop_graph('issue_1399', true); SELECT drop_graph('issue_1393', true); +SELECT drop_graph('issue_1964', true); -- -- End diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 446e97b3f..6f06bbb82 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -4261,6 +4261,38 @@ static Node *create_property_constraints(cypher_parsestate *cpstate, } else { + /* + * Map decomposition into individual index lookups requires known + * keys at parse time. When the property constraint is a parameter + * (cypher_param), the keys are not available until execution, so + * fall back to the containment operator. + */ + if (is_ag_node(property_constraints, cypher_param)) + { + /* + * Use @>> (top-level containment) for =properties form, + * @> (deep containment) otherwise — matching the + * enable_containment=on path above. + */ + if ((entity->type == ENT_VERTEX && + entity->entity.node->use_equals) || + ((entity->type == ENT_EDGE || + entity->type == ENT_VLE_EDGE) && + entity->entity.rel->use_equals)) + { + return (Node *)make_op(pstate, + list_make1(makeString("@>>")), + prop_expr, const_expr, + last_srf, -1); + } + else + { + return (Node *)make_op(pstate, + list_make1(makeString("@>")), + prop_expr, const_expr, + last_srf, -1); + } + } return (Node *)transform_map_to_ind( cpstate, entity, (cypher_map *)property_constraints); } @@ -4690,7 +4722,10 @@ static List *transform_match_entities(cypher_parsestate *cpstate, Query *query, -1); } - ((cypher_map*)node->props)->keep_null = true; + if (is_ag_node(node->props, cypher_map)) + { + ((cypher_map*)node->props)->keep_null = true; + } n = create_property_constraints(cpstate, entity, node->props, prop_expr); @@ -4819,7 +4854,10 @@ static List *transform_match_entities(cypher_parsestate *cpstate, Query *query, false, -1); } - ((cypher_map*)rel->props)->keep_null = true; + if (is_ag_node(rel->props, cypher_map)) + { + ((cypher_map*)rel->props)->keep_null = true; + } r = create_property_constraints(cpstate, entity, rel->props, prop_expr); From 90c33eb6b7c36f8280893bdc2b64666debb836de Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Mon, 6 Apr 2026 03:27:03 -0700 Subject: [PATCH 040/100] Add extension upgrade template regression test (#2364) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note: This PR was created with AI tools and a human. Add a version-agnostic regression test (age_upgrade) that validates the upgrade template (age----y.y.y.sql) works correctly by simulating a full extension version upgrade within "make installcheck". Add full upgrade scripts to the install path (DATA) in the Makefile, excluding template upgrade files. This enables the install to copy all version upgrade files into the PG AGE install. This is needed for ALTER EXTENSION Adjusted installcheck.yaml to allow git commit history for this test. Makefile infrastructure: - Build the install SQL (age--.sql) from the initial version-bump commit in git history, so CREATE EXTENSION installs "day-one" SQL while the .so comes from current HEAD — implicitly testing backward compat. - Build a synthetic "next" version (age--.sql) from HEAD and stamp the upgrade template to produce age----.sql. - Add an installcheck prerequisite that temporarily installs both synthetic files into the PG extension directory; a generated cleanup script removes them at the end of the test via \! shell escape. EXTRA_CLEAN catches stragglers on "make clean". - Skip the test automatically when: (a) no git history (tarball builds), (b) no upgrade template exists, or (c) a real upgrade script from the current version is already committed (detected via git ls-files).o Regression test (regress/sql/age_upgrade.sql): - Creates 3 graphs (company, network, routes) with 8 vertex labels, 8 edge labels, 23 vertices, 28 edges, and 4 GIN indexes. - Records integrity checksums (agtype sums), vertex/edge counts, and label counts before the upgrade; repeats all checks after ALTER EXTENSION UPDATE to the synthetic next version. - Verifies structural queries: VLE management chains, circular follow chains, flight distances with edge properties. - Verifies all 4 GIN indexes survive the upgrade via pg_indexes. - Uses ORDER BY on all multi-row queries for deterministic output. - Returns agtype natively (no ::numeric casts) for portability. - Avoids version-dependent output (checks boolean IS NOT NULL instead of printing the version string). - Uses JOIN-based label counts to avoid NULL comparison bugs with the internal _ag_catalog graph. - Cleans up all 3 graphs and restores the default AGE version. modified: Makefile new file: regress/expected/age_upgrade.out new file: regress/sql/age_upgrade.sql modified: .github/workflows/installcheck.yaml --- .github/workflows/installcheck.yaml | 2 + Makefile | 136 ++++++++- regress/expected/age_upgrade.out | 443 ++++++++++++++++++++++++++++ regress/sql/age_upgrade.sql | 296 +++++++++++++++++++ 4 files changed, 875 insertions(+), 2 deletions(-) create mode 100644 regress/expected/age_upgrade.out create mode 100644 regress/sql/age_upgrade.sql diff --git a/.github/workflows/installcheck.yaml b/.github/workflows/installcheck.yaml index dcb2057df..0cbb33b90 100644 --- a/.github/workflows/installcheck.yaml +++ b/.github/workflows/installcheck.yaml @@ -42,6 +42,8 @@ jobs: make PG_CONFIG=$HOME/pg18/bin/pg_config install -j$(nproc) > /dev/null - uses: actions/checkout@v3 + with: + fetch-depth: 75 - name: Build AGE id: build diff --git a/Makefile b/Makefile index 394951ca0..5c0a8e1ef 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,64 @@ MODULE_big = age age_sql = age--1.7.0.sql +# --- Extension upgrade regression test support --- +# +# Validates the upgrade template (age----y.y.y.sql) by simulating an +# extension version upgrade entirely within "make installcheck". The test: +# +# 1. Builds the install SQL from the INITIAL version-bump commit (the "from" +# version). This is the age--.sql used by CREATE EXTENSION age. +# 2. Builds the install SQL from current HEAD as a synthetic "next" version +# (the "to" version). This is age--.sql where NEXT = CURR.minor+1. +# 3. Stamps the upgrade template with the synthetic version, producing the +# upgrade script age----.sql. +# 4. Temporarily installs both synthetic files into the PG extension directory +# so that ALTER EXTENSION age UPDATE TO '' can find them. +# 5. The age_upgrade regression test exercises the full upgrade path: install +# at CURR, create data, ALTER EXTENSION UPDATE to NEXT, verify data. +# 6. The test SQL cleans up the synthetic files via a generated shell script. +# +# This forces developers to keep the upgrade template in sync: any SQL object +# added after the version-bump commit must also appear in the template, or the +# upgrade test will fail (the object will be missing after ALTER EXTENSION UPDATE). +# +# The .so (shared library) is always built from current HEAD, so C-level changes +# in the PR are tested by every regression test, not just the upgrade test. +# +# Graceful degradation — the upgrade test is silently skipped when: +# - No git history (tarball build): AGE_VER_COMMIT is empty. +# - No upgrade template: age----y.y.y.sql does not exist. +# - A real (git-tracked) upgrade script from already exists +# (e.g., age--1.7.0--1.8.0.sql is committed): the synthetic test is +# redundant because the real script ships with the extension. +# Current version from age.control (e.g., "1.7.0") +AGE_CURR_VER := $(shell awk -F"'" '/default_version/ {print $$2}' age.control 2>/dev/null) +# Git commit that last changed age.control — the "initial release" commit +AGE_VER_COMMIT := $(shell git log -1 --format=%H -- age.control 2>/dev/null) +# Synthetic next version: current minor + 1 (e.g., 1.7.0 -> 1.8.0) +AGE_NEXT_VER := $(shell echo $(AGE_CURR_VER) | awk -F. '{printf "%s.%s.%s", $$1, $$2+1, $$3}') +# The upgrade template file (e.g., age--1.7.0--y.y.y.sql); empty if not present +AGE_UPGRADE_TEMPLATE := $(wildcard age--$(AGE_CURR_VER)--y.y.y.sql) + +# Synthetic filenames — these are NOT installed permanently; they are temporarily +# placed in $(SHAREDIR)/extension/ during installcheck and removed after. +age_next_sql = $(if $(AGE_NEXT_VER),age--$(AGE_NEXT_VER).sql) +age_upgrade_test_sql = $(if $(AGE_NEXT_VER),age--$(AGE_CURR_VER)--$(AGE_NEXT_VER).sql) + +# Real (git-tracked, non-template) upgrade scripts FROM the current version. +# If any exist (e.g., age--1.7.0--1.8.0.sql is committed), the synthetic +# upgrade test is redundant because a real upgrade path already ships. +# Uses git ls-files so untracked synthetic files are NOT matched. +AGE_REAL_UPGRADE := $(shell git ls-files 'age--$(AGE_CURR_VER)--*.sql' 2>/dev/null | grep -v 'y\.y\.y') + +# Non-empty when ALL of these hold: +# 1. Git history is available (AGE_VER_COMMIT non-empty) +# 2. The upgrade template exists (AGE_UPGRADE_TEMPLATE non-empty) +# 3. No real upgrade script from current version exists (AGE_REAL_UPGRADE empty) +# When a real upgrade script ships, the test is skipped — the real script +# supersedes the synthetic one and has its own validation path. +AGE_HAS_UPGRADE_TEST = $(and $(AGE_VER_COMMIT),$(AGE_UPGRADE_TEMPLATE),$(if $(AGE_REAL_UPGRADE),,yes)) + OBJS = src/backend/age.o \ src/backend/catalog/ag_catalog.o \ src/backend/catalog/ag_graph.o \ @@ -83,6 +141,10 @@ SQLS := $(addsuffix .sql,$(SQLS)) DATA_built = $(age_sql) +# Git-tracked upgrade scripts shipped with the extension (e.g., age--1.6.0--1.7.0.sql). +# Excludes the upgrade template (y.y.y) and the synthetic stamped test file. +DATA = $(filter-out age--%-y.y.y.sql $(age_upgrade_test_sql),$(wildcard age--*--*.sql)) + # sorted in dependency order REGRESS = scan \ graphid \ @@ -119,6 +181,13 @@ ifneq ($(EXTRA_TESTS),) REGRESS += $(EXTRA_TESTS) endif +# Extension upgrade test — included when git history is available, the upgrade +# template exists, and no real upgrade script from the current version is +# committed. Runs between "security" and "drop" in test order. +ifneq ($(AGE_HAS_UPGRADE_TEST),) + REGRESS += age_upgrade +endif + REGRESS += drop srcdir=`pwd` @@ -127,7 +196,7 @@ ag_regress_dir = $(srcdir)/regress REGRESS_OPTS = --load-extension=age --inputdir=$(ag_regress_dir) --outputdir=$(ag_regress_dir) --temp-instance=$(ag_regress_dir)/instance --port=61958 --encoding=UTF-8 --temp-config $(ag_regress_dir)/age_regression.conf ag_regress_out = instance/ log/ results/ regression.* -EXTRA_CLEAN = $(addprefix $(ag_regress_dir)/, $(ag_regress_out)) src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/include/parser/cypher_kwlist_d.h $(all_age_sql) +EXTRA_CLEAN = $(addprefix $(ag_regress_dir)/, $(ag_regress_out)) src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/include/parser/cypher_kwlist_d.h $(all_age_sql) $(age_next_sql) $(age_upgrade_test_sql) $(ag_regress_dir)/age_upgrade_cleanup.sh GEN_KEYWORDLIST = $(PERL) -I ./tools/ ./tools/gen_keywordlist.pl GEN_KEYWORDLIST_DEPS = ./tools/gen_keywordlist.pl tools/PerfectHash.pm @@ -157,7 +226,27 @@ src/backend/parser/cypher_parser.bc: src/backend/parser/cypher_gram.c src/includ src/backend/parser/cypher_keywords.o: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/backend/parser/cypher_keywords.bc: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h -# Strip PASSEDBYVALUE on 32-bit (SIZEOF_DATUM=4) for graphid pass-by-reference +# Build the default install SQL (age--.sql) from the INITIAL version-bump +# commit in git history. This means CREATE EXTENSION age installs the "day-one +# release" SQL — the state of sql/ at the moment the version was bumped in +# age.control. All other regression tests run against this SQL + the current +# HEAD .so, implicitly validating that the .so is backward-compatible with the +# initial release SQL. +# +# The current HEAD SQL goes into the synthetic next version (age--.sql) +# which is only reachable via ALTER EXTENSION UPDATE in the upgrade test. +ifneq ($(AGE_VER_COMMIT),) +$(age_sql): age.control + @echo "Building initial-release install SQL: $@ from commit $(AGE_VER_COMMIT)" + @for f in $$(git show $(AGE_VER_COMMIT):sql/sql_files 2>/dev/null); do \ + git show $(AGE_VER_COMMIT):sql/$${f}.sql 2>/dev/null; \ + done > $@ +ifeq ($(SIZEOF_DATUM),4) + @echo "32-bit build: removing PASSEDBYVALUE from graphid type" + @sed 's/^ PASSEDBYVALUE,$$/ -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@ +endif +else +# Fallback: no git history (tarball build) — use current HEAD SQL fragments $(age_sql): $(SQLS) @cat $(SQLS) > $@ ifeq ($(SIZEOF_DATUM),4) @@ -165,7 +254,50 @@ ifeq ($(SIZEOF_DATUM),4) @sed 's/^ PASSEDBYVALUE,$$/ -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@ @grep -q 'PASSEDBYVALUE removed for 32-bit' $@ || { echo "Error: PASSEDBYVALUE replacement failed in $@"; exit 1; } endif +endif + +# Build synthetic "next" version install SQL from current HEAD's sql/sql_files. +# This represents what the extension SQL looks like in the PR being tested. +ifneq ($(AGE_HAS_UPGRADE_TEST),) +$(age_next_sql): $(SQLS) + @echo "Building synthetic next version install SQL: $@ ($(AGE_NEXT_VER))" + @cat $(SQLS) > $@ +ifeq ($(SIZEOF_DATUM),4) + @sed 's/^ PASSEDBYVALUE,$$/ -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@ +endif + +# Stamp upgrade template as upgrade from current to synthetic next version +$(age_upgrade_test_sql): $(AGE_UPGRADE_TEMPLATE) + @echo "Stamping upgrade template: $< -> $@" + @sed -e "s/1\.X\.0/$(AGE_NEXT_VER)/g" -e "s/y\.y\.y/$(AGE_NEXT_VER)/g" $< > $@ +endif src/backend/parser/ag_scanner.c: FLEX_NO_BACKUP=yes +# --- Upgrade test file lifecycle during installcheck --- +# +# Problem: The upgrade test needs age--.sql and age----.sql +# in the PG extension directory for ALTER EXTENSION UPDATE to find them, but +# we must not leave them installed permanently (they would confuse users). +# +# Solution: A Make prerequisite installs them before pg_regress runs, and the +# test SQL removes them at the end via \! (shell escape in psql). A generated +# cleanup script (regress/age_upgrade_cleanup.sh) contains the exact absolute +# paths so the removal works regardless of the working directory. EXTRA_CLEAN +# also removes them on "make clean" as a safety net. +# +# This adds a prerequisite to "installcheck" but does NOT override the PGXS +# recipe, so there are no Makefile warnings. +SHAREDIR = $(shell $(PG_CONFIG) --sharedir) + installcheck: export LC_COLLATE=C +ifneq ($(AGE_HAS_UPGRADE_TEST),) +.PHONY: _install_upgrade_test_files +_install_upgrade_test_files: $(age_next_sql) $(age_upgrade_test_sql) ## Build, install synthetic files, generate cleanup script + @echo "Installing upgrade test files to $(SHAREDIR)/extension/" + @$(INSTALL_DATA) $(age_next_sql) $(age_upgrade_test_sql) '$(SHAREDIR)/extension/' + @printf '#!/bin/sh\nrm -f "$(SHAREDIR)/extension/$(age_next_sql)" "$(SHAREDIR)/extension/$(age_upgrade_test_sql)"\n' > $(ag_regress_dir)/age_upgrade_cleanup.sh + @chmod +x $(ag_regress_dir)/age_upgrade_cleanup.sh + +installcheck: _install_upgrade_test_files +endif diff --git a/regress/expected/age_upgrade.out b/regress/expected/age_upgrade.out new file mode 100644 index 000000000..39073b460 --- /dev/null +++ b/regress/expected/age_upgrade.out @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +-- +-- Extension upgrade regression test +-- +-- This test validates the upgrade template (age----y.y.y.sql) by: +-- 1. Installing AGE at the current version (built from the initial +-- version-bump commit's SQL — the "day-one release" state) +-- 2. Creating three graphs with multiple labels, edges, GIN indexes, +-- and numeric properties that serve as integrity checksums +-- 3. Upgrading to a synthetic "next" version via the stamped template +-- 4. Verifying all data, structure, and checksums survived the upgrade +-- +-- The Makefile builds: +-- age--.sql from the initial version-bump commit (git history) +-- age--.sql from current HEAD's sql/sql_files +-- age----.sql stamped from the upgrade template +-- +-- All version discovery is dynamic — no hardcoded versions anywhere. +-- This test is version-agnostic and works on any branch for any version. +-- +LOAD 'age'; +SET search_path TO ag_catalog; +-- Step 1: Clean up any state left by prior tests, then drop AGE entirely. +-- The --load-extension=age flag installed AGE at the current (default) version. +-- We need to remove it so we can cleanly re-create for this test. +SELECT drop_graph(name, true) FROM ag_graph ORDER BY name; + drop_graph +------------ +(0 rows) + +DROP EXTENSION age; +-- Step 2: Verify we have multiple installable versions. +SELECT count(*) > 1 AS has_upgrade_path +FROM pg_available_extension_versions WHERE name = 'age'; + has_upgrade_path +------------------ + t +(1 row) + +-- Step 3: Install AGE at the default version (the initial release SQL). +CREATE EXTENSION age; +SELECT extversion IS NOT NULL AS version_installed FROM pg_extension WHERE extname = 'age'; + version_installed +------------------- + t +(1 row) + +-- Step 4: Create three test graphs with diverse labels, edges, and data. +LOAD 'age'; +SET search_path TO ag_catalog, "$user", public; +-- +-- Graph 1: "company" — organization hierarchy with numeric checksums. +-- Labels: Employee, Department, Project +-- Edges: WORKS_IN, MANAGES, ASSIGNED_TO +-- Each vertex has a "val" property (float) for checksum validation. +-- +SELECT create_graph('company'); +NOTICE: graph "company" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('company', $$ + CREATE (e1:Employee {name: 'Alice', role: 'VP', val: 3.14159}) + CREATE (e2:Employee {name: 'Bob', role: 'Manager', val: 2.71828}) + CREATE (e3:Employee {name: 'Charlie', role: 'Engineer', val: 1.41421}) + CREATE (e4:Employee {name: 'Diana', role: 'Engineer', val: 1.73205}) + CREATE (d1:Department {name: 'Engineering', budget: 500000, val: 42.0}) + CREATE (d2:Department {name: 'Research', budget: 300000, val: 17.5}) + CREATE (p1:Project {name: 'Atlas', priority: 1, val: 99.99}) + CREATE (p2:Project {name: 'Beacon', priority: 2, val: 88.88}) + CREATE (p3:Project {name: 'Cipher', priority: 3, val: 77.77}) + CREATE (e1)-[:WORKS_IN {since: 2019}]->(d1) + CREATE (e2)-[:WORKS_IN {since: 2020}]->(d1) + CREATE (e3)-[:WORKS_IN {since: 2021}]->(d1) + CREATE (e4)-[:WORKS_IN {since: 2022}]->(d2) + CREATE (e1)-[:MANAGES {level: 1}]->(e2) + CREATE (e2)-[:MANAGES {level: 2}]->(e3) + CREATE (e3)-[:ASSIGNED_TO {hours: 40}]->(p1) + CREATE (e3)-[:ASSIGNED_TO {hours: 20}]->(p2) + CREATE (e4)-[:ASSIGNED_TO {hours: 30}]->(p2) + CREATE (e4)-[:ASSIGNED_TO {hours: 10}]->(p3) + RETURN 'company graph created' +$$) AS (result agtype); + result +------------------------- + "company graph created" +(1 row) + +-- GIN index on Employee properties in company graph +CREATE INDEX company_employee_gin ON company."Employee" USING GIN (properties); +-- +-- Graph 2: "network" — social network with weighted edges. +-- Labels: User, Post +-- Edges: FOLLOWS, AUTHORED, LIKES +-- +SELECT create_graph('network'); +NOTICE: graph "network" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('network', $$ + CREATE (u1:User {handle: '@alpha', score: 1000.01}) + CREATE (u2:User {handle: '@beta', score: 2000.02}) + CREATE (u3:User {handle: '@gamma', score: 3000.03}) + CREATE (u4:User {handle: '@delta', score: 4000.04}) + CREATE (u5:User {handle: '@epsilon', score: 5000.05}) + CREATE (p1:Post {title: 'Hello World', views: 150}) + CREATE (p2:Post {title: 'Graph Databases 101', views: 890}) + CREATE (p3:Post {title: 'AGE is awesome', views: 2200}) + CREATE (u1)-[:FOLLOWS {weight: 0.9}]->(u2) + CREATE (u2)-[:FOLLOWS {weight: 0.8}]->(u3) + CREATE (u3)-[:FOLLOWS {weight: 0.7}]->(u4) + CREATE (u4)-[:FOLLOWS {weight: 0.6}]->(u5) + CREATE (u5)-[:FOLLOWS {weight: 0.5}]->(u1) + CREATE (u1)-[:AUTHORED]->(p1) + CREATE (u2)-[:AUTHORED]->(p2) + CREATE (u3)-[:AUTHORED]->(p3) + CREATE (u4)-[:LIKES]->(p1) + CREATE (u5)-[:LIKES]->(p2) + CREATE (u1)-[:LIKES]->(p3) + CREATE (u2)-[:LIKES]->(p3) + RETURN 'network graph created' +$$) AS (result agtype); + result +------------------------- + "network graph created" +(1 row) + +-- GIN indexes on network graph +CREATE INDEX network_user_gin ON network."User" USING GIN (properties); +CREATE INDEX network_post_gin ON network."Post" USING GIN (properties); +-- +-- Graph 3: "routes" — geographic routing with precise coordinates. +-- Labels: City, Airport +-- Edges: ROAD, FLIGHT +-- Coordinates use precise decimals that are easy to checksum. +-- +SELECT create_graph('routes'); +NOTICE: graph "routes" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('routes', $$ + CREATE (c1:City {name: 'Portland', lat: 45.5152, lon: -122.6784, pop: 652503}) + CREATE (c2:City {name: 'Seattle', lat: 47.6062, lon: -122.3321, pop: 749256}) + CREATE (c3:City {name: 'Vancouver', lat: 49.2827, lon: -123.1207, pop: 631486}) + CREATE (a1:Airport {code: 'PDX', elev: 30.5}) + CREATE (a2:Airport {code: 'SEA', elev: 131.7}) + CREATE (a3:Airport {code: 'YVR', elev: 4.3}) + CREATE (c1)-[:ROAD {distance_km: 279.5, toll: 0.0}]->(c2) + CREATE (c2)-[:ROAD {distance_km: 225.3, toll: 5.0}]->(c3) + CREATE (c1)-[:ROAD {distance_km: 502.1, toll: 5.0}]->(c3) + CREATE (a1)-[:FLIGHT {distance_km: 229.0, duration_min: 55}]->(a2) + CREATE (a2)-[:FLIGHT {distance_km: 198.0, duration_min: 50}]->(a3) + CREATE (a1)-[:FLIGHT {distance_km: 426.0, duration_min: 75}]->(a3) + RETURN 'routes graph created' +$$) AS (result agtype); + result +------------------------ + "routes graph created" +(1 row) + +-- GIN index on routes graph +CREATE INDEX routes_city_gin ON routes."City" USING GIN (properties); +-- Step 5: Record pre-upgrade integrity checksums. +-- These sums use the "val" / "score" / coordinate properties as fingerprints. +-- company: sum of all val properties (should be a precise known value) +SELECT * FROM cypher('company', $$ + MATCH (n) WHERE n.val IS NOT NULL RETURN sum(n.val) +$$) AS (company_val_sum_before agtype); + company_val_sum_before +------------------------ + 335.14612999999997 +(1 row) + +-- network: sum of all score properties +SELECT * FROM cypher('network', $$ + MATCH (n:User) RETURN sum(n.score) +$$) AS (network_score_sum_before agtype); + network_score_sum_before +-------------------------- + 15000.149999999998 +(1 row) + +-- routes: sum of all latitude values +SELECT * FROM cypher('routes', $$ + MATCH (c:City) RETURN sum(c.lat) +$$) AS (routes_lat_sum_before agtype); + routes_lat_sum_before +----------------------- + 142.4041 +(1 row) + +-- Total vertex and edge counts across all three graphs +SELECT sum(cnt)::int AS total_vertices_before FROM ( + SELECT count(*) AS cnt FROM cypher('company', $$ MATCH (n) RETURN n $$) AS (n agtype) + UNION ALL + SELECT count(*) FROM cypher('network', $$ MATCH (n) RETURN n $$) AS (n agtype) + UNION ALL + SELECT count(*) FROM cypher('routes', $$ MATCH (n) RETURN n $$) AS (n agtype) +) sub; + total_vertices_before +----------------------- + 23 +(1 row) + +SELECT sum(cnt)::int AS total_edges_before FROM ( + SELECT count(*) AS cnt FROM cypher('company', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) + UNION ALL + SELECT count(*) FROM cypher('network', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) + UNION ALL + SELECT count(*) FROM cypher('routes', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) +) sub; + total_edges_before +-------------------- + 28 +(1 row) + +-- Count of distinct labels (ag_label entries) across all graphs +SELECT count(*)::int AS total_labels_before +FROM ag_label al JOIN ag_graph ag ON al.graph = ag.graphid +WHERE ag.name <> '_ag_catalog'; + total_labels_before +--------------------- + 21 +(1 row) + +-- Step 6: Upgrade AGE to the synthetic next version via the stamped template. +DO $$ +DECLARE next_ver text; +BEGIN + SELECT version INTO next_ver + FROM pg_available_extension_versions + WHERE name = 'age' AND version <> ( + SELECT default_version FROM pg_available_extensions WHERE name = 'age' + ) + ORDER BY string_to_array(version, '.')::int[] DESC + LIMIT 1; + + IF next_ver IS NULL THEN + RAISE EXCEPTION 'No next version available for upgrade test'; + END IF; + + EXECUTE format('ALTER EXTENSION age UPDATE TO %L', next_ver); +END; +$$; +-- Step 7: Confirm version changed. +SELECT installed_version <> default_version AS upgraded_past_default +FROM pg_available_extensions WHERE name = 'age'; + upgraded_past_default +----------------------- + t +(1 row) + +-- Step 8: Verify all data survived — reload and recheck. +LOAD 'age'; +SET search_path TO ag_catalog, "$user", public; +-- Repeat integrity checksums — must match pre-upgrade values exactly. +SELECT * FROM cypher('company', $$ + MATCH (n) WHERE n.val IS NOT NULL RETURN sum(n.val) +$$) AS (company_val_sum_after agtype); + company_val_sum_after +----------------------- + 335.14612999999997 +(1 row) + +SELECT * FROM cypher('network', $$ + MATCH (n:User) RETURN sum(n.score) +$$) AS (network_score_sum_after agtype); + network_score_sum_after +------------------------- + 15000.149999999998 +(1 row) + +SELECT * FROM cypher('routes', $$ + MATCH (c:City) RETURN sum(c.lat) +$$) AS (routes_lat_sum_after agtype); + routes_lat_sum_after +---------------------- + 142.4041 +(1 row) + +SELECT sum(cnt)::int AS total_vertices_after FROM ( + SELECT count(*) AS cnt FROM cypher('company', $$ MATCH (n) RETURN n $$) AS (n agtype) + UNION ALL + SELECT count(*) FROM cypher('network', $$ MATCH (n) RETURN n $$) AS (n agtype) + UNION ALL + SELECT count(*) FROM cypher('routes', $$ MATCH (n) RETURN n $$) AS (n agtype) +) sub; + total_vertices_after +---------------------- + 23 +(1 row) + +SELECT sum(cnt)::int AS total_edges_after FROM ( + SELECT count(*) AS cnt FROM cypher('company', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) + UNION ALL + SELECT count(*) FROM cypher('network', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) + UNION ALL + SELECT count(*) FROM cypher('routes', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) +) sub; + total_edges_after +------------------- + 28 +(1 row) + +SELECT count(*)::int AS total_labels_after +FROM ag_label al JOIN ag_graph ag ON al.graph = ag.graphid +WHERE ag.name <> '_ag_catalog'; + total_labels_after +-------------------- + 21 +(1 row) + +-- Step 9: Verify specific structural queries across all three graphs. +-- company: management chain +SELECT * FROM cypher('company', $$ + MATCH (boss:Employee)-[:MANAGES*]->(report:Employee) + RETURN boss.name, report.name + ORDER BY boss.name, report.name +$$) AS (boss agtype, report agtype); + boss | report +---------+----------- + "Alice" | "Bob" + "Alice" | "Charlie" + "Bob" | "Charlie" +(3 rows) + +-- network: circular follow chain (proves full cycle survived) +SELECT * FROM cypher('network', $$ + MATCH (a:User)-[:FOLLOWS]->(b:User) + RETURN a.handle, b.handle + ORDER BY a.handle +$$) AS (follower agtype, followed agtype); + follower | followed +------------+------------ + "@alpha" | "@beta" + "@beta" | "@gamma" + "@delta" | "@epsilon" + "@epsilon" | "@alpha" + "@gamma" | "@delta" +(5 rows) + +-- routes: all flights with distances (proves edge properties intact) +SELECT * FROM cypher('routes', $$ + MATCH (a:Airport)-[f:FLIGHT]->(b:Airport) + RETURN a.code, b.code, f.distance_km + ORDER BY a.code, b.code +$$) AS (origin agtype, dest agtype, dist agtype); + origin | dest | dist +--------+-------+------- + "PDX" | "SEA" | 229.0 + "PDX" | "YVR" | 426.0 + "SEA" | "YVR" | 198.0 +(3 rows) + +-- Step 10: Verify GIN indexes still exist after upgrade. +SELECT indexname FROM pg_indexes +WHERE schemaname IN ('company', 'network', 'routes') + AND tablename IN ('Employee', 'User', 'Post', 'City') + AND indexdef LIKE '%gin%' +ORDER BY indexname; + indexname +---------------------- + company_employee_gin + network_post_gin + network_user_gin + routes_city_gin +(4 rows) + +-- Step 11: Cleanup and restore AGE at the default version for subsequent tests. +SELECT drop_graph('routes', true); +NOTICE: drop cascades to 6 other objects +DETAIL: drop cascades to table routes._ag_label_vertex +drop cascades to table routes._ag_label_edge +drop cascades to table routes."City" +drop cascades to table routes."Airport" +drop cascades to table routes."ROAD" +drop cascades to table routes."FLIGHT" +NOTICE: graph "routes" has been dropped + drop_graph +------------ + +(1 row) + +SELECT drop_graph('network', true); +NOTICE: drop cascades to 7 other objects +DETAIL: drop cascades to table network._ag_label_vertex +drop cascades to table network._ag_label_edge +drop cascades to table network."User" +drop cascades to table network."Post" +drop cascades to table network."FOLLOWS" +drop cascades to table network."AUTHORED" +drop cascades to table network."LIKES" +NOTICE: graph "network" has been dropped + drop_graph +------------ + +(1 row) + +SELECT drop_graph('company', true); +NOTICE: drop cascades to 8 other objects +DETAIL: drop cascades to table company._ag_label_vertex +drop cascades to table company._ag_label_edge +drop cascades to table company."Employee" +drop cascades to table company."Department" +drop cascades to table company."Project" +drop cascades to table company."WORKS_IN" +drop cascades to table company."MANAGES" +drop cascades to table company."ASSIGNED_TO" +NOTICE: graph "company" has been dropped + drop_graph +------------ + +(1 row) + +DROP EXTENSION age; +CREATE EXTENSION age; +-- Step 12: Remove synthetic upgrade test files from the extension directory. +\! sh ./regress/age_upgrade_cleanup.sh diff --git a/regress/sql/age_upgrade.sql b/regress/sql/age_upgrade.sql new file mode 100644 index 000000000..649349cdf --- /dev/null +++ b/regress/sql/age_upgrade.sql @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +-- +-- Extension upgrade regression test +-- +-- This test validates the upgrade template (age----y.y.y.sql) by: +-- 1. Installing AGE at the current version (built from the initial +-- version-bump commit's SQL — the "day-one release" state) +-- 2. Creating three graphs with multiple labels, edges, GIN indexes, +-- and numeric properties that serve as integrity checksums +-- 3. Upgrading to a synthetic "next" version via the stamped template +-- 4. Verifying all data, structure, and checksums survived the upgrade +-- +-- The Makefile builds: +-- age--.sql from the initial version-bump commit (git history) +-- age--.sql from current HEAD's sql/sql_files +-- age----.sql stamped from the upgrade template +-- +-- All version discovery is dynamic — no hardcoded versions anywhere. +-- This test is version-agnostic and works on any branch for any version. +-- + +LOAD 'age'; +SET search_path TO ag_catalog; + +-- Step 1: Clean up any state left by prior tests, then drop AGE entirely. +-- The --load-extension=age flag installed AGE at the current (default) version. +-- We need to remove it so we can cleanly re-create for this test. +SELECT drop_graph(name, true) FROM ag_graph ORDER BY name; +DROP EXTENSION age; + +-- Step 2: Verify we have multiple installable versions. +SELECT count(*) > 1 AS has_upgrade_path +FROM pg_available_extension_versions WHERE name = 'age'; + +-- Step 3: Install AGE at the default version (the initial release SQL). +CREATE EXTENSION age; +SELECT extversion IS NOT NULL AS version_installed FROM pg_extension WHERE extname = 'age'; + +-- Step 4: Create three test graphs with diverse labels, edges, and data. +LOAD 'age'; +SET search_path TO ag_catalog, "$user", public; + +-- +-- Graph 1: "company" — organization hierarchy with numeric checksums. +-- Labels: Employee, Department, Project +-- Edges: WORKS_IN, MANAGES, ASSIGNED_TO +-- Each vertex has a "val" property (float) for checksum validation. +-- +SELECT create_graph('company'); + +SELECT * FROM cypher('company', $$ + CREATE (e1:Employee {name: 'Alice', role: 'VP', val: 3.14159}) + CREATE (e2:Employee {name: 'Bob', role: 'Manager', val: 2.71828}) + CREATE (e3:Employee {name: 'Charlie', role: 'Engineer', val: 1.41421}) + CREATE (e4:Employee {name: 'Diana', role: 'Engineer', val: 1.73205}) + CREATE (d1:Department {name: 'Engineering', budget: 500000, val: 42.0}) + CREATE (d2:Department {name: 'Research', budget: 300000, val: 17.5}) + CREATE (p1:Project {name: 'Atlas', priority: 1, val: 99.99}) + CREATE (p2:Project {name: 'Beacon', priority: 2, val: 88.88}) + CREATE (p3:Project {name: 'Cipher', priority: 3, val: 77.77}) + CREATE (e1)-[:WORKS_IN {since: 2019}]->(d1) + CREATE (e2)-[:WORKS_IN {since: 2020}]->(d1) + CREATE (e3)-[:WORKS_IN {since: 2021}]->(d1) + CREATE (e4)-[:WORKS_IN {since: 2022}]->(d2) + CREATE (e1)-[:MANAGES {level: 1}]->(e2) + CREATE (e2)-[:MANAGES {level: 2}]->(e3) + CREATE (e3)-[:ASSIGNED_TO {hours: 40}]->(p1) + CREATE (e3)-[:ASSIGNED_TO {hours: 20}]->(p2) + CREATE (e4)-[:ASSIGNED_TO {hours: 30}]->(p2) + CREATE (e4)-[:ASSIGNED_TO {hours: 10}]->(p3) + RETURN 'company graph created' +$$) AS (result agtype); + +-- GIN index on Employee properties in company graph +CREATE INDEX company_employee_gin ON company."Employee" USING GIN (properties); + +-- +-- Graph 2: "network" — social network with weighted edges. +-- Labels: User, Post +-- Edges: FOLLOWS, AUTHORED, LIKES +-- +SELECT create_graph('network'); + +SELECT * FROM cypher('network', $$ + CREATE (u1:User {handle: '@alpha', score: 1000.01}) + CREATE (u2:User {handle: '@beta', score: 2000.02}) + CREATE (u3:User {handle: '@gamma', score: 3000.03}) + CREATE (u4:User {handle: '@delta', score: 4000.04}) + CREATE (u5:User {handle: '@epsilon', score: 5000.05}) + CREATE (p1:Post {title: 'Hello World', views: 150}) + CREATE (p2:Post {title: 'Graph Databases 101', views: 890}) + CREATE (p3:Post {title: 'AGE is awesome', views: 2200}) + CREATE (u1)-[:FOLLOWS {weight: 0.9}]->(u2) + CREATE (u2)-[:FOLLOWS {weight: 0.8}]->(u3) + CREATE (u3)-[:FOLLOWS {weight: 0.7}]->(u4) + CREATE (u4)-[:FOLLOWS {weight: 0.6}]->(u5) + CREATE (u5)-[:FOLLOWS {weight: 0.5}]->(u1) + CREATE (u1)-[:AUTHORED]->(p1) + CREATE (u2)-[:AUTHORED]->(p2) + CREATE (u3)-[:AUTHORED]->(p3) + CREATE (u4)-[:LIKES]->(p1) + CREATE (u5)-[:LIKES]->(p2) + CREATE (u1)-[:LIKES]->(p3) + CREATE (u2)-[:LIKES]->(p3) + RETURN 'network graph created' +$$) AS (result agtype); + +-- GIN indexes on network graph +CREATE INDEX network_user_gin ON network."User" USING GIN (properties); +CREATE INDEX network_post_gin ON network."Post" USING GIN (properties); + +-- +-- Graph 3: "routes" — geographic routing with precise coordinates. +-- Labels: City, Airport +-- Edges: ROAD, FLIGHT +-- Coordinates use precise decimals that are easy to checksum. +-- +SELECT create_graph('routes'); + +SELECT * FROM cypher('routes', $$ + CREATE (c1:City {name: 'Portland', lat: 45.5152, lon: -122.6784, pop: 652503}) + CREATE (c2:City {name: 'Seattle', lat: 47.6062, lon: -122.3321, pop: 749256}) + CREATE (c3:City {name: 'Vancouver', lat: 49.2827, lon: -123.1207, pop: 631486}) + CREATE (a1:Airport {code: 'PDX', elev: 30.5}) + CREATE (a2:Airport {code: 'SEA', elev: 131.7}) + CREATE (a3:Airport {code: 'YVR', elev: 4.3}) + CREATE (c1)-[:ROAD {distance_km: 279.5, toll: 0.0}]->(c2) + CREATE (c2)-[:ROAD {distance_km: 225.3, toll: 5.0}]->(c3) + CREATE (c1)-[:ROAD {distance_km: 502.1, toll: 5.0}]->(c3) + CREATE (a1)-[:FLIGHT {distance_km: 229.0, duration_min: 55}]->(a2) + CREATE (a2)-[:FLIGHT {distance_km: 198.0, duration_min: 50}]->(a3) + CREATE (a1)-[:FLIGHT {distance_km: 426.0, duration_min: 75}]->(a3) + RETURN 'routes graph created' +$$) AS (result agtype); + +-- GIN index on routes graph +CREATE INDEX routes_city_gin ON routes."City" USING GIN (properties); + +-- Step 5: Record pre-upgrade integrity checksums. +-- These sums use the "val" / "score" / coordinate properties as fingerprints. + +-- company: sum of all val properties (should be a precise known value) +SELECT * FROM cypher('company', $$ + MATCH (n) WHERE n.val IS NOT NULL RETURN sum(n.val) +$$) AS (company_val_sum_before agtype); + +-- network: sum of all score properties +SELECT * FROM cypher('network', $$ + MATCH (n:User) RETURN sum(n.score) +$$) AS (network_score_sum_before agtype); + +-- routes: sum of all latitude values +SELECT * FROM cypher('routes', $$ + MATCH (c:City) RETURN sum(c.lat) +$$) AS (routes_lat_sum_before agtype); + +-- Total vertex and edge counts across all three graphs +SELECT sum(cnt)::int AS total_vertices_before FROM ( + SELECT count(*) AS cnt FROM cypher('company', $$ MATCH (n) RETURN n $$) AS (n agtype) + UNION ALL + SELECT count(*) FROM cypher('network', $$ MATCH (n) RETURN n $$) AS (n agtype) + UNION ALL + SELECT count(*) FROM cypher('routes', $$ MATCH (n) RETURN n $$) AS (n agtype) +) sub; + +SELECT sum(cnt)::int AS total_edges_before FROM ( + SELECT count(*) AS cnt FROM cypher('company', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) + UNION ALL + SELECT count(*) FROM cypher('network', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) + UNION ALL + SELECT count(*) FROM cypher('routes', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) +) sub; + +-- Count of distinct labels (ag_label entries) across all graphs +SELECT count(*)::int AS total_labels_before +FROM ag_label al JOIN ag_graph ag ON al.graph = ag.graphid +WHERE ag.name <> '_ag_catalog'; + +-- Step 6: Upgrade AGE to the synthetic next version via the stamped template. +DO $$ +DECLARE next_ver text; +BEGIN + SELECT version INTO next_ver + FROM pg_available_extension_versions + WHERE name = 'age' AND version <> ( + SELECT default_version FROM pg_available_extensions WHERE name = 'age' + ) + ORDER BY string_to_array(version, '.')::int[] DESC + LIMIT 1; + + IF next_ver IS NULL THEN + RAISE EXCEPTION 'No next version available for upgrade test'; + END IF; + + EXECUTE format('ALTER EXTENSION age UPDATE TO %L', next_ver); +END; +$$; + +-- Step 7: Confirm version changed. +SELECT installed_version <> default_version AS upgraded_past_default +FROM pg_available_extensions WHERE name = 'age'; + +-- Step 8: Verify all data survived — reload and recheck. +LOAD 'age'; +SET search_path TO ag_catalog, "$user", public; + +-- Repeat integrity checksums — must match pre-upgrade values exactly. +SELECT * FROM cypher('company', $$ + MATCH (n) WHERE n.val IS NOT NULL RETURN sum(n.val) +$$) AS (company_val_sum_after agtype); + +SELECT * FROM cypher('network', $$ + MATCH (n:User) RETURN sum(n.score) +$$) AS (network_score_sum_after agtype); + +SELECT * FROM cypher('routes', $$ + MATCH (c:City) RETURN sum(c.lat) +$$) AS (routes_lat_sum_after agtype); + +SELECT sum(cnt)::int AS total_vertices_after FROM ( + SELECT count(*) AS cnt FROM cypher('company', $$ MATCH (n) RETURN n $$) AS (n agtype) + UNION ALL + SELECT count(*) FROM cypher('network', $$ MATCH (n) RETURN n $$) AS (n agtype) + UNION ALL + SELECT count(*) FROM cypher('routes', $$ MATCH (n) RETURN n $$) AS (n agtype) +) sub; + +SELECT sum(cnt)::int AS total_edges_after FROM ( + SELECT count(*) AS cnt FROM cypher('company', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) + UNION ALL + SELECT count(*) FROM cypher('network', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) + UNION ALL + SELECT count(*) FROM cypher('routes', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) +) sub; + +SELECT count(*)::int AS total_labels_after +FROM ag_label al JOIN ag_graph ag ON al.graph = ag.graphid +WHERE ag.name <> '_ag_catalog'; + +-- Step 9: Verify specific structural queries across all three graphs. + +-- company: management chain +SELECT * FROM cypher('company', $$ + MATCH (boss:Employee)-[:MANAGES*]->(report:Employee) + RETURN boss.name, report.name + ORDER BY boss.name, report.name +$$) AS (boss agtype, report agtype); + +-- network: circular follow chain (proves full cycle survived) +SELECT * FROM cypher('network', $$ + MATCH (a:User)-[:FOLLOWS]->(b:User) + RETURN a.handle, b.handle + ORDER BY a.handle +$$) AS (follower agtype, followed agtype); + +-- routes: all flights with distances (proves edge properties intact) +SELECT * FROM cypher('routes', $$ + MATCH (a:Airport)-[f:FLIGHT]->(b:Airport) + RETURN a.code, b.code, f.distance_km + ORDER BY a.code, b.code +$$) AS (origin agtype, dest agtype, dist agtype); + +-- Step 10: Verify GIN indexes still exist after upgrade. +SELECT indexname FROM pg_indexes +WHERE schemaname IN ('company', 'network', 'routes') + AND tablename IN ('Employee', 'User', 'Post', 'City') + AND indexdef LIKE '%gin%' +ORDER BY indexname; + +-- Step 11: Cleanup and restore AGE at the default version for subsequent tests. +SELECT drop_graph('routes', true); +SELECT drop_graph('network', true); +SELECT drop_graph('company', true); +DROP EXTENSION age; +CREATE EXTENSION age; + +-- Step 12: Remove synthetic upgrade test files from the extension directory. +\! sh ./regress/age_upgrade_cleanup.sh From a29e2810b1ac5a9e4c3e553879b683a6a1902c8b Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Mon, 6 Apr 2026 03:28:46 -0700 Subject: [PATCH 041/100] Fix nondeterministic age_global_graph regression test (#2365) The age_global_graph test had two issues that could cause intermittent failures: 1. Nondeterministic warning output: The graph_stats() call on a graph with deliberately deleted vertices produces WARNING messages for dangling edges. These warnings are emitted by iterating edge label tables (knows, stalks), and the iteration order is not guaranteed. Since PostgreSQL WARNING messages cannot be caught or counted from SQL (only ERROR and above are catchable via PL/pgSQL exception handling), we suppress them with SET client_min_messages = error. The suppressed warnings are documented verbatim in comments. The graph_stats() result row still validates correct dangling-edge handling. 2. Nondeterministic row ordering: Multiple MATCH...RETURN queries returned multi-row results without ORDER BY, relying on scan order. Added ORDER BY id(u), id(v), id(e), id(n), or id(a) as appropriate to all MATCH...RETURN queries for future-proofing, even those currently returning a single row. Files changed: regress/sql/age_global_graph.sql regress/expected/age_global_graph.out Co-authored-by: GitHub Copilot --- regress/expected/age_global_graph.out | 59 +++++++++++++++------------ regress/sql/age_global_graph.sql | 43 +++++++++++++------ 2 files changed, 64 insertions(+), 38 deletions(-) diff --git a/regress/expected/age_global_graph.out b/regress/expected/age_global_graph.out index f2d9b059f..478637800 100644 --- a/regress/expected/age_global_graph.out +++ b/regress/expected/age_global_graph.out @@ -44,19 +44,19 @@ SELECT * FROM cypher('ag_graph_3', $$ CREATE (v:vertex3) RETURN v $$) AS (v agt (1 row) -- load contexts using the vertex_stats command -SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); result ----------------------------------------------------------------------------------------------- {"id": 844424930131969, "label": "vertex3", "in_degree": 0, "out_degree": 0, "self_loops": 0} (1 row) -SELECT * FROM cypher('ag_graph_2', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_2', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); result ----------------------------------------------------------------------------------------------- {"id": 844424930131969, "label": "vertex2", "in_degree": 0, "out_degree": 0, "self_loops": 0} (1 row) -SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); result ----------------------------------------------------------------------------------------------- {"id": 844424930131969, "label": "vertex1", "in_degree": 0, "out_degree": 0, "self_loops": 0} @@ -64,7 +64,7 @@ SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (r --- loading undefined contexts --- should throw exception - graph "ag_graph_4" does not exist -SELECT * FROM cypher('ag_graph_4', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_4', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); ERROR: graph "ag_graph_4" does not exist LINE 1: SELECT * FROM cypher('ag_graph_4', $$ MATCH (u) RETURN verte... ^ @@ -130,19 +130,19 @@ LINE 1: SELECT * FROM cypher('ag_graph_4', $$ RETURN delete_global_g... -- delete_GRAPH_global_contexts -- -- load contexts again -SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); result ----------------------------------------------------------------------------------------------- {"id": 844424930131969, "label": "vertex3", "in_degree": 0, "out_degree": 0, "self_loops": 0} (1 row) -SELECT * FROM cypher('ag_graph_2', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_2', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); result ----------------------------------------------------------------------------------------------- {"id": 844424930131969, "label": "vertex2", "in_degree": 0, "out_degree": 0, "self_loops": 0} (1 row) -SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); result ----------------------------------------------------------------------------------------------- {"id": 844424930131969, "label": "vertex1", "in_degree": 0, "out_degree": 0, "self_loops": 0} @@ -193,14 +193,14 @@ SELECT * FROM cypher('ag_graph_2', $$ CREATE (:Person) $$) as (v agtype); (0 rows) ---adding edges between nodes -SELECT * FROM cypher('ag_graph_2', $$ MATCH (a:Person), (b:Person) WHERE a.name = 'A' AND b.name = 'B' CREATE (a)-[e:RELTYPE]->(b) RETURN e $$) as (e agtype); +SELECT * FROM cypher('ag_graph_2', $$ MATCH (a:Person), (b:Person) WHERE a.name = 'A' AND b.name = 'B' CREATE (a)-[e:RELTYPE]->(b) RETURN e ORDER BY id(e) $$) as (e agtype); e --- (0 rows) --checking if vertex stats have been updated along with the new label --should return 3 vertices -SELECT * FROM cypher('ag_graph_1', $$ MATCH (n) RETURN vertex_stats(n) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_1', $$ MATCH (n) RETURN vertex_stats(n) ORDER BY id(n) $$) AS (result agtype); result ----------------------------------------------------------------------------------------------- {"id": 281474976710657, "label": "", "in_degree": 0, "out_degree": 0, "self_loops": 0} @@ -209,7 +209,7 @@ SELECT * FROM cypher('ag_graph_1', $$ MATCH (n) RETURN vertex_stats(n) $$) AS (r (3 rows) --should return 1 vertice and 1 label -SELECT * FROM cypher('ag_graph_2', $$ MATCH (a) RETURN vertex_stats(a) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_2', $$ MATCH (a) RETURN vertex_stats(a) ORDER BY id(a) $$) AS (result agtype); result ----------------------------------------------------------------------------------------------- {"id": 844424930131969, "label": "vertex2", "in_degree": 0, "out_degree": 0, "self_loops": 0} @@ -271,7 +271,7 @@ SELECT * FROM cypher('ag_graph_1', $$ MATCH (u)-[]->(v) SET u.id = id(u) SET v.id = id(v) SET u.name = 'u' SET v.name = 'v' - RETURN u,v $$) AS (u agtype, v agtype); + RETURN u,v ORDER BY id(u), id(v) $$) AS (u agtype, v agtype); u | v --------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------- {"id": 281474976710659, "label": "", "properties": {"id": 281474976710659, "name": "u"}}::vertex | {"id": 281474976710660, "label": "", "properties": {"id": 281474976710660, "name": "v"}}::vertex @@ -285,17 +285,17 @@ SELECT * FROM cypher('ag_graph_1', $$ MATCH (u)-[]->(v) MERGE (v)-[:stalks]->(u) -------- (0 rows) -SELECT * FROM cypher('ag_graph_1', $$ MATCH (u)-[e]->(v) RETURN u, e, v $$) AS (u agtype, e agtype, v agtype); +SELECT * FROM cypher('ag_graph_1', $$ MATCH (u)-[e]->(v) RETURN u, e, v ORDER BY id(e) $$) AS (u agtype, e agtype, v agtype); u | e | v --------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------- - {"id": 281474976710660, "label": "", "properties": {"id": 281474976710660, "name": "v"}}::vertex | {"id": 1407374883553281, "label": "stalks", "end_id": 281474976710659, "start_id": 281474976710660, "properties": {}}::edge | {"id": 281474976710659, "label": "", "properties": {"id": 281474976710659, "name": "u"}}::vertex {"id": 281474976710659, "label": "", "properties": {"id": 281474976710659, "name": "u"}}::vertex | {"id": 1125899906842625, "label": "knows", "end_id": 281474976710660, "start_id": 281474976710659, "properties": {}}::edge | {"id": 281474976710660, "label": "", "properties": {"id": 281474976710660, "name": "v"}}::vertex - {"id": 281474976710662, "label": "", "properties": {"id": 281474976710662, "name": "v"}}::vertex | {"id": 1407374883553282, "label": "stalks", "end_id": 281474976710661, "start_id": 281474976710662, "properties": {}}::edge | {"id": 281474976710661, "label": "", "properties": {"id": 281474976710661, "name": "u"}}::vertex {"id": 281474976710661, "label": "", "properties": {"id": 281474976710661, "name": "u"}}::vertex | {"id": 1125899906842626, "label": "knows", "end_id": 281474976710662, "start_id": 281474976710661, "properties": {}}::edge | {"id": 281474976710662, "label": "", "properties": {"id": 281474976710662, "name": "v"}}::vertex - {"id": 281474976710664, "label": "", "properties": {"id": 281474976710664, "name": "v"}}::vertex | {"id": 1407374883553283, "label": "stalks", "end_id": 281474976710663, "start_id": 281474976710664, "properties": {}}::edge | {"id": 281474976710663, "label": "", "properties": {"id": 281474976710663, "name": "u"}}::vertex {"id": 281474976710663, "label": "", "properties": {"id": 281474976710663, "name": "u"}}::vertex | {"id": 1125899906842627, "label": "knows", "end_id": 281474976710664, "start_id": 281474976710663, "properties": {}}::edge | {"id": 281474976710664, "label": "", "properties": {"id": 281474976710664, "name": "v"}}::vertex - {"id": 281474976710666, "label": "", "properties": {"id": 281474976710666, "name": "v"}}::vertex | {"id": 1407374883553284, "label": "stalks", "end_id": 281474976710665, "start_id": 281474976710666, "properties": {}}::edge | {"id": 281474976710665, "label": "", "properties": {"id": 281474976710665, "name": "u"}}::vertex {"id": 281474976710665, "label": "", "properties": {"id": 281474976710665, "name": "u"}}::vertex | {"id": 1125899906842628, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710665, "properties": {}}::edge | {"id": 281474976710666, "label": "", "properties": {"id": 281474976710666, "name": "v"}}::vertex + {"id": 281474976710660, "label": "", "properties": {"id": 281474976710660, "name": "v"}}::vertex | {"id": 1407374883553281, "label": "stalks", "end_id": 281474976710659, "start_id": 281474976710660, "properties": {}}::edge | {"id": 281474976710659, "label": "", "properties": {"id": 281474976710659, "name": "u"}}::vertex + {"id": 281474976710662, "label": "", "properties": {"id": 281474976710662, "name": "v"}}::vertex | {"id": 1407374883553282, "label": "stalks", "end_id": 281474976710661, "start_id": 281474976710662, "properties": {}}::edge | {"id": 281474976710661, "label": "", "properties": {"id": 281474976710661, "name": "u"}}::vertex + {"id": 281474976710664, "label": "", "properties": {"id": 281474976710664, "name": "v"}}::vertex | {"id": 1407374883553283, "label": "stalks", "end_id": 281474976710663, "start_id": 281474976710664, "properties": {}}::edge | {"id": 281474976710663, "label": "", "properties": {"id": 281474976710663, "name": "u"}}::vertex + {"id": 281474976710666, "label": "", "properties": {"id": 281474976710666, "name": "v"}}::vertex | {"id": 1407374883553284, "label": "stalks", "end_id": 281474976710665, "start_id": 281474976710666, "properties": {}}::edge | {"id": 281474976710665, "label": "", "properties": {"id": 281474976710665, "name": "u"}}::vertex (8 rows) -- what is there now? @@ -351,21 +351,30 @@ SELECT * FROM ag_graph_1._ag_label_edge; 1407374883553284 | 281474976710666 | 281474976710665 | {} (8 rows) --- there should be warning messages +-- The graph_stats query below will produce warnings for the dangling edges +-- created by the DELETE commands above. The warnings appear in nondeterministic +-- order because they come from iterating edge label tables (knows, stalks), +-- so we suppress them with client_min_messages. Without suppression, the +-- output would include these warnings (in some order): +-- +-- WARNING: edge: [id: 1125899906842626, start: 281474976710661, end: 281474976710662, label: knows] start and end vertices not found +-- WARNING: ignored malformed or dangling edge +-- WARNING: edge: [id: 1125899906842627, start: 281474976710663, end: 281474976710664, label: knows] end vertex not found +-- WARNING: ignored malformed or dangling edge +-- WARNING: edge: [id: 1407374883553282, start: 281474976710662, end: 281474976710661, label: stalks] start and end vertices not found +-- WARNING: ignored malformed or dangling edge +-- WARNING: edge: [id: 1407374883553283, start: 281474976710664, end: 281474976710663, label: stalks] start vertex not found +-- WARNING: ignored malformed or dangling edge +-- +-- The result row validates that graph_stats handled the dangling edges correctly. +SET client_min_messages = error; SELECT * FROM cypher('ag_graph_1', $$ RETURN graph_stats('ag_graph_1') $$) AS (result agtype); -WARNING: edge: [id: 1125899906842626, start: 281474976710661, end: 281474976710662, label: knows] start and end vertices not found -WARNING: ignored malformed or dangling edge -WARNING: edge: [id: 1125899906842627, start: 281474976710663, end: 281474976710664, label: knows] end vertex not found -WARNING: ignored malformed or dangling edge -WARNING: edge: [id: 1407374883553282, start: 281474976710662, end: 281474976710661, label: stalks] start and end vertices not found -WARNING: ignored malformed or dangling edge -WARNING: edge: [id: 1407374883553283, start: 281474976710664, end: 281474976710663, label: stalks] start vertex not found -WARNING: ignored malformed or dangling edge result -------------------------------------------------------------------------- {"graph": "ag_graph_1", "num_loaded_edges": 8, "num_loaded_vertices": 8} (1 row) +RESET client_min_messages; --drop graphs SELECT * FROM drop_graph('ag_graph_1', true); NOTICE: drop cascades to 5 other objects diff --git a/regress/sql/age_global_graph.sql b/regress/sql/age_global_graph.sql index acd95fbf0..376681a8b 100644 --- a/regress/sql/age_global_graph.sql +++ b/regress/sql/age_global_graph.sql @@ -16,13 +16,13 @@ SELECT * FROM create_graph('ag_graph_3'); SELECT * FROM cypher('ag_graph_3', $$ CREATE (v:vertex3) RETURN v $$) AS (v agtype); -- load contexts using the vertex_stats command -SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); -SELECT * FROM cypher('ag_graph_2', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); -SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_2', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); --- loading undefined contexts --- should throw exception - graph "ag_graph_4" does not exist -SELECT * FROM cypher('ag_graph_4', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_4', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); --- delete with invalid parameter ---should return false @@ -55,9 +55,9 @@ SELECT * FROM cypher('ag_graph_4', $$ RETURN delete_global_graphs('ag_graph_4') -- -- load contexts again -SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); -SELECT * FROM cypher('ag_graph_2', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); -SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_2', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); -- delete all graph contexts -- should return true @@ -81,14 +81,14 @@ SELECT * FROM cypher('ag_graph_1', $$ CREATE (n), (m) $$) as (v agtype); SELECT * FROM cypher('ag_graph_2', $$ CREATE (:Person) $$) as (v agtype); ---adding edges between nodes -SELECT * FROM cypher('ag_graph_2', $$ MATCH (a:Person), (b:Person) WHERE a.name = 'A' AND b.name = 'B' CREATE (a)-[e:RELTYPE]->(b) RETURN e $$) as (e agtype); +SELECT * FROM cypher('ag_graph_2', $$ MATCH (a:Person), (b:Person) WHERE a.name = 'A' AND b.name = 'B' CREATE (a)-[e:RELTYPE]->(b) RETURN e ORDER BY id(e) $$) as (e agtype); --checking if vertex stats have been updated along with the new label --should return 3 vertices -SELECT * FROM cypher('ag_graph_1', $$ MATCH (n) RETURN vertex_stats(n) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_1', $$ MATCH (n) RETURN vertex_stats(n) ORDER BY id(n) $$) AS (result agtype); --should return 1 vertice and 1 label -SELECT * FROM cypher('ag_graph_2', $$ MATCH (a) RETURN vertex_stats(a) $$) AS (result agtype); +SELECT * FROM cypher('ag_graph_2', $$ MATCH (a) RETURN vertex_stats(a) ORDER BY id(a) $$) AS (result agtype); -- -- graph_stats command @@ -109,9 +109,9 @@ SELECT * FROM cypher('ag_graph_1', $$ MATCH (u)-[]->(v) SET u.id = id(u) SET v.id = id(v) SET u.name = 'u' SET v.name = 'v' - RETURN u,v $$) AS (u agtype, v agtype); + RETURN u,v ORDER BY id(u), id(v) $$) AS (u agtype, v agtype); SELECT * FROM cypher('ag_graph_1', $$ MATCH (u)-[]->(v) MERGE (v)-[:stalks]->(u) $$) AS (result agtype); -SELECT * FROM cypher('ag_graph_1', $$ MATCH (u)-[e]->(v) RETURN u, e, v $$) AS (u agtype, e agtype, v agtype); +SELECT * FROM cypher('ag_graph_1', $$ MATCH (u)-[e]->(v) RETURN u, e, v ORDER BY id(e) $$) AS (u agtype, e agtype, v agtype); -- what is there now? SELECT * FROM cypher('ag_graph_1', $$ RETURN graph_stats('ag_graph_1') $$) AS (result agtype); -- remove some vertices @@ -121,8 +121,25 @@ DELETE FROM ag_graph_1._ag_label_vertex WHERE id::text = '281474976710662'; DELETE FROM ag_graph_1._ag_label_vertex WHERE id::text = '281474976710664'; SELECT * FROM ag_graph_1._ag_label_vertex; SELECT * FROM ag_graph_1._ag_label_edge; --- there should be warning messages +-- The graph_stats query below will produce warnings for the dangling edges +-- created by the DELETE commands above. The warnings appear in nondeterministic +-- order because they come from iterating edge label tables (knows, stalks), +-- so we suppress them with client_min_messages. Without suppression, the +-- output would include these warnings (in some order): +-- +-- WARNING: edge: [id: 1125899906842626, start: 281474976710661, end: 281474976710662, label: knows] start and end vertices not found +-- WARNING: ignored malformed or dangling edge +-- WARNING: edge: [id: 1125899906842627, start: 281474976710663, end: 281474976710664, label: knows] end vertex not found +-- WARNING: ignored malformed or dangling edge +-- WARNING: edge: [id: 1407374883553282, start: 281474976710662, end: 281474976710661, label: stalks] start and end vertices not found +-- WARNING: ignored malformed or dangling edge +-- WARNING: edge: [id: 1407374883553283, start: 281474976710664, end: 281474976710663, label: stalks] start vertex not found +-- WARNING: ignored malformed or dangling edge +-- +-- The result row validates that graph_stats handled the dangling edges correctly. +SET client_min_messages = error; SELECT * FROM cypher('ag_graph_1', $$ RETURN graph_stats('ag_graph_1') $$) AS (result agtype); +RESET client_min_messages; --drop graphs From d6f1b7f67087bd86f2e0fe6e1b6c897b7e569926 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Mon, 6 Apr 2026 11:34:47 -0400 Subject: [PATCH 042/100] Fix MATCH on brand-new label after CREATE returning 0 rows (#2341) * Fix MATCH on brand-new label after CREATE returning 0 rows (issue #2193) When CREATE introduces a new label and a subsequent MATCH references it (e.g., CREATE (:Person) WITH ... MATCH (p:Person)), the query returns 0 rows on first execution but works on the second. Root cause: match_check_valid_label() in transform_cypher_match() runs before transform_prev_cypher_clause() processes the predecessor chain. Since CREATE has not yet executed its transform (which creates the label table as a side effect), the label is not in the cache and the check generates a One-Time Filter: false plan that returns no rows. Fix: Skip the early label validity check when the predecessor clause chain contains a data-modifying operation (CREATE, SET, DELETE, MERGE). After transform_prev_cypher_clause() completes and any new labels exist in the cache, run a deferred label check. If the labels are still invalid at that point, generate an empty result via makeBoolConst(false). This preserves the existing behavior for MATCH without DML predecessors (e.g., MATCH-MATCH chains still get the early check and proper error messages for invalid labels). Depends on: PR #2340 (clause_chain_has_dml helper) Co-Authored-By: Claude Opus 4.6 * Address review feedback: fix variable registration for deferred label check When the deferred label validity check (DML predecessor + non-existent label) found an invalid label, the code skipped transform_match_pattern() entirely, which meant MATCH-introduced variables were never registered in the namespace. This would cause errors if a later clause referenced those variables (e.g., RETURN p). Fix: mirror the early-check strategy by injecting a paradoxical WHERE (true = false) and always calling transform_match_pattern(). Variables get registered normally; zero rows are returned via the impossible qual. Also add ORDER BY to multi-row regression tests for deterministic output, and add a test case for DML predecessor + non-existent label + returning a MATCH-introduced variable. Co-Authored-By: Claude Opus 4.6 * Address Copilot review: DRY false-where helper, cache has_dml, ORDER BY in tests - Factor duplicated WHERE true=false construction into make_false_where_clause() helper (used in both early and deferred label validation paths) - Compute clause_chain_has_dml() once and reuse, avoiding repeated clause chain traversal - Add ORDER BY to the single-CREATE City regression test for deterministic result ordering * Address Copilot review: volatile false predicate, DML side-effect test 1. Prevent plan elimination of DML predecessor: replace constant (true = false) with volatile (random() IS NULL) in the deferred label check path. PG's planner can constant-fold the former into a One-Time Filter: false, skipping the DML scan entirely. 2. Unify make_false_where_clause(bool volatile_needed): merge the constant and volatile variants into a single parameterized function. Call sites are now self-documenting: - make_false_where_clause(false) for non-DML path - make_false_where_clause(true) for DML predecessor path 3. Document why add_volatile_wrapper() cannot be reused here (it operates post-transform at the Expr level and returns agtype, while the WHERE clause is built at the parse-tree level). 4. Add regression test verifying CREATE side effects persist when MATCH references a non-existent label after a DML predecessor. All regression tests pass (cypher_match: ok). Co-Authored-By: Claude Opus 4.6 * Replace non-ASCII em dashes with -- in C comments ASCII-only codebase convention; avoids encoding/tooling issues. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- regress/expected/cypher_match.out | 97 +++++++++++++++++++++++++++ regress/sql/cypher_match.sql | 58 ++++++++++++++++ src/backend/parser/cypher_clause.c | 103 +++++++++++++++++++++++++---- 3 files changed, 244 insertions(+), 14 deletions(-) diff --git a/regress/expected/cypher_match.out b/regress/expected/cypher_match.out index 94315f134..2f01d5163 100644 --- a/regress/expected/cypher_match.out +++ b/regress/expected/cypher_match.out @@ -3737,6 +3737,103 @@ EXECUTE issue_1964_vertex_eq_on('{"props": {"name": "Alice", "age": 25}}'); (0 rows) DEALLOCATE issue_1964_vertex_eq_on; +-- +-- Issue 2193: CREATE ... WITH ... MATCH on brand-new label returns 0 rows +-- on first execution because match_check_valid_label() runs before +-- transform_prev_cypher_clause() creates the label table. +-- +SELECT create_graph('issue_2193'); +NOTICE: graph "issue_2193" has been created + create_graph +-------------- + +(1 row) + +-- Reporter's exact case: CREATE two Person nodes, then MATCH on Person +-- Should return 2 rows on the very first execution +SELECT * FROM cypher('issue_2193', $$ + CREATE (a:Person {name: 'Jane', livesIn: 'London'}), + (b:Person {name: 'Tom', livesIn: 'Copenhagen'}) + WITH a, b + MATCH (p:Person) + RETURN p.name ORDER BY p.name +$$) AS (result agtype); + result +-------- + "Jane" + "Tom" +(2 rows) + +-- Single CREATE + MATCH on brand-new label +SELECT * FROM cypher('issue_2193', $$ + CREATE (a:City {name: 'Berlin'}) + WITH a + MATCH (c:City) + RETURN c.name ORDER BY c.name +$$) AS (result agtype); + result +---------- + "Berlin" +(1 row) + +-- MATCH on a label that now exists (second execution) still works +SELECT * FROM cypher('issue_2193', $$ + CREATE (a:City {name: 'Paris'}) + WITH a + MATCH (c:City) + RETURN c.name ORDER BY c.name +$$) AS (result agtype); + result +---------- + "Berlin" + "Paris" +(2 rows) + +-- MATCH on non-existent label without DML predecessor still returns 0 rows +SELECT * FROM cypher('issue_2193', $$ + MATCH (x:NonExistentLabel) + RETURN x +$$) AS (result agtype); + result +-------- +(0 rows) + +-- MATCH on non-existent label after DML predecessor still returns 0 rows +-- and MATCH-introduced variable (p) is properly registered +SELECT * FROM cypher('issue_2193', $$ + CREATE (a:Person {name: 'Alice'}) + WITH a + MATCH (p:NonExistentLabel) + RETURN p +$$) AS (result agtype); + result +-------- +(0 rows) + +-- Verify that the CREATE side effect was preserved even though MATCH +-- returned 0 rows (guards against plan-elimination regressions where +-- a constant-false predicate causes PG to skip the DML predecessor) +SELECT * FROM cypher('issue_2193', $$ + MATCH (a:Person {name: 'Alice'}) + RETURN a.name +$$) AS (result agtype); + result +--------- + "Alice" +(1 row) + +SELECT drop_graph('issue_2193', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table issue_2193._ag_label_vertex +drop cascades to table issue_2193._ag_label_edge +drop cascades to table issue_2193."Person" +drop cascades to table issue_2193."City" +NOTICE: graph "issue_2193" has been dropped + drop_graph +------------ + +(1 row) + -- -- Clean up -- diff --git a/regress/sql/cypher_match.sql b/regress/sql/cypher_match.sql index d14f45f10..410d097bb 100644 --- a/regress/sql/cypher_match.sql +++ b/regress/sql/cypher_match.sql @@ -1558,6 +1558,64 @@ PREPARE issue_1964_vertex_eq_on(agtype) AS EXECUTE issue_1964_vertex_eq_on('{"props": {"name": "Alice", "age": 25}}'); DEALLOCATE issue_1964_vertex_eq_on; +-- +-- Issue 2193: CREATE ... WITH ... MATCH on brand-new label returns 0 rows +-- on first execution because match_check_valid_label() runs before +-- transform_prev_cypher_clause() creates the label table. +-- +SELECT create_graph('issue_2193'); + +-- Reporter's exact case: CREATE two Person nodes, then MATCH on Person +-- Should return 2 rows on the very first execution +SELECT * FROM cypher('issue_2193', $$ + CREATE (a:Person {name: 'Jane', livesIn: 'London'}), + (b:Person {name: 'Tom', livesIn: 'Copenhagen'}) + WITH a, b + MATCH (p:Person) + RETURN p.name ORDER BY p.name +$$) AS (result agtype); + +-- Single CREATE + MATCH on brand-new label +SELECT * FROM cypher('issue_2193', $$ + CREATE (a:City {name: 'Berlin'}) + WITH a + MATCH (c:City) + RETURN c.name ORDER BY c.name +$$) AS (result agtype); + +-- MATCH on a label that now exists (second execution) still works +SELECT * FROM cypher('issue_2193', $$ + CREATE (a:City {name: 'Paris'}) + WITH a + MATCH (c:City) + RETURN c.name ORDER BY c.name +$$) AS (result agtype); + +-- MATCH on non-existent label without DML predecessor still returns 0 rows +SELECT * FROM cypher('issue_2193', $$ + MATCH (x:NonExistentLabel) + RETURN x +$$) AS (result agtype); + +-- MATCH on non-existent label after DML predecessor still returns 0 rows +-- and MATCH-introduced variable (p) is properly registered +SELECT * FROM cypher('issue_2193', $$ + CREATE (a:Person {name: 'Alice'}) + WITH a + MATCH (p:NonExistentLabel) + RETURN p +$$) AS (result agtype); + +-- Verify that the CREATE side effect was preserved even though MATCH +-- returned 0 rows (guards against plan-elimination regressions where +-- a constant-false predicate causes PG to skip the DML predecessor) +SELECT * FROM cypher('issue_2193', $$ + MATCH (a:Person {name: 'Alice'}) + RETURN a.name +$$) AS (result agtype); + +SELECT drop_graph('issue_2193', true); + -- -- Clean up -- diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 6f06bbb82..e5540aa3e 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -346,6 +346,7 @@ static bool isa_special_VLE_case(cypher_path *path); static ParseNamespaceItem *find_pnsi(cypher_parsestate *cpstate, char *varname); static bool has_list_comp_or_subquery(Node *expr, void *context); static bool clause_chain_has_dml(cypher_clause *clause); +static Node *make_false_where_clause(bool volatile_needed); /* * Add required permissions to the RTEPermissionInfo for a relation. @@ -2639,20 +2640,19 @@ static Query *transform_cypher_match(cypher_parsestate *cpstate, cypher_match *match_self = (cypher_match*) clause->self; Node *where = match_self->where; - if(!match_check_valid_label(match_self, cpstate)) + /* + * Check label validity early unless the predecessor clause chain + * contains a data-modifying operation (CREATE, SET, DELETE, MERGE). + * DML predecessors may create new labels that are not yet in the + * cache, so the check is deferred to after transform_prev_cypher_clause() + * for those cases. + */ + if (!clause_chain_has_dml(clause->prev) && + !match_check_valid_label(match_self, cpstate)) { - cypher_bool_const *l = make_ag_node(cypher_bool_const); - cypher_bool_const *r = make_ag_node(cypher_bool_const); - - l->boolean = true; - l->location = -1; - r->boolean = false; - r->location = -1; - - /*if the label is invalid, create a paradoxical where to get null*/ - match_self->where = (Node *)makeSimpleA_Expr(AEXPR_OP, "=", - (Node *)l, - (Node *)r, -1); + /* Label is invalid -- inject a false WHERE so the MATCH returns + * zero rows. No DML predecessor here, so constant-foldable is fine. */ + match_self->where = make_false_where_clause(false); } if (has_list_comp_or_subquery((Node *)match_self->where, NULL)) @@ -2915,6 +2915,7 @@ static Query *transform_cypher_match_pattern(cypher_parsestate *cpstate, RangeTblEntry *rte; int rtindex; ParseNamespaceItem *pnsi; + bool has_dml; pnsi = transform_prev_cypher_clause(cpstate, clause->prev, true); rte = pnsi->p_rte; @@ -2928,7 +2929,9 @@ static Query *transform_cypher_match_pattern(cypher_parsestate *cpstate, * DML executes -- resulting in quals checking NULL values * and filtering out all rows. */ - if (clause_chain_has_dml(clause->prev)) + has_dml = clause_chain_has_dml(clause->prev); + + if (has_dml) { rte->security_barrier = true; } @@ -2949,6 +2952,25 @@ static Query *transform_cypher_match_pattern(cypher_parsestate *cpstate, */ pnsi = get_namespace_item(pstate, rte); query->targetList = expandNSItemAttrs(pstate, pnsi, 0, true, -1); + + /* + * Now that the predecessor chain is fully transformed and + * any CREATE-generated labels exist in the cache, check + * whether the MATCH pattern references valid labels. This + * deferred check is only needed when the chain has DML, + * since labels created by CREATE are not in the cache at + * the time of the early check in transform_cypher_match(). + * + * We use a volatile false predicate (random() IS NULL) + * instead of a constant one (true = false) because PG's + * planner can constant-fold the latter into a One-Time + * Filter: false, eliminating the entire plan subtree -- + * including the DML predecessor scan -- without executing it. + */ + if (has_dml && !match_check_valid_label(self, cpstate)) + { + where = make_false_where_clause(true); + } } transform_match_pattern(cpstate, query, self->pattern, where); @@ -6621,6 +6643,59 @@ static bool clause_chain_has_dml(cypher_clause *clause) return false; } +/* + * Build a false WHERE clause that forces a MATCH to return zero rows. + * Used when the MATCH pattern references a label that does not exist. + * + * When volatile_needed is false, returns a constant (true = false) that + * PG's planner may constant-fold -- this is fine when there is no DML + * predecessor whose execution must be preserved. + * + * When volatile_needed is true, returns (random() IS NULL) instead. + * random() is VOLATILE, so eval_const_expressions() cannot fold this, + * preventing PG from creating a One-Time Filter: false that would + * eliminate the DML predecessor scan without executing it. + * + * Note: AGE's add_volatile_wrapper() serves a similar anti-fold purpose + * but operates at the Expr level (post-transform) and returns agtype, + * so it cannot be used here in the parse-tree WHERE clause context. + */ +static Node *make_false_where_clause(bool volatile_needed) +{ + if (volatile_needed) + { + FuncCall *random_fn; + NullTest *nt; + + random_fn = makeFuncCall( + list_make2(makeString("pg_catalog"), makeString("random")), + NIL, + COERCE_EXPLICIT_CALL, + -1); + + nt = makeNode(NullTest); + nt->arg = (Expr *)random_fn; + nt->nulltesttype = IS_NULL; + nt->argisrow = false; + nt->location = -1; + + return (Node *)nt; + } + else + { + cypher_bool_const *l = make_ag_node(cypher_bool_const); + cypher_bool_const *r = make_ag_node(cypher_bool_const); + + l->boolean = true; + l->location = -1; + r->boolean = false; + r->location = -1; + + return (Node *)makeSimpleA_Expr(AEXPR_OP, "=", + (Node *)l, (Node *)r, -1); + } +} + static Query *analyze_cypher_clause(transform_method transform, cypher_clause *clause, cypher_parsestate *parent_cpstate) From b3a00eea32c3faa93c43e0d91c0a75671e48ce65 Mon Sep 17 00:00:00 2001 From: a_bondar Date: Wed, 15 Apr 2026 01:23:53 +0700 Subject: [PATCH 043/100] Add index scan (#2351) * Add index scan This commit fixes performance degradation during insertion scenarios by replacing SeqScan with IndexScan. Motivation / Problem: As a result of load testing, a significant performance degradation was found in insertion scenarios. The scenarios used were taken from an open-source benchmark and rewritten in pure SQL. Examples of the queries can be found here: https://github.com/ldbc/ldbc_snb_interactive_v1_impls/blob/main/cypher/queries/interactive-update-1.cypher https://github.com/ldbc/ldbc_snb_interactive_v1_impls/blob/main/cypher/queries/interactive-update-6.cypher https://github.com/ldbc/ldbc_snb_interactive_v1_impls/blob/main/cypher/queries/interactive-update-7.cypher Perf analysis showed that the main bottleneck is the entity_exists function. The root cause lies in the use of a Sequential Scan (SeqScan) to check for the existence of an entity prior to insertion. The time complexity of a SeqScan is O(N), meaning the search time grows linearly as the number of rows in the table increases. The larger the graph became, the longer each individual insertion took. This led to a drop in TPS regardless of the concurrency level (the issue was consistently reproduced with both 1 and 30 threads). Co-authored-by: Daria Barsukova Co-authored-by: Alexandra Bondar * Address code review commets * Address code review commets * Add brackets --------- Co-authored-by: Alexandra Bondar Co-authored-by: Daria Barsukova --- src/backend/catalog/ag_label.c | 123 +++++-- src/backend/executor/cypher_delete.c | 411 ++++++++++++++++------- src/backend/executor/cypher_set.c | 189 ++++++++--- src/backend/executor/cypher_utils.c | 47 ++- src/backend/utils/adt/age_global_graph.c | 124 +++++-- src/backend/utils/adt/agtype.c | 116 ++++++- src/include/executor/cypher_utils.h | 6 + src/include/utils/agtype.h | 1 + 8 files changed, 781 insertions(+), 236 deletions(-) diff --git a/src/backend/catalog/ag_label.c b/src/backend/catalog/ag_label.c index 54c31ef36..d407626fd 100644 --- a/src/backend/catalog/ag_label.c +++ b/src/backend/catalog/ag_label.c @@ -21,6 +21,7 @@ #include "access/genam.h" #include "catalog/indexing.h" +#include "catalog/namespace.h" #include "executor/executor.h" #include "nodes/makefuncs.h" #include "utils/builtins.h" @@ -297,49 +298,115 @@ List *get_all_edge_labels_per_graph(EState *estate, Oid graph_oid) HeapTuple tuple; TupleTableSlot *slot; ResultRelInfo *resultRelInfo; - - /* setup scan keys to get all edges for the given graph oid */ - ScanKeyInit(&scan_keys[1], Anum_ag_label_graph, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(graph_oid)); - ScanKeyInit(&scan_keys[0], Anum_ag_label_kind, BTEqualStrategyNumber, - F_CHAREQ, CharGetDatum(LABEL_TYPE_EDGE)); + Oid index_oid; /* setup the table to be scanned */ - ag_label = table_open(ag_label_relation_id(), RowExclusiveLock); - scan_desc = table_beginscan(ag_label, estate->es_snapshot, 2, scan_keys); + ag_label = table_open(ag_label_relation_id(), AccessShareLock); + + index_oid = find_usable_btree_index_for_attr(ag_label, Anum_ag_label_graph); resultRelInfo = create_entity_result_rel_info(estate, "ag_catalog", "ag_label"); - slot = ExecInitExtraTupleSlot( - estate, RelationGetDescr(resultRelInfo->ri_RelationDesc), - &TTSOpsHeapTuple); - - /* scan through the results and get all the label names. */ - while(true) + if (OidIsValid(index_oid)) + { + Relation index_rel; + IndexScanDesc index_scan_desc; + + slot = ExecInitExtraTupleSlot( + estate, RelationGetDescr(resultRelInfo->ri_RelationDesc), + &TTSOpsBufferHeapTuple); + + index_rel = index_open(index_oid, AccessShareLock); + + /* + * Use 1 as the attribute number because 'graph' is the 1st column + * in the ag_label_graph_oid_index + */ + ScanKeyInit(&scan_keys[0], 1, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(graph_oid)); + + index_scan_desc = index_beginscan(ag_label, index_rel, estate->es_snapshot, NULL, 1, 0); + index_rescan(index_scan_desc, scan_keys, 1, NULL, 0); + + while (index_getnext_slot(index_scan_desc, ForwardScanDirection, slot)) + { + Name label; + Name lval; + bool isNull; + Datum datum; + char kind; + + /*There isn't field kind in index. So we should check it by hands*/ + datum = slot_getattr(slot, Anum_ag_label_kind, &isNull); + if (isNull) + { + continue; + } + + kind = DatumGetChar(datum); + + if (kind != LABEL_TYPE_EDGE) + { + continue; + } + + datum = slot_getattr(slot, Anum_ag_label_name, &isNull); + if (!isNull) + { + label = DatumGetName(datum); + lval = (Name) palloc(NAMEDATALEN); + namestrcpy(lval, NameStr(*label)); + labels = lappend(labels, lval); + } + } + + index_endscan(index_scan_desc); + index_close(index_rel, AccessShareLock); + } + else { - Name label; - bool isNull; - Datum datum; + slot = ExecInitExtraTupleSlot( + estate, RelationGetDescr(resultRelInfo->ri_RelationDesc), + &TTSOpsHeapTuple); - tuple = heap_getnext(scan_desc, ForwardScanDirection); + /* setup scan keys to get all edges for the given graph oid */ + ScanKeyInit(&scan_keys[1], Anum_ag_label_graph, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(graph_oid)); + ScanKeyInit(&scan_keys[0], Anum_ag_label_kind, BTEqualStrategyNumber, + F_CHAREQ, CharGetDatum(LABEL_TYPE_EDGE)); - /* no more labels to process */ - if (!HeapTupleIsValid(tuple)) - break; + scan_desc = table_beginscan(ag_label, estate->es_snapshot, 2, scan_keys); - ExecStoreHeapTuple(tuple, slot, false); + /* scan through the results and get all the label names. */ + while(true) + { + Name label; + Name lval; + bool isNull; + Datum datum; - datum = slot_getattr(slot, Anum_ag_label_name, &isNull); - label = DatumGetName(datum); + tuple = heap_getnext(scan_desc, ForwardScanDirection); - labels = lappend(labels, label); - } + /* no more labels to process */ + if (!HeapTupleIsValid(tuple)) + break; + + ExecStoreHeapTuple(tuple, slot, false); + + datum = slot_getattr(slot, Anum_ag_label_name, &isNull); + label = DatumGetName(datum); - table_endscan(scan_desc); + lval = (Name) palloc(NAMEDATALEN); + namestrcpy(lval, NameStr(*label)); + labels = lappend(labels, lval); + } + + table_endscan(scan_desc); + } destroy_entity_result_rel_info(resultRelInfo); - table_close(resultRelInfo->ri_RelationDesc, RowExclusiveLock); + table_close(resultRelInfo->ri_RelationDesc, AccessShareLock); return labels; } diff --git a/src/backend/executor/cypher_delete.c b/src/backend/executor/cypher_delete.c index 0b486ad5e..58503ec27 100644 --- a/src/backend/executor/cypher_delete.c +++ b/src/backend/executor/cypher_delete.c @@ -376,6 +376,8 @@ static void process_delete_list(CustomScanState *node) EState *estate = node->ss.ps.state; HTAB *qual_cache = NULL; HASHCTL hashctl; + HTAB *index_cache = NULL; + HASHCTL idx_hashctl; /* Hash table for caching compiled security quals per label */ MemSet(&hashctl, 0, sizeof(hashctl)); @@ -385,18 +387,34 @@ static void process_delete_list(CustomScanState *node) qual_cache = hash_create("delete_qual_cache", 8, &hashctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + MemSet(&idx_hashctl, 0, sizeof(idx_hashctl)); + idx_hashctl.keysize = sizeof(Oid); + idx_hashctl.entrysize = sizeof(IndexCacheEntry); + idx_hashctl.hcxt = CurrentMemoryContext; + index_cache = hash_create("delete_index_cache", 8, &idx_hashctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + foreach(lc, css->delete_data->delete_items) { cypher_delete_item *item; agtype_value *original_entity_value, *id, *label; ScanKeyData scan_keys[1]; - TableScanDesc scan_desc; + TableScanDesc scan_desc = NULL; ResultRelInfo *resultRelInfo; - HeapTuple heap_tuple; + HeapTuple heap_tuple = NULL; char *label_name; Integer *pos; int entity_position; Oid relid; + Relation rel; + int id_attr_num; + Oid index_oid = InvalidOid; + TupleTableSlot *slot = NULL; + Relation index_rel = NULL; + IndexScanDesc index_scan_desc = NULL; + bool shouldFree = false; + IndexCacheEntry *idx_entry; + bool found_idx_entry; item = lfirst(lc); @@ -415,7 +433,8 @@ static void process_delete_list(CustomScanState *node) label_name = pnstrdup(label->val.string.val, label->val.string.len); resultRelInfo = create_entity_result_rel_info(estate, css->delete_data->graph_name, label_name); - relid = RelationGetRelid(resultRelInfo->ri_RelationDesc); + rel = resultRelInfo->ri_RelationDesc; + relid = RelationGetRelid(rel); /* * Setup the scan key to require the id field on-disc to match the @@ -423,12 +442,14 @@ static void process_delete_list(CustomScanState *node) */ if (original_entity_value->type == AGTV_VERTEX) { + id_attr_num = Anum_ag_label_vertex_table_id; ScanKeyInit(&scan_keys[0], Anum_ag_label_vertex_table_id, BTEqualStrategyNumber, F_GRAPHIDEQ, GRAPHID_GET_DATUM(id->val.int_value)); } else if (original_entity_value->type == AGTV_EDGE) { + id_attr_num = Anum_ag_label_edge_table_id; ScanKeyInit(&scan_keys[0], Anum_ag_label_edge_table_id, BTEqualStrategyNumber, F_GRAPHIDEQ, GRAPHID_GET_DATUM(id->val.int_value)); @@ -439,82 +460,214 @@ static void process_delete_list(CustomScanState *node) errmsg("DELETE clause can only delete vertices and edges"))); } + idx_entry = hash_search(index_cache, &relid, HASH_ENTER, &found_idx_entry); + + if (!found_idx_entry) + { + idx_entry->index_oid = find_usable_btree_index_for_attr(rel, id_attr_num); + } + + index_oid = idx_entry->index_oid; + /* * Setup the scan description, with the correct snapshot and scan keys. */ estate->es_snapshot->curcid = GetCurrentCommandId(false); estate->es_output_cid = GetCurrentCommandId(false); - scan_desc = table_beginscan(resultRelInfo->ri_RelationDesc, - estate->es_snapshot, 1, scan_keys); - /* Retrieve the tuple. */ - heap_tuple = heap_getnext(scan_desc, ForwardScanDirection); - - /* - * If the heap tuple still exists (It wasn't deleted after this variable - * was created) we can delete it. Otherwise, its safe to skip this - * delete. - */ - if (!HeapTupleIsValid(heap_tuple)) + if (OidIsValid(index_oid)) { - table_endscan(scan_desc); - destroy_entity_result_rel_info(resultRelInfo); + slot = table_slot_create(rel, NULL); - continue; + index_rel = index_open(index_oid, RowExclusiveLock); + index_scan_desc = index_beginscan(rel, index_rel, estate->es_snapshot, NULL, 1, 0); + index_rescan(index_scan_desc, scan_keys, 1, NULL, 0); + + if (index_getnext_slot(index_scan_desc, ForwardScanDirection, slot)) + { + heap_tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); + } + } + else + { + scan_desc = table_beginscan(rel, estate->es_snapshot, 1, scan_keys); + /* Retrieve the tuple. */ + heap_tuple = heap_getnext(scan_desc, ForwardScanDirection); } - /* Check RLS security quals (USING policy) before delete */ - if (check_enable_rls(relid, InvalidOid, true) == RLS_ENABLED) + if (HeapTupleIsValid(heap_tuple)) { - RLSCacheEntry *entry; - bool found; + bool passed_rls = true; - /* Get cached security quals and slot for this label */ - entry = hash_search(qual_cache, &relid, HASH_ENTER, &found); - if (!found) + /* Check RLS security quals (USING policy) before delete */ + if (check_enable_rls(relid, InvalidOid, true) == RLS_ENABLED) { - entry->qualExprs = setup_security_quals(resultRelInfo, estate, - node, CMD_DELETE); - entry->slot = ExecInitExtraTupleSlot( - estate, RelationGetDescr(resultRelInfo->ri_RelationDesc), - &TTSOpsHeapTuple); - entry->withCheckOptions = NIL; - entry->withCheckOptionExprs = NIL; + RLSCacheEntry *entry; + bool found_rls; + + entry = hash_search(qual_cache, &relid, HASH_ENTER, &found_rls); + if (!found_rls) + { + entry->qualExprs = setup_security_quals(resultRelInfo, estate, node, CMD_DELETE); + entry->slot = ExecInitExtraTupleSlot(estate, RelationGetDescr(rel), &TTSOpsHeapTuple); + } + + ExecStoreHeapTuple(heap_tuple, entry->slot, false); + + if (!check_security_quals(entry->qualExprs, entry->slot, econtext)) + { + passed_rls = false; + } } - ExecStoreHeapTuple(heap_tuple, entry->slot, false); + if (passed_rls) + { + /* + * For vertices, we insert the vertex ID in the hashtable + * vertex_id_htab. This hashtable is used later to process + * connected edges. + */ + if (original_entity_value->type == AGTV_VERTEX) + { + bool found; + hash_search(css->vertex_id_htab, (void *)&(id->val.int_value), + HASH_ENTER, &found); + } - /* Silently skip if USING policy filters out this row */ - if (!check_security_quals(entry->qualExprs, entry->slot, econtext)) + /* At this point, we are ready to delete the node/vertex. */ + delete_entity(estate, resultRelInfo, heap_tuple); + } + + if (shouldFree) { - table_endscan(scan_desc); - destroy_entity_result_rel_info(resultRelInfo); - continue; + heap_freetuple(heap_tuple); } } - /* - * For vertices, we insert the vertex ID in the hashtable - * vertex_id_htab. This hashtable is used later to process - * connected edges. - */ - if (original_entity_value->type == AGTV_VERTEX) + if (OidIsValid(index_oid)) { - bool found; - hash_search(css->vertex_id_htab, (void *)&(id->val.int_value), - HASH_ENTER, &found); + ExecDropSingleTupleTableSlot(slot); + index_endscan(index_scan_desc); + index_close(index_rel, RowExclusiveLock); + } + else + { + table_endscan(scan_desc); } - /* At this point, we are ready to delete the node/vertex. */ - delete_entity(estate, resultRelInfo, heap_tuple); - - /* Close the scan and the relation. */ - table_endscan(scan_desc); destroy_entity_result_rel_info(resultRelInfo); } /* Clean up the cache */ hash_destroy(qual_cache); + hash_destroy(index_cache); +} + +/* + * Helper function to scan an edge table using a specific index (start_id or end_id) + * and delete the connected edges if the vertex is being deleted. + */ +static void process_edges_by_index(Oid index_oid, + Relation rel, + EState *estate, + cypher_delete_custom_scan_state *css, + ResultRelInfo *resultRelInfo, + Oid relid, + char *label_name, + bool rls_enabled, + List *qualExprs, + ExprContext *econtext, + bool is_pass_two) +{ + HASH_SEQ_STATUS hash_status; + graphid *vid; + Relation index_rel; + IndexScanDesc scan; + ScanKeyData key; + TupleTableSlot *slot; + + slot = table_slot_create(rel, NULL); + + index_rel = index_open(index_oid, RowExclusiveLock); + scan = index_beginscan(rel, index_rel, estate->es_snapshot, NULL, 1, 0); + + /* Initialize ScanKey with a dummy argument (0), updated in the loop */ + ScanKeyInit(&key, 1, BTEqualStrategyNumber, F_GRAPHIDEQ, 0); + hash_seq_init(&hash_status, css->vertex_id_htab); + + while ((vid = (graphid *) hash_seq_search(&hash_status)) != NULL) + { + /* Update search key with the current ID of the vertex being deleted */ + key.sk_argument = GRAPHID_GET_DATUM(*vid); + index_rescan(scan, &key, 1, NULL, 0); + + while (index_getnext_slot(scan, ForwardScanDirection, slot)) + { + if (is_pass_two) + { + bool is_null; + graphid startid; + bool found_startid = false; + + startid = GRAPHID_GET_DATUM(slot_getattr(slot, Anum_ag_label_edge_table_start_id, &is_null)); + + hash_search(css->vertex_id_htab, (void *)&startid, HASH_FIND, &found_startid); + + if (found_startid) + { + ExecClearTuple(slot); + continue; + } + } + + /* If edge found - delete it (or error if not DETACH) */ + if (css->delete_data->detach) + { + AclResult aclresult; + bool shouldFree; + HeapTuple tuple; + + /* Check that the user has DELETE permission on the edge table */ + aclresult = pg_class_aclcheck(relid, GetUserId(), ACL_DELETE); + if (aclresult != ACLCHECK_OK) + { + aclcheck_error(aclresult, OBJECT_TABLE, label_name); + } + + /* Check RLS security quals (USING policy) before delete */ + if (rls_enabled) + { + if (!check_security_quals(qualExprs, slot, econtext)) + { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot delete edge due to row-level security policy on \"%s\"", + label_name), + errhint("DETACH DELETE requires permission to delete all connected edges."))); + } + } + + tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); + delete_entity(estate, resultRelInfo, tuple); + + if (shouldFree) + { + heap_freetuple(tuple); + } + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Cannot delete a vertex that has edge(s). " + "Delete the edge(s) first, or try DETACH DELETE."))); + } + ExecClearTuple(slot); + } + } + index_endscan(scan); + index_close(index_rel, RowExclusiveLock); + ExecDropSingleTupleTableSlot(slot); } /* @@ -542,17 +695,16 @@ static void check_for_connected_edges(CustomScanState *node) bool rls_enabled = false; List *qualExprs = NIL; ExprContext *econtext = NULL; + Oid start_index_oid = InvalidOid; + Oid end_index_oid = InvalidOid; + Relation rel; resultRelInfo = create_entity_result_rel_info(estate, graph_name, label_name); - relid = RelationGetRelid(resultRelInfo->ri_RelationDesc); + rel = resultRelInfo->ri_RelationDesc; + relid = RelationGetRelid(rel); estate->es_snapshot->curcid = GetCurrentCommandId(false); estate->es_output_cid = GetCurrentCommandId(false); - scan_desc = table_beginscan(resultRelInfo->ri_RelationDesc, - estate->es_snapshot, 0, NULL); - slot = ExecInitExtraTupleSlot( - estate, RelationGetDescr(resultRelInfo->ri_RelationDesc), - &TTSOpsHeapTuple); /* * For DETACH DELETE with RLS enabled, compile the security qual @@ -570,86 +722,109 @@ static void check_for_connected_edges(CustomScanState *node) } } - /* for each row */ - while (true) - { - graphid startid; - graphid endid; - bool isNull; - bool found_startid = false; - bool found_endid = false; + /* Look for indexes on start_id and end_id columns. */ + start_index_oid = find_usable_btree_index_for_attr(rel, Anum_ag_label_edge_table_start_id); + end_index_oid = find_usable_btree_index_for_attr(rel, Anum_ag_label_edge_table_end_id); - tuple = heap_getnext(scan_desc, ForwardScanDirection); + if (OidIsValid(start_index_oid) && OidIsValid(end_index_oid)) + { + /* PASS 1: Find edges where the deleted vertex is the START_ID. */ + process_edges_by_index(start_index_oid, rel, estate, css, resultRelInfo, + relid, label_name, rls_enabled, qualExprs, econtext, false); + + /* PASS 2: Find edges where the deleted vertex is the END_ID. */ + process_edges_by_index(end_index_oid, rel, estate, css, resultRelInfo, + relid, label_name, rls_enabled, qualExprs, econtext, true); + } + else + { + scan_desc = table_beginscan(rel, estate->es_snapshot, 0, NULL); + slot = ExecInitExtraTupleSlot( + estate, RelationGetDescr(rel), + &TTSOpsHeapTuple); - /* no more tuples to process, break and scan the next label. */ - if (!HeapTupleIsValid(tuple)) + /* for each row */ + while (true) { - break; - } + graphid startid; + graphid endid; + bool isNull; + bool found_startid = false; + bool found_endid = false; - ExecStoreHeapTuple(tuple, slot, false); + tuple = heap_getnext(scan_desc, ForwardScanDirection); - startid = GRAPHID_GET_DATUM(slot_getattr( - slot, Anum_ag_label_edge_table_start_id, &isNull)); - endid = GRAPHID_GET_DATUM( - slot_getattr(slot, Anum_ag_label_edge_table_end_id, &isNull)); + /* no more tuples to process, break and scan the next label. */ + if (!HeapTupleIsValid(tuple)) + { + break; + } - hash_search(css->vertex_id_htab, (void *)&startid, HASH_FIND, - &found_startid); + ExecStoreHeapTuple(tuple, slot, false); - if (!found_startid) - { - hash_search(css->vertex_id_htab, (void *)&endid, HASH_FIND, - &found_endid); - } + startid = GRAPHID_GET_DATUM(slot_getattr( + slot, Anum_ag_label_edge_table_start_id, &isNull)); + endid = GRAPHID_GET_DATUM( + slot_getattr(slot, Anum_ag_label_edge_table_end_id, &isNull)); - if (found_startid || found_endid) - { - if (css->delete_data->detach) + hash_search(css->vertex_id_htab, (void *)&startid, HASH_FIND, + &found_startid); + + if (!found_startid) { - AclResult aclresult; + hash_search(css->vertex_id_htab, (void *)&endid, HASH_FIND, + &found_endid); + } - /* Check that the user has DELETE permission on the edge table */ - aclresult = pg_class_aclcheck(relid, GetUserId(), ACL_DELETE); - if (aclresult != ACLCHECK_OK) + if (found_startid || found_endid) + { + if (css->delete_data->detach) { - aclcheck_error(aclresult, OBJECT_TABLE, label_name); - } + AclResult aclresult; - /* Check RLS security quals (USING policy) before delete */ - if (rls_enabled) - { - /* - * For DETACH DELETE, error out if edge RLS check fails. - * Unlike normal DELETE which silently skips, we cannot - * silently skip edges here as it would leave dangling - * edges pointing to deleted vertices. - */ - if (!check_security_quals(qualExprs, slot, econtext)) + /* Check that the user has DELETE permission on the edge table */ + aclresult = pg_class_aclcheck(relid, GetUserId(), ACL_DELETE); + if (aclresult != ACLCHECK_OK) { - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("cannot delete edge due to row-level security policy on \"%s\"", - label_name), - errhint("DETACH DELETE requires permission to delete all connected edges."))); + aclcheck_error(aclresult, OBJECT_TABLE, label_name); } - } - delete_entity(estate, resultRelInfo, tuple); - } - else - { - ereport( - ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg( - "Cannot delete a vertex that has edge(s). " - "Delete the edge(s) first, or try DETACH DELETE."))); + /* Check RLS security quals (USING policy) before delete */ + if (rls_enabled) + { + /* + * For DETACH DELETE, error out if edge RLS check fails. + * Unlike normal DELETE which silently skips, we cannot + * silently skip edges here as it would leave dangling + * edges pointing to deleted vertices. + */ + if (!check_security_quals(qualExprs, slot, econtext)) + { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot delete edge due to row-level security policy on \"%s\"", + label_name), + errhint("DETACH DELETE requires permission to delete all connected edges."))); + } + } + + delete_entity(estate, resultRelInfo, tuple); + } + else + { + ereport( + ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg( + "Cannot delete a vertex that has edge(s). " + "Delete the edge(s) first, or try DETACH DELETE."))); + } } } + + table_endscan(scan_desc); } - table_endscan(scan_desc); destroy_entity_result_rel_info(resultRelInfo); } } diff --git a/src/backend/executor/cypher_set.c b/src/backend/executor/cypher_set.c index a1063af32..09d3d3b54 100644 --- a/src/backend/executor/cypher_set.c +++ b/src/backend/executor/cypher_set.c @@ -383,6 +383,8 @@ static void process_update_list(CustomScanState *node) int lidx = 0; HTAB *qual_cache = NULL; HASHCTL hashctl; + HTAB *index_cache = NULL; + HASHCTL idx_hashctl; /* allocate an array to hold the last update index of each 'entity' */ luindex = palloc0(sizeof(int) * scanTupleSlot->tts_nvalid); @@ -395,6 +397,13 @@ static void process_update_list(CustomScanState *node) qual_cache = hash_create("update_qual_cache", 8, &hashctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + MemSet(&idx_hashctl, 0, sizeof(idx_hashctl)); + idx_hashctl.keysize = sizeof(Oid); + idx_hashctl.entrysize = sizeof(IndexCacheEntry); + idx_hashctl.hcxt = CurrentMemoryContext; + index_cache = hash_create("update_index_cache", 8, &idx_hashctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + /* * Iterate through the SET items list and store the loop index of each * 'entity' update. As there is only one entry for each entity, this will @@ -439,6 +448,11 @@ static void process_update_list(CustomScanState *node) HeapTuple heap_tuple; char *clause_name = css->set_list->clause_name; int cid; + Oid index_oid = InvalidOid; + Relation rel; + Oid relid; + IndexCacheEntry *idx_entry; + bool found_idx_entry; update_item = (cypher_update_item *)lfirst(lc); @@ -538,6 +552,17 @@ static void process_update_list(CustomScanState *node) resultRelInfo = create_entity_result_rel_info( estate, css->set_list->graph_name, label_name); + rel = resultRelInfo->ri_RelationDesc; + relid = RelationGetRelid(rel); + + idx_entry = hash_search(index_cache, &relid, HASH_ENTER, &found_idx_entry); + if (!found_idx_entry) + { + /* Check if there is a valid index on the 'id' column */ + idx_entry->index_oid = find_usable_btree_index_for_attr(rel, 1); + } + index_oid = idx_entry->index_oid; + slot = ExecInitExtraTupleSlot( estate, RelationGetDescr(resultRelInfo->ri_RelationDesc), &TTSOpsHeapTuple); @@ -546,7 +571,6 @@ static void process_update_list(CustomScanState *node) if (check_enable_rls(resultRelInfo->ri_RelationDesc->rd_id, InvalidOid, true) == RLS_ENABLED) { - Oid relid = RelationGetRelid(resultRelInfo->ri_RelationDesc); RLSCacheEntry *entry; bool found; @@ -628,60 +652,132 @@ static void process_update_list(CustomScanState *node) if (luindex[update_item->entity_position - 1] == lidx) { - /* - * Setup the scan key to require the id field on-disc to match the - * entity's graphid. - */ - ScanKeyInit(&scan_keys[0], 1, BTEqualStrategyNumber, F_GRAPHIDEQ, - GRAPHID_GET_DATUM(id->val.int_value)); - /* - * Setup the scan description, with the correct snapshot and scan - * keys. - */ - scan_desc = table_beginscan(resultRelInfo->ri_RelationDesc, - estate->es_snapshot, 1, scan_keys); - /* Retrieve the tuple. */ - heap_tuple = heap_getnext(scan_desc, ForwardScanDirection); - - /* - * If the heap tuple still exists (It wasn't deleted between the - * match and this SET/REMOVE) update the heap_tuple. - */ - if (HeapTupleIsValid(heap_tuple)) - { - bool should_update = true; - Oid relid = RelationGetRelid(resultRelInfo->ri_RelationDesc); - - /* Check RLS security quals (USING policy) before update */ - if (check_enable_rls(relid, InvalidOid, true) == RLS_ENABLED) + if (OidIsValid(index_oid)) + { + Relation index_rel; + IndexScanDesc idx_scan_desc; + TupleTableSlot *index_slot; + + index_rel = index_open(index_oid, RowExclusiveLock); + + /* + * Setup the scan key to require the id field on-disc to match the + * entity's graphid. + */ + ScanKeyInit(&scan_keys[0], 1, BTEqualStrategyNumber, F_GRAPHIDEQ, + GRAPHID_GET_DATUM(id->val.int_value)); + + index_slot = table_slot_create(rel, NULL); + idx_scan_desc = index_beginscan(rel, index_rel, estate->es_snapshot, NULL, 1, 0); + index_rescan(idx_scan_desc, scan_keys, 1, NULL, 0); + + if (index_getnext_slot(idx_scan_desc, ForwardScanDirection, index_slot)) { - RLSCacheEntry *entry; + bool shouldFree; + + /* Retrieve the tuple from the slot */ + heap_tuple = ExecFetchSlotHeapTuple(index_slot, true, &shouldFree); - /* Entry was already created earlier when setting up WCOs */ - entry = hash_search(qual_cache, &relid, HASH_FIND, NULL); - if (!entry) + if (HeapTupleIsValid(heap_tuple)) { - ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("missing RLS cache entry for relation %u", - relid))); + bool should_update = true; + HeapTuple original_tuple = heap_tuple; + + /* Check RLS security quals (USING policy) before update */ + if (check_enable_rls(relid, InvalidOid, true) == RLS_ENABLED) + { + RLSCacheEntry *entry; + + /* Entry was already created earlier when setting up WCOs */ + entry = hash_search(qual_cache, &relid, HASH_FIND, NULL); + if (!entry) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("missing RLS cache entry for relation %u", + relid))); + } + + ExecStoreHeapTuple(heap_tuple, entry->slot, false); + should_update = check_security_quals(entry->qualExprs, + entry->slot, + econtext); + } + + /* Silently skip if USING policy filters out this row */ + if (should_update) + { + heap_tuple = update_entity_tuple(resultRelInfo, slot, estate, + original_tuple); + } + + if (shouldFree) + { + heap_freetuple(original_tuple); + } } - - ExecStoreHeapTuple(heap_tuple, entry->slot, false); - should_update = check_security_quals(entry->qualExprs, - entry->slot, - econtext); } - /* Silently skip if USING policy filters out this row */ - if (should_update) + ExecDropSingleTupleTableSlot(index_slot); + index_endscan(idx_scan_desc); + index_close(index_rel, RowExclusiveLock); + } + else + { + /* + * Setup the scan key to require the id field on-disc to match the + * entity's graphid. + */ + ScanKeyInit(&scan_keys[0], 1, BTEqualStrategyNumber, F_GRAPHIDEQ, + GRAPHID_GET_DATUM(id->val.int_value)); + /* + * Setup the scan description, with the correct snapshot and scan + * keys. + */ + scan_desc = table_beginscan(resultRelInfo->ri_RelationDesc, + estate->es_snapshot, 1, scan_keys); + /* Retrieve the tuple. */ + heap_tuple = heap_getnext(scan_desc, ForwardScanDirection); + + /* + * If the heap tuple still exists (It wasn't deleted between the + * match and this SET/REMOVE) update the heap_tuple. + */ + if (HeapTupleIsValid(heap_tuple)) { - heap_tuple = update_entity_tuple(resultRelInfo, slot, estate, - heap_tuple); + bool should_update = true; + + /* Check RLS security quals (USING policy) before update */ + if (check_enable_rls(relid, InvalidOid, true) == RLS_ENABLED) + { + RLSCacheEntry *entry; + + /* Entry was already created earlier when setting up WCOs */ + entry = hash_search(qual_cache, &relid, HASH_FIND, NULL); + if (!entry) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("missing RLS cache entry for relation %u", + relid))); + } + + ExecStoreHeapTuple(heap_tuple, entry->slot, false); + should_update = check_security_quals(entry->qualExprs, + entry->slot, + econtext); + } + + /* Silently skip if USING policy filters out this row */ + if (should_update) + { + heap_tuple = update_entity_tuple(resultRelInfo, slot, estate, + heap_tuple); + } } + /* close the ScanDescription */ + table_endscan(scan_desc); } - /* close the ScanDescription */ - table_endscan(scan_desc); } estate->es_snapshot->curcid = cid; @@ -695,6 +791,7 @@ static void process_update_list(CustomScanState *node) /* Clean up the cache */ hash_destroy(qual_cache); + hash_destroy(index_cache); /* free our lookup array */ pfree_if_not_null(luindex); diff --git a/src/backend/executor/cypher_utils.c b/src/backend/executor/cypher_utils.c index 940284234..8237cdcce 100644 --- a/src/backend/executor/cypher_utils.c +++ b/src/backend/executor/cypher_utils.c @@ -208,6 +208,8 @@ bool entity_exists(EState *estate, Oid graph_oid, graphid id) HeapTuple tuple; Relation rel; bool result = true; + TupleTableSlot *slot; + Oid index_oid = InvalidOid; CommandId saved_curcid; /* @@ -238,20 +240,47 @@ bool entity_exists(EState *estate, Oid graph_oid, graphid id) GetCurrentCommandId(false)); rel = table_open(label->relation, RowExclusiveLock); - scan_desc = table_beginscan(rel, estate->es_snapshot, 1, scan_keys); - tuple = heap_getnext(scan_desc, ForwardScanDirection); + index_oid = find_usable_btree_index_for_attr(rel, 1); - /* - * If a single tuple was returned, the tuple is still valid, otherwise' - * set to false. - */ - if (!HeapTupleIsValid(tuple)) + if (OidIsValid(index_oid)) { - result = false; + IndexScanDesc index_scan_desc; + Relation index_rel; + + slot = table_slot_create(rel, NULL); + + index_rel = index_open(index_oid, AccessShareLock); + + index_scan_desc = index_beginscan(rel, index_rel, estate->es_snapshot, NULL, 1, 0); + index_rescan(index_scan_desc, scan_keys, 1, NULL, 0); + + if (!index_getnext_slot(index_scan_desc, ForwardScanDirection, slot)) + { + result = false; + } + + index_endscan(index_scan_desc); + index_close(index_rel, AccessShareLock); + ExecDropSingleTupleTableSlot(slot); + } + else + { + scan_desc = table_beginscan(rel, estate->es_snapshot, 1, scan_keys); + tuple = heap_getnext(scan_desc, ForwardScanDirection); + + /* + * If a single tuple was returned, the tuple is still valid, otherwise' + * set to false. + */ + if (!HeapTupleIsValid(tuple)) + { + result = false; + } + + table_endscan(scan_desc); } - table_endscan(scan_desc); table_close(rel, RowExclusiveLock); /* Restore the original curcid */ diff --git a/src/backend/utils/adt/age_global_graph.c b/src/backend/utils/adt/age_global_graph.c index 4e5b58632..a9308970f 100644 --- a/src/backend/utils/adt/age_global_graph.c +++ b/src/backend/utils/adt/age_global_graph.c @@ -204,43 +204,123 @@ static List *get_ag_labels_names(Snapshot snapshot, Oid graph_oid, TableScanDesc scan_desc; HeapTuple tuple; TupleDesc tupdesc; + Oid index_oid = InvalidOid; /* we need a valid snapshot */ Assert(snapshot != NULL); - /* setup scan keys to get all edges for the given graph oid */ - ScanKeyInit(&scan_keys[1], Anum_ag_label_graph, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(graph_oid)); - ScanKeyInit(&scan_keys[0], Anum_ag_label_kind, BTEqualStrategyNumber, - F_CHAREQ, CharGetDatum(label_type)); - /* setup the table to be scanned, ag_label in this case */ ag_label = table_open(ag_label_relation_id(), AccessShareLock); - scan_desc = table_beginscan(ag_label, snapshot, 2, scan_keys); /* get the tupdesc - we don't need to release this one */ tupdesc = RelationGetDescr(ag_label); /* bail if the number of columns differs - this table has 5 */ Assert(tupdesc->natts == Natts_ag_label); - /* get all of the label names */ - while((tuple = heap_getnext(scan_desc, ForwardScanDirection)) != NULL) + /* + * Find a usable index whose first key column is ag_label.graph + * (Anum_ag_label_graph) + */ + index_oid = find_usable_btree_index_for_attr(ag_label, Anum_ag_label_graph); + + if (OidIsValid(index_oid)) { - Name label; - bool is_null = false; - - /* something is wrong if this tuple isn't valid */ - Assert(HeapTupleIsValid(tuple)); - /* get the label name */ - label = DatumGetName(heap_getattr(tuple, Anum_ag_label_name, tupdesc, - &is_null)); - Assert(!is_null); - /* add it to our list */ - labels = lappend(labels, label); + Relation index_rel; + IndexScanDesc idx_scan_desc; + ScanKeyData key; + TupleTableSlot *slot; + + index_rel = index_open(index_oid, AccessShareLock); + slot = table_slot_create(ag_label, NULL); + + /* + * Setup ScanKey: ag_label.graph = graph_oid + * Note: We CANNOT filter by 'kind' here because it is not in the index. + */ + ScanKeyInit(&key, 1, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(graph_oid)); + + idx_scan_desc = index_beginscan(ag_label, index_rel, snapshot, NULL, 1, 0); + index_rescan(idx_scan_desc, &key, 1, NULL, 0); + + while (index_getnext_slot(idx_scan_desc, ForwardScanDirection, slot)) + { + bool shouldFree; + + tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); + + if (HeapTupleIsValid(tuple)) + { + bool is_null; + Datum kind_datum; + + /* + * Since the index only gave us rows for the correct graph, + * we must now check if the label 'kind' matches (vertex 'v' or edge 'e'). + */ + kind_datum = heap_getattr(tuple, Anum_ag_label_kind, tupdesc, &is_null); + + if (!is_null && DatumGetChar(kind_datum) == label_type) + { + Datum name_datum = heap_getattr(tuple, Anum_ag_label_name, tupdesc, &is_null); + if (!is_null) + { + Name label_name_ptr; + Name lval; + + label_name_ptr = DatumGetName(name_datum); + lval = (Name) palloc(NAMEDATALEN); + namestrcpy(lval, NameStr(*label_name_ptr)); + labels = lappend(labels, lval); + } + } + } + + if (shouldFree) + { + heap_freetuple(tuple); + } + ExecClearTuple(slot); + } + + ExecDropSingleTupleTableSlot(slot); + index_endscan(idx_scan_desc); + index_close(index_rel, AccessShareLock); + } + else + { + /* setup scan keys to get all edges for the given graph oid */ + ScanKeyInit(&scan_keys[1], Anum_ag_label_graph, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(graph_oid)); + ScanKeyInit(&scan_keys[0], Anum_ag_label_kind, BTEqualStrategyNumber, + F_CHAREQ, CharGetDatum(label_type)); + + scan_desc = table_beginscan(ag_label, snapshot, 2, scan_keys); + + /* get all of the label names */ + while((tuple = heap_getnext(scan_desc, ForwardScanDirection)) != NULL) + { + Name label; + Name lval; + bool is_null = false; + + /* something is wrong if this tuple isn't valid */ + Assert(HeapTupleIsValid(tuple)); + /* get the label name */ + label = DatumGetName(heap_getattr(tuple, Anum_ag_label_name, tupdesc, + &is_null)); + + Assert(!is_null); + /* add it to our list */ + lval = (Name) palloc(NAMEDATALEN); + namestrcpy(lval, NameStr(*label)); + labels = lappend(labels, lval); + } + + /* close up scan */ + table_endscan(scan_desc); } - /* close up scan */ - table_endscan(scan_desc); table_close(ag_label, AccessShareLock); return labels; diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index c0c54e5a4..386219556 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -38,6 +38,7 @@ #include "access/genam.h" #include "access/heapam.h" #include "catalog/namespace.h" +#include "catalog/pg_am_d.h" #include "catalog/pg_collation_d.h" #include "catalog/pg_operator_d.h" #include "funcapi.h" @@ -5624,11 +5625,14 @@ static Datum get_vertex(const char *graph, const char *vertex_label, { ScanKeyData scan_keys[1]; Relation graph_vertex_label; - TableScanDesc scan_desc; - HeapTuple tuple; + TableScanDesc scan_desc = NULL; + HeapTuple tuple = NULL; TupleDesc tupdesc; Datum id, properties, result; AclResult aclresult; + TupleTableSlot *slot; + Oid index_oid; + bool should_free_tuple = false; /* get the specific graph namespace (schema) */ Oid graph_namespace_oid = get_namespace_oid(graph, false); @@ -5646,19 +5650,54 @@ static Datum get_vertex(const char *graph, const char *vertex_label, aclcheck_error(aclresult, OBJECT_TABLE, vertex_label); } - /* initialize the scan key */ - ScanKeyInit(&scan_keys[0], 1, BTEqualStrategyNumber, F_OIDEQ, - Int64GetDatum(graphid)); - - /* open the relation (table), begin the scan, and get the tuple */ + /* open the relation (table) */ graph_vertex_label = table_open(vertex_label_table_oid, ShareLock); - scan_desc = table_beginscan(graph_vertex_label, snapshot, 1, scan_keys); - tuple = heap_getnext(scan_desc, ForwardScanDirection); + + index_oid = find_usable_btree_index_for_attr(graph_vertex_label, 1); + + if (OidIsValid(index_oid)) + { + IndexScanDesc index_scan_desc; + Relation index_rel; + + index_rel = index_open(index_oid, ShareLock); + slot = table_slot_create(graph_vertex_label, NULL); + + /* initialize the scan key using GRAPHIDEQ for index */ + ScanKeyInit(&scan_keys[0], 1, BTEqualStrategyNumber, + F_GRAPHIDEQ, Int64GetDatum(graphid)); + + index_scan_desc = index_beginscan(graph_vertex_label, index_rel, + snapshot, NULL, 1, 0); + index_rescan(index_scan_desc, scan_keys, 1, NULL, 0); + + if (index_getnext_slot(index_scan_desc, ForwardScanDirection, slot)) + { + tuple = ExecCopySlotHeapTuple(slot); + should_free_tuple = true; + } + + index_endscan(index_scan_desc); + index_close(index_rel, ShareLock); + ExecDropSingleTupleTableSlot(slot); + } + else + { + /* fallback to sequential scan */ + ScanKeyInit(&scan_keys[0], 1, BTEqualStrategyNumber, F_GRAPHIDEQ, + GRAPHID_GET_DATUM(graphid)); + + scan_desc = table_beginscan(graph_vertex_label, snapshot, 1, scan_keys); + tuple = heap_getnext(scan_desc, ForwardScanDirection); + } /* bail if the tuple isn't valid */ if (!HeapTupleIsValid(tuple)) { - table_endscan(scan_desc); + if (scan_desc) + { + table_endscan(scan_desc); + } table_close(graph_vertex_label, ShareLock); ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), @@ -5668,7 +5707,15 @@ static Datum get_vertex(const char *graph, const char *vertex_label, /* Check RLS policies - error if filtered out */ if (!check_rls_for_tuple(graph_vertex_label, tuple, CMD_SELECT)) { - table_endscan(scan_desc); + if (scan_desc) + { + table_endscan(scan_desc); + } + if (should_free_tuple && tuple != NULL) + { + heap_freetuple(tuple); + } + table_close(graph_vertex_label, ShareLock); ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -5693,8 +5740,17 @@ static Datum get_vertex(const char *graph, const char *vertex_label, /* reconstruct the vertex */ result = DirectFunctionCall3(_agtype_build_vertex, id, CStringGetDatum(vertex_label), properties); - /* end the scan and close the relation */ - table_endscan(scan_desc); + + /* end the scan and close the relation with new cleanup logic */ + if (scan_desc) + { + table_endscan(scan_desc); + } + if (should_free_tuple && tuple != NULL) + { + heap_freetuple(tuple); + } + table_close(graph_vertex_label, ShareLock); /* return the vertex datum */ return result; @@ -12020,6 +12076,40 @@ static int64 get_int64_from_int_datums(Datum d, Oid type, char *funcname, return result; } +/* + * Helper function to find a valid index for a specific attribute. + * Returns the OID of the index, or InvalidOid if none is found. + */ +Oid find_usable_btree_index_for_attr(Relation rel, AttrNumber attnum) +{ +List *index_list = RelationGetIndexList(rel); + ListCell *ilc; + Oid index_oid = InvalidOid; + + foreach(ilc, index_list) + { + Oid curr_idx_oid = lfirst_oid(ilc); + Relation curr_idx_rel = index_open(curr_idx_oid, AccessShareLock); + + if (curr_idx_rel->rd_index->indisvalid && + curr_idx_rel->rd_index->indnatts >= 1 && + curr_idx_rel->rd_index->indkey.values[0] == attnum && + curr_idx_rel->rd_rel->relam == BTREE_AM_OID && + RelationGetIndexPredicate(curr_idx_rel) == NIL) + { + index_oid = curr_idx_oid; + index_close(curr_idx_rel, AccessShareLock); + break; + } + + index_close(curr_idx_rel, AccessShareLock); + } + + list_free(index_list); + + return index_oid; +} + PG_FUNCTION_INFO_V1(age_range); /* * Execution function to implement openCypher range() function diff --git a/src/include/executor/cypher_utils.h b/src/include/executor/cypher_utils.h index 278094e07..cdd8fa33e 100644 --- a/src/include/executor/cypher_utils.h +++ b/src/include/executor/cypher_utils.h @@ -152,4 +152,10 @@ typedef struct RLSCacheEntry List *withCheckOptionExprs; } RLSCacheEntry; +/* Hash table entry for caching index OIDs per label */ +typedef struct IndexCacheEntry { + Oid relid; /* hash key */ + Oid index_oid; +} IndexCacheEntry; + #endif diff --git a/src/include/utils/agtype.h b/src/include/utils/agtype.h index ec9125073..807d3795e 100644 --- a/src/include/utils/agtype.h +++ b/src/include/utils/agtype.h @@ -593,6 +593,7 @@ bool is_numeric_result(agtype_value *lhs, agtype_value *rhs); void copy_agtype_value(agtype_parse_state* pstate, agtype_value* original_agtype_value, agtype_value **copied_agtype_value, bool is_top_level); +Oid find_usable_btree_index_for_attr(Relation rel, AttrNumber attnum); /* agtype.c support functions */ /* From 945a259d86d42204f186f79dec758585afde0eaa Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Tue, 14 Apr 2026 12:01:26 -0700 Subject: [PATCH 044/100] Add missing include for PR: Add index scan (#2351) (#2379) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI build did not fail the PR (we need to verify why and correct it) due to the missing include - src/backend/utils/adt/age_global_graph.c:273:25: error: implicit declaration of function ‘namestrcpy’; did you mean ‘strcpy’? [-Wimplicit-function-declaration] 273 | namestrcpy(lval, NameStr(*label_name_ptr)); | ^~~~~~~~~~ | strcpy The build still works as the linker was able to resolve it. This PR will add it in to correct the error going forward. --- src/backend/utils/adt/age_global_graph.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/utils/adt/age_global_graph.c b/src/backend/utils/adt/age_global_graph.c index a9308970f..f99d75abe 100644 --- a/src/backend/utils/adt/age_global_graph.c +++ b/src/backend/utils/adt/age_global_graph.c @@ -27,6 +27,7 @@ #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/snapmgr.h" +#include "utils/builtins.h" #include "utils/age_global_graph.h" #include "catalog/ag_graph.h" From 1847644cf7d2f52486f3a6dba90317ad72f178c7 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Wed, 15 Apr 2026 01:09:50 -0700 Subject: [PATCH 045/100] Improve extension upgrade regression test (addendum to #2364) (#2377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note: This PR was created with AI tools and a human. This is an addendum to PR #2364 with three improvements. Makefile: - Replace awk-based synthetic version (minor+1) with an _upgrade_test suffix (e.g., 1.7.0 -> 1.7.0_upgrade_test). The awk approach produced numeric versions like 1.8.0 that could collide with real future upgrade scripts, and the ::int[] cast in the SQL version lookup fails on non-numeric version strings. The _upgrade_test suffix avoids both issues and is unambiguously synthetic. - Extend the generated cleanup script to also remove repo-root copies of the synthetic files and to self-delete, preventing stale artifacts from accumulating across repeated test runs. Regression test (regress/sql/age_upgrade.sql): - Simplify version lookup to directly select the _upgrade_test version via LIKE '%_upgrade_test' instead of picking the highest non-default version with string_to_array(version, '.')::int[] DESC. The old approach would fail with a cast error on the _upgrade_test suffix and was unnecessarily indirect — the test knows exactly what synthetic version the Makefile installed. modified: Makefile modified: regress/expected/age_upgrade.out modified: regress/sql/age_upgrade.sql Co-authored-by: Claude --- Makefile | 8 ++++---- regress/expected/age_upgrade.out | 7 ++----- regress/sql/age_upgrade.sql | 7 ++----- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 5c0a8e1ef..f07e1b710 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ age_sql = age--1.7.0.sql # 1. Builds the install SQL from the INITIAL version-bump commit (the "from" # version). This is the age--.sql used by CREATE EXTENSION age. # 2. Builds the install SQL from current HEAD as a synthetic "next" version -# (the "to" version). This is age--.sql where NEXT = CURR.minor+1. +# (the "to" version). This is age--.sql where NEXT = CURR_upgrade_test. # 3. Stamps the upgrade template with the synthetic version, producing the # upgrade script age----.sql. # 4. Temporarily installs both synthetic files into the PG extension directory @@ -53,8 +53,8 @@ age_sql = age--1.7.0.sql AGE_CURR_VER := $(shell awk -F"'" '/default_version/ {print $$2}' age.control 2>/dev/null) # Git commit that last changed age.control — the "initial release" commit AGE_VER_COMMIT := $(shell git log -1 --format=%H -- age.control 2>/dev/null) -# Synthetic next version: current minor + 1 (e.g., 1.7.0 -> 1.8.0) -AGE_NEXT_VER := $(shell echo $(AGE_CURR_VER) | awk -F. '{printf "%s.%s.%s", $$1, $$2+1, $$3}') +# Synthetic next version: current version with _upgrade_test suffix (e.g., 1.7.0 -> 1.7.0_upgrade_test) +AGE_NEXT_VER := $(AGE_CURR_VER)_upgrade_test # The upgrade template file (e.g., age--1.7.0--y.y.y.sql); empty if not present AGE_UPGRADE_TEMPLATE := $(wildcard age--$(AGE_CURR_VER)--y.y.y.sql) @@ -296,7 +296,7 @@ ifneq ($(AGE_HAS_UPGRADE_TEST),) _install_upgrade_test_files: $(age_next_sql) $(age_upgrade_test_sql) ## Build, install synthetic files, generate cleanup script @echo "Installing upgrade test files to $(SHAREDIR)/extension/" @$(INSTALL_DATA) $(age_next_sql) $(age_upgrade_test_sql) '$(SHAREDIR)/extension/' - @printf '#!/bin/sh\nrm -f "$(SHAREDIR)/extension/$(age_next_sql)" "$(SHAREDIR)/extension/$(age_upgrade_test_sql)"\n' > $(ag_regress_dir)/age_upgrade_cleanup.sh + @printf '#!/bin/sh\nrm -f "$(SHAREDIR)/extension/$(age_next_sql)" "$(SHAREDIR)/extension/$(age_upgrade_test_sql)"\nrm -f "$(age_next_sql)" "$(age_upgrade_test_sql)" "$(ag_regress_dir)/age_upgrade_cleanup.sh"\n' > $(ag_regress_dir)/age_upgrade_cleanup.sh @chmod +x $(ag_regress_dir)/age_upgrade_cleanup.sh installcheck: _install_upgrade_test_files diff --git a/regress/expected/age_upgrade.out b/regress/expected/age_upgrade.out index 39073b460..d9855cd7a 100644 --- a/regress/expected/age_upgrade.out +++ b/regress/expected/age_upgrade.out @@ -254,12 +254,9 @@ DECLARE next_ver text; BEGIN SELECT version INTO next_ver FROM pg_available_extension_versions - WHERE name = 'age' AND version <> ( - SELECT default_version FROM pg_available_extensions WHERE name = 'age' - ) - ORDER BY string_to_array(version, '.')::int[] DESC + WHERE name = 'age' AND version LIKE '%_upgrade_test' + ORDER BY version DESC LIMIT 1; - IF next_ver IS NULL THEN RAISE EXCEPTION 'No next version available for upgrade test'; END IF; diff --git a/regress/sql/age_upgrade.sql b/regress/sql/age_upgrade.sql index 649349cdf..db6362e1b 100644 --- a/regress/sql/age_upgrade.sql +++ b/regress/sql/age_upgrade.sql @@ -200,12 +200,9 @@ DECLARE next_ver text; BEGIN SELECT version INTO next_ver FROM pg_available_extension_versions - WHERE name = 'age' AND version <> ( - SELECT default_version FROM pg_available_extensions WHERE name = 'age' - ) - ORDER BY string_to_array(version, '.')::int[] DESC + WHERE name = 'age' AND version LIKE '%_upgrade_test' + ORDER BY version DESC LIMIT 1; - IF next_ver IS NULL THEN RAISE EXCEPTION 'No next version available for upgrade test'; END IF; From f1a9b1d9b8b171656efd82674dec2e6ce06b8f9a Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Thu, 16 Apr 2026 11:15:34 -0700 Subject: [PATCH 046/100] Fix upgrade test: build default install SQL from HEAD, not initial commit (#2397) The upgrade test previously built age--.sql from the initial version-bump commit, meaning CREATE EXTENSION installed 'day-one' SQL. This caused all 31 non-upgrade regression tests to run WITHOUT SQL functions added after the version bump (e.g., age_invalidate_graph_cache, age_prepare_pg_upgrade, age_vertex_stats, etc.). These functions were never registered in pg_proc, so features depending on them (like VLE cache invalidation triggers) were silently disabled during testing. Fix by inverting the upgrade test direction: Before (installs incomplete SQL, upgrades to complete): age--1.7.0.sql built from version-bump commit (incomplete) age--1.7.0_upgrade_test.sql built from HEAD (complete) age--1.7.0--1.7.0_upgrade_test.sql stamped from template Test: CREATE EXTENSION age -> data -> ALTER EXTENSION UPDATE TO '1.7.0_upgrade_test' After (installs complete SQL, upgrades from synthetic initial): age--1.7.0.sql built from HEAD (complete) age--1.7.0_initial.sql built from version-bump commit (synthetic) age--1.7.0_initial--1.7.0.sql stamped from template Test: CREATE EXTENSION age VERSION '1.7.0_initial' -> data -> ALTER EXTENSION UPDATE TO '1.7.0' This ensures: - All 31 non-upgrade tests run with every SQL function registered - The upgrade template is still validated (initial -> current) - The test ends at the default version (clean state for drop test) - Tarball builds (no git) work identically (cat sql/sql_files) - All synthetic files are cleaned up after the test Makefile changes: - age_sql rule: cat current HEAD sql/sql_files (was: git show from commit) - New age_init_sql rule: git show from version-bump commit - age_upgrade_test_sql: stamps template as INIT->CURR (was CURR->NEXT) - EXTRA_CLEAN, _install_upgrade_test_files: updated filenames - Removed AGE_NEXT_VER, age_next_sql; added AGE_INIT_VER, age_init_sql Test SQL changes: - Step 3: CREATE EXTENSION age VERSION '' (was: CREATE EXTENSION age) - Step 6: ALTER EXTENSION age UPDATE TO default_version (was: LIKE '%_upgrade_test') - Step 7: Check installed = default (was: installed <> default) - Comments updated throughout to reflect inverted direction All 32 regression tests pass. modified: Makefile modified: regress/expected/age_upgrade.out modified: regress/sql/age_upgrade.sql --- Makefile | 92 ++++++++++++++------------------ regress/expected/age_upgrade.out | 59 ++++++++++++-------- regress/sql/age_upgrade.sql | 55 +++++++++++-------- 3 files changed, 109 insertions(+), 97 deletions(-) diff --git a/Makefile b/Makefile index f07e1b710..e358acf38 100644 --- a/Makefile +++ b/Makefile @@ -24,24 +24,25 @@ age_sql = age--1.7.0.sql # Validates the upgrade template (age----y.y.y.sql) by simulating an # extension version upgrade entirely within "make installcheck". The test: # -# 1. Builds the install SQL from the INITIAL version-bump commit (the "from" -# version). This is the age--.sql used by CREATE EXTENSION age. -# 2. Builds the install SQL from current HEAD as a synthetic "next" version -# (the "to" version). This is age--.sql where NEXT = CURR_upgrade_test. -# 3. Stamps the upgrade template with the synthetic version, producing the -# upgrade script age----.sql. -# 4. Temporarily installs both synthetic files into the PG extension directory -# so that ALTER EXTENSION age UPDATE TO '' can find them. +# 1. Builds the default install SQL (age--.sql) from current HEAD's +# sql/sql_files. This is what CREATE EXTENSION age installs. +# 2. Builds a synthetic "initial" version install SQL from the version-bump +# commit in git history. This represents the pre-upgrade state. +# 3. Stamps the upgrade template to upgrade from the initial version to the +# current version. +# 4. Temporarily installs the synthetic files into the PG extension directory +# so that CREATE EXTENSION age VERSION '' and ALTER EXTENSION +# age UPDATE TO '' can find them. # 5. The age_upgrade regression test exercises the full upgrade path: install -# at CURR, create data, ALTER EXTENSION UPDATE to NEXT, verify data. +# at INIT, create data, ALTER EXTENSION UPDATE to CURR, verify data. # 6. The test SQL cleans up the synthetic files via a generated shell script. # # This forces developers to keep the upgrade template in sync: any SQL object # added after the version-bump commit must also appear in the template, or the # upgrade test will fail (the object will be missing after ALTER EXTENSION UPDATE). # -# The .so (shared library) is always built from current HEAD, so C-level changes -# in the PR are tested by every regression test, not just the upgrade test. +# Because the default install SQL comes from HEAD, all 31 non-upgrade tests +# run with every SQL function registered — no functions are missing. # # Graceful degradation — the upgrade test is silently skipped when: # - No git history (tarball build): AGE_VER_COMMIT is empty. @@ -53,15 +54,15 @@ age_sql = age--1.7.0.sql AGE_CURR_VER := $(shell awk -F"'" '/default_version/ {print $$2}' age.control 2>/dev/null) # Git commit that last changed age.control — the "initial release" commit AGE_VER_COMMIT := $(shell git log -1 --format=%H -- age.control 2>/dev/null) -# Synthetic next version: current version with _upgrade_test suffix (e.g., 1.7.0 -> 1.7.0_upgrade_test) -AGE_NEXT_VER := $(AGE_CURR_VER)_upgrade_test +# Synthetic initial version: current version with _initial suffix +AGE_INIT_VER := $(AGE_CURR_VER)_initial # The upgrade template file (e.g., age--1.7.0--y.y.y.sql); empty if not present AGE_UPGRADE_TEMPLATE := $(wildcard age--$(AGE_CURR_VER)--y.y.y.sql) # Synthetic filenames — these are NOT installed permanently; they are temporarily # placed in $(SHAREDIR)/extension/ during installcheck and removed after. -age_next_sql = $(if $(AGE_NEXT_VER),age--$(AGE_NEXT_VER).sql) -age_upgrade_test_sql = $(if $(AGE_NEXT_VER),age--$(AGE_CURR_VER)--$(AGE_NEXT_VER).sql) +age_init_sql = $(if $(AGE_INIT_VER),age--$(AGE_INIT_VER).sql) +age_upgrade_test_sql = $(if $(AGE_INIT_VER),age--$(AGE_INIT_VER)--$(AGE_CURR_VER).sql) # Real (git-tracked, non-template) upgrade scripts FROM the current version. # If any exist (e.g., age--1.7.0--1.8.0.sql is committed), the synthetic @@ -196,7 +197,7 @@ ag_regress_dir = $(srcdir)/regress REGRESS_OPTS = --load-extension=age --inputdir=$(ag_regress_dir) --outputdir=$(ag_regress_dir) --temp-instance=$(ag_regress_dir)/instance --port=61958 --encoding=UTF-8 --temp-config $(ag_regress_dir)/age_regression.conf ag_regress_out = instance/ log/ results/ regression.* -EXTRA_CLEAN = $(addprefix $(ag_regress_dir)/, $(ag_regress_out)) src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/include/parser/cypher_kwlist_d.h $(all_age_sql) $(age_next_sql) $(age_upgrade_test_sql) $(ag_regress_dir)/age_upgrade_cleanup.sh +EXTRA_CLEAN = $(addprefix $(ag_regress_dir)/, $(ag_regress_out)) src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/include/parser/cypher_kwlist_d.h $(all_age_sql) $(age_init_sql) $(age_upgrade_test_sql) $(ag_regress_dir)/age_upgrade_cleanup.sh GEN_KEYWORDLIST = $(PERL) -I ./tools/ ./tools/gen_keywordlist.pl GEN_KEYWORDLIST_DEPS = ./tools/gen_keywordlist.pl tools/PerfectHash.pm @@ -226,59 +227,44 @@ src/backend/parser/cypher_parser.bc: src/backend/parser/cypher_gram.c src/includ src/backend/parser/cypher_keywords.o: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/backend/parser/cypher_keywords.bc: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h -# Build the default install SQL (age--.sql) from the INITIAL version-bump -# commit in git history. This means CREATE EXTENSION age installs the "day-one -# release" SQL — the state of sql/ at the moment the version was bumped in -# age.control. All other regression tests run against this SQL + the current -# HEAD .so, implicitly validating that the .so is backward-compatible with the -# initial release SQL. -# -# The current HEAD SQL goes into the synthetic next version (age--.sql) -# which is only reachable via ALTER EXTENSION UPDATE in the upgrade test. -ifneq ($(AGE_VER_COMMIT),) -$(age_sql): age.control - @echo "Building initial-release install SQL: $@ from commit $(AGE_VER_COMMIT)" - @for f in $$(git show $(AGE_VER_COMMIT):sql/sql_files 2>/dev/null); do \ - git show $(AGE_VER_COMMIT):sql/$${f}.sql 2>/dev/null; \ - done > $@ -ifeq ($(SIZEOF_DATUM),4) - @echo "32-bit build: removing PASSEDBYVALUE from graphid type" - @sed 's/^ PASSEDBYVALUE,$$/ -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@ -endif -else -# Fallback: no git history (tarball build) — use current HEAD SQL fragments +# Build the default install SQL (age--.sql) from current HEAD's sql/sql_files. +# This is what CREATE EXTENSION age installs — it contains ALL current functions. +# All 31 non-upgrade regression tests run against this complete SQL. $(age_sql): $(SQLS) + @echo "Building install SQL: $@ from HEAD" @cat $(SQLS) > $@ ifeq ($(SIZEOF_DATUM),4) @echo "32-bit build: removing PASSEDBYVALUE from graphid type" @sed 's/^ PASSEDBYVALUE,$$/ -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@ - @grep -q 'PASSEDBYVALUE removed for 32-bit' $@ || { echo "Error: PASSEDBYVALUE replacement failed in $@"; exit 1; } -endif endif -# Build synthetic "next" version install SQL from current HEAD's sql/sql_files. -# This represents what the extension SQL looks like in the PR being tested. +# Build synthetic "initial" version install SQL from the version-bump commit. +# This represents the pre-upgrade state — the SQL at the time the version was +# bumped in age.control. Used only by the upgrade test. ifneq ($(AGE_HAS_UPGRADE_TEST),) -$(age_next_sql): $(SQLS) - @echo "Building synthetic next version install SQL: $@ ($(AGE_NEXT_VER))" - @cat $(SQLS) > $@ +$(age_init_sql): age.control + @echo "Building synthetic initial version install SQL: $@ from commit $(AGE_VER_COMMIT)" + @for f in $$(git show $(AGE_VER_COMMIT):sql/sql_files 2>/dev/null); do \ + git show $(AGE_VER_COMMIT):sql/$${f}.sql 2>/dev/null; \ + done > $@ ifeq ($(SIZEOF_DATUM),4) @sed 's/^ PASSEDBYVALUE,$$/ -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@ endif -# Stamp upgrade template as upgrade from current to synthetic next version +# Stamp upgrade template as upgrade from initial to current version $(age_upgrade_test_sql): $(AGE_UPGRADE_TEMPLATE) - @echo "Stamping upgrade template: $< -> $@" - @sed -e "s/1\.X\.0/$(AGE_NEXT_VER)/g" -e "s/y\.y\.y/$(AGE_NEXT_VER)/g" $< > $@ + @echo "Stamping upgrade template: $< -> $@ ($(AGE_INIT_VER) -> $(AGE_CURR_VER))" + @sed -e "s/1\.X\.0/$(AGE_CURR_VER)/g" -e "s/y\.y\.y/$(AGE_CURR_VER)/g" $< > $@ endif src/backend/parser/ag_scanner.c: FLEX_NO_BACKUP=yes # --- Upgrade test file lifecycle during installcheck --- # -# Problem: The upgrade test needs age--.sql and age----.sql -# in the PG extension directory for ALTER EXTENSION UPDATE to find them, but -# we must not leave them installed permanently (they would confuse users). +# Problem: The upgrade test needs age--.sql and age----.sql +# in the PG extension directory for CREATE EXTENSION VERSION and ALTER +# EXTENSION UPDATE to find them, but we must not leave them installed +# permanently (they would confuse users). # # Solution: A Make prerequisite installs them before pg_regress runs, and the # test SQL removes them at the end via \! (shell escape in psql). A generated @@ -293,10 +279,10 @@ SHAREDIR = $(shell $(PG_CONFIG) --sharedir) installcheck: export LC_COLLATE=C ifneq ($(AGE_HAS_UPGRADE_TEST),) .PHONY: _install_upgrade_test_files -_install_upgrade_test_files: $(age_next_sql) $(age_upgrade_test_sql) ## Build, install synthetic files, generate cleanup script +_install_upgrade_test_files: $(age_init_sql) $(age_upgrade_test_sql) ## Build, install synthetic files, generate cleanup script @echo "Installing upgrade test files to $(SHAREDIR)/extension/" - @$(INSTALL_DATA) $(age_next_sql) $(age_upgrade_test_sql) '$(SHAREDIR)/extension/' - @printf '#!/bin/sh\nrm -f "$(SHAREDIR)/extension/$(age_next_sql)" "$(SHAREDIR)/extension/$(age_upgrade_test_sql)"\nrm -f "$(age_next_sql)" "$(age_upgrade_test_sql)" "$(ag_regress_dir)/age_upgrade_cleanup.sh"\n' > $(ag_regress_dir)/age_upgrade_cleanup.sh + @$(INSTALL_DATA) $(age_init_sql) $(age_upgrade_test_sql) '$(SHAREDIR)/extension/' + @printf '#!/bin/sh\nrm -f "$(SHAREDIR)/extension/$(age_init_sql)" "$(SHAREDIR)/extension/$(age_upgrade_test_sql)"\nrm -f "$(age_init_sql)" "$(age_upgrade_test_sql)" "$(ag_regress_dir)/age_upgrade_cleanup.sh"\n' > $(ag_regress_dir)/age_upgrade_cleanup.sh @chmod +x $(ag_regress_dir)/age_upgrade_cleanup.sh installcheck: _install_upgrade_test_files diff --git a/regress/expected/age_upgrade.out b/regress/expected/age_upgrade.out index d9855cd7a..7c3da0150 100644 --- a/regress/expected/age_upgrade.out +++ b/regress/expected/age_upgrade.out @@ -20,17 +20,17 @@ -- Extension upgrade regression test -- -- This test validates the upgrade template (age----y.y.y.sql) by: --- 1. Installing AGE at the current version (built from the initial --- version-bump commit's SQL — the "day-one release" state) +-- 1. Dropping AGE and reinstalling at the synthetic "initial" version +-- (built from the version-bump commit's SQL — the "day-one" state) -- 2. Creating three graphs with multiple labels, edges, GIN indexes, -- and numeric properties that serve as integrity checksums --- 3. Upgrading to a synthetic "next" version via the stamped template +-- 3. Upgrading to the current (default) version via the stamped template -- 4. Verifying all data, structure, and checksums survived the upgrade -- -- The Makefile builds: --- age--.sql from the initial version-bump commit (git history) --- age--.sql from current HEAD's sql/sql_files --- age----.sql stamped from the upgrade template +-- age--.sql from current HEAD's sql/sql_files (default) +-- age--_initial.sql from the version-bump commit (synthetic) +-- age--_initial--.sql stamped from the upgrade template -- -- All version discovery is dynamic — no hardcoded versions anywhere. -- This test is version-agnostic and works on any branch for any version. @@ -39,7 +39,7 @@ LOAD 'age'; SET search_path TO ag_catalog; -- Step 1: Clean up any state left by prior tests, then drop AGE entirely. -- The --load-extension=age flag installed AGE at the current (default) version. --- We need to remove it so we can cleanly re-create for this test. +-- We need to remove it so we can reinstall at the synthetic initial version. SELECT drop_graph(name, true) FROM ag_graph ORDER BY name; drop_graph ------------ @@ -54,8 +54,22 @@ FROM pg_available_extension_versions WHERE name = 'age'; t (1 row) --- Step 3: Install AGE at the default version (the initial release SQL). -CREATE EXTENSION age; +-- Step 3: Install AGE at the synthetic initial version (pre-upgrade state). +DO $$ +DECLARE init_ver text; +BEGIN + SELECT version INTO init_ver + FROM pg_available_extension_versions + WHERE name = 'age' AND version LIKE '%_initial' + ORDER BY version DESC + LIMIT 1; + IF init_ver IS NULL THEN + RAISE EXCEPTION 'No initial version available for upgrade test'; + END IF; + + EXECUTE format('CREATE EXTENSION age VERSION %L', init_ver); +END; +$$; SELECT extversion IS NOT NULL AS version_installed FROM pg_extension WHERE extname = 'age'; version_installed ------------------- @@ -248,27 +262,26 @@ WHERE ag.name <> '_ag_catalog'; 21 (1 row) --- Step 6: Upgrade AGE to the synthetic next version via the stamped template. +-- Step 6: Upgrade AGE from the initial version to the current (default) version +-- via the stamped upgrade template. DO $$ -DECLARE next_ver text; +DECLARE curr_ver text; BEGIN - SELECT version INTO next_ver - FROM pg_available_extension_versions - WHERE name = 'age' AND version LIKE '%_upgrade_test' - ORDER BY version DESC - LIMIT 1; - IF next_ver IS NULL THEN - RAISE EXCEPTION 'No next version available for upgrade test'; + SELECT default_version INTO curr_ver + FROM pg_available_extensions + WHERE name = 'age'; + IF curr_ver IS NULL THEN + RAISE EXCEPTION 'No default version found for upgrade test'; END IF; - EXECUTE format('ALTER EXTENSION age UPDATE TO %L', next_ver); + EXECUTE format('ALTER EXTENSION age UPDATE TO %L', curr_ver); END; $$; --- Step 7: Confirm version changed. -SELECT installed_version <> default_version AS upgraded_past_default +-- Step 7: Confirm version is now the default (current HEAD) version. +SELECT installed_version = default_version AS upgraded_to_current FROM pg_available_extensions WHERE name = 'age'; - upgraded_past_default ------------------------ + upgraded_to_current +--------------------- t (1 row) diff --git a/regress/sql/age_upgrade.sql b/regress/sql/age_upgrade.sql index db6362e1b..70d45064a 100644 --- a/regress/sql/age_upgrade.sql +++ b/regress/sql/age_upgrade.sql @@ -21,17 +21,17 @@ -- Extension upgrade regression test -- -- This test validates the upgrade template (age----y.y.y.sql) by: --- 1. Installing AGE at the current version (built from the initial --- version-bump commit's SQL — the "day-one release" state) +-- 1. Dropping AGE and reinstalling at the synthetic "initial" version +-- (built from the version-bump commit's SQL — the "day-one" state) -- 2. Creating three graphs with multiple labels, edges, GIN indexes, -- and numeric properties that serve as integrity checksums --- 3. Upgrading to a synthetic "next" version via the stamped template +-- 3. Upgrading to the current (default) version via the stamped template -- 4. Verifying all data, structure, and checksums survived the upgrade -- -- The Makefile builds: --- age--.sql from the initial version-bump commit (git history) --- age--.sql from current HEAD's sql/sql_files --- age----.sql stamped from the upgrade template +-- age--.sql from current HEAD's sql/sql_files (default) +-- age--_initial.sql from the version-bump commit (synthetic) +-- age--_initial--.sql stamped from the upgrade template -- -- All version discovery is dynamic — no hardcoded versions anywhere. -- This test is version-agnostic and works on any branch for any version. @@ -42,7 +42,7 @@ SET search_path TO ag_catalog; -- Step 1: Clean up any state left by prior tests, then drop AGE entirely. -- The --load-extension=age flag installed AGE at the current (default) version. --- We need to remove it so we can cleanly re-create for this test. +-- We need to remove it so we can reinstall at the synthetic initial version. SELECT drop_graph(name, true) FROM ag_graph ORDER BY name; DROP EXTENSION age; @@ -50,8 +50,22 @@ DROP EXTENSION age; SELECT count(*) > 1 AS has_upgrade_path FROM pg_available_extension_versions WHERE name = 'age'; --- Step 3: Install AGE at the default version (the initial release SQL). -CREATE EXTENSION age; +-- Step 3: Install AGE at the synthetic initial version (pre-upgrade state). +DO $$ +DECLARE init_ver text; +BEGIN + SELECT version INTO init_ver + FROM pg_available_extension_versions + WHERE name = 'age' AND version LIKE '%_initial' + ORDER BY version DESC + LIMIT 1; + IF init_ver IS NULL THEN + RAISE EXCEPTION 'No initial version available for upgrade test'; + END IF; + + EXECUTE format('CREATE EXTENSION age VERSION %L', init_ver); +END; +$$; SELECT extversion IS NOT NULL AS version_installed FROM pg_extension WHERE extname = 'age'; -- Step 4: Create three test graphs with diverse labels, edges, and data. @@ -194,25 +208,24 @@ SELECT count(*)::int AS total_labels_before FROM ag_label al JOIN ag_graph ag ON al.graph = ag.graphid WHERE ag.name <> '_ag_catalog'; --- Step 6: Upgrade AGE to the synthetic next version via the stamped template. +-- Step 6: Upgrade AGE from the initial version to the current (default) version +-- via the stamped upgrade template. DO $$ -DECLARE next_ver text; +DECLARE curr_ver text; BEGIN - SELECT version INTO next_ver - FROM pg_available_extension_versions - WHERE name = 'age' AND version LIKE '%_upgrade_test' - ORDER BY version DESC - LIMIT 1; - IF next_ver IS NULL THEN - RAISE EXCEPTION 'No next version available for upgrade test'; + SELECT default_version INTO curr_ver + FROM pg_available_extensions + WHERE name = 'age'; + IF curr_ver IS NULL THEN + RAISE EXCEPTION 'No default version found for upgrade test'; END IF; - EXECUTE format('ALTER EXTENSION age UPDATE TO %L', next_ver); + EXECUTE format('ALTER EXTENSION age UPDATE TO %L', curr_ver); END; $$; --- Step 7: Confirm version changed. -SELECT installed_version <> default_version AS upgraded_past_default +-- Step 7: Confirm version is now the default (current HEAD) version. +SELECT installed_version = default_version AS upgraded_to_current FROM pg_available_extensions WHERE name = 'age'; -- Step 8: Verify all data survived — reload and recheck. From ce5004593ea5acb2cec3a2dc94a06241d7f23c17 Mon Sep 17 00:00:00 2001 From: Ueslei Lima Date: Sun, 19 Apr 2026 09:10:39 -0300 Subject: [PATCH 047/100] Python driver: Add `skip_load` parameter to skip `LOAD 'age'` statement (#2366) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add skip_load parameter to connect and setUpAge functions to control plugin loading. * Add tests and README docs for skip_load parameter - Add 4 unit tests for setUpAge() skip_load behavior: skip_load=True skips LOAD, skip_load=False executes LOAD, load_from_plugins integration, and search_path always set. - Document skip_load in README under new "Managed PostgreSQL Usage" section for Azure/AWS RDS/etc. environments. - Fix syntax in existing load_from_plugins code example. Made-with: Cursor * Address review feedback: ValueError for contradictory flags, e2e test - Raise ValueError when skip_load=True and load_from_plugins=True are both set (contradictory combination) - Add end-to-end test verifying skip_load is forwarded through the full age.connect() → Age.connect() → setUpAge() call chain - Replace fragile string assertions with assert_called_with/assert_any_call - README: mention configure_connection() as the pool-based alternative for managed PostgreSQL environments Made-with: Cursor * Fix README: use setUpAge(skip_load=True) for pool example configure_connection() is not part of this PR; use the available setUpAge() API with skip_load=True for the connection pool example. Made-with: Cursor * fix(python-driver): use quoted $user in search_path and run TestSetUpAge in CI PostgreSQL treats single-quoted '$user' as a literal schema name; use "$user" so the session user schema is included. Include TestSetUpAge in the test_age_py __main__ suite so skip_load tests run in GitHub Actions. Made-with: Cursor --- drivers/python/README.md | 20 +++++++++- drivers/python/age/__init__.py | 4 +- drivers/python/age/age.py | 24 +++++++---- drivers/python/test_age_py.py | 73 ++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 12 deletions(-) diff --git a/drivers/python/README.md b/drivers/python/README.md index e64f9de67..f4fa43919 100644 --- a/drivers/python/README.md +++ b/drivers/python/README.md @@ -89,9 +89,25 @@ SET search_path = ag_catalog, "$user", public; * Make sure to give your non-superuser db account proper permissions to the graph schemas and corresponding objects * Make sure to initiate the Apache Age python driver with the ```load_from_plugins``` parameter. This parameter tries to load the Apache Age extension from the PostgreSQL plugins directory located at ```$libdir/plugins/age```. Example: - ```python. + ```python ag = age.connect(host='localhost', port=5432, user='dbuser', password='strong_password', - dbname=postgres, load_from_plugins=True, graph='graph_name) + dbname='postgres', load_from_plugins=True, graph='graph_name') + ``` + +### Managed PostgreSQL Usage (Azure, AWS RDS, etc.) +* On managed PostgreSQL services where the AGE extension is loaded server-side via ```shared_preload_libraries```, + the ```LOAD 'age'``` command may fail because the binary is not at the expected file path. Use the ```skip_load``` + parameter to skip the ```LOAD``` statement while still performing all other setup: + ```python + ag = age.connect(host='myserver.postgres.database.azure.com', port=5432, + user='dbuser', password='strong_password', + dbname='postgres', skip_load=True, graph='graph_name') + ``` +* **Connection pools:** If you manage connections externally (e.g. via ```psycopg_pool.ConnectionPool```), + you can call ```setUpAge()``` with ```skip_load=True``` on each pooled connection: + ```python + from age.age import setUpAge + setUpAge(conn, 'graph_name', skip_load=True) ``` ### License diff --git a/drivers/python/age/__init__.py b/drivers/python/age/__init__.py index 685f0fe74..caee6a43c 100644 --- a/drivers/python/age/__init__.py +++ b/drivers/python/age/__init__.py @@ -25,13 +25,13 @@ def version(): def connect(dsn=None, graph=None, connection_factory=None, cursor_factory=ClientCursor, load_from_plugins=False, - **kwargs): + skip_load=False, **kwargs): dsn = conninfo.make_conninfo('' if dsn is None else dsn, **kwargs) ag = Age() ag.connect(dsn=dsn, graph=graph, connection_factory=connection_factory, cursor_factory=cursor_factory, - load_from_plugins=load_from_plugins, **kwargs) + load_from_plugins=load_from_plugins, skip_load=skip_load, **kwargs) return ag # Dummy ResultHandler diff --git a/drivers/python/age/age.py b/drivers/python/age/age.py index fad1f27b1..f85b1ab19 100644 --- a/drivers/python/age/age.py +++ b/drivers/python/age/age.py @@ -137,14 +137,22 @@ def load(self, data: bytes | bytearray | memoryview) -> Any | None: return parseAgeValue(data_bytes.decode('utf-8')) -def setUpAge(conn:psycopg.connection, graphName:str, load_from_plugins:bool=False): +def setUpAge(conn:psycopg.connection, graphName:str, load_from_plugins:bool=False, skip_load:bool=False): + if skip_load and load_from_plugins: + raise ValueError( + "skip_load=True and load_from_plugins=True are contradictory. " + "Set skip_load=False to load the extension from the plugins path, " + "or remove load_from_plugins to skip loading entirely." + ) + with conn.cursor() as cursor: - if load_from_plugins: - cursor.execute("LOAD '$libdir/plugins/age';") - else: - cursor.execute("LOAD 'age';") + if not skip_load: + if load_from_plugins: + cursor.execute("LOAD '$libdir/plugins/age';") + else: + cursor.execute("LOAD 'age';") - cursor.execute("SET search_path = ag_catalog, '$user', public;") + cursor.execute('SET search_path = ag_catalog, "$user", public;') ag_info = TypeInfo.fetch(conn, 'agtype') @@ -333,9 +341,9 @@ def __init__(self): # Connect to PostgreSQL Server and establish session and type extension environment. def connect(self, graph:str=None, dsn:str=None, connection_factory=None, cursor_factory=ClientCursor, - load_from_plugins:bool=False, **kwargs): + load_from_plugins:bool=False, skip_load:bool=False, **kwargs): conn = psycopg.connect(dsn, cursor_factory=cursor_factory, **kwargs) - setUpAge(conn, graph, load_from_plugins) + setUpAge(conn, graph, load_from_plugins, skip_load=skip_load) self.connection = conn self.graphName = graph return self diff --git a/drivers/python/test_age_py.py b/drivers/python/test_age_py.py index f904fb9e3..960d28d4d 100644 --- a/drivers/python/test_age_py.py +++ b/drivers/python/test_age_py.py @@ -16,6 +16,7 @@ from age.models import Vertex import unittest +import unittest.mock import decimal import age import argparse @@ -28,6 +29,76 @@ TEST_GRAPH_NAME = "test_graph" +class TestSetUpAge(unittest.TestCase): + """Unit tests for setUpAge() skip_load parameter — no DB required.""" + + def _make_mock_conn(self): + mock_conn = unittest.mock.MagicMock() + mock_cursor = unittest.mock.MagicMock() + mock_conn.cursor.return_value.__enter__ = unittest.mock.Mock(return_value=mock_cursor) + mock_conn.cursor.return_value.__exit__ = unittest.mock.Mock(return_value=False) + mock_conn.adapters = unittest.mock.MagicMock() + mock_type_info = unittest.mock.MagicMock() + mock_type_info.oid = 1 + mock_type_info.array_oid = 2 + return mock_conn, mock_cursor, mock_type_info + + def test_skip_load_true_does_not_execute_load(self): + """When skip_load=True, LOAD 'age' must not be executed.""" + mock_conn, mock_cursor, mock_type_info = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=mock_type_info), \ + unittest.mock.patch("age.age.checkGraphCreated"): + age.age.setUpAge(mock_conn, "test_graph", skip_load=True) + mock_cursor.execute.assert_called_once_with( + 'SET search_path = ag_catalog, "$user", public;' + ) + + def test_skip_load_false_executes_load(self): + """When skip_load=False (default), LOAD 'age' must be executed.""" + mock_conn, mock_cursor, mock_type_info = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=mock_type_info), \ + unittest.mock.patch("age.age.checkGraphCreated"): + age.age.setUpAge(mock_conn, "test_graph", skip_load=False) + mock_cursor.execute.assert_any_call("LOAD 'age';") + + def test_skip_load_with_load_from_plugins(self): + """When skip_load=False and load_from_plugins=True, LOAD from plugins path.""" + mock_conn, mock_cursor, mock_type_info = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=mock_type_info), \ + unittest.mock.patch("age.age.checkGraphCreated"): + age.age.setUpAge(mock_conn, "test_graph", load_from_plugins=True, skip_load=False) + mock_cursor.execute.assert_any_call("LOAD '$libdir/plugins/age';") + + def test_skip_load_true_still_sets_search_path(self): + """When skip_load=True, search_path must still be set.""" + mock_conn, mock_cursor, mock_type_info = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=mock_type_info), \ + unittest.mock.patch("age.age.checkGraphCreated"): + age.age.setUpAge(mock_conn, "test_graph", skip_load=True) + mock_cursor.execute.assert_any_call( + 'SET search_path = ag_catalog, "$user", public;' + ) + + def test_contradictory_skip_load_and_load_from_plugins_raises(self): + """skip_load=True + load_from_plugins=True must raise ValueError.""" + mock_conn, _, _ = self._make_mock_conn() + with self.assertRaises(ValueError): + age.age.setUpAge(mock_conn, "test_graph", load_from_plugins=True, skip_load=True) + + def test_connect_forwards_skip_load_to_setup(self): + """age.connect(skip_load=True) must forward skip_load through the full call chain.""" + with unittest.mock.patch("age.age.psycopg.connect") as mock_psycopg, \ + unittest.mock.patch("age.age.setUpAge") as mock_setup: + mock_psycopg.return_value = unittest.mock.MagicMock() + age.connect(dsn="host=localhost", graph="test_graph", skip_load=True) + mock_setup.assert_called_once() + _, kwargs = mock_setup.call_args + self.assertTrue( + kwargs.get("skip_load", False), + "skip_load must be forwarded from age.connect() to setUpAge()" + ) + + class TestAgeBasic(unittest.TestCase): ag = None args: argparse.Namespace = argparse.Namespace( @@ -485,6 +556,8 @@ def testSerialization(self): args = parser.parse_args() suite = unittest.TestSuite() + loader = unittest.TestLoader() + suite.addTests(loader.loadTestsFromTestCase(TestSetUpAge)) suite.addTest(TestAgeBasic("testExec")) suite.addTest(TestAgeBasic("testQuery")) suite.addTest(TestAgeBasic("testChangeData")) From 15030a038278bf5ad0355ce2b80ccacc27bb3dc2 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Sun, 19 Apr 2026 08:48:25 -0400 Subject: [PATCH 048/100] Fix OPTIONAL MATCH dropping null-preserving rows with subquery WHERE (#2380) * Fix OPTIONAL MATCH dropping null-preserving rows with subquery WHERE Cypher OPTIONAL MATCH semantics require that when no right-hand row survives the WHERE predicate, the outer row is still emitted with NULLs in the optional columns. Before this fix, a WHERE containing a list comprehension or sub-pattern predicate (EXISTS { ... }, COUNT { ... }) would take the transform_cypher_clause_with_where rewrite path, which detaches the WHERE, transforms the match clause as a subquery, and then attaches the WHERE as an outer filter on that subquery. For OPTIONAL MATCH, the inner subquery already produced a LATERAL LEFT JOIN with null-preserving rows; the outer filter then ran against those nulled rows and dropped them when the predicate evaluated NULL or false on the nulled side, producing zero rows where Cypher semantics require one null-filled row per outer match. Fix: in transform_cypher_match, the has_list_comp_or_subquery rewrite now only applies to non-optional MATCH. In the OPTIONAL MATCH path, transform_cypher_optional_match_clause detaches the WHERE from the cypher_match node before recursively transforming the right-hand side (so the inner transform does not double-apply or misresolve the predicate in a fresh namespace), and re-attaches the transformed predicate as the LEFT JOIN's ON condition after both sides are in the namespace. A LEFT JOIN with a failing ON condition correctly preserves left rows with null right columns, which matches Cypher OPTIONAL MATCH ... WHERE semantics. Regression tests cover: - EXISTS { (friend)-[...]->(...) } referencing the optional variable - EXISTS { (p)-[...]->(...) } referencing the outer variable - non-correlated EXISTS (previously-working guard) - plain scalar predicate on the optional variable (guard) - constant-false WHERE (guard) Fixes issue #2378. Co-Authored-By: Claude Opus 4.6 (1M context) --- regress/expected/cypher_match.out | 124 +++++++++++++++++++++++++++++ regress/sql/cypher_match.sql | 73 +++++++++++++++++ src/backend/parser/cypher_clause.c | 63 ++++++++++++++- 3 files changed, 259 insertions(+), 1 deletion(-) diff --git a/regress/expected/cypher_match.out b/regress/expected/cypher_match.out index 2f01d5163..e55ed23c3 100644 --- a/regress/expected/cypher_match.out +++ b/regress/expected/cypher_match.out @@ -3834,6 +3834,130 @@ NOTICE: graph "issue_2193" has been dropped (1 row) +-- +-- Issue 2378: OPTIONAL MATCH may incorrectly drop null-preserving outer +-- rows when its WHERE clause contains a correlated sub-pattern predicate. +-- +-- Cypher OPTIONAL MATCH semantics: the WHERE applies to the optional +-- binding; when no right-hand row survives the predicate, the outer row +-- is still emitted with NULLs in the optional columns. Before the fix, +-- a WHERE containing EXISTS { ... } or COUNT { ... } was attached as an +-- outer filter on the transformed subquery, so it ran after the LATERAL +-- LEFT JOIN produced null-preserving rows and then incorrectly dropped +-- them when the predicate evaluated NULL/false on the nulled side. +-- +SELECT create_graph('issue_2378'); +NOTICE: graph "issue_2378" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('issue_2378', $$ + CREATE (a:Person {name: 'Alice'}), + (b:Person {name: 'Bob'}), + (c:Person {name: 'Charlie'}), + (a)-[:KNOWS]->(b), + (a)-[:KNOWS]->(c) +$$) AS (v agtype); + v +--- +(0 rows) + +-- Correlated EXISTS referencing the optional variable (friend). +-- Neither Bob nor Charlie knows anyone, so for every outer p the +-- predicate fails on all optional matches; expect one row per person +-- with friend = NULL. +SELECT * FROM cypher('issue_2378', $$ + MATCH (p:Person) + OPTIONAL MATCH (p)-[:KNOWS]->(friend:Person) + WHERE EXISTS { (friend)-[:KNOWS]->(:Person) } + RETURN p.name AS name, friend.name AS friend + ORDER BY name +$$) AS (name agtype, friend agtype); + name | friend +-----------+-------- + "Alice" | + "Bob" | + "Charlie" | +(3 rows) + +-- Correlated EXISTS referencing the outer variable (p). +-- Alice knows someone so her optional matches pass; Bob and Charlie +-- don't, so they are emitted with NULL friend. +SELECT * FROM cypher('issue_2378', $$ + MATCH (p:Person) + OPTIONAL MATCH (p)-[:KNOWS]->(friend:Person) + WHERE EXISTS { (p)-[:KNOWS]->(:Person) } + RETURN p.name AS name, friend.name AS friend + ORDER BY name, friend +$$) AS (name agtype, friend agtype); + name | friend +-----------+----------- + "Alice" | "Bob" + "Alice" | "Charlie" + "Bob" | + "Charlie" | +(4 rows) + +-- Non-correlated EXISTS (was already working; kept as a regression guard). +SELECT * FROM cypher('issue_2378', $$ + MATCH (p:Person) + OPTIONAL MATCH (p)-[:KNOWS]->(friend:Person) + WHERE EXISTS { MATCH (x:Person) RETURN x } + RETURN p.name AS name, friend.name AS friend + ORDER BY name, friend +$$) AS (name agtype, friend agtype); + name | friend +-----------+----------- + "Alice" | "Bob" + "Alice" | "Charlie" + "Bob" | + "Charlie" | +(4 rows) + +-- Plain scalar predicate on the optional variable (was already working). +SELECT * FROM cypher('issue_2378', $$ + MATCH (p:Person) + OPTIONAL MATCH (p)-[:KNOWS]->(friend:Person) + WHERE friend.name = 'Bob' + RETURN p.name AS name, friend.name AS friend + ORDER BY name +$$) AS (name agtype, friend agtype); + name | friend +-----------+-------- + "Alice" | "Bob" + "Bob" | + "Charlie" | +(3 rows) + +-- Constant-false WHERE on the optional side (was already working). +SELECT * FROM cypher('issue_2378', $$ + MATCH (p:Person) + OPTIONAL MATCH (p)-[:KNOWS]->(friend:Person) + WHERE false + RETURN p.name AS name, friend.name AS friend + ORDER BY name +$$) AS (name agtype, friend agtype); + name | friend +-----------+-------- + "Alice" | + "Bob" | + "Charlie" | +(3 rows) + +SELECT drop_graph('issue_2378', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table issue_2378._ag_label_vertex +drop cascades to table issue_2378._ag_label_edge +drop cascades to table issue_2378."Person" +drop cascades to table issue_2378."KNOWS" +NOTICE: graph "issue_2378" has been dropped + drop_graph +------------ + +(1 row) + -- -- Clean up -- diff --git a/regress/sql/cypher_match.sql b/regress/sql/cypher_match.sql index 410d097bb..e56aafac8 100644 --- a/regress/sql/cypher_match.sql +++ b/regress/sql/cypher_match.sql @@ -1616,6 +1616,79 @@ $$) AS (result agtype); SELECT drop_graph('issue_2193', true); +-- +-- Issue 2378: OPTIONAL MATCH may incorrectly drop null-preserving outer +-- rows when its WHERE clause contains a correlated sub-pattern predicate. +-- +-- Cypher OPTIONAL MATCH semantics: the WHERE applies to the optional +-- binding; when no right-hand row survives the predicate, the outer row +-- is still emitted with NULLs in the optional columns. Before the fix, +-- a WHERE containing EXISTS { ... } or COUNT { ... } was attached as an +-- outer filter on the transformed subquery, so it ran after the LATERAL +-- LEFT JOIN produced null-preserving rows and then incorrectly dropped +-- them when the predicate evaluated NULL/false on the nulled side. +-- +SELECT create_graph('issue_2378'); +SELECT * FROM cypher('issue_2378', $$ + CREATE (a:Person {name: 'Alice'}), + (b:Person {name: 'Bob'}), + (c:Person {name: 'Charlie'}), + (a)-[:KNOWS]->(b), + (a)-[:KNOWS]->(c) +$$) AS (v agtype); + +-- Correlated EXISTS referencing the optional variable (friend). +-- Neither Bob nor Charlie knows anyone, so for every outer p the +-- predicate fails on all optional matches; expect one row per person +-- with friend = NULL. +SELECT * FROM cypher('issue_2378', $$ + MATCH (p:Person) + OPTIONAL MATCH (p)-[:KNOWS]->(friend:Person) + WHERE EXISTS { (friend)-[:KNOWS]->(:Person) } + RETURN p.name AS name, friend.name AS friend + ORDER BY name +$$) AS (name agtype, friend agtype); + +-- Correlated EXISTS referencing the outer variable (p). +-- Alice knows someone so her optional matches pass; Bob and Charlie +-- don't, so they are emitted with NULL friend. +SELECT * FROM cypher('issue_2378', $$ + MATCH (p:Person) + OPTIONAL MATCH (p)-[:KNOWS]->(friend:Person) + WHERE EXISTS { (p)-[:KNOWS]->(:Person) } + RETURN p.name AS name, friend.name AS friend + ORDER BY name, friend +$$) AS (name agtype, friend agtype); + +-- Non-correlated EXISTS (was already working; kept as a regression guard). +SELECT * FROM cypher('issue_2378', $$ + MATCH (p:Person) + OPTIONAL MATCH (p)-[:KNOWS]->(friend:Person) + WHERE EXISTS { MATCH (x:Person) RETURN x } + RETURN p.name AS name, friend.name AS friend + ORDER BY name, friend +$$) AS (name agtype, friend agtype); + +-- Plain scalar predicate on the optional variable (was already working). +SELECT * FROM cypher('issue_2378', $$ + MATCH (p:Person) + OPTIONAL MATCH (p)-[:KNOWS]->(friend:Person) + WHERE friend.name = 'Bob' + RETURN p.name AS name, friend.name AS friend + ORDER BY name +$$) AS (name agtype, friend agtype); + +-- Constant-false WHERE on the optional side (was already working). +SELECT * FROM cypher('issue_2378', $$ + MATCH (p:Person) + OPTIONAL MATCH (p)-[:KNOWS]->(friend:Person) + WHERE false + RETURN p.name AS name, friend.name AS friend + ORDER BY name +$$) AS (name agtype, friend agtype); + +SELECT drop_graph('issue_2378', true); + -- -- Clean up -- diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index e5540aa3e..95b205c07 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -2640,6 +2640,7 @@ static Query *transform_cypher_match(cypher_parsestate *cpstate, cypher_match *match_self = (cypher_match*) clause->self; Node *where = match_self->where; + /* * Check label validity early unless the predecessor clause chain * contains a data-modifying operation (CREATE, SET, DELETE, MERGE). @@ -2655,7 +2656,23 @@ static Query *transform_cypher_match(cypher_parsestate *cpstate, match_self->where = make_false_where_clause(false); } - if (has_list_comp_or_subquery((Node *)match_self->where, NULL)) + /* + * For a non-optional MATCH with a list comprehension or subquery in + * its WHERE clause, transform the match pattern as a subquery and + * then apply the WHERE as an outer filter. This keeps the parent's + * namespace available to the subquery-bearing predicate. + * + * This rewrite is NOT safe for OPTIONAL MATCH: wrapping the WHERE + * around the transformed clause turns it into a post-filter on the + * LATERAL LEFT JOIN produced by transform_cypher_optional_match_clause, + * which incorrectly drops the null-preserving outer rows that the + * LEFT JOIN generates when no right-hand match exists. For the + * optional case we fall through to the normal transform, which + * places the WHERE inside the right-hand subquery of the LEFT JOIN + * where it correctly scopes to the optional binding (issue #2378). + */ + if (!match_self->optional && + has_list_comp_or_subquery((Node *)match_self->where, NULL)) { match_self->where = NULL; return transform_cypher_clause_with_where(cpstate, @@ -2794,10 +2811,28 @@ static RangeTblEntry *transform_cypher_optional_match_clause(cypher_parsestate * List *res_colnames = NIL, *res_colvars = NIL; Alias *l_alias, *r_alias; ParseNamespaceItem *jnsitem; + cypher_match *match_self = (cypher_match *) clause->self; + Node *saved_where = match_self->where; int i = 0; j->jointype = JOIN_LEFT; + /* + * If the OPTIONAL MATCH carries a WHERE clause, temporarily detach + * it so that the recursive right-hand transform does NOT try to + * apply it inside the inner subquery. We re-apply the predicate + * below as a LEFT JOIN ON condition, which is the only placement + * that both (a) scopes the predicate to the optional binding and + * (b) preserves null-filled outer rows when the predicate fails. + * Without this, a WHERE that contains a sub-pattern predicate + * (e.g. EXISTS {...} referencing the optional variable) either + * gets silently dropped during the inner transform (namespace + * mismatch re-binds the variable in a fresh scope) or gets pulled + * up by the containing wrapper and filters out the null-preserving + * rows. See issue #2378. + */ + match_self->where = NULL; + l_alias = makeAlias(PREV_CYPHER_CLAUSE_ALIAS, NIL); r_alias = makeAlias(CYPHER_OPT_RIGHT_ALIAS, NIL); @@ -2819,6 +2854,32 @@ static RangeTblEntry *transform_cypher_optional_match_clause(cypher_parsestate * j->rarg = transform_clause_for_join(cpstate, clause, &r_rte, &r_nsitem, r_alias); + /* add right-side nsitem so the re-attached WHERE below can resolve + * newly-bound variables from the optional pattern */ + pstate->p_namespace = lappend(pstate->p_namespace, r_nsitem); + + /* + * Now that both sides are visible in the namespace, re-attach the + * OPTIONAL MATCH's WHERE predicate as the LEFT JOIN's ON clause. + * PostgreSQL correctly preserves left rows whose right side fails + * an ON condition (LEFT JOIN semantics), which is exactly what + * Cypher OPTIONAL MATCH ... WHERE requires: if the WHERE filters + * out all matches for a given outer row, that outer row is still + * emitted with nulls in the optional columns. + */ + if (saved_where != NULL) + { + Node *where_qual; + + where_qual = transform_cypher_expr(cpstate, saved_where, + EXPR_KIND_WHERE); + where_qual = coerce_to_boolean(pstate, where_qual, "WHERE"); + j->quals = where_qual; + } + + /* restore the WHERE on the node so we don't mutate caller state */ + match_self->where = saved_where; + /* * Since this is a left join, we need to mark j->rarg as it may potentially * emit NULL. The jindex argument holds rtindex of the join's RTE, which is From 4f9db377cc5576403e891ba67f2c7d103aaebcd7 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Sun, 19 Apr 2026 09:26:47 -0400 Subject: [PATCH 049/100] Implement predicate functions: all(), any(), none(), single() (#2359) * Implement predicate functions: all(), any(), none(), single() Implement the four openCypher predicate functions (issues #552, #553, #555, #556) that test list elements against a predicate: all(x IN list WHERE predicate) -- true if all elements match any(x IN list WHERE predicate) -- true if at least one matches none(x IN list WHERE predicate) -- true if no elements match single(x IN list WHERE predicate) -- true if exactly one matches Implementation approach: - Add cypher_predicate_function node type with CPFK_ALL/ANY/NONE/SINGLE kind enum, reusing the list comprehension's unnest-based transformation - Grammar rules in expr_func_subexpr (alongside EXISTS, COALESCE, COUNT) - Transform to efficient SQL sublinks: all() -> NOT EXISTS (SELECT 1 FROM unnest WHERE NOT pred) any() -> EXISTS (SELECT 1 FROM unnest WHERE pred) none() -> NOT EXISTS (SELECT 1 FROM unnest WHERE pred) single() -> (SELECT count(*) FROM unnest WHERE pred) = 1 - Three new keywords (ANY_P, NONE, SINGLE) added to safe_keywords for backward compatibility as property keys and label names - Shared extract_iter_variable_name() helper for variable validation All 32 regression tests pass. New predicate_functions test covers basic semantics, empty lists, graph data integration, boolean combinations, nested predicates, and keyword backward compatibility. Co-Authored-By: Claude Opus 4.6 (1M context) * Address Copilot review: NULL semantics, iterator validation, single() perf, tests - Rewrite predicate functions from EXISTS_SUBLINK to EXPR_SUBLINK with aggregate-based CASE expressions (bool_or + IS TRUE/FALSE/NULL) to preserve three-valued Cypher NULL semantics - Add list_length check in extract_iter_variable_name() to reject qualified names like x.y as iterator variables - Add copy/read support for cypher_predicate_function ExtensibleNode to prevent query rewriter crashes - Use IS TRUE filtering in single() count (LIMIT 2 optimization breaks correlated variable refs in graph contexts -- documented) - Add 13 NULL regression tests: null list input, null elements, null predicates for all four functions Co-Authored-By: Claude Opus 4.6 (1M context) * Address Copilot round 2: NULL-list guard, single() comment, pg_aggregate.h 1. Add NULL-list guard for all predicate functions (all/any/none/single). Wraps the result with CASE WHEN list IS NULL THEN NULL ELSE END in the grammar layer. This fixes single(x IN null WHERE ...) returning false instead of NULL. The expr pointer is safely shared between the NullTest and the predicate function node because AGE's expression transformer creates new nodes without modifying the parse tree in-place. 2. Fix single() block comment in transform_cypher_predicate_function: described LIMIT 2 optimization but implementation uses plain count(*). Updated comment to match actual implementation. 3. Keep #include "catalog/pg_aggregate.h" -- Copilot suggested removal but AGGKIND_NORMAL macro requires it (build fails without it). Regression test: predicate_functions OK. Co-Authored-By: Claude Opus 4.6 (1M context) * Address Copilot round 3: reuse extract_iter_variable_name for list comprehensions - Refactor build_list_comprehension_node() to reuse the shared extract_iter_variable_name() helper, so `var IN list` validation is consistent between list comprehensions and predicate functions (all/any/none/single). Qualified ColumnRefs like `x.y IN list` are now rejected in list comprehensions the same way they are in predicate functions. - Update list_comprehension expected output for the normalized lowercase "syntax error at or near IN" message. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- Makefile | 1 + regress/expected/list_comprehension.out | 4 +- regress/expected/predicate_functions.out | 409 +++++++++++++++++++++++ regress/sql/predicate_functions.sql | 250 ++++++++++++++ src/backend/nodes/ag_nodes.c | 6 +- src/backend/nodes/cypher_copyfuncs.c | 12 + src/backend/nodes/cypher_outfuncs.c | 11 + src/backend/nodes/cypher_readfuncs.c | 14 + src/backend/parser/cypher_analyze.c | 16 + src/backend/parser/cypher_clause.c | 334 ++++++++++++++++++ src/backend/parser/cypher_gram.y | 197 ++++++++++- src/include/nodes/ag_nodes.h | 4 +- src/include/nodes/cypher_copyfuncs.h | 4 + src/include/nodes/cypher_nodes.h | 21 ++ src/include/nodes/cypher_outfuncs.h | 1 + src/include/nodes/cypher_readfuncs.h | 3 + src/include/parser/cypher_kwlist.h | 3 + 17 files changed, 1272 insertions(+), 18 deletions(-) create mode 100644 regress/expected/predicate_functions.out create mode 100644 regress/sql/predicate_functions.sql diff --git a/Makefile b/Makefile index e358acf38..76f63c248 100644 --- a/Makefile +++ b/Makefile @@ -174,6 +174,7 @@ REGRESS = scan \ name_validation \ jsonb_operators \ list_comprehension \ + predicate_functions \ map_projection \ direct_field_access \ security diff --git a/regress/expected/list_comprehension.out b/regress/expected/list_comprehension.out index 5a3756422..e12ad621d 100644 --- a/regress/expected/list_comprehension.out +++ b/regress/expected/list_comprehension.out @@ -577,11 +577,11 @@ LINE 1: ...$$ RETURN [i IN range(0, 10, 2) WHERE i>5 | i^2], i $$) AS (... HINT: variable i does not exist within scope of usage -- Invalid list comprehension SELECT * FROM cypher('list_comprehension', $$ RETURN [1 IN range(0, 10, 2) WHERE 2>5] $$) AS (result agtype); -ERROR: Syntax error at or near IN +ERROR: syntax error at or near IN LINE 1: SELECT * FROM cypher('list_comprehension', $$ RETURN [1 IN r... ^ SELECT * FROM cypher('list_comprehension', $$ RETURN [1 IN range(0, 10, 2) | 1] $$) AS (result agtype); -ERROR: Syntax error at or near IN +ERROR: syntax error at or near IN LINE 1: SELECT * FROM cypher('list_comprehension', $$ RETURN [1 IN r... ^ -- Issue - error using list comprehension with WITH * diff --git a/regress/expected/predicate_functions.out b/regress/expected/predicate_functions.out new file mode 100644 index 000000000..47226453d --- /dev/null +++ b/regress/expected/predicate_functions.out @@ -0,0 +1,409 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +SELECT create_graph('predicate_functions'); +NOTICE: graph "predicate_functions" has been created + create_graph +-------------- + +(1 row) + +-- +-- all() predicate function +-- +-- all elements satisfy predicate -> true +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1, 2, 3] WHERE x > 0) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- not all elements satisfy predicate -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1, 2, 3] WHERE x > 1) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- empty list -> true (vacuous truth) +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [] WHERE x > 0) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- +-- any() predicate function +-- +-- at least one element satisfies -> true +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [1, 2, 3] WHERE x > 2) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- no element satisfies -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [1, 2, 3] WHERE x > 5) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- empty list -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [] WHERE x > 0) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- +-- none() predicate function +-- +-- no element satisfies predicate -> true +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [1, 2, 3] WHERE x > 5) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- at least one satisfies -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [1, 2, 3] WHERE x > 2) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- empty list -> true +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [] WHERE x > 0) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- +-- single() predicate function +-- +-- exactly one element satisfies -> true +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [1, 2, 3] WHERE x > 2) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- more than one satisfies -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [1, 2, 3] WHERE x > 1) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- none satisfies -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [1, 2, 3] WHERE x > 5) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- empty list -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [] WHERE x > 0) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- +-- NULL list input: all four return null +-- (NULL-list guard in the grammar produces CASE WHEN expr IS NULL +-- THEN NULL ELSE END) +-- +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN null WHERE x > 0) +$$) AS (result agtype); + result +-------- + +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN null WHERE x > 0) +$$) AS (result agtype); + result +-------- + +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN null WHERE x > 0) +$$) AS (result agtype); + result +-------- + +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN null WHERE x > 0) +$$) AS (result agtype); + result +-------- + +(1 row) + +-- +-- NULL predicate results: three-valued logic +-- +-- Note: In AGE's agtype, null is a first-class value. The comparison +-- agtype_null > agtype_integer evaluates to true (not SQL NULL). +-- Three-valued logic only applies when the predicate itself is a +-- literal null constant, which becomes SQL NULL after coercion. +-- agtype null in list: null > 0 = true in AGE, so any() = true +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [null] WHERE x > 0) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- agtype null + real values: all comparisons are true +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [null, 1, 2] WHERE x > 0) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- literal null predicate: pred = SQL NULL -> three-valued logic +-- all([1] WHERE null) = null (unknown) +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1] WHERE null) +$$) AS (result agtype); + result +-------- + +(1 row) + +-- agtype null in list: null > 0 = true in AGE, so all() = true +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1, null, 2] WHERE x > 0) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- -1 > 0 = false, so all() = false +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1, null, -1] WHERE x > 0) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- agtype null > 0 = true in AGE, so none() = false +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [null] WHERE x > 0) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- 5 > 0 = true, so none() = false +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [null, 5] WHERE x > 0) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- agtype null > 0 = true AND 5 > 0 = true: 2 matches, single = false +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [null, 5] WHERE x > 0) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- single() with null list: NULL (same as other predicate functions) +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN null WHERE x > 0) +$$) AS (result agtype); + result +-------- + +(1 row) + +-- +-- Integration with graph data +-- +SELECT * FROM cypher('predicate_functions', $$ + CREATE ({name: 'even', vals: [2, 4, 6, 8]}) +$$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('predicate_functions', $$ + CREATE ({name: 'mixed', vals: [1, 2, 3, 4]}) +$$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('predicate_functions', $$ + CREATE ({name: 'odd', vals: [1, 3, 5, 7]}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- all() with graph properties +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WHERE all(x IN u.vals WHERE x % 2 = 0) + RETURN u.name + ORDER BY u.name +$$) AS (result agtype); + result +-------- + "even" +(1 row) + +-- any() with graph properties +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WHERE any(x IN u.vals WHERE x > 6) + RETURN u.name + ORDER BY u.name +$$) AS (result agtype); + result +-------- + "even" + "odd" +(2 rows) + +-- none() with graph properties +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WHERE none(x IN u.vals WHERE x < 0) + RETURN u.name + ORDER BY u.name +$$) AS (result agtype); + result +--------- + "even" + "mixed" + "odd" +(3 rows) + +-- single() with graph properties +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WHERE single(x IN u.vals WHERE x = 8) + RETURN u.name + ORDER BY u.name +$$) AS (result agtype); + result +-------- + "even" +(1 row) + +-- +-- Predicate functions in boolean expressions +-- +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [1, 2, 3] WHERE x > 2) + AND all(y IN [4, 5, 6] WHERE y > 0) +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [1, 2, 3] WHERE x > 5) + OR single(y IN [1, 2, 3] WHERE y = 2) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- +-- Nested predicate functions +-- +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [1, 2, 3] WHERE all(y IN [1, 2] WHERE y < x)) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- +-- Keywords as property key names (safe_keywords backward compatibility) +-- +SELECT * FROM cypher('predicate_functions', $$ + RETURN {any: 1, none: 2, single: 3} +$$) AS (result agtype); + result +------------------------------------ + {"any": 1, "none": 2, "single": 3} +(1 row) + +-- +-- Cleanup +-- +SELECT * FROM drop_graph('predicate_functions', true); +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table predicate_functions._ag_label_vertex +drop cascades to table predicate_functions._ag_label_edge +NOTICE: graph "predicate_functions" has been dropped + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/predicate_functions.sql b/regress/sql/predicate_functions.sql new file mode 100644 index 000000000..7466cc2a4 --- /dev/null +++ b/regress/sql/predicate_functions.sql @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +SELECT create_graph('predicate_functions'); + +-- +-- all() predicate function +-- +-- all elements satisfy predicate -> true +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1, 2, 3] WHERE x > 0) +$$) AS (result agtype); + +-- not all elements satisfy predicate -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1, 2, 3] WHERE x > 1) +$$) AS (result agtype); + +-- empty list -> true (vacuous truth) +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [] WHERE x > 0) +$$) AS (result agtype); + +-- +-- any() predicate function +-- +-- at least one element satisfies -> true +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [1, 2, 3] WHERE x > 2) +$$) AS (result agtype); + +-- no element satisfies -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [1, 2, 3] WHERE x > 5) +$$) AS (result agtype); + +-- empty list -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [] WHERE x > 0) +$$) AS (result agtype); + +-- +-- none() predicate function +-- +-- no element satisfies predicate -> true +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [1, 2, 3] WHERE x > 5) +$$) AS (result agtype); + +-- at least one satisfies -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [1, 2, 3] WHERE x > 2) +$$) AS (result agtype); + +-- empty list -> true +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [] WHERE x > 0) +$$) AS (result agtype); + +-- +-- single() predicate function +-- +-- exactly one element satisfies -> true +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [1, 2, 3] WHERE x > 2) +$$) AS (result agtype); + +-- more than one satisfies -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [1, 2, 3] WHERE x > 1) +$$) AS (result agtype); + +-- none satisfies -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [1, 2, 3] WHERE x > 5) +$$) AS (result agtype); + +-- empty list -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [] WHERE x > 0) +$$) AS (result agtype); + +-- +-- NULL list input: all four return null +-- (NULL-list guard in the grammar produces CASE WHEN expr IS NULL +-- THEN NULL ELSE END) +-- +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN null WHERE x > 0) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN null WHERE x > 0) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN null WHERE x > 0) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN null WHERE x > 0) +$$) AS (result agtype); + +-- +-- NULL predicate results: three-valued logic +-- +-- Note: In AGE's agtype, null is a first-class value. The comparison +-- agtype_null > agtype_integer evaluates to true (not SQL NULL). +-- Three-valued logic only applies when the predicate itself is a +-- literal null constant, which becomes SQL NULL after coercion. + +-- agtype null in list: null > 0 = true in AGE, so any() = true +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [null] WHERE x > 0) +$$) AS (result agtype); + +-- agtype null + real values: all comparisons are true +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [null, 1, 2] WHERE x > 0) +$$) AS (result agtype); + +-- literal null predicate: pred = SQL NULL -> three-valued logic +-- all([1] WHERE null) = null (unknown) +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1] WHERE null) +$$) AS (result agtype); + +-- agtype null in list: null > 0 = true in AGE, so all() = true +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1, null, 2] WHERE x > 0) +$$) AS (result agtype); + +-- -1 > 0 = false, so all() = false +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1, null, -1] WHERE x > 0) +$$) AS (result agtype); + +-- agtype null > 0 = true in AGE, so none() = false +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [null] WHERE x > 0) +$$) AS (result agtype); + +-- 5 > 0 = true, so none() = false +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [null, 5] WHERE x > 0) +$$) AS (result agtype); + +-- agtype null > 0 = true AND 5 > 0 = true: 2 matches, single = false +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [null, 5] WHERE x > 0) +$$) AS (result agtype); + +-- single() with null list: NULL (same as other predicate functions) +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN null WHERE x > 0) +$$) AS (result agtype); + +-- +-- Integration with graph data +-- +SELECT * FROM cypher('predicate_functions', $$ + CREATE ({name: 'even', vals: [2, 4, 6, 8]}) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + CREATE ({name: 'mixed', vals: [1, 2, 3, 4]}) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + CREATE ({name: 'odd', vals: [1, 3, 5, 7]}) +$$) AS (result agtype); + +-- all() with graph properties +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WHERE all(x IN u.vals WHERE x % 2 = 0) + RETURN u.name + ORDER BY u.name +$$) AS (result agtype); + +-- any() with graph properties +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WHERE any(x IN u.vals WHERE x > 6) + RETURN u.name + ORDER BY u.name +$$) AS (result agtype); + +-- none() with graph properties +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WHERE none(x IN u.vals WHERE x < 0) + RETURN u.name + ORDER BY u.name +$$) AS (result agtype); + +-- single() with graph properties +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WHERE single(x IN u.vals WHERE x = 8) + RETURN u.name + ORDER BY u.name +$$) AS (result agtype); + +-- +-- Predicate functions in boolean expressions +-- +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [1, 2, 3] WHERE x > 2) + AND all(y IN [4, 5, 6] WHERE y > 0) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [1, 2, 3] WHERE x > 5) + OR single(y IN [1, 2, 3] WHERE y = 2) +$$) AS (result agtype); + +-- +-- Nested predicate functions +-- +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [1, 2, 3] WHERE all(y IN [1, 2] WHERE y < x)) +$$) AS (result agtype); + +-- +-- Keywords as property key names (safe_keywords backward compatibility) +-- +SELECT * FROM cypher('predicate_functions', $$ + RETURN {any: 1, none: 2, single: 3} +$$) AS (result agtype); + +-- +-- Cleanup +-- +SELECT * FROM drop_graph('predicate_functions', true); diff --git a/src/backend/nodes/ag_nodes.c b/src/backend/nodes/ag_nodes.c index 7aaaecaa5..bd78549ca 100644 --- a/src/backend/nodes/ag_nodes.c +++ b/src/backend/nodes/ag_nodes.c @@ -64,7 +64,8 @@ const char *node_names[] = { "cypher_update_item", "cypher_delete_information", "cypher_delete_item", - "cypher_merge_information" + "cypher_merge_information", + "cypher_predicate_function" }; /* @@ -132,7 +133,8 @@ const ExtensibleNodeMethods node_methods[] = { DEFINE_NODE_METHODS_EXTENDED(cypher_update_item), DEFINE_NODE_METHODS_EXTENDED(cypher_delete_information), DEFINE_NODE_METHODS_EXTENDED(cypher_delete_item), - DEFINE_NODE_METHODS_EXTENDED(cypher_merge_information) + DEFINE_NODE_METHODS_EXTENDED(cypher_merge_information), + DEFINE_NODE_METHODS_EXTENDED(cypher_predicate_function) }; static bool equal_ag_node(const ExtensibleNode *a, const ExtensibleNode *b) diff --git a/src/backend/nodes/cypher_copyfuncs.c b/src/backend/nodes/cypher_copyfuncs.c index 56895ee06..420ab1d22 100644 --- a/src/backend/nodes/cypher_copyfuncs.c +++ b/src/backend/nodes/cypher_copyfuncs.c @@ -169,3 +169,15 @@ void copy_cypher_merge_information(ExtensibleNode *newnode, const ExtensibleNode COPY_SCALAR_FIELD(merge_function_attr); COPY_NODE_FIELD(path); } + +/* copy function for cypher_predicate_function */ +void copy_cypher_predicate_function(ExtensibleNode *newnode, + const ExtensibleNode *from) +{ + COPY_LOCALS(cypher_predicate_function); + + COPY_SCALAR_FIELD(kind); + COPY_STRING_FIELD(varname); + COPY_NODE_FIELD(expr); + COPY_NODE_FIELD(where); +} diff --git a/src/backend/nodes/cypher_outfuncs.c b/src/backend/nodes/cypher_outfuncs.c index 4772621c9..cf8a400fc 100644 --- a/src/backend/nodes/cypher_outfuncs.c +++ b/src/backend/nodes/cypher_outfuncs.c @@ -189,6 +189,17 @@ void out_cypher_list_comprehension(StringInfo str, const ExtensibleNode *node) } +/* serialization function for the cypher_predicate_function ExtensibleNode. */ +void out_cypher_predicate_function(StringInfo str, const ExtensibleNode *node) +{ + DEFINE_AG_NODE(cypher_predicate_function); + + WRITE_ENUM_FIELD(kind, cypher_predicate_function_kind); + WRITE_STRING_FIELD(varname); + WRITE_NODE_FIELD(expr); + WRITE_NODE_FIELD(where); +} + /* serialization function for the cypher_delete ExtensibleNode. */ void out_cypher_merge(StringInfo str, const ExtensibleNode *node) { diff --git a/src/backend/nodes/cypher_readfuncs.c b/src/backend/nodes/cypher_readfuncs.c index a58b90cbc..14b553dbb 100644 --- a/src/backend/nodes/cypher_readfuncs.c +++ b/src/backend/nodes/cypher_readfuncs.c @@ -311,3 +311,17 @@ void read_cypher_merge_information(struct ExtensibleNode *node) READ_INT_FIELD(merge_function_attr); READ_NODE_FIELD(path); } + +/* + * Deserialize a string representing the cypher_predicate_function + * data structure. + */ +void read_cypher_predicate_function(struct ExtensibleNode *node) +{ + READ_LOCALS(cypher_predicate_function); + + READ_ENUM_FIELD(kind, cypher_predicate_function_kind); + READ_STRING_FIELD(varname); + READ_NODE_FIELD(expr); + READ_NODE_FIELD(where); +} diff --git a/src/backend/parser/cypher_analyze.c b/src/backend/parser/cypher_analyze.c index a408eea6e..7844af2f0 100644 --- a/src/backend/parser/cypher_analyze.c +++ b/src/backend/parser/cypher_analyze.c @@ -843,6 +843,22 @@ bool cypher_raw_expr_tree_walker_impl(Node *node, return true; } } + else if (is_ag_node(node, cypher_predicate_function)) + { + cypher_predicate_function *pf; + + pf = (cypher_predicate_function *)node; + + if (WALK(pf->expr)) + { + return true; + } + + if (WALK(pf->where)) + { + return true; + } + } /* Add more node types here as needed */ else { diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 95b205c07..0ab574fe6 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -25,6 +25,7 @@ #include "postgres.h" #include "access/heapam.h" +#include "catalog/pg_aggregate.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" @@ -254,6 +255,10 @@ static Query *transform_cypher_unwind(cypher_parsestate *cpstate, static Query *transform_cypher_list_comprehension(cypher_parsestate *cpstate, cypher_clause *clause); +/* predicate functions */ +static Query *transform_cypher_predicate_function(cypher_parsestate *cpstate, + cypher_clause *clause); + /* merge */ static Query *transform_cypher_merge(cypher_parsestate *cpstate, cypher_clause *clause); @@ -513,6 +518,10 @@ Query *transform_cypher_clause(cypher_parsestate *cpstate, { result = transform_cypher_list_comprehension(cpstate, clause); } + else if (is_ag_node(self, cypher_predicate_function)) + { + result = transform_cypher_predicate_function(cpstate, clause); + } else { ereport(ERROR, (errmsg_internal("unexpected Node for cypher_clause"))); @@ -1603,6 +1612,331 @@ static Query *transform_cypher_list_comprehension(cypher_parsestate *cpstate, return query; } +/* + * Helper: build a BooleanTest node (pred IS TRUE, pred IS FALSE, etc.) + */ +static Node *make_boolean_test(Node *arg, BoolTestType testtype) +{ + BooleanTest *bt = makeNode(BooleanTest); + + bt->arg = (Expr *) arg; + bt->booltesttype = testtype; + bt->location = -1; + + return (Node *) bt; +} + +/* + * Helper: build a fully-transformed bool_or(expr) Aggref node. + * + * The argument must already be a transformed boolean expression. + * We construct the Aggref manually to avoid going through FuncCall + * + transformExpr, which expects raw parse tree nodes. + */ +static Node *make_bool_or_agg(ParseState *pstate, Node *arg) +{ + Aggref *agg; + TargetEntry *te; + Oid bool_or_oid; + Oid argtypes[1] = { BOOLOID }; + + /* Look up bool_or(boolean) */ + bool_or_oid = LookupFuncName(list_make1(makeString("bool_or")), + 1, argtypes, false); + + /* Build the TargetEntry for the aggregate argument */ + te = makeTargetEntry((Expr *) arg, 1, NULL, false); + + /* Construct the Aggref */ + agg = makeNode(Aggref); + agg->aggfnoid = bool_or_oid; + agg->aggtype = BOOLOID; + agg->aggcollid = InvalidOid; + agg->inputcollid = InvalidOid; + agg->aggtranstype = InvalidOid; /* filled by planner */ + agg->aggargtypes = list_make1_oid(BOOLOID); + agg->aggdirectargs = NIL; + agg->args = list_make1(te); + agg->aggorder = NIL; + agg->aggdistinct = NIL; + agg->aggfilter = NULL; + agg->aggstar = false; + agg->aggvariadic = false; + agg->aggkind = AGGKIND_NORMAL; + agg->aggpresorted = false; + agg->agglevelsup = 0; + agg->aggsplit = AGGSPLIT_SIMPLE; + agg->aggno = -1; + agg->aggtransno = -1; + agg->location = -1; + + /* Register the aggregate with the parse state */ + pstate->p_hasAggs = true; + + return (Node *) agg; +} + +/* + * Helper: build a transformed CASE expression implementing three-valued + * predicate logic for all(), any(), and none(). + * + * any(): CASE WHEN bool_or(pred IS TRUE) THEN true + * WHEN bool_or(pred IS NULL) THEN NULL + * ELSE false END + * + * all(): CASE WHEN bool_or(pred IS FALSE) THEN false + * WHEN bool_or(pred IS NULL) THEN NULL + * ELSE true END + * + * none(): CASE WHEN bool_or(pred IS TRUE) THEN false + * WHEN bool_or(pred IS NULL) THEN NULL + * ELSE true END + * + * Empty list: both bool_or calls return NULL (no rows), so the CASE + * falls through to the default: false for any(), true for all()/none(). + * This matches Cypher's vacuous truth semantics. + */ +static Node *make_predicate_case_expr(ParseState *pstate, Node *pred, + cypher_predicate_function_kind kind) +{ + CaseExpr *cexpr; + CaseWhen *when1, *when2; + Node *bool_or_first, *bool_or_null; + Node *true_const, *false_const, *null_const; + + /* boolean constants */ + true_const = makeBoolConst(true, false); + false_const = makeBoolConst(false, false); + null_const = makeBoolConst(false, true); /* isnull = true */ + + /* Second branch is common to all: bool_or(pred IS NULL) -> NULL */ + bool_or_null = make_bool_or_agg(pstate, + make_boolean_test(pred, IS_UNKNOWN)); + + when2 = makeNode(CaseWhen); + when2->expr = (Expr *) bool_or_null; + when2->result = (Expr *) null_const; + when2->location = -1; + + if (kind == CPFK_ALL) + { + /* bool_or(pred IS FALSE) -> false */ + bool_or_first = make_bool_or_agg(pstate, + make_boolean_test(pred, IS_FALSE)); + when1 = makeNode(CaseWhen); + when1->expr = (Expr *) bool_or_first; + when1->result = (Expr *) false_const; + when1->location = -1; + + cexpr = makeNode(CaseExpr); + cexpr->casetype = BOOLOID; + cexpr->arg = NULL; + cexpr->args = list_make2(when1, when2); + cexpr->defresult = (Expr *) true_const; + cexpr->location = -1; + } + else if (kind == CPFK_ANY) + { + /* bool_or(pred IS TRUE) -> true */ + bool_or_first = make_bool_or_agg(pstate, + make_boolean_test(pred, IS_TRUE)); + when1 = makeNode(CaseWhen); + when1->expr = (Expr *) bool_or_first; + when1->result = (Expr *) true_const; + when1->location = -1; + + cexpr = makeNode(CaseExpr); + cexpr->casetype = BOOLOID; + cexpr->arg = NULL; + cexpr->args = list_make2(when1, when2); + cexpr->defresult = (Expr *) false_const; + cexpr->location = -1; + } + else /* CPFK_NONE */ + { + /* bool_or(pred IS TRUE) -> false */ + bool_or_first = make_bool_or_agg(pstate, + make_boolean_test(pred, IS_TRUE)); + when1 = makeNode(CaseWhen); + when1->expr = (Expr *) bool_or_first; + when1->result = (Expr *) false_const; + when1->location = -1; + + cexpr = makeNode(CaseExpr); + cexpr->casetype = BOOLOID; + cexpr->arg = NULL; + cexpr->args = list_make2(when1, when2); + cexpr->defresult = (Expr *) true_const; + cexpr->location = -1; + } + + return (Node *) cexpr; +} + +/* + * Transform a cypher_predicate_function node into a query tree. + * + * Generates aggregate-based queries that preserve Cypher's three-valued + * NULL semantics. The grammar layer wraps the SubLink with a + * CASE WHEN list IS NULL THEN NULL ELSE (subquery) END guard so all + * four functions return NULL when the input list is NULL. + * + * For all()/any()/none(): + * SELECT CASE WHEN bool_or(pred IS TRUE/FALSE) THEN ... + * WHEN bool_or(pred IS NULL) THEN NULL + * ELSE ... END + * FROM unnest(list) AS x + * + * For single(): + * SELECT count(*) + * FROM unnest(list) AS x + * WHERE pred IS TRUE + * + * All four use EXPR_SUBLINK so the subquery returns a scalar value. + */ +static Query *transform_cypher_predicate_function(cypher_parsestate *cpstate, + cypher_clause *clause) +{ + Query *query; + RangeFunction *rf; + cypher_predicate_function *pred_func; + FuncCall *func_call; + Node *pred, *n; + RangeTblEntry *rte = NULL; + int rtindex; + List *namespace = NULL; + TargetEntry *te; + cypher_parsestate *child_cpstate = make_cypher_parsestate(cpstate); + ParseState *child_pstate = (ParseState *) child_cpstate; + + pred_func = (cypher_predicate_function *) clause->self; + + query = makeNode(Query); + query->commandType = CMD_SELECT; + + /* FROM unnest(expr) AS varname */ + func_call = makeFuncCall(list_make1(makeString("unnest")), + list_make1(pred_func->expr), + COERCE_SQL_SYNTAX, -1); + + rf = makeNode(RangeFunction); + rf->lateral = false; + rf->ordinality = false; + rf->is_rowsfrom = false; + rf->functions = list_make1(list_make2((Node *) func_call, NIL)); + rf->alias = makeAlias(pred_func->varname, NIL); + rf->coldeflist = NIL; + + n = transform_from_clause_item(child_cpstate, (Node *) rf, + &rte, &rtindex, &namespace); + checkNameSpaceConflicts(child_pstate, child_pstate->p_namespace, namespace); + child_pstate->p_joinlist = lappend(child_pstate->p_joinlist, n); + child_pstate->p_namespace = list_concat(child_pstate->p_namespace, + namespace); + + /* make all namespace items unconditionally visible */ + setNamespaceLateralState(child_pstate->p_namespace, false, true); + + /* Transform the predicate expression */ + pred = transform_cypher_expr(child_cpstate, pred_func->where, + EXPR_KIND_WHERE); + if (pred) + { + pred = coerce_to_boolean(child_pstate, pred, "WHERE"); + } + + if (pred_func->kind == CPFK_SINGLE) + { + /* + * single(): SELECT count(*) FROM unnest(list) AS x + * WHERE pred IS TRUE + * + * Using IS TRUE ensures NULL predicates are not counted as + * matches, preserving correct semantics. The grammar layer + * compares the result = 1. + * + * Note: a LIMIT 2 optimization (to short-circuit after two + * matches) would require a nested subquery that breaks + * correlated variable references. Deferred to a future + * optimization pass. + */ + FuncCall *count_call; + Node *count_expr; + Node *is_true_qual; + + /* WHERE pred IS TRUE -- NULLs are not counted */ + is_true_qual = make_boolean_test(pred, IS_TRUE); + + count_call = makeFuncCall(list_make1(makeString("count")), + NIL, COERCE_SQL_SYNTAX, -1); + count_call->agg_star = true; + + count_expr = transformExpr(child_pstate, (Node *) count_call, + EXPR_KIND_SELECT_TARGET); + + te = makeTargetEntry((Expr *) count_expr, + (AttrNumber) child_pstate->p_next_resno++, + "count", false); + + query->targetList = lappend(query->targetList, te); + query->jointree = makeFromExpr(child_pstate->p_joinlist, + is_true_qual); + query->rtable = child_pstate->p_rtable; + query->rteperminfos = child_pstate->p_rteperminfos; + query->hasAggs = child_pstate->p_hasAggs; + query->hasSubLinks = child_pstate->p_hasSubLinks; + query->hasTargetSRFs = child_pstate->p_hasTargetSRFs; + + assign_query_collations(child_pstate, query); + + if (child_pstate->p_hasAggs || + query->groupClause || query->groupingSets || query->havingQual) + { + parse_check_aggregates(child_pstate, query); + } + + free_cypher_parsestate(child_cpstate); + + return query; + } + else + { + /* + * all()/any()/none(): Build a CASE expression with bool_or() + * aggregates that preserves three-valued NULL semantics. + * No WHERE clause -- the logic is entirely in the SELECT list. + */ + Node *case_expr; + + case_expr = make_predicate_case_expr(child_pstate, pred, + pred_func->kind); + + te = makeTargetEntry((Expr *) case_expr, + (AttrNumber) child_pstate->p_next_resno++, + "result", false); + + query->targetList = lappend(query->targetList, te); + query->jointree = makeFromExpr(child_pstate->p_joinlist, NULL); + query->rtable = child_pstate->p_rtable; + query->rteperminfos = child_pstate->p_rteperminfos; + query->hasAggs = child_pstate->p_hasAggs; + query->hasSubLinks = child_pstate->p_hasSubLinks; + query->hasTargetSRFs = child_pstate->p_hasTargetSRFs; + + assign_query_collations(child_pstate, query); + + if (child_pstate->p_hasAggs || + query->groupClause || query->groupingSets || query->havingQual) + { + parse_check_aggregates(child_pstate, query); + } + + free_cypher_parsestate(child_cpstate); + + return query; + } +} + /* * Iterate through the list of items to delete and extract the variable name. * Then find the resno that the variable name belongs to. diff --git a/src/backend/parser/cypher_gram.y b/src/backend/parser/cypher_gram.y index 5ba1e6354..e14c6b481 100644 --- a/src/backend/parser/cypher_gram.y +++ b/src/backend/parser/cypher_gram.y @@ -79,7 +79,7 @@ %token NOT_EQ LT_EQ GT_EQ DOT_DOT TYPECAST PLUS_EQ /* keywords in alphabetical order */ -%token ALL ANALYZE AND AS ASC ASCENDING +%token ALL ANALYZE AND ANY_P AS ASC ASCENDING BY CALL CASE COALESCE CONTAINS COUNT CREATE DELETE DESC DESCENDING DETACH DISTINCT @@ -88,10 +88,10 @@ IN IS LIMIT MATCH MERGE - NOT NULL_P + NONE NOT NULL_P OPERATOR OPTIONAL OR ORDER REMOVE RETURN - SET SKIP STARTS + SET SINGLE SKIP STARTS THEN TRUE_P UNION UNWIND VERBOSE @@ -274,11 +274,19 @@ static Node *build_comparison_expression(Node *left_grammar_node, Node *right_grammar_node, char *opr_name, int location); +/* shared helper for list iteration constructs */ +static char *extract_iter_variable_name(Node *var); + /* list_comprehension */ static Node *build_list_comprehension_node(Node *var, Node *expr, Node *where, Node *mapping_expr, int location); +/* predicate functions: all(), any(), none(), single() */ +static Node *build_predicate_function_node(cypher_predicate_function_kind kind, + Node *var, Node *expr, + Node *where, int location); + /* helper functions */ static ExplainStmt *make_explain_stmt(List *options); static void validate_return_item_aliases(List *items, ag_scanner_t scanner); @@ -1853,6 +1861,22 @@ expr_func_subexpr: list_make1(makeString("count")), NIL, @1); $$ = (Node *)n; } + | ALL '(' expr IN expr WHERE expr ')' + { + $$ = build_predicate_function_node(CPFK_ALL, $3, $5, $7, @1); + } + | ANY_P '(' expr IN expr WHERE expr ')' + { + $$ = build_predicate_function_node(CPFK_ANY, $3, $5, $7, @1); + } + | NONE '(' expr IN expr WHERE expr ')' + { + $$ = build_predicate_function_node(CPFK_NONE, $3, $5, $7, @1); + } + | SINGLE '(' expr IN expr WHERE expr ')' + { + $$ = build_predicate_function_node(CPFK_SINGLE, $3, $5, $7, @1); + } ; expr_subquery: @@ -2372,6 +2396,7 @@ safe_keywords: ALL { $$ = KEYWORD_STRDUP($1); } | ANALYZE { $$ = KEYWORD_STRDUP($1); } | AND { $$ = KEYWORD_STRDUP($1); } + | ANY_P { $$ = KEYWORD_STRDUP($1); } | AS { $$ = KEYWORD_STRDUP($1); } | ASC { $$ = KEYWORD_STRDUP($1); } | ASCENDING { $$ = KEYWORD_STRDUP($1); } @@ -2396,6 +2421,7 @@ safe_keywords: | LIMIT { $$ = KEYWORD_STRDUP($1); } | MATCH { $$ = KEYWORD_STRDUP($1); } | MERGE { $$ = KEYWORD_STRDUP($1); } + | NONE { $$ = KEYWORD_STRDUP($1); } | NOT { $$ = KEYWORD_STRDUP($1); } | OPERATOR { $$ = KEYWORD_STRDUP($1); } | OPTIONAL { $$ = KEYWORD_STRDUP($1); } @@ -2404,6 +2430,7 @@ safe_keywords: | REMOVE { $$ = KEYWORD_STRDUP($1); } | RETURN { $$ = KEYWORD_STRDUP($1); } | SET { $$ = KEYWORD_STRDUP($1); } + | SINGLE { $$ = KEYWORD_STRDUP($1); } | SKIP { $$ = KEYWORD_STRDUP($1); } | STARTS { $$ = KEYWORD_STRDUP($1); } | THEN { $$ = KEYWORD_STRDUP($1); } @@ -3268,35 +3295,66 @@ static cypher_relationship *build_VLE_relation(List *left_arg, return cr; } -/* helper function to build a list_comprehension grammar node */ -static Node *build_list_comprehension_node(Node *var, Node *expr, - Node *where, Node *mapping_expr, - int location) +/* + * Extract and validate the iterator variable name from a ColumnRef node. + * Used by predicate functions (all/any/none/single) which share the + * "variable IN list" syntax with list comprehensions. + */ +static char *extract_iter_variable_name(Node *var) { - SubLink *sub; + ColumnRef *cref; String *val; - ColumnRef *cref = NULL; - cypher_list_comprehension *list_comp = NULL; if (!IsA(var, ColumnRef)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("Syntax error at or near IN"))); + errmsg("syntax error at or near IN"))); } cref = (ColumnRef *)var; + + /* The iterator must be a simple unqualified name (single field) */ + if (list_length(cref->fields) != 1) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("qualified name not allowed as iterator variable"))); + } + val = linitial(cref->fields); if (!IsA(val, String)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("Invalid list comprehension variable name"))); + errmsg("invalid iterator variable name"))); } + return val->sval; +} + +/* helper function to build a list_comprehension grammar node */ +static Node *build_list_comprehension_node(Node *var, Node *expr, + Node *where, Node *mapping_expr, + int location) +{ + SubLink *sub; + ColumnRef *cref; + char *varname; + cypher_list_comprehension *list_comp = NULL; + + /* + * Reuse the shared iterator-name helper so list comprehensions and + * predicate functions (all/any/none/single) validate `var IN list` + * the same way — in particular, qualified ColumnRefs like `x.y` are + * rejected rather than silently treated as iterator `x`. + */ + varname = extract_iter_variable_name(var); + cref = (ColumnRef *) var; + /* build the list comprehension node */ list_comp = make_ag_node(cypher_list_comprehension); - list_comp->varname = val->sval; + list_comp->varname = varname; list_comp->expr = expr; list_comp->where = where; list_comp->mapping_expr = (mapping_expr != NULL) ? mapping_expr : @@ -3317,6 +3375,119 @@ static Node *build_list_comprehension_node(Node *var, Node *expr, return (Node *) node_to_agtype((Node *)sub, "agtype[]", location); } +/* + * Helper function to build a predicate function grammar node. + * + * Predicate functions follow the openCypher syntax: + * all(x IN list WHERE predicate) + * any(x IN list WHERE predicate) + * none(x IN list WHERE predicate) + * single(x IN list WHERE predicate) + * + * All four use EXPR_SUBLINK (scalar subquery). The transform layer + * generates aggregate-based queries (using bool_or + CASE) for + * all/any/none to preserve three-valued NULL semantics, and + * count(*) with IS TRUE filtering for single(). + * + * For single(), the subquery result is compared = 1. + */ +static Node *build_predicate_function_node(cypher_predicate_function_kind kind, + Node *var, Node *expr, + Node *where, int location) +{ + SubLink *sub; + cypher_predicate_function *pred_func = NULL; + Node *result; + + /* build the predicate function node */ + pred_func = make_ag_node(cypher_predicate_function); + pred_func->kind = kind; + pred_func->varname = extract_iter_variable_name(var); + pred_func->expr = expr; + pred_func->where = where; + + /* + * Wrap the predicate function in a SubLink. PostgreSQL's SubLink is + * reused here as the carrier for our custom subquery node -- the + * predicate function node is stored as the subselect and will be + * transformed into a real Query by transform_cypher_predicate_function() + * in cypher_clause.c. + * + * All predicate functions now use EXPR_SUBLINK: the transform layer + * generates aggregate-based queries that return a scalar boolean + * (for all/any/none) or integer (for single). + * + * The transform layer also wraps the result with a NULL-list guard + * (CASE WHEN list IS NULL THEN NULL ELSE END) to ensure + * all four functions return NULL when the input list is NULL. + */ + sub = makeNode(SubLink); + sub->subLinkId = 0; + sub->testexpr = NULL; + sub->operName = NIL; + sub->subselect = (Node *) pred_func; + sub->location = location; + sub->subLinkType = EXPR_SUBLINK; + + if (kind == CPFK_SINGLE) + { + /* + * single() -> (subquery) = 1 + * The subquery returns count(*) with IS TRUE filtering. + */ + Node *eq_expr; + + eq_expr = (Node *) makeSimpleA_Expr(AEXPR_OP, "=", + (Node *) sub, + make_int_const(1, location), + location); + result = (Node *) node_to_agtype(eq_expr, "boolean", location); + } + else + { + /* + * all()/any()/none(): the subquery returns a boolean directly + * from the CASE+bool_or() aggregate expression. + */ + result = (Node *) node_to_agtype((Node *) sub, "boolean", location); + } + + /* + * NULL-list guard: CASE WHEN expr IS NULL THEN NULL ELSE result END + * + * Without this, unnest(NULL) produces zero rows, causing all/any/none + * to accidentally return NULL (via bool_or over empty input) and + * single to return false (count(*) = 0). Cypher semantics require + * all four to return NULL when the input list is NULL. + * + * The expr pointer is shared with pred_func->expr. This is safe + * because AGE's expression transformer (transform_cypher_expr_recurse) + * creates new nodes rather than modifying the parse tree in-place, + * so the two references are transformed independently. + */ + { + NullTest *null_test = makeNode(NullTest); + CaseWhen *case_when = makeNode(CaseWhen); + CaseExpr *guard = makeNode(CaseExpr); + + null_test->arg = (Expr *) expr; + null_test->nulltesttype = IS_NULL; + null_test->argisrow = false; + null_test->location = location; + + case_when->expr = (Expr *) null_test; + case_when->result = (Expr *) make_null_const(location); + case_when->location = location; + + guard->arg = NULL; + guard->args = list_make1(case_when); + guard->defresult = (Expr *) result; + guard->location = location; + + return (Node *) guard; + } +} + /* Helper function to create an ExplainStmt node */ static ExplainStmt *make_explain_stmt(List *options) { diff --git a/src/include/nodes/ag_nodes.h b/src/include/nodes/ag_nodes.h index 121832c01..47c55041b 100644 --- a/src/include/nodes/ag_nodes.h +++ b/src/include/nodes/ag_nodes.h @@ -75,7 +75,9 @@ typedef enum ag_node_tag /* delete data structures */ cypher_delete_information_t, cypher_delete_item_t, - cypher_merge_information_t + cypher_merge_information_t, + /* predicate functions */ + cypher_predicate_function_t } ag_node_tag; extern const char *node_names[]; diff --git a/src/include/nodes/cypher_copyfuncs.h b/src/include/nodes/cypher_copyfuncs.h index d7ed7eff7..e770cebe2 100644 --- a/src/include/nodes/cypher_copyfuncs.h +++ b/src/include/nodes/cypher_copyfuncs.h @@ -52,4 +52,8 @@ void copy_cypher_delete_item(ExtensibleNode *newnode, /* merge data structure */ void copy_cypher_merge_information(ExtensibleNode *newnode, const ExtensibleNode *from); + +/* predicate function data structure */ +void copy_cypher_predicate_function(ExtensibleNode *newnode, + const ExtensibleNode *from); #endif diff --git a/src/include/nodes/cypher_nodes.h b/src/include/nodes/cypher_nodes.h index db47eb313..93bbe01de 100644 --- a/src/include/nodes/cypher_nodes.h +++ b/src/include/nodes/cypher_nodes.h @@ -224,6 +224,27 @@ typedef struct cypher_map_projection Node *mapping_expr; } cypher_list_comprehension; +/* + * Predicate function kinds for all(), any(), none(), single(). + * These take the form: func(variable IN list WHERE predicate) + */ +typedef enum cypher_predicate_function_kind +{ + CPFK_ALL = 0, + CPFK_ANY, + CPFK_NONE, + CPFK_SINGLE +} cypher_predicate_function_kind; + +typedef struct cypher_predicate_function +{ + ExtensibleNode extensible; + cypher_predicate_function_kind kind; + char *varname; + Node *expr; /* the list to iterate over */ + Node *where; /* the predicate to test */ +} cypher_predicate_function; + typedef enum cypher_map_projection_element_type { PROPERTY_SELECTOR = 0, /* map_var { .key } */ diff --git a/src/include/nodes/cypher_outfuncs.h b/src/include/nodes/cypher_outfuncs.h index 418d35f4e..55285bdba 100644 --- a/src/include/nodes/cypher_outfuncs.h +++ b/src/include/nodes/cypher_outfuncs.h @@ -49,6 +49,7 @@ void out_cypher_map(StringInfo str, const ExtensibleNode *node); void out_cypher_map_projection(StringInfo str, const ExtensibleNode *node); void out_cypher_list(StringInfo str, const ExtensibleNode *node); void out_cypher_list_comprehension(StringInfo str, const ExtensibleNode *node); +void out_cypher_predicate_function(StringInfo str, const ExtensibleNode *node); /* comparison expression */ void out_cypher_comparison_aexpr(StringInfo str, const ExtensibleNode *node); diff --git a/src/include/nodes/cypher_readfuncs.h b/src/include/nodes/cypher_readfuncs.h index 5792332ac..9202ba511 100644 --- a/src/include/nodes/cypher_readfuncs.h +++ b/src/include/nodes/cypher_readfuncs.h @@ -51,4 +51,7 @@ void read_cypher_delete_item(struct ExtensibleNode *node); void read_cypher_merge_information(struct ExtensibleNode *node); +/* predicate function data structure */ +void read_cypher_predicate_function(struct ExtensibleNode *node); + #endif diff --git a/src/include/parser/cypher_kwlist.h b/src/include/parser/cypher_kwlist.h index e4c4437ba..0de294979 100644 --- a/src/include/parser/cypher_kwlist.h +++ b/src/include/parser/cypher_kwlist.h @@ -1,6 +1,7 @@ PG_KEYWORD("all", ALL, RESERVED_KEYWORD) PG_KEYWORD("analyze", ANALYZE, RESERVED_KEYWORD) PG_KEYWORD("and", AND, RESERVED_KEYWORD) +PG_KEYWORD("any", ANY_P, RESERVED_KEYWORD) PG_KEYWORD("as", AS, RESERVED_KEYWORD) PG_KEYWORD("asc", ASC, RESERVED_KEYWORD) PG_KEYWORD("ascending", ASCENDING, RESERVED_KEYWORD) @@ -27,6 +28,7 @@ PG_KEYWORD("is", IS, RESERVED_KEYWORD) PG_KEYWORD("limit", LIMIT, RESERVED_KEYWORD) PG_KEYWORD("match", MATCH, RESERVED_KEYWORD) PG_KEYWORD("merge", MERGE, RESERVED_KEYWORD) +PG_KEYWORD("none", NONE, RESERVED_KEYWORD) PG_KEYWORD("not", NOT, RESERVED_KEYWORD) PG_KEYWORD("null", NULL_P, RESERVED_KEYWORD) PG_KEYWORD("operator", OPERATOR, RESERVED_KEYWORD) @@ -36,6 +38,7 @@ PG_KEYWORD("order", ORDER, RESERVED_KEYWORD) PG_KEYWORD("remove", REMOVE, RESERVED_KEYWORD) PG_KEYWORD("return", RETURN, RESERVED_KEYWORD) PG_KEYWORD("set", SET, RESERVED_KEYWORD) +PG_KEYWORD("single", SINGLE, RESERVED_KEYWORD) PG_KEYWORD("skip", SKIP, RESERVED_KEYWORD) PG_KEYWORD("starts", STARTS, RESERVED_KEYWORD) PG_KEYWORD("then", THEN, RESERVED_KEYWORD) From bdc8b6ddabe210ec02a77d232b742ced9c0d438e Mon Sep 17 00:00:00 2001 From: Ueslei Lima Date: Mon, 20 Apr 2026 06:16:33 -0300 Subject: [PATCH 050/100] fix(python-driver): quote-escape column aliases in buildCypher() (#2373) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(python-driver): quote-escape column aliases in buildCypher() Always double-quote column names in the AS (...) clause generated by buildCypher() and _validate_column(). This prevents PostgreSQL parse errors when column aliases happen to be reserved words (e.g. 'count', 'order', 'type', 'group', 'select'). Before: SELECT * from cypher(NULL,NULL) as (count agtype); After: SELECT * from cypher(NULL,NULL) as ("count" agtype); PostgreSQL always accepts double-quoted identifiers, so quoting unconditionally is safe for all names — no reserved word list needed. Closes #2370 * Address review feedback: document type quoting, improve tests - _validate_column: add comment explaining why type_name is intentionally left unquoted (PG type names are case-insensitive) - buildCypher: fix graphName == None to idiomatic 'is None' - test_reserved_word_count: replace fragile assertNotIn with regex - Add test for reserved word in "name type" pair (e.g. "order agtype") - Add comment explaining intentional _validate_column private import * Fix test_security.py for quoted column names, improve comment - Update test_security.py assertions to expect double-quoted column names (consistent with the always-quote-column-names change) - Reword type_name comment to clarify case-folding semantics Made-with: Cursor --- drivers/python/age/age.py | 23 +++++++---- drivers/python/test_age_py.py | 73 +++++++++++++++++++++++++++++++++ drivers/python/test_security.py | 14 +++---- 3 files changed, 95 insertions(+), 15 deletions(-) diff --git a/drivers/python/age/age.py b/drivers/python/age/age.py index f85b1ab19..ae76bcf50 100644 --- a/drivers/python/age/age.py +++ b/drivers/python/age/age.py @@ -216,14 +216,19 @@ def _validate_column(col: str) -> str: name, type_name = parts validate_identifier(name, "Column name") validate_identifier(type_name, "Column type") - return f"{name} {type_name}" + # Only the column name is double-quoted. The type name is left + # unquoted so PostgreSQL applies its default identifier folding + # for type names in column definitions. Double-quoting would + # make the type name case-sensitive and could change type + # resolution in surprising ways for user-defined types. + return f'"{name}" {type_name}' else: validate_identifier(col, "Column name") - return f"{col} agtype" + return f'"{col}" agtype' def buildCypher(graphName:str, cypherStmt:str, columns:list) ->str: - if graphName == None: + if graphName is None: raise _EXCEPTION_GraphNotSet columnExp=[] @@ -233,16 +238,18 @@ def buildCypher(graphName:str, cypherStmt:str, columns:list) ->str: if validated: columnExp.append(validated) else: - columnExp.append('v agtype') + columnExp.append('"v" agtype') # Design note: String concatenation is used here instead of # psycopg.sql.Identifier() because column specifications are - # "name type" pairs (e.g. "v agtype") that don't map directly to + # "name type" pairs (e.g. '"v" agtype') that don't map directly to # sql.Identifier(). Each component has already been validated by # _validate_column() → validate_identifier(), which restricts - # names to ^[A-Za-z_][A-Za-z0-9_]*$ and max 63 chars. The - # graphName and cypherStmt are NOT embedded here — this template - # only contains the validated column list and static SQL keywords. + # names to ^[A-Za-z_][A-Za-z0-9_]*$ and max 63 chars. Column names + # are always double-quoted to avoid conflicts with PostgreSQL + # reserved words (e.g. "count", "order", "type"). The graphName + # and cypherStmt are NOT embedded here — this template only + # contains the validated column list and static SQL keywords. stmtArr = [] stmtArr.append("SELECT * from cypher(NULL,NULL) as (") stmtArr.append(','.join(columnExp)) diff --git a/drivers/python/test_age_py.py b/drivers/python/test_age_py.py index 960d28d4d..12dd7bd55 100644 --- a/drivers/python/test_age_py.py +++ b/drivers/python/test_age_py.py @@ -13,12 +13,18 @@ # specific language governing permissions and limitations # under the License. import json +import re from age.models import Vertex import unittest import unittest.mock import decimal import age +# _validate_column is private but tested directly because its quoting +# behavior is security-relevant and the public surface (buildCypher) +# makes it difficult to isolate quoting assertions. +from age.age import buildCypher, _validate_column +from age.exceptions import InvalidIdentifier import argparse TEST_HOST = "localhost" @@ -99,6 +105,72 @@ def test_connect_forwards_skip_load_to_setup(self): ) +class TestBuildCypher(unittest.TestCase): + """Unit tests for buildCypher() and _validate_column() — no DB required.""" + + def test_simple_column(self): + result = buildCypher("g", "MATCH (n) RETURN n", ["n"]) + self.assertIn('"n" agtype', result) + + def test_column_with_type(self): + result = buildCypher("g", "MATCH (n) RETURN n", ["n agtype"]) + self.assertIn('"n" agtype', result) + + def test_reserved_word_count(self): + """Issue #2370: 'count' is a PostgreSQL reserved word.""" + result = buildCypher("g", "MATCH (n) RETURN count(n)", ["count"]) + self.assertIn('"count" agtype', result) + # Verify 'count' never appears unquoted as a column name + self.assertIsNone( + re.search(r'(?() RETURN type(r)", ["type"]) + self.assertIn('"type" agtype', result) + + def test_reserved_word_select(self): + """Issue #2370: 'select' is a PostgreSQL reserved word.""" + result = buildCypher("g", "MATCH (n) RETURN n", ["select"]) + self.assertIn('"select" agtype', result) + + def test_reserved_word_group(self): + """Issue #2370: 'group' is a PostgreSQL reserved word.""" + result = buildCypher("g", "MATCH (n) RETURN n", ["group"]) + self.assertIn('"group" agtype', result) + + def test_multiple_columns(self): + result = buildCypher("g", "MATCH (n) RETURN n.name, count(n)", ["name", "count"]) + self.assertIn('"name" agtype', result) + self.assertIn('"count" agtype', result) + + def test_default_column(self): + result = buildCypher("g", "MATCH (n) RETURN n", None) + self.assertIn('"v" agtype', result) + + def test_invalid_column_rejected(self): + with self.assertRaises(InvalidIdentifier): + buildCypher("g", "MATCH (n) RETURN n", ["invalid;col"]) + + def test_reserved_word_in_name_type_pair(self): + """Quoting applies even when the column is specified as 'name type'.""" + result = buildCypher("g", "MATCH (n) RETURN n.order", ["order agtype"]) + self.assertIn('"order" agtype', result) + + def test_validate_column_quoting(self): + self.assertEqual(_validate_column("v"), '"v" agtype') + self.assertEqual(_validate_column("v agtype"), '"v" agtype') + self.assertEqual(_validate_column("count"), '"count" agtype') + self.assertEqual(_validate_column("my_col"), '"my_col" agtype') + + class TestAgeBasic(unittest.TestCase): ag = None args: argparse.Namespace = argparse.Namespace( @@ -558,6 +630,7 @@ def testSerialization(self): suite = unittest.TestSuite() loader = unittest.TestLoader() suite.addTests(loader.loadTestsFromTestCase(TestSetUpAge)) + suite.addTests(loader.loadTestsFromTestCase(TestBuildCypher)) suite.addTest(TestAgeBasic("testExec")) suite.addTest(TestAgeBasic("testQuery")) suite.addTest(TestAgeBasic("testChangeData")) diff --git a/drivers/python/test_security.py b/drivers/python/test_security.py index 55347868e..01afee793 100644 --- a/drivers/python/test_security.py +++ b/drivers/python/test_security.py @@ -167,10 +167,10 @@ class TestColumnValidation(unittest.TestCase): """Test _validate_column prevents injection through column specs.""" def test_plain_column_name(self): - self.assertEqual(_validate_column('v'), 'v agtype') + self.assertEqual(_validate_column('v'), '"v" agtype') def test_column_with_type(self): - self.assertEqual(_validate_column('n agtype'), 'n agtype') + self.assertEqual(_validate_column('n agtype'), '"n" agtype') def test_empty_column(self): self.assertEqual(_validate_column(''), '') @@ -198,20 +198,20 @@ class TestBuildCypher(unittest.TestCase): def test_default_column(self): result = buildCypher('test_graph', 'MATCH (n) RETURN n', None) - self.assertIn('v agtype', result) + self.assertIn('"v" agtype', result) def test_single_column(self): result = buildCypher('test_graph', 'MATCH (n) RETURN n', ['n']) - self.assertIn('n agtype', result) + self.assertIn('"n" agtype', result) def test_typed_column(self): result = buildCypher('test_graph', 'MATCH (n) RETURN n', ['n agtype']) - self.assertIn('n agtype', result) + self.assertIn('"n" agtype', result) def test_multiple_columns(self): result = buildCypher('test_graph', 'MATCH (n) RETURN n', ['a', 'b']) - self.assertIn('a agtype', result) - self.assertIn('b agtype', result) + self.assertIn('"a" agtype', result) + self.assertIn('"b" agtype', result) def test_rejects_injection_in_column(self): with self.assertRaises(InvalidIdentifier): From 6c4083862fbf59309a7735aafa7ab1def1bda13d Mon Sep 17 00:00:00 2001 From: Sai Asish Y Date: Mon, 20 Apr 2026 02:19:01 -0700 Subject: [PATCH 051/100] Fix substring() crash when start-offset is NULL and length is supplied (#2401) age_substring() reads the null map produced by extract_variadic_args() and rejects null offset/length with this guard: if ((nargs == 2 && nulls[1]) || (nargs == 3 && nulls[2])) { ereport(ERROR, ..., errmsg("substring() offset or length cannot be null")); } The condition only checks nulls[1] in the 2-argument form. When the caller passes `substring(str, null, len)` the function takes nargs = 3, nulls[1] = true, but the guard above does not fire. Execution reaches the numeric-parameter loop below, which reads args[1] through DatumGetInt32 / DATUM_GET_AGTYPE_P without ever re-checking nulls[i]. The Datum in that slot is undefined, the dereference segfaults, and the PostgreSQL backend terminates - not a query error but a connection-level crash (#2386). Widen the guard to nargs >= 2 && nulls[1] so it catches start-is-null in both the 2-arg and 3-arg forms. nulls[2] is still only checked when nargs == 3. No behaviour change on any non-null path; the connection-crash case is now reported as a normal query error, matching the intent the existing error message already implies. --- src/backend/utils/adt/agtype.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index 386219556..b2215d016 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -8639,8 +8639,15 @@ Datum age_substring(PG_FUNCTION_ARGS) PG_RETURN_NULL(); } - /* neither offset or length can be null if there is a valid string */ - if ((nargs == 2 && nulls[1]) || + /* + * neither offset nor length may be null when there is a valid string. + * Both arg positions must be checked whenever they are supplied; the + * previous condition missed the `start is null, length is provided` + * case (nargs == 3 && nulls[1]), which fell through to the numeric + * parser below and dereferenced an undefined Datum - crashing the + * backend (#2386). + */ + if ((nargs >= 2 && nulls[1]) || (nargs == 3 && nulls[2])) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), From 908d7b29811a23d7b01c024af5d6697fe670e123 Mon Sep 17 00:00:00 2001 From: Sai Asish Y Date: Mon, 20 Apr 2026 02:32:54 -0700 Subject: [PATCH 052/100] Return an empty list from tail() for empty and singleton lists (#2399) age_tail() in src/backend/utils/adt/agtype.c had an explicit early return that mapped both the empty-list and the singleton-list cases to SQL NULL: count = AGT_ROOT_COUNT(agt_arg); /* if we have an empty list or only one element in the list, return null */ if (count <= 1) { PG_RETURN_NULL(); } The Cypher specification (and Neo4j / Memgraph) both define tail(list) as "the list without its first element". For a zero-or-one-element input that is an empty list [], not null. Differential tests against Neo4j surface this divergence immediately. Remove the early return. The loop below already iterates from i=1 so it naturally emits [] for count <= 1 and emits the correct tail for the general case. No behaviour change for count >= 2. Tests and expected output are updated to match the new, spec-correct result. --- regress/expected/expr.out | 6 +++--- regress/sql/expr.sql | 2 +- src/backend/utils/adt/agtype.c | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/regress/expected/expr.out b/regress/expected/expr.out index 6d9341451..e95fe14e3 100644 --- a/regress/expected/expr.out +++ b/regress/expected/expr.out @@ -8039,17 +8039,17 @@ SELECT * FROM cypher('list', $$ RETURN tail(["a","b","c","d","e"]) $$) AS (tail ["b", "c", "d", "e"] (1 row) --- should return null +-- should return an empty list SELECT * FROM cypher('list', $$ RETURN tail([1]) $$) AS (tail agtype); tail ------ - + [] (1 row) SELECT * FROM cypher('list', $$ RETURN tail([]) $$) AS (tail agtype); tail ------ - + [] (1 row) -- should throw errors diff --git a/regress/sql/expr.sql b/regress/sql/expr.sql index 445e2d237..7304d4d65 100644 --- a/regress/sql/expr.sql +++ b/regress/sql/expr.sql @@ -3269,7 +3269,7 @@ SELECT * from cypher('list', $$RETURN range(0, -10.0, -3.0)$$) as (range agtype) -- should return the last elements of the list SELECT * FROM cypher('list', $$ RETURN tail([1,2,3,4,5]) $$) AS (tail agtype); SELECT * FROM cypher('list', $$ RETURN tail(["a","b","c","d","e"]) $$) AS (tail agtype); --- should return null +-- should return an empty list SELECT * FROM cypher('list', $$ RETURN tail([1]) $$) AS (tail agtype); SELECT * FROM cypher('list', $$ RETURN tail([]) $$) AS (tail agtype); -- should throw errors diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index b2215d016..8fad00bdf 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -6056,11 +6056,11 @@ Datum age_tail(PG_FUNCTION_ARGS) count = AGT_ROOT_COUNT(agt_arg); - /* if we have an empty list or only one element in the list, return null */ - if (count <= 1) - { - PG_RETURN_NULL(); - } + /* + * For an empty or singleton list, tail() returns an empty list. The loop + * below already produces that result (i starts at 1 so nothing is pushed + * when count <= 1), so we do not special-case the count here. + */ /* clear the result structure */ MemSet(&agis_result, 0, sizeof(agtype_in_state)); From 26bd4f4d05e519cc340630bb71dc82e419485f31 Mon Sep 17 00:00:00 2001 From: Ueslei Lima Date: Mon, 20 Apr 2026 07:41:03 -0300 Subject: [PATCH 053/100] fix(python-driver): add null-guards in ANTLR parser and relax runtime version pin (#2372) Fix two related issues in the Python driver's ANTLR4 parsing pipeline: - Add null-guards in ResultVisitor methods (visitAgValue, visitFloatLiteral, visitPair, visitObj, handleAnnotatedValue) to prevent AttributeError crashes when the ANTLR4 parse tree contains None child nodes. This occurs with vertices that have complex properties (large arrays, special characters, deeply nested structures). - Relax antlr4-python3-runtime version constraint from ==4.11.1 to >=4.11.1,<5.0 in both pyproject.toml and requirements.txt. The 4.11.1 pin is incompatible with Python >= 3.13. The ANTLR ATN serialized format is unchanged between 4.11 and 4.13, so the generated lexer/parser files are compatible. Validated with antlr4-python3-runtime==4.13.2 on Python 3.11-3.14. - Also replaces shadowing of builtin 'dict' in handleAnnotatedValue with 'd', and uses .get() for safer key access on parsed vertex/edge dicts. - Add tests for malformed/truncated agtype input handling. Verify that malformed and truncated agtype strings raise AGTypeError (or recover gracefully) rather than crashing with AttributeError. This tests the null-guards added to the ANTLR parser visitor. - visitFloatLiteral: raise AGTypeError on malformed child node instead of silently returning a fallback value - visitObj: add comment documenting that visitPair's validation makes the None-guard defensive-only - handleAnnotatedValue: add comment explaining partial-construction behavior on type-check failure - pyproject.toml: add comment explaining ANTLR4 version range rationale - Tests: assert AGTypeError (or graceful recovery) for malformed and truncated inputs, not just absence of AttributeError - handleAnnotatedValue: default properties to {} when missing from parsed dict, preventing __getitem__ crashes on access - Tests: replace weak assertNotIsInstance with structural type checks - Fix truncated test docstring to match actual assertion behavior - Use PostgreSQL "$user" placeholder in SET search_path. - Exercise real escapes and Unicode in special-characters vertex test (json.dumps). - Add Python 3.9 trove classifier to match requires-python and dependency comment. - Build vertex agtype with string concat to avoid invalid f-string braces. - Assert stored description matches parser behavior: JSON escapes remain literal, UTF-8 decodes normally (ensure_ascii=False on json.dumps). - Regenerate parser with ANTLR 4.13.2 to silence runtime-version-mismatch warning. The generated lexer/parser were hardcoded to check for ANTLR runtime 4.11.1, which triggered a noisy 'ANTLR runtime and generated code versions disagree' warning when installed against a newer runtime like 4.13.2. Regenerating from Agtype.g4 with the 4.13.2 tool aligns the generated checkVersion() call with the default-installed runtime in the allowed dependency range and eliminates the warning. - Bumps the declared floor of antlr4-python3-runtime to 4.13.2 so the default install path is warning-free. Made-with: Cursor --- drivers/python/age/builder.py | 71 ++++++--- drivers/python/age/gen/AgtypeLexer.py | 7 +- drivers/python/age/gen/AgtypeListener.py | 7 +- drivers/python/age/gen/AgtypeParser.py | 7 +- drivers/python/age/gen/AgtypeVisitor.py | 7 +- drivers/python/pyproject.toml | 7 +- drivers/python/requirements.txt | 2 +- drivers/python/test_agtypes.py | 176 ++++++++++++++++++++++- 8 files changed, 245 insertions(+), 39 deletions(-) diff --git a/drivers/python/age/builder.py b/drivers/python/age/builder.py index f1e7a2ce8..08a40c252 100644 --- a/drivers/python/age/builder.py +++ b/drivers/python/age/builder.py @@ -92,9 +92,16 @@ def visitAgValue(self, ctx:AgtypeParser.AgValueContext): if annoCtx is not None: annoCtx.accept(self) - anno = annoCtx.IDENT().getText() + identNode = annoCtx.IDENT() + if identNode is None: + raise AGTypeError(ctx.getText(), "Missing type annotation identifier") + anno = identNode.getText() + if valueCtx is None: + raise AGTypeError(ctx.getText(), "Missing value for annotated type") return self.handleAnnotatedValue(anno, valueCtx) else: + if valueCtx is None: + return None return valueCtx.accept(self) @@ -109,9 +116,14 @@ def visitIntegerValue(self, ctx:AgtypeParser.IntegerValueContext): # Visit a parse tree produced by AgtypeParser#floatLiteral. def visitFloatLiteral(self, ctx:AgtypeParser.FloatLiteralContext): + text = ctx.getText() c = ctx.getChild(0) + if c is None or not hasattr(c, 'symbol') or c.symbol is None: + raise AGTypeError( + str(text), + "Malformed float literal: missing or invalid child node" + ) tp = c.symbol.type - text = ctx.getText() if tp == AgtypeParser.RegularFloat: return float(text) elif tp == AgtypeParser.ExponentFloat: @@ -150,15 +162,27 @@ def visitObj(self, ctx:AgtypeParser.ObjContext): namVal = self.visitPair(c) name = namVal[0] valCtx = namVal[1] - val = valCtx.accept(self) - obj[name] = val + # visitPair() raises AGTypeError when the value node is + # missing, so valCtx should never be None here. The + # guard is kept as a defensive fallback only. + if valCtx is not None: + val = valCtx.accept(self) + obj[name] = val + else: + obj[name] = None return obj # Visit a parse tree produced by AgtypeParser#pair. def visitPair(self, ctx:AgtypeParser.PairContext): self.visitChildren(ctx) - return (ctx.STRING().getText().strip('"') , ctx.agValue()) + strNode = ctx.STRING() + agValNode = ctx.agValue() + if strNode is None: + raise AGTypeError(ctx.getText(), "Missing key in object pair") + if agValNode is None: + raise AGTypeError(ctx.getText(), "Missing value in object pair") + return (strNode.getText().strip('"') , agValNode) # Visit a parse tree produced by AgtypeParser#array. @@ -171,38 +195,49 @@ def visitArray(self, ctx:AgtypeParser.ArrayContext): return li def handleAnnotatedValue(self, anno:str, ctx:ParserRuleContext): + # Each branch below constructs a model object (Vertex, Edge, Path) + # and populates it from the parsed dict/list. If a type check + # fails (e.g. the parsed value is not a dict), AGTypeError is + # raised and the partially-constructed object is discarded — no + # cleanup is needed because the caller propagates the exception. if anno == "numeric": return Decimal(ctx.getText()) elif anno == "vertex": - dict = ctx.accept(self) - vid = dict["id"] + d = ctx.accept(self) + if not isinstance(d, dict): + raise AGTypeError(str(ctx.getText()), "Expected dict for vertex, got " + type(d).__name__) + vid = d.get("id") vertex = None - if self.vertexCache != None and vid in self.vertexCache : + if self.vertexCache is not None and vid in self.vertexCache: vertex = self.vertexCache[vid] else: vertex = Vertex() - vertex.id = dict["id"] - vertex.label = dict["label"] - vertex.properties = dict["properties"] + vertex.id = d.get("id") + vertex.label = d.get("label") + vertex.properties = d.get("properties") or {} - if self.vertexCache != None: + if self.vertexCache is not None: self.vertexCache[vid] = vertex return vertex elif anno == "edge": edge = Edge() - dict = ctx.accept(self) - edge.id = dict["id"] - edge.label = dict["label"] - edge.end_id = dict["end_id"] - edge.start_id = dict["start_id"] - edge.properties = dict["properties"] + d = ctx.accept(self) + if not isinstance(d, dict): + raise AGTypeError(str(ctx.getText()), "Expected dict for edge, got " + type(d).__name__) + edge.id = d.get("id") + edge.label = d.get("label") + edge.end_id = d.get("end_id") + edge.start_id = d.get("start_id") + edge.properties = d.get("properties") or {} return edge elif anno == "path": arr = ctx.accept(self) + if not isinstance(arr, list): + raise AGTypeError(str(ctx.getText()), "Expected list for path, got " + type(arr).__name__) path = Path(arr) return path diff --git a/drivers/python/age/gen/AgtypeLexer.py b/drivers/python/age/gen/AgtypeLexer.py index 54f19c7db..f94dc828b 100644 --- a/drivers/python/age/gen/AgtypeLexer.py +++ b/drivers/python/age/gen/AgtypeLexer.py @@ -12,8 +12,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# Generated from ../Agtype.g4 by ANTLR 4.11.1 - +# Generated from ../Agtype.g4 by ANTLR 4.13.2 from antlr4 import * from io import StringIO import sys @@ -142,9 +141,7 @@ class AgtypeLexer(Lexer): def __init__(self, input=None, output:TextIO = sys.stdout): super().__init__(input, output) - self.checkVersion("4.11.1") + self.checkVersion("4.13.2") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None - - diff --git a/drivers/python/age/gen/AgtypeListener.py b/drivers/python/age/gen/AgtypeListener.py index 6e60da421..39751ac51 100644 --- a/drivers/python/age/gen/AgtypeListener.py +++ b/drivers/python/age/gen/AgtypeListener.py @@ -12,10 +12,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# Generated from ../Agtype.g4 by ANTLR 4.11.1 - +# Generated from ../Agtype.g4 by ANTLR 4.13.2 from antlr4 import * -if __name__ is not None and "." in __name__: +if "." in __name__: from .AgtypeParser import AgtypeParser else: from AgtypeParser import AgtypeParser @@ -159,4 +158,4 @@ def exitFloatLiteral(self, ctx:AgtypeParser.FloatLiteralContext): -del AgtypeParser \ No newline at end of file +del AgtypeParser diff --git a/drivers/python/age/gen/AgtypeParser.py b/drivers/python/age/gen/AgtypeParser.py index daaf578e8..19a43d08f 100644 --- a/drivers/python/age/gen/AgtypeParser.py +++ b/drivers/python/age/gen/AgtypeParser.py @@ -12,9 +12,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# Generated from ../Agtype.g4 by ANTLR 4.11.1 +# Generated from ../Agtype.g4 by ANTLR 4.13.2 # encoding: utf-8 - from antlr4 import * from io import StringIO import sys @@ -108,7 +107,7 @@ class AgtypeParser ( Parser ): def __init__(self, input:TokenStream, output:TextIO = sys.stdout): super().__init__(input, output) - self.checkVersion("4.11.1") + self.checkVersion("4.13.2") self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache) self._predicates = None @@ -854,5 +853,3 @@ def floatLiteral(self): finally: self.exitRule() return localctx - - diff --git a/drivers/python/age/gen/AgtypeVisitor.py b/drivers/python/age/gen/AgtypeVisitor.py index 824eb512e..8ea2f1479 100644 --- a/drivers/python/age/gen/AgtypeVisitor.py +++ b/drivers/python/age/gen/AgtypeVisitor.py @@ -12,10 +12,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# Generated from ../Agtype.g4 by ANTLR 4.11.1 - +# Generated from ../Agtype.g4 by ANTLR 4.13.2 from antlr4 import * -if __name__ is not None and "." in __name__: +if "." in __name__: from .AgtypeParser import AgtypeParser else: from AgtypeParser import AgtypeParser @@ -100,4 +99,4 @@ def visitFloatLiteral(self, ctx:AgtypeParser.FloatLiteralContext): -del AgtypeParser \ No newline at end of file +del AgtypeParser diff --git a/drivers/python/pyproject.toml b/drivers/python/pyproject.toml index 18112381c..651fc2057 100644 --- a/drivers/python/pyproject.toml +++ b/drivers/python/pyproject.toml @@ -29,6 +29,7 @@ authors = [ {name = "Ikchan Kwon, Apache AGE", email = "dev-subscribe@age.apache.org"} ] classifiers = [ + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -37,7 +38,11 @@ classifiers = [ ] dependencies = [ "psycopg", - "antlr4-python3-runtime==4.11.1", + # Parser is generated with ANTLR 4.13.2. Runtime is forward- and + # backward-compatible within the 4.x series (checkVersion() emits a + # warning on mismatch but parsing still works); tested on 4.11.1–4.13.2 + # with Python 3.9–3.14. + "antlr4-python3-runtime>=4.13.2,<5.0", ] [project.urls] diff --git a/drivers/python/requirements.txt b/drivers/python/requirements.txt index 449d38c67..ceadc06ce 100644 --- a/drivers/python/requirements.txt +++ b/drivers/python/requirements.txt @@ -1,4 +1,4 @@ psycopg -antlr4-python3-runtime==4.11.1 +antlr4-python3-runtime>=4.11.1,<5.0 setuptools networkx diff --git a/drivers/python/test_agtypes.py b/drivers/python/test_agtypes.py index 4e9752e61..97f7972d1 100644 --- a/drivers/python/test_agtypes.py +++ b/drivers/python/test_agtypes.py @@ -13,9 +13,11 @@ # specific language governing permissions and limitations # under the License. +import json +import math import unittest from decimal import Decimal -import math + import age @@ -127,6 +129,178 @@ def test_path(self): self.assertEqual(vertexEnd.label, "Person") self.assertEqual(vertexEnd["name"], "Joe") + def test_vertex_large_array_properties(self): + """Issue #2367: Parser should handle vertices with large array properties.""" + vertexExp = ( + '{"id": 1125899906842625, "label": "TestNode", ' + '"properties": {"name": "test", ' + '"tags": ["tag1", "tag2", "tag3", "tag4", "tag5", "tag6", "tag7", ' + '"tag8", "tag9", "tag10", "tag11", "tag12"]}}::vertex' + ) + vertex = self.parse(vertexExp) + self.assertEqual(vertex.id, 1125899906842625) + self.assertEqual(vertex.label, "TestNode") + self.assertEqual(vertex["name"], "test") + self.assertEqual(len(vertex["tags"]), 12) + self.assertEqual(vertex["tags"][0], "tag1") + self.assertEqual(vertex["tags"][11], "tag12") + + def test_vertex_special_characters_in_properties(self): + """Issue #2367: Parser accepts JSON-escaped property strings and UTF-8.""" + # Input uses json.dumps so quotes, backslashes, and newlines are valid JSON. + logical_description = 'Quoted "text", path C:\\tmp\\file, line1\nline2, café 雪' + props = json.dumps( + {"name": "test", "description": logical_description}, + ensure_ascii=False, + ) + vertexExp = ( + '{"id": 1125899906842626, "label": "TestNode", ' + '"properties": ' + props + '}::vertex' + ) + vertex = self.parse(vertexExp) + self.assertEqual(vertex.id, 1125899906842626) + self.assertEqual(vertex["name"], "test") + # The agtype visitor keeps JSON string escapes as literal characters + # (except UTF-8 code points, which decode normally). + self.assertEqual( + vertex["description"], + 'Quoted \\"text\\", path C:\\\\tmp\\\\file, line1\\nline2, café 雪', + ) + + def test_vertex_nested_properties(self): + """Issue #2367: Parser should handle deeply nested property structures.""" + vertexExp = ( + '{"id": 1125899906842627, "label": "TestNode", ' + '"properties": {"name": "test", ' + '"metadata": {"level1": {"level2": {"level3": "deep_value"}}}}}::vertex' + ) + vertex = self.parse(vertexExp) + self.assertEqual(vertex.id, 1125899906842627) + self.assertEqual(vertex["name"], "test") + self.assertEqual(vertex["metadata"]["level1"]["level2"]["level3"], "deep_value") + + def test_vertex_empty_properties(self): + """Parser should handle vertices with empty properties dict.""" + vertexExp = '{"id": 1125899906842628, "label": "EmptyNode", "properties": {}}::vertex' + vertex = self.parse(vertexExp) + self.assertEqual(vertex.id, 1125899906842628) + self.assertEqual(vertex.label, "EmptyNode") + self.assertEqual(vertex.properties, {}) + + def test_vertex_null_property_values(self): + """Parser should handle vertices with null property values.""" + vertexExp = ( + '{"id": 1125899906842629, "label": "TestNode", ' + '"properties": {"name": "test", "optional": null, "also_null": null}}::vertex' + ) + vertex = self.parse(vertexExp) + self.assertEqual(vertex["name"], "test") + self.assertIsNone(vertex["optional"]) + self.assertIsNone(vertex["also_null"]) + + def test_edge_with_complex_properties(self): + """Parser should handle edges with complex property structures.""" + edgeExp = ( + '{"id": 2533274790396577, "label": "HAS_RELATION", ' + '"end_id": 1125899906842625, "start_id": 1125899906842626, ' + '"properties": {"weight": 3, "tags": ["a", "b", "c"], "active": true}}::edge' + ) + edge = self.parse(edgeExp) + self.assertEqual(edge.id, 2533274790396577) + self.assertEqual(edge.label, "HAS_RELATION") + self.assertEqual(edge.start_id, 1125899906842626) + self.assertEqual(edge.end_id, 1125899906842625) + self.assertEqual(edge["weight"], 3) + self.assertEqual(edge["tags"], ["a", "b", "c"]) + self.assertEqual(edge["active"], True) + + def test_path_with_multiple_edges(self): + """Parser should handle paths with multiple edges and complex properties.""" + pathExp = ( + '[{"id": 1, "label": "A", "properties": {"name": "start"}}::vertex, ' + '{"id": 10, "label": "r1", "end_id": 2, "start_id": 1, "properties": {"w": 1}}::edge, ' + '{"id": 2, "label": "B", "properties": {"name": "middle"}}::vertex, ' + '{"id": 11, "label": "r2", "end_id": 3, "start_id": 2, "properties": {"w": 2}}::edge, ' + '{"id": 3, "label": "C", "properties": {"name": "end"}}::vertex]::path' + ) + path = self.parse(pathExp) + self.assertEqual(len(path), 5) + self.assertEqual(path[0]["name"], "start") + self.assertEqual(path[2]["name"], "middle") + self.assertEqual(path[4]["name"], "end") + + def test_empty_input(self): + """Parser should handle empty/null input gracefully.""" + self.assertIsNone(self.parse('')) + self.assertIsNone(self.parse(None)) + + def test_array_of_mixed_types(self): + """Parser should handle arrays with mixed types including nested arrays.""" + arrStr = '["str", 42, true, null, [1, 2, 3], {"key": "val"}]' + result = self.parse(arrStr) + self.assertEqual(result[0], "str") + self.assertEqual(result[1], 42) + self.assertEqual(result[2], True) + self.assertIsNone(result[3]) + self.assertEqual(result[4], [1, 2, 3]) + self.assertEqual(result[5], {"key": "val"}) + + def test_malformed_vertex_raises_agtypeerror_or_recovers(self): + """Issue #2367: Malformed agtype must raise AGTypeError or recover gracefully.""" + from age.exceptions import AGTypeError + + malformed_inputs = [ + '{"id": 1, "label":}::vertex', + '{"id": 1, "label": "X", "properties": {}::vertex', + '{::vertex', + '{"id": 1, "label": "X", "properties": {"key":}}::vertex', + ] + for inp in malformed_inputs: + try: + result = self.parse(inp) + # Parser recovery is acceptable — verify the result is a + # usable Python value (None, container, or model object). + self.assertTrue( + result is None + or isinstance(result, (dict, list, tuple)) + or hasattr(result, "__dict__"), + f"Recovered to unexpected type {type(result).__name__}: {inp}" + ) + except AGTypeError: + pass # expected + except AttributeError: + self.fail( + f"Malformed input raised AttributeError instead of " + f"AGTypeError: {inp}" + ) + + def test_truncated_agtype_does_not_crash(self): + """Issue #2367: Truncated agtype must raise AGTypeError or recover, never AttributeError.""" + from age.exceptions import AGTypeError + + truncated_inputs = [ + '{"id": 1, "label": "X", "properties": {"name": "te', + '{"id": 1, "label": "X"', + '[{"id": 1}::vertex, {"id": 2', + ] + for inp in truncated_inputs: + try: + result = self.parse(inp) + # Recovery is acceptable for truncated input + self.assertTrue( + result is None + or isinstance(result, (dict, list, tuple)) + or hasattr(result, "__dict__"), + f"Recovered to unexpected type {type(result).__name__}: {inp}" + ) + except AGTypeError: + pass # expected + except AttributeError: + self.fail( + f"Truncated input raised AttributeError instead of " + f"AGTypeError: {inp}" + ) + if __name__ == '__main__': unittest.main() From ee25e2a381271d60d0fae9ff21a78f8a539bca3d Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Mon, 20 Apr 2026 20:31:39 +0500 Subject: [PATCH 054/100] CI: fail build on compiler and bison warnings (#2398) - installcheck.yaml: build with COPT=-Werror so compiler warnings fail CI - Makefile: add -Werror to BISONFLAGS for cypher_gram.c so bison conflicts/warnings fail the build --- .github/workflows/installcheck.yaml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/installcheck.yaml b/.github/workflows/installcheck.yaml index 0cbb33b90..886dc08f1 100644 --- a/.github/workflows/installcheck.yaml +++ b/.github/workflows/installcheck.yaml @@ -48,7 +48,7 @@ jobs: - name: Build AGE id: build run: | - make PG_CONFIG=$HOME/pg18/bin/pg_config install -j$(nproc) + make PG_CONFIG=$HOME/pg18/bin/pg_config COPT=-Werror install -j$(nproc) - name: Pull and build pgvector id: pgvector diff --git a/Makefile b/Makefile index 76f63c248..5405665d8 100644 --- a/Makefile +++ b/Makefile @@ -221,7 +221,7 @@ src/include/parser/cypher_kwlist_d.h: src/include/parser/cypher_kwlist.h $(GEN_K src/include/parser/cypher_gram_def.h: src/backend/parser/cypher_gram.c -src/backend/parser/cypher_gram.c: BISONFLAGS += --defines=src/include/parser/cypher_gram_def.h +src/backend/parser/cypher_gram.c: BISONFLAGS += --defines=src/include/parser/cypher_gram_def.h -Werror src/backend/parser/cypher_parser.o: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/backend/parser/cypher_parser.bc: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h From 798917c23e88b5297f5b8f1862418302e7953988 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Tue, 21 Apr 2026 00:50:23 -0700 Subject: [PATCH 055/100] VLE cache + performance improvements (#2376) VLE cache + perf: version counter, thin entries, variadic elimination, edge cleanup list removal Replace snapshot-based VLE cache invalidation with per-graph version counters and add performance optimizations for VLE traversal. Cache invalidation: - Add DSM (PG 17+) and shmem (PG <17) per-graph monotonic version counters for cross-backend cache invalidation - Replace snapshot xmin/xmax/curcid comparison with version counter check in is_ggctx_invalid(), with snapshot fallback for safety - Add executor hooks in CREATE/DELETE/SET/MERGE to increment the graph version counter on mutations - Add SQL trigger function (age_invalidate_graph_cache) for catching SQL-level mutations (INSERT/UPDATE/DELETE/TRUNCATE) - Auto-install trigger on new label tables via label_commands.c with LookupFuncName check for backward compatibility - Add TRUNCATE interception in ProcessUtility hook (ag_catalog.c) - Add shmem_request/startup hooks for PG <17 (age.c) - Use search_label_relation_cache() in get_graph_oid_for_table() for fast label-to-graph lookup instead of catalog table scan Thin entries (lazy property fetch): - Replace Datum edge_properties/vertex_properties with 6-byte ItemPointerData TID in vertex_entry and edge_entry - Add get_vertex_entry_properties() and get_edge_entry_properties() that do heap_fetch via stored TID on demand - Add is_an_edge_match() fast path: skip property access entirely for label-only VLE patterns (the common case) Edge cleanup list removal: - Remove ggctx->edges linked list from GRAPH_global_context. With thin entries, edge_entry has no palloc'd sub-structures to free (TIDs are inline). The cleanup list existed to pfree datumCopy'd edge_properties Datums, which no longer exist. hash_destroy() handles all edge hash table memory. Saves ~5.6 GB at SF10. Performance optimizations: - Reduce VERTEX/EDGE_HTAB_INITIAL_SIZE from 1,000,000 to 10,000. PG dynahash grows automatically; large initial size wastes memory. - Eliminate extract_variadic_args in age_match_vle_terminal_edge: direct PG_GETARG_DATUM + cached arg types via fn_extra - Eliminate extract_variadic_args in age_match_vle_edge_to_id_qual: same pattern - Add agtype_access_operator 2-arg fast path: bypasses extract_variadic_args_min for the common property access case - Add age_tointeger 1-arg fast path: bypasses extract_variadic_args with cached arg type - Add GraphIdStack (flat array-based) DFS stacks replacing ListGraphId linked-list stacks for push/pop without palloc/pfree Upgrade script: - Add trigger installation on pre-existing label tables during extension upgrade via DO block in age--1.7.0--y.y.y.sql. Tables created after upgrade get triggers automatically via label_commands.c. Regression tests: - Add VLE cache invalidation tests (CREATE, DELETE, SET mutations) - Add thin entry edge property fetch tests (RETURN p, UNWIND) - Add direct SQL trigger tests (INSERT, UPDATE, DELETE, TRUNCATE on label tables with VLE cache invalidation verification) All 32 regression tests pass. SF3 Benchmarks (9.3M vertices, 52.7M edges, warm cache, median): Total IC1-IC12: 1,530s -> 1,159s (-24.3%, 371s saved) VLE-heavy queries: IC3 (KNOWS*1..2 + 2 countries): 34.9s -> 20.7s (-40.6%) IC5 (forum members, KNOWS*1..2): 46.2s -> 29.9s (-35.4%) IC6 (tag co-occurrence, KNOWS*1..2): 28.2s -> 19.2s (-31.8%) IC9 (recent messages, KNOWS*1..2): 86.0s -> 60.4s (-29.7%) IC11 (KNOWS*1..2 + WORK_AT): 18.7s -> 11.0s (-41.5%) IC1 (KNOWS*1..3 + profile): 1269.4s -> 974.9s (-23.2%) Short Reads (IS1-IS7): No meaningful change -- non-VLE queries, sub-millisecond to sub-second. Within run-to-run variance. Updates (IU1-IU8, SF3, median, 50 ops each): IU2 (Like Post): 99.5ms -> 6.3ms (-93.6%) IU3 (Like Comment): 318.3ms -> 5.7ms (-98.2%) IU7 (Add Comment): 344.4ms -> 11.3ms (-96.7%) IU5 (Forum Member): 12.3ms -> 5.9ms (-52.0%) Version counter eliminates redundant VLE cache rebuilds on mutations. Previously, every INSERT/UPDATE/DELETE invalidated the cache via snapshot comparison, forcing a full rebuild on the next VLE query. Now, mutations just bump a counter; rebuild only occurs when VLE actually runs and finds the counter changed. Graph cache memory (SF3, calculated): Total: ~15.7 GB -> ~8.7 GB (-45%, 7.0 GB saved) Thin entries (TID replaces datumCopy'd properties): -5.3 GB Edge cleanup list removal (no longer needed): -1.7 GB Co-authored-by: Claude Opus modified: age--1.7.0--y.y.y.sql modified: regress/expected/age_global_graph.out modified: regress/sql/age_global_graph.sql modified: sql/age_main.sql modified: src/backend/age.c modified: src/backend/catalog/ag_catalog.c modified: src/backend/commands/label_commands.c modified: src/backend/executor/cypher_create.c modified: src/backend/executor/cypher_delete.c modified: src/backend/executor/cypher_merge.c modified: src/backend/executor/cypher_set.c modified: src/backend/utils/adt/age_global_graph.c modified: src/backend/utils/adt/age_graphid_ds.c modified: src/backend/utils/adt/age_vle.c modified: src/backend/utils/adt/agtype.c modified: src/include/utils/age_global_graph.h modified: src/include/utils/age_graphid_ds.h --- age--1.7.0--y.y.y.sql | 51 ++ regress/expected/age_global_graph.out | 288 +++++++++++ regress/sql/age_global_graph.sql | 176 +++++++ sql/age_main.sql | 11 + src/backend/age.c | 36 ++ src/backend/catalog/ag_catalog.c | 55 ++- src/backend/commands/label_commands.c | 60 +++ src/backend/executor/cypher_create.c | 4 + src/backend/executor/cypher_delete.c | 7 + src/backend/executor/cypher_merge.c | 7 + src/backend/executor/cypher_set.c | 8 + src/backend/utils/adt/age_global_graph.c | 581 +++++++++++++++++++---- src/backend/utils/adt/age_graphid_ds.c | 91 ++++ src/backend/utils/adt/age_vle.c | 316 ++++++------ src/backend/utils/adt/agtype.c | 167 ++++++- src/include/utils/age_global_graph.h | 12 + src/include/utils/age_graphid_ds.h | 25 + 17 files changed, 1629 insertions(+), 266 deletions(-) diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index 4f8c54a30..a3cdea279 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -408,3 +408,54 @@ $function$; COMMENT ON FUNCTION ag_catalog.age_pg_upgrade_status() IS 'Returns the current pg_upgrade readiness status of the AGE installation.'; + +-- +-- VLE cache invalidation trigger function +-- Installed on graph label tables to catch SQL-level mutations +-- and increment the per-graph version counter for VLE cache invalidation. +-- +CREATE FUNCTION ag_catalog.age_invalidate_graph_cache() + RETURNS trigger + LANGUAGE c +AS 'MODULE_PATHNAME'; + +-- +-- Install the cache invalidation trigger on all pre-existing label tables. +-- New label tables created after this upgrade will get the trigger +-- automatically via label_commands.c. This DO block handles tables +-- that were created before the upgrade. +-- +DO $$ +DECLARE + r RECORD; +BEGIN + FOR r IN + SELECT n.nspname AS schema_name, c.relname AS table_name + FROM ag_catalog.ag_label l + JOIN pg_catalog.pg_class c ON c.oid = l.relation + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE l.name != '_ag_label_vertex' + AND l.name != '_ag_label_edge' + LOOP + -- Skip if trigger already exists on this table + IF NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_trigger t + JOIN pg_catalog.pg_class c ON c.oid = t.tgrelid + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = r.schema_name + AND c.relname = r.table_name + AND t.tgname = '_age_cache_invalidate' + ) + THEN + EXECUTE format( + 'CREATE TRIGGER _age_cache_invalidate ' + 'AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ' + 'ON %I.%I ' + 'FOR EACH STATEMENT ' + 'EXECUTE FUNCTION ag_catalog.age_invalidate_graph_cache()', + r.schema_name, r.table_name + ); + END IF; + END LOOP; +END; +$$; diff --git a/regress/expected/age_global_graph.out b/regress/expected/age_global_graph.out index 478637800..cbfeb6f3c 100644 --- a/regress/expected/age_global_graph.out +++ b/regress/expected/age_global_graph.out @@ -413,6 +413,294 @@ NOTICE: graph "ag_graph_3" has been dropped (1 row) +----------------------------------------------------------------------------------------------------------------------------- +-- +-- VLE cache invalidation tests +-- +-- These tests verify that the graph version counter properly invalidates +-- the VLE hash table cache when the graph is mutated, and that thin +-- entry lazy property fetch returns correct data. +-- +-- Setup: create a graph with a chain a->b->c->d +SELECT * FROM create_graph('vle_cache_test'); +NOTICE: graph "vle_cache_test" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('vle_cache_test', $$ + CREATE (a:Node {name: 'a'})-[:Edge]->(b:Node {name: 'b'})-[:Edge]->(c:Node {name: 'c'})-[:Edge]->(d:Node {name: 'd'}) +$$) AS (v agtype); + v +--- +(0 rows) + +-- VLE query: find all paths from a's neighbors (should find b, b->c, b->c->d) +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + name +------ + "b" + "c" + "d" +(3 rows) + +-- Now add a new node e connected to d. This should invalidate the cache. +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (d:Node {name: 'd'}) + CREATE (d)-[:Edge]->(:Node {name: 'e'}) +$$) AS (v agtype); + v +--- +(0 rows) + +-- VLE query again: should now also find e via a->b->c->d->e (4 hops won't reach, +-- but d->e is 1 hop from d, and a->b->c->d->e would be 4 hops from a). +-- Increase range to *1..4 to include e +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..4]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + name +------ + "b" + "c" + "d" + "e" +(4 rows) + +-- Test cache invalidation on DELETE: remove node c and its edges +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (c:Node {name: 'c'}) + DETACH DELETE c +$$) AS (v agtype); + v +--- +(0 rows) + +-- VLE query: should only find b now (c is gone, so b->c path is broken) +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..4]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + name +------ + "b" +(1 row) + +-- Test cache invalidation on SET: change b's name property +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (b:Node {name: 'b'}) + SET b.name = 'b_modified' + RETURN b.name +$$) AS (name agtype); + name +-------------- + "b_modified" +(1 row) + +-- VLE query: verify the updated property is returned via lazy fetch +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..4]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + name +-------------- + "b_modified" +(1 row) + +-- Test VLE with edge properties (exercises thin entry edge property fetch) +SELECT * FROM drop_graph('vle_cache_test', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table vle_cache_test._ag_label_vertex +drop cascades to table vle_cache_test._ag_label_edge +drop cascades to table vle_cache_test."Node" +drop cascades to table vle_cache_test."Edge" +NOTICE: graph "vle_cache_test" has been dropped + drop_graph +------------ + +(1 row) + +SELECT * FROM create_graph('vle_cache_test2'); +NOTICE: graph "vle_cache_test2" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('vle_cache_test2', $$ + CREATE (a:N {name: 'a'})-[:E {weight: 1}]->(b:N {name: 'b'})-[:E {weight: 2}]->(c:N {name: 'c'}) +$$) AS (v agtype); + v +--- +(0 rows) + +-- VLE path output to verify edge properties are fetched correctly via +-- thin entry lazy fetch. Returning the full path forces build_path() +-- to call get_edge_entry_properties() for each edge in the result. +-- The output must contain the correct weight values (1 and 2). +SELECT * FROM cypher('vle_cache_test2', $$ + MATCH p=(a:N {name: 'a'})-[:E *1..2]->(n:N) + RETURN p + ORDER BY n.name +$$) AS (p agtype); + p +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + [{"id": 844424930131969, "label": "N", "properties": {"name": "a"}}::vertex, {"id": 1125899906842626, "label": "E", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {"weight": 1}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "b"}}::vertex]::path + [{"id": 844424930131969, "label": "N", "properties": {"name": "a"}}::vertex, {"id": 1125899906842626, "label": "E", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {"weight": 1}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "b"}}::vertex, {"id": 1125899906842625, "label": "E", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {"weight": 2}}::edge, {"id": 844424930131971, "label": "N", "properties": {"name": "c"}}::vertex]::path +(2 rows) + +-- VLE edge properties via UNWIND + relationships() to individually verify +-- each edge's properties are correctly fetched from the heap via TID. +SELECT * FROM cypher('vle_cache_test2', $$ + MATCH p=(a:N {name: 'a'})-[:E *1..2]->(n:N) + WITH p, n + UNWIND relationships(p) AS e + RETURN n.name, e.weight + ORDER BY n.name, e.weight +$$) AS (name agtype, weight agtype); + name | weight +------+-------- + "b" | 1 + "c" | 1 + "c" | 2 +(3 rows) + +-- Cleanup +SELECT * FROM drop_graph('vle_cache_test2', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table vle_cache_test2._ag_label_vertex +drop cascades to table vle_cache_test2._ag_label_edge +drop cascades to table vle_cache_test2."N" +drop cascades to table vle_cache_test2."E" +NOTICE: graph "vle_cache_test2" has been dropped + drop_graph +------------ + +(1 row) + +----------------------------------------------------------------------------------------------------------------------------- +-- +-- VLE cache invalidation via direct SQL (trigger tests) +-- +-- These tests verify that the SQL trigger (age_invalidate_graph_cache) +-- properly invalidates the VLE cache when label tables are mutated +-- via direct SQL INSERT, UPDATE, DELETE, and TRUNCATE — not via Cypher. +-- +-- Setup: create graph with a chain a->b->c using Cypher +SELECT * FROM create_graph('vle_trigger_test'); +NOTICE: graph "vle_trigger_test" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('vle_trigger_test', $$ + CREATE (a:Node {name: 'a'})-[:Edge]->(b:Node {name: 'b'})-[:Edge]->(c:Node {name: 'c'}) +$$) AS (v agtype); + v +--- +(0 rows) + +-- Prime the VLE cache: find all nodes reachable from a +SELECT * FROM cypher('vle_trigger_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + name +------ + "b" + "c" +(2 rows) + +-- Direct SQL INSERT on vertex: add a new vertex via SQL. +-- This should fire the trigger and invalidate the VLE cache. +-- Use _graphid() with the label's id and nextval for the entry id. +INSERT INTO vle_trigger_test."Node" (id, properties) +SELECT ag_catalog._graphid(l.id, + nextval('vle_trigger_test."Node_id_seq"'::regclass)), + '{"name": "d"}'::agtype +FROM ag_catalog.ag_label l +JOIN ag_catalog.ag_graph g ON g.graphid = l.graph +WHERE g.name = 'vle_trigger_test' AND l.name = 'Node'; +-- VLE query: results should be unchanged (d has no edges) but cache was rebuilt +SELECT * FROM cypher('vle_trigger_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + name +------ + "b" + "c" +(2 rows) + +-- Direct SQL UPDATE on vertex: change b's name to 'b_updated' +-- This should fire the trigger and invalidate the VLE cache. +UPDATE vle_trigger_test."Node" +SET properties = '{"name": "b_updated"}'::agtype +WHERE properties @> '{"name": "b"}'::agtype; +-- VLE query: verify updated property is visible (cache invalidated by trigger) +SELECT * FROM cypher('vle_trigger_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + name +------------- + "b_updated" + "c" +(2 rows) + +-- Direct SQL DELETE on edge: remove the edge from b_updated to c +DELETE FROM vle_trigger_test."Edge" +WHERE end_id = (SELECT id FROM vle_trigger_test."Node" + WHERE properties @> '{"name": "c"}'::agtype); +-- VLE query: only b_updated reachable now (edge to c is gone) +SELECT * FROM cypher('vle_trigger_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + name +------------- + "b_updated" +(1 row) + +-- Direct SQL TRUNCATE on edge table: remove all edges +TRUNCATE vle_trigger_test."Edge"; +-- VLE query: no edges exist, should return no rows +SELECT * FROM cypher('vle_trigger_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + name +------ +(0 rows) + +-- Cleanup +SELECT * FROM drop_graph('vle_trigger_test', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table vle_trigger_test._ag_label_vertex +drop cascades to table vle_trigger_test._ag_label_edge +drop cascades to table vle_trigger_test."Node" +drop cascades to table vle_trigger_test."Edge" +NOTICE: graph "vle_trigger_test" has been dropped + drop_graph +------------ + +(1 row) + ----------------------------------------------------------------------------------------------------------------------------- -- -- End of tests diff --git a/regress/sql/age_global_graph.sql b/regress/sql/age_global_graph.sql index 376681a8b..6ee25e1f3 100644 --- a/regress/sql/age_global_graph.sql +++ b/regress/sql/age_global_graph.sql @@ -146,6 +146,182 @@ RESET client_min_messages; SELECT * FROM drop_graph('ag_graph_1', true); SELECT * FROM drop_graph('ag_graph_2', true); SELECT * FROM drop_graph('ag_graph_3', true); + +----------------------------------------------------------------------------------------------------------------------------- +-- +-- VLE cache invalidation tests +-- +-- These tests verify that the graph version counter properly invalidates +-- the VLE hash table cache when the graph is mutated, and that thin +-- entry lazy property fetch returns correct data. +-- + +-- Setup: create a graph with a chain a->b->c->d +SELECT * FROM create_graph('vle_cache_test'); + +SELECT * FROM cypher('vle_cache_test', $$ + CREATE (a:Node {name: 'a'})-[:Edge]->(b:Node {name: 'b'})-[:Edge]->(c:Node {name: 'c'})-[:Edge]->(d:Node {name: 'd'}) +$$) AS (v agtype); + +-- VLE query: find all paths from a's neighbors (should find b, b->c, b->c->d) +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + +-- Now add a new node e connected to d. This should invalidate the cache. +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (d:Node {name: 'd'}) + CREATE (d)-[:Edge]->(:Node {name: 'e'}) +$$) AS (v agtype); + +-- VLE query again: should now also find e via a->b->c->d->e (4 hops won't reach, +-- but d->e is 1 hop from d, and a->b->c->d->e would be 4 hops from a). +-- Increase range to *1..4 to include e +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..4]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + +-- Test cache invalidation on DELETE: remove node c and its edges +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (c:Node {name: 'c'}) + DETACH DELETE c +$$) AS (v agtype); + +-- VLE query: should only find b now (c is gone, so b->c path is broken) +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..4]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + +-- Test cache invalidation on SET: change b's name property +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (b:Node {name: 'b'}) + SET b.name = 'b_modified' + RETURN b.name +$$) AS (name agtype); + +-- VLE query: verify the updated property is returned via lazy fetch +SELECT * FROM cypher('vle_cache_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..4]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + +-- Test VLE with edge properties (exercises thin entry edge property fetch) +SELECT * FROM drop_graph('vle_cache_test', true); +SELECT * FROM create_graph('vle_cache_test2'); + +SELECT * FROM cypher('vle_cache_test2', $$ + CREATE (a:N {name: 'a'})-[:E {weight: 1}]->(b:N {name: 'b'})-[:E {weight: 2}]->(c:N {name: 'c'}) +$$) AS (v agtype); + +-- VLE path output to verify edge properties are fetched correctly via +-- thin entry lazy fetch. Returning the full path forces build_path() +-- to call get_edge_entry_properties() for each edge in the result. +-- The output must contain the correct weight values (1 and 2). +SELECT * FROM cypher('vle_cache_test2', $$ + MATCH p=(a:N {name: 'a'})-[:E *1..2]->(n:N) + RETURN p + ORDER BY n.name +$$) AS (p agtype); + +-- VLE edge properties via UNWIND + relationships() to individually verify +-- each edge's properties are correctly fetched from the heap via TID. +SELECT * FROM cypher('vle_cache_test2', $$ + MATCH p=(a:N {name: 'a'})-[:E *1..2]->(n:N) + WITH p, n + UNWIND relationships(p) AS e + RETURN n.name, e.weight + ORDER BY n.name, e.weight +$$) AS (name agtype, weight agtype); + +-- Cleanup +SELECT * FROM drop_graph('vle_cache_test2', true); + +----------------------------------------------------------------------------------------------------------------------------- +-- +-- VLE cache invalidation via direct SQL (trigger tests) +-- +-- These tests verify that the SQL trigger (age_invalidate_graph_cache) +-- properly invalidates the VLE cache when label tables are mutated +-- via direct SQL INSERT, UPDATE, DELETE, and TRUNCATE — not via Cypher. +-- + +-- Setup: create graph with a chain a->b->c using Cypher +SELECT * FROM create_graph('vle_trigger_test'); + +SELECT * FROM cypher('vle_trigger_test', $$ + CREATE (a:Node {name: 'a'})-[:Edge]->(b:Node {name: 'b'})-[:Edge]->(c:Node {name: 'c'}) +$$) AS (v agtype); + +-- Prime the VLE cache: find all nodes reachable from a +SELECT * FROM cypher('vle_trigger_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + +-- Direct SQL INSERT on vertex: add a new vertex via SQL. +-- This should fire the trigger and invalidate the VLE cache. +-- Use _graphid() with the label's id and nextval for the entry id. +INSERT INTO vle_trigger_test."Node" (id, properties) +SELECT ag_catalog._graphid(l.id, + nextval('vle_trigger_test."Node_id_seq"'::regclass)), + '{"name": "d"}'::agtype +FROM ag_catalog.ag_label l +JOIN ag_catalog.ag_graph g ON g.graphid = l.graph +WHERE g.name = 'vle_trigger_test' AND l.name = 'Node'; + +-- VLE query: results should be unchanged (d has no edges) but cache was rebuilt +SELECT * FROM cypher('vle_trigger_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + +-- Direct SQL UPDATE on vertex: change b's name to 'b_updated' +-- This should fire the trigger and invalidate the VLE cache. +UPDATE vle_trigger_test."Node" +SET properties = '{"name": "b_updated"}'::agtype +WHERE properties @> '{"name": "b"}'::agtype; + +-- VLE query: verify updated property is visible (cache invalidated by trigger) +SELECT * FROM cypher('vle_trigger_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + +-- Direct SQL DELETE on edge: remove the edge from b_updated to c +DELETE FROM vle_trigger_test."Edge" +WHERE end_id = (SELECT id FROM vle_trigger_test."Node" + WHERE properties @> '{"name": "c"}'::agtype); + +-- VLE query: only b_updated reachable now (edge to c is gone) +SELECT * FROM cypher('vle_trigger_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + +-- Direct SQL TRUNCATE on edge table: remove all edges +TRUNCATE vle_trigger_test."Edge"; + +-- VLE query: no edges exist, should return no rows +SELECT * FROM cypher('vle_trigger_test', $$ + MATCH (a:Node {name: 'a'})-[:Edge*1..3]->(n:Node) + RETURN n.name + ORDER BY n.name +$$) AS (name agtype); + +-- Cleanup +SELECT * FROM drop_graph('vle_trigger_test', true); + ----------------------------------------------------------------------------------------------------------------------------- -- -- End of tests diff --git a/sql/age_main.sql b/sql/age_main.sql index 59ada0f9f..3e9a71c92 100644 --- a/sql/age_main.sql +++ b/sql/age_main.sql @@ -381,3 +381,14 @@ CREATE FUNCTION ag_catalog._extract_label_id(graphid) STABLE PARALLEL SAFE AS 'MODULE_PATHNAME'; + +-- +-- VLE cache invalidation trigger function. +-- Installed on graph label tables to catch SQL-level mutations +-- (INSERT/UPDATE/DELETE/TRUNCATE) and increment the graph's +-- version counter so VLE caches are properly invalidated. +-- +CREATE FUNCTION ag_catalog.age_invalidate_graph_cache() + RETURNS trigger + LANGUAGE c +AS 'MODULE_PATHNAME'; diff --git a/src/backend/age.c b/src/backend/age.c index 2c016b021..18085302c 100644 --- a/src/backend/age.c +++ b/src/backend/age.c @@ -22,6 +22,33 @@ #include "optimizer/cypher_paths.h" #include "parser/cypher_analyze.h" #include "utils/ag_guc.h" +#include "utils/age_global_graph.h" + +#if PG_VERSION_NUM < 170000 +#include "miscadmin.h" + +/* saved hook pointers for PG < 17 shmem path */ +static shmem_request_hook_type prev_shmem_request_hook = NULL; +static shmem_startup_hook_type prev_shmem_startup_hook = NULL; + +static void age_shmem_request_hook(void) +{ + if (prev_shmem_request_hook) + { + prev_shmem_request_hook(); + } + age_graph_version_shmem_request(); +} + +static void age_shmem_startup_hook(void) +{ + if (prev_shmem_startup_hook) + { + prev_shmem_startup_hook(); + } + age_graph_version_shmem_startup(); +} +#endif /* PG_VERSION_NUM < 170000 */ PG_MODULE_MAGIC; @@ -35,6 +62,15 @@ void _PG_init(void) process_utility_hook_init(); post_parse_analyze_init(); define_config_params(); + +#if PG_VERSION_NUM < 170000 + /* Register shared memory hooks for graph version tracking. + * On PG 17+, DSM is used instead (no hooks needed). */ + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = age_shmem_request_hook; + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = age_shmem_startup_hook; +#endif } void _PG_fini(void); diff --git a/src/backend/catalog/ag_catalog.c b/src/backend/catalog/ag_catalog.c index f4887f445..79beeb42e 100644 --- a/src/backend/catalog/ag_catalog.c +++ b/src/backend/catalog/ag_catalog.c @@ -25,12 +25,14 @@ #include "catalog/pg_class_d.h" #include "catalog/pg_namespace_d.h" #include "commands/defrem.h" +#include "nodes/parsenodes.h" #include "tcop/utility.h" #include "utils/lsyscache.h" #include "catalog/ag_graph.h" #include "catalog/ag_label.h" #include "utils/ag_cache.h" +#include "utils/age_global_graph.h" static object_access_hook_type prev_object_access_hook; static ProcessUtility_hook_type prev_process_utility_hook; @@ -94,19 +96,50 @@ void ag_ProcessUtility_hook(PlannedStmt *pstmt, const char *queryString, { drop_age_extension((DropStmt *)pstmt->utilityStmt); } - else if (prev_process_utility_hook) - { - (*prev_process_utility_hook) (pstmt, queryString, readOnlyTree, context, - params, queryEnv, dest, qc); - } else { - Assert(IsA(pstmt, PlannedStmt)); - Assert(pstmt->commandType == CMD_UTILITY); - Assert(queryString != NULL); /* required as of 8.4 */ - Assert(qc == NULL || qc->commandTag == CMDTAG_UNKNOWN); - standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, - params, queryEnv, dest, qc); + /* + * Check for TRUNCATE on graph label tables. If any truncated + * table is a graph label table, increment the version counter + * for that graph to invalidate VLE caches. We do this before + * the truncate executes so the cache is invalidated regardless. + */ + if (IsA(pstmt->utilityStmt, TruncateStmt)) + { + TruncateStmt *tstmt = (TruncateStmt *) pstmt->utilityStmt; + ListCell *lc; + + foreach(lc, tstmt->relations) + { + RangeVar *rv = (RangeVar *) lfirst(lc); + Oid rel_oid = RangeVarGetRelid(rv, AccessShareLock, true); + + if (OidIsValid(rel_oid)) + { + Oid graph_oid = get_graph_oid_for_table(rel_oid); + + if (OidIsValid(graph_oid)) + { + increment_graph_version(graph_oid); + } + } + } + } + + if (prev_process_utility_hook) + { + (*prev_process_utility_hook) (pstmt, queryString, readOnlyTree, + context, params, queryEnv, dest, qc); + } + else + { + Assert(IsA(pstmt, PlannedStmt)); + Assert(pstmt->commandType == CMD_UTILITY); + Assert(queryString != NULL); + Assert(qc == NULL || qc->commandTag == CMDTAG_UNKNOWN); + standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, + params, queryEnv, dest, qc); + } } } diff --git a/src/backend/commands/label_commands.c b/src/backend/commands/label_commands.c index 051bbc8a0..ac789ecce 100644 --- a/src/backend/commands/label_commands.c +++ b/src/backend/commands/label_commands.c @@ -25,9 +25,11 @@ #include "commands/defrem.h" #include "commands/sequence.h" #include "commands/tablecmds.h" +#include "catalog/pg_trigger.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parser.h" +#include "parser/parse_func.h" #include "tcop/utility.h" #include "utils/acl.h" #include "utils/builtins.h" @@ -432,6 +434,64 @@ static void create_table_for_label(char *graph_name, char *label_name, create_index_on_column(schema_name, rel_name, "start_id", false); create_index_on_column(schema_name, rel_name, "end_id", false); } + + /* + * Install a cache invalidation trigger on the new label table, if the + * trigger function exists. The function is registered in the extension + * SQL (age_main.sql). It may not exist if running against an older + * version of the extension SQL that hasn't been upgraded yet. + * + * When installed, the trigger fires AFTER INSERT/UPDATE/DELETE/TRUNCATE + * (FOR EACH STATEMENT) and increments the graph's version counter so + * VLE caches are properly invalidated when the table is modified via SQL. + */ + { + Oid func_oid; + + /* check if the trigger function is registered in the catalog */ + func_oid = LookupFuncName( + list_make2(makeString("ag_catalog"), + makeString("age_invalidate_graph_cache")), + 0, NULL, true); + + if (OidIsValid(func_oid)) + { + CreateTrigStmt *trigger_stmt = makeNode(CreateTrigStmt); + PlannedStmt *trigger_wrapper; + + trigger_stmt->replace = false; + trigger_stmt->isconstraint = false; + trigger_stmt->trigname = "_age_cache_invalidate"; + trigger_stmt->relation = makeRangeVar(schema_name, rel_name, -1); + trigger_stmt->funcname = list_make2(makeString("ag_catalog"), + makeString("age_invalidate_graph_cache")); + trigger_stmt->args = NIL; + trigger_stmt->row = false; + trigger_stmt->timing = TRIGGER_TYPE_AFTER; + trigger_stmt->events = TRIGGER_TYPE_INSERT | TRIGGER_TYPE_UPDATE | + TRIGGER_TYPE_DELETE | TRIGGER_TYPE_TRUNCATE; + trigger_stmt->columns = NIL; + trigger_stmt->whenClause = NULL; + trigger_stmt->transitionRels = NIL; + trigger_stmt->deferrable = false; + trigger_stmt->initdeferred = false; + trigger_stmt->constrrel = NULL; + + trigger_wrapper = makeNode(PlannedStmt); + trigger_wrapper->commandType = CMD_UTILITY; + trigger_wrapper->canSetTag = false; + trigger_wrapper->utilityStmt = (Node *) trigger_stmt; + trigger_wrapper->stmt_location = -1; + trigger_wrapper->stmt_len = 0; + + ProcessUtility(trigger_wrapper, + "(generated CREATE TRIGGER command)", + false, PROCESS_UTILITY_SUBCOMMAND, + NULL, NULL, None_Receiver, NULL); + + CommandCounterIncrement(); + } + } } static void create_index_on_column(char *schema_name, diff --git a/src/backend/executor/cypher_create.c b/src/backend/executor/cypher_create.c index 495eb3a08..876b6f250 100644 --- a/src/backend/executor/cypher_create.c +++ b/src/backend/executor/cypher_create.c @@ -25,6 +25,7 @@ #include "catalog/ag_label.h" #include "executor/cypher_executor.h" #include "executor/cypher_utils.h" +#include "utils/age_global_graph.h" static void begin_cypher_create(CustomScanState *node, EState *estate, int eflags); @@ -249,6 +250,9 @@ static TupleTableSlot *exec_cypher_create(CustomScanState *node) /* update the current command Id */ CommandCounterIncrement(); + /* invalidate VLE cache — graph was mutated */ + increment_graph_version(css->graph_oid); + /* if this was a terminal CREATE just return NULL */ if (terminal) { diff --git a/src/backend/executor/cypher_delete.c b/src/backend/executor/cypher_delete.c index 58503ec27..19eca3ad6 100644 --- a/src/backend/executor/cypher_delete.c +++ b/src/backend/executor/cypher_delete.c @@ -28,6 +28,7 @@ #include "catalog/ag_label.h" #include "executor/cypher_executor.h" +#include "utils/age_global_graph.h" #include "executor/cypher_utils.h" static void begin_cypher_delete(CustomScanState *node, EState *estate, @@ -193,8 +194,14 @@ static TupleTableSlot *exec_cypher_delete(CustomScanState *node) */ static void end_cypher_delete(CustomScanState *node) { + cypher_delete_custom_scan_state *css = + (cypher_delete_custom_scan_state *)node; + check_for_connected_edges(node); + /* invalidate VLE cache — graph was mutated */ + increment_graph_version(css->delete_data->graph_oid); + hash_destroy(((cypher_delete_custom_scan_state *)node)->vertex_id_htab); ExecEndNode(node->ss.ps.lefttree); diff --git a/src/backend/executor/cypher_merge.c b/src/backend/executor/cypher_merge.c index 1edfc812d..c2db878b7 100644 --- a/src/backend/executor/cypher_merge.c +++ b/src/backend/executor/cypher_merge.c @@ -26,6 +26,7 @@ #include "catalog/ag_label.h" #include "executor/cypher_executor.h" #include "executor/cypher_utils.h" +#include "utils/age_global_graph.h" /* * The following structure is used to hold a single vertex or edge component @@ -936,6 +937,12 @@ static void end_cypher_merge(CustomScanState *node) /* increment the command counter */ CommandCounterIncrement(); + /* invalidate VLE cache if merge created anything */ + if (css->created_new_path) + { + increment_graph_version(css->graph_oid); + } + ExecEndNode(node->ss.ps.lefttree); foreach (lc, path->target_nodes) diff --git a/src/backend/executor/cypher_set.c b/src/backend/executor/cypher_set.c index 09d3d3b54..8dfa63268 100644 --- a/src/backend/executor/cypher_set.c +++ b/src/backend/executor/cypher_set.c @@ -26,6 +26,8 @@ #include "executor/cypher_executor.h" #include "executor/cypher_utils.h" +#include "utils/age_global_graph.h" +#include "catalog/ag_graph.h" static void begin_cypher_set(CustomScanState *node, EState *estate, int eflags); @@ -829,6 +831,9 @@ static TupleTableSlot *exec_cypher_set(CustomScanState *node) /* increment the command counter to reflect the updates */ CommandCounterIncrement(); + /* invalidate VLE cache — graph was mutated */ + increment_graph_version(get_graph_oid(css->set_list->graph_name)); + return NULL; } @@ -837,6 +842,9 @@ static TupleTableSlot *exec_cypher_set(CustomScanState *node) /* increment the command counter to reflect the updates */ CommandCounterIncrement(); + /* invalidate VLE cache — graph was mutated */ + increment_graph_version(get_graph_oid(css->set_list->graph_name)); + estate->es_result_relations = saved_resultRels; econtext->ecxt_scantuple = ExecProject(node->ss.ps.lefttree->ps_ProjInfo); diff --git a/src/backend/utils/adt/age_global_graph.c b/src/backend/utils/adt/age_global_graph.c index f99d75abe..a9b9b7111 100644 --- a/src/backend/utils/adt/age_global_graph.c +++ b/src/backend/utils/adt/age_global_graph.c @@ -21,25 +21,82 @@ #include "access/heapam.h" #include "catalog/namespace.h" +#include "commands/trigger.h" #include "common/hashfn.h" #include "commands/label_commands.h" +#include "port/atomics.h" +#include "storage/lwlock.h" #include "utils/datum.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/snapmgr.h" #include "utils/builtins.h" +#if PG_VERSION_NUM >= 170000 +#include "storage/dsm_registry.h" +#else +#include "storage/ipc.h" +#include "storage/shmem.h" +#endif + #include "utils/age_global_graph.h" #include "catalog/ag_graph.h" #include "catalog/ag_label.h" +#include "utils/ag_cache.h" #include /* defines */ #define VERTEX_HTAB_NAME "Vertex to edge lists " /* added a space at end for */ #define EDGE_HTAB_NAME "Edge to vertex mapping " /* the graph name to follow */ -#define VERTEX_HTAB_INITIAL_SIZE 1000000 -#define EDGE_HTAB_INITIAL_SIZE 1000000 +#define VERTEX_HTAB_INITIAL_SIZE 10000 +#define EDGE_HTAB_INITIAL_SIZE 10000 + +/* Maximum number of graphs tracked for version counting */ +#define AGE_MAX_GRAPHS 128 + +/* + * Graph version counter entry. Stored in shared memory (DSM or shmem) + * so that all backends can see mutation events. The version counter is + * incremented by Cypher mutations (CREATE/DELETE/SET/MERGE) and by + * SQL triggers on label tables. VLE cache invalidation checks this + * counter instead of snapshot xmin/xmax/curcid. + */ +typedef struct GraphVersionEntry +{ + Oid graph_oid; /* graph identifier (0 = unused slot) */ + pg_atomic_uint64 version; /* monotonic change counter */ +} GraphVersionEntry; + +/* + * Shared memory state for graph version tracking. + * Contains a fixed-size array of per-graph version counters. + */ +typedef struct GraphVersionState +{ + LWLock lock; /* protects slot allocation only */ + int num_entries; /* number of active entries */ + GraphVersionEntry entries[AGE_MAX_GRAPHS]; +} GraphVersionState; + +/* + * Version mode detection — determined once per backend on first use. + * DSM: PG 17+ GetNamedDSMSegment (no shared_preload_libraries needed) + * SHMEM: PG < 17 with shared_preload_libraries + * SNAPSHOT: PG < 17 without shared_preload_libraries (current behavior) + */ +typedef enum +{ + VERSION_MODE_UNKNOWN = 0, + VERSION_MODE_DSM, + VERSION_MODE_SHMEM, + VERSION_MODE_SNAPSHOT +} VersionMode; + +static VersionMode version_mode = VERSION_MODE_UNKNOWN; + +/* For PG < 17 shmem path */ +static GraphVersionState *shmem_version_state = NULL; /* internal data structures implementation */ @@ -51,7 +108,7 @@ typedef struct vertex_entry ListGraphId *edges_out; /* List of exiting edges graphids (int64) */ ListGraphId *edges_self; /* List of selfloop edges graphids (int64) */ Oid vertex_label_table_oid; /* the label table oid */ - Datum vertex_properties; /* datum property value */ + ItemPointerData tid; /* physical tuple location for lazy fetch */ } vertex_entry; /* edge entry for the edge_hashtable */ @@ -59,7 +116,7 @@ typedef struct edge_entry { graphid edge_id; /* edge id, it is also the hash key */ Oid edge_label_table_oid; /* the label table oid */ - Datum edge_properties; /* datum property value */ + ItemPointerData tid; /* physical tuple location for lazy fetch */ graphid start_vertex_id; /* start vertex */ graphid end_vertex_id; /* end vertex */ } edge_entry; @@ -75,13 +132,13 @@ typedef struct GRAPH_global_context Oid graph_oid; /* graph oid for searching */ HTAB *vertex_hashtable; /* hashtable to hold vertex edge lists */ HTAB *edge_hashtable; /* hashtable to hold edge to vertex map */ - TransactionId xmin; /* transaction ids for this graph */ - TransactionId xmax; - CommandId curcid; /* currentCommandId graph was created with */ + uint64 graph_version; /* version counter for cache invalidation */ + TransactionId xmin; /* snapshot fallback: transaction xmin */ + TransactionId xmax; /* snapshot fallback: transaction xmax */ + CommandId curcid; /* snapshot fallback: command id */ int64 num_loaded_vertices; /* number of loaded vertices in this graph */ int64 num_loaded_edges; /* number of loaded edges in this graph */ ListGraphId *vertices; /* vertices for vertex hashtable cleanup */ - ListGraphId *edges; /* edges for edge hashtable cleanup */ struct GRAPH_global_context *next; /* next graph */ } GRAPH_global_context; @@ -111,35 +168,59 @@ static void freeze_GRAPH_global_hashtables(GRAPH_global_context *ggctx); static List *get_ag_labels_names(Snapshot snapshot, Oid graph_oid, char label_type); static bool insert_edge_entry(GRAPH_global_context *ggctx, graphid edge_id, - Datum edge_properties, graphid start_vertex_id, + ItemPointerData tid, graphid start_vertex_id, graphid end_vertex_id, Oid edge_label_table_oid); static bool insert_vertex_edge(GRAPH_global_context *ggctx, graphid start_vertex_id, graphid end_vertex_id, graphid edge_id, char *edge_label_name); static bool insert_vertex_entry(GRAPH_global_context *ggctx, graphid vertex_id, Oid vertex_label_table_oid, - Datum vertex_properties); + ItemPointerData tid); /* definitions */ /* * Helper function to determine validity of the passed GRAPH_global_context. - * This is based off of the current active snapshot, to see if the graph could - * have been modified. Ideally, we should find a way to more accurately know - * whether the particular graph was modified. + * + * Uses graph-specific version counters (via DSM or shmem) when available. + * Falls back to snapshot-based invalidation when shared memory is not + * initialized (PG < 17 without shared_preload_libraries). + * + * The version counter approach only invalidates when the specific graph + * has been mutated (via Cypher operations or SQL triggers), avoiding false + * invalidation from unrelated transactions on the server. */ bool is_ggctx_invalid(GRAPH_global_context *ggctx) { - Snapshot snap = GetActiveSnapshot(); + /* use version counter if DSM or SHMEM mode is active */ + if (version_mode == VERSION_MODE_DSM || version_mode == VERSION_MODE_SHMEM) + { + uint64 current_version = get_graph_version(ggctx->graph_oid); + + /* + * If current_version is 0, no mutations have been tracked through + * the version counter system yet. Fall through to snapshot-based + * checking for safety — the graph may have been mutated via paths + * that don't increment the counter (e.g., before executor hooks + * are in place, or via direct SQL without triggers). + * + * Once current_version > 0, we know the counter is actively + * tracking this graph and can rely on it exclusively. + */ + if (current_version > 0) + { + return (ggctx->graph_version != current_version); + } + /* fall through to snapshot check */ + } - /* - * If the transaction ids (xmin or xmax) or currentCommandId (curcid) have - * changed, then we have a graph that was updated. This means that the - * global context for this graph is no longer valid. - */ - return (ggctx->xmin != snap->xmin || - ggctx->xmax != snap->xmax || - ggctx->curcid != snap->curcid); + /* SNAPSHOT fallback: original behavior — check snapshot ids */ + { + Snapshot snap = GetActiveSnapshot(); + return (ggctx->xmin != snap->xmin || + ggctx->xmax != snap->xmax || + ggctx->curcid != snap->curcid); + } } /* * Helper function to create the global vertex and edge hashtables. One @@ -332,7 +413,7 @@ static List *get_ag_labels_names(Snapshot snapshot, Oid graph_oid, * current GRAPH global edge hashtable. */ static bool insert_edge_entry(GRAPH_global_context *ggctx, graphid edge_id, - Datum edge_properties, graphid start_vertex_id, + ItemPointerData tid, graphid start_vertex_id, graphid end_vertex_id, Oid edge_label_table_oid) { edge_entry *ee = NULL; @@ -378,14 +459,11 @@ static bool insert_edge_entry(GRAPH_global_context *ggctx, graphid edge_id, * for hash function collisions. */ ee->edge_id = edge_id; - ee->edge_properties = edge_properties; + ee->tid = tid; ee->start_vertex_id = start_vertex_id; ee->end_vertex_id = end_vertex_id; ee->edge_label_table_oid = edge_label_table_oid; - /* we also need to store the edge id for clean up of edge property datums */ - ggctx->edges = append_graphid(ggctx->edges, edge_id); - /* increment the number of loaded edges */ ggctx->num_loaded_edges++; @@ -398,7 +476,7 @@ static bool insert_edge_entry(GRAPH_global_context *ggctx, graphid edge_id, */ static bool insert_vertex_entry(GRAPH_global_context *ggctx, graphid vertex_id, Oid vertex_label_table_oid, - Datum vertex_properties) + ItemPointerData tid) { vertex_entry *ve = NULL; bool found = false; @@ -440,8 +518,8 @@ static bool insert_vertex_entry(GRAPH_global_context *ggctx, graphid vertex_id, ve->vertex_id = vertex_id; /* set the label table oid for this vertex */ ve->vertex_label_table_oid = vertex_label_table_oid; - /* set the datum vertex properties */ - ve->vertex_properties = vertex_properties; + /* set the TID for lazy property fetch */ + ve->tid = tid; /* set the NIL edge list */ ve->edges_in = NULL; ve->edges_out = NULL; @@ -590,7 +668,6 @@ static void load_vertex_hashtable(GRAPH_global_context *ggctx) while((tuple = heap_getnext(scan_desc, ForwardScanDirection)) != NULL) { graphid vertex_id; - Datum vertex_properties; bool inserted = false; /* something is wrong if this isn't true */ @@ -603,16 +680,11 @@ static void load_vertex_hashtable(GRAPH_global_context *ggctx) /* get the vertex id */ vertex_id = DatumGetInt64(column_get_datum(tupdesc, tuple, 0, "id", GRAPHIDOID, true)); - /* get the vertex properties datum */ - vertex_properties = column_get_datum(tupdesc, tuple, 1, - "properties", AGTYPEOID, true); - /* we need to make a copy of the properties datum */ - vertex_properties = datumCopy(vertex_properties, false, -1); - /* insert vertex into vertex hashtable */ + /* insert vertex into vertex hashtable with TID (no property copy) */ inserted = insert_vertex_entry(ggctx, vertex_id, vertex_label_table_oid, - vertex_properties); + tuple->t_self); /* warn if there is a duplicate */ if (!inserted) @@ -700,7 +772,6 @@ static void load_edge_hashtable(GRAPH_global_context *ggctx) graphid edge_id; graphid edge_vertex_start_id; graphid edge_vertex_end_id; - Datum edge_properties; bool inserted = false; /* something is wrong if this isn't true */ @@ -724,15 +795,9 @@ static void load_edge_hashtable(GRAPH_global_context *ggctx) 2, "end_id", GRAPHIDOID, true)); - /* get the edge properties datum */ - edge_properties = column_get_datum(tupdesc, tuple, 3, "properties", - AGTYPEOID, true); - /* we need to make a copy of the properties datum */ - edge_properties = datumCopy(edge_properties, false, -1); - - /* insert edge into edge hashtable */ - inserted = insert_edge_entry(ggctx, edge_id, edge_properties, + /* insert edge into edge hashtable with TID (no property copy) */ + inserted = insert_edge_entry(ggctx, edge_id, tuple->t_self, edge_vertex_start_id, edge_vertex_end_id, edge_label_table_oid); @@ -781,7 +846,6 @@ static void freeze_GRAPH_global_hashtables(GRAPH_global_context *ggctx) static bool free_specific_GRAPH_global_context(GRAPH_global_context *ggctx) { GraphIdNode *curr_vertex = NULL; - GraphIdNode *curr_edge = NULL; /* don't do anything if NULL */ if (ggctx == NULL) @@ -821,10 +885,6 @@ static bool free_specific_GRAPH_global_context(GRAPH_global_context *ggctx) return false; } - /* free the vertex's datumCopy properties */ - pfree_if_not_null(DatumGetPointer(value->vertex_properties)); - value->vertex_properties = 0; - /* free the edge list associated with this vertex */ free_ListGraphId(value->edges_in); free_ListGraphId(value->edges_out); @@ -838,47 +898,10 @@ static bool free_specific_GRAPH_global_context(GRAPH_global_context *ggctx) curr_vertex = next_vertex; } - /* free the edge properties, starting with the head */ - curr_edge = peek_stack_head(ggctx->edges); - while (curr_edge != NULL) - { - GraphIdNode *next_edge = NULL; - edge_entry *value = NULL; - bool found = false; - graphid edge_id; - - /* get the next edge in the list, if any */ - next_edge = next_GraphIdNode(curr_edge); - - /* get the current edge id */ - edge_id = get_graphid(curr_edge); - - /* retrieve the edge entry */ - value = (edge_entry *)hash_search(ggctx->edge_hashtable, - (void *)&edge_id, HASH_FIND, - &found); - /* this is bad if it isn't found, but leave that to the caller */ - if (found == false) - { - return false; - } - - /* free the edge's datumCopy properties */ - pfree_if_not_null(DatumGetPointer(value->edge_properties)); - value->edge_properties = 0; - - /* move to the next edge */ - curr_edge = next_edge; - } - /* free the vertices list */ free_ListGraphId(ggctx->vertices); ggctx->vertices = NULL; - /* free the edges list */ - free_ListGraphId(ggctx->edges); - ggctx->edges = NULL; - /* free the hashtables */ hash_destroy(ggctx->vertex_hashtable); hash_destroy(ggctx->edge_hashtable); @@ -1011,15 +1034,16 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char *graph_name, new_ggctx->graph_name = pstrdup(graph_name); new_ggctx->graph_oid = graph_oid; - /* set the transaction ids */ + /* set the graph version counter for cache invalidation */ + new_ggctx->graph_version = get_graph_version(graph_oid); + + /* set snapshot fields for SNAPSHOT fallback mode */ new_ggctx->xmin = GetActiveSnapshot()->xmin; new_ggctx->xmax = GetActiveSnapshot()->xmax; new_ggctx->curcid = GetActiveSnapshot()->curcid; /* initialize our vertices list */ new_ggctx->vertices = NULL; - /* initialize our edges list */ - new_ggctx->edges = NULL; /* build the hashtables for this graph */ create_GRAPH_global_hashtables(new_ggctx); @@ -1261,9 +1285,59 @@ Oid get_vertex_entry_label_table_oid(vertex_entry *ve) return ve->vertex_label_table_oid; } +/* + * Fetch vertex properties on demand from the heap via stored TID. + * + * Returns a datumCopy of the properties in the current memory context. + * The caller does not need to free the result explicitly — it will be + * freed when the memory context is reset (typically the SRF multi-call + * context for VLE, which is cleaned up when the SRF completes). + * + * If the tuple is no longer visible (e.g., concurrent mutation between + * cache build and fetch), the version counter should have invalidated + * the cache. If we get here with a stale TID, it indicates a bug in + * the invalidation logic. + */ Datum get_vertex_entry_properties(vertex_entry *ve) { - return ve->vertex_properties; + Relation rel; + HeapTupleData tuple; + Buffer buffer; + Datum result = (Datum) 0; + + rel = table_open(ve->vertex_label_table_oid, AccessShareLock); + tuple.t_self = ve->tid; + + if (heap_fetch(rel, GetActiveSnapshot(), &tuple, &buffer, true)) + { + TupleDesc tupdesc = RelationGetDescr(rel); + bool isnull; + Datum props; + + /* properties is column 2 (1-indexed) */ + props = heap_getattr(&tuple, 2, tupdesc, &isnull); + if (!isnull) + { + result = datumCopy(props, false, -1); + } + + ReleaseBuffer(buffer); + } + + table_close(rel, AccessShareLock); + + /* + * If heap_fetch failed, the tuple is no longer visible. This should + * not happen under normal operation because the version counter + * invalidates the cache when the graph is mutated. + */ + if (result == (Datum) 0) + { + elog(ERROR, "get_vertex_entry_properties: stale TID - " + "vertex entry references a tuple that is no longer visible"); + } + + return result; } /* edge_entry accessor functions */ @@ -1277,9 +1351,45 @@ Oid get_edge_entry_label_table_oid(edge_entry *ee) return ee->edge_label_table_oid; } +/* + * Fetch edge properties on demand from the heap via stored TID. + * See get_vertex_entry_properties for memory and safety notes. + */ Datum get_edge_entry_properties(edge_entry *ee) { - return ee->edge_properties; + Relation rel; + HeapTupleData tuple; + Buffer buffer; + Datum result = (Datum) 0; + + rel = table_open(ee->edge_label_table_oid, AccessShareLock); + tuple.t_self = ee->tid; + + if (heap_fetch(rel, GetActiveSnapshot(), &tuple, &buffer, true)) + { + TupleDesc tupdesc = RelationGetDescr(rel); + bool isnull; + Datum props; + + /* properties is column 4 (1-indexed) */ + props = heap_getattr(&tuple, 4, tupdesc, &isnull); + if (!isnull) + { + result = datumCopy(props, false, -1); + } + + ReleaseBuffer(buffer); + } + + table_close(rel, AccessShareLock); + + if (result == (Datum) 0) + { + elog(ERROR, "get_edge_entry_properties: stale TID - " + "edge entry references a tuple that is no longer visible"); + } + + return result; } graphid get_edge_entry_start_vertex_id(edge_entry *ee) @@ -1531,3 +1641,286 @@ Datum age_graph_stats(PG_FUNCTION_ARGS) PG_RETURN_POINTER(agtype_value_to_agtype(result.res)); } + +/* + * ============================================================================ + * Graph Version Counter Implementation + * + * Provides per-graph monotonic version counters in shared memory for + * cross-backend VLE cache invalidation. Three modes are supported: + * + * DSM (PG 17+): Uses GetNamedDSMSegment — works without shared_preload_libs + * SHMEM (PG <17): Uses shmem_request/startup hooks — needs shared_preload_libs + * SNAPSHOT: Falls back to original snapshot-based invalidation + * ============================================================================ + */ + +#if PG_VERSION_NUM >= 170000 +/* + * DSM path: GetNamedDSMSegment init callback. + * Called once when the DSM segment is first created. + */ +static void age_dsm_init_callback(void *ptr) +{ + GraphVersionState *state = (GraphVersionState *) ptr; + + LWLockInitialize(&state->lock, + LWLockNewTrancheId()); + LWLockRegisterTranche(state->lock.tranche, "age_graph_version"); + state->num_entries = 0; + memset(state->entries, 0, sizeof(state->entries)); +} + +/* + * Get the shared GraphVersionState via DSM registry. + * The segment is created on first access and persists until server shutdown. + */ +static GraphVersionState *get_version_state_dsm(void) +{ + bool found; + + return (GraphVersionState *) + GetNamedDSMSegment("age_graph_versions", + sizeof(GraphVersionState), + age_dsm_init_callback, + &found); +} +#endif /* PG_VERSION_NUM >= 170000 */ + +/* + * SHMEM path: request and startup hooks for PG < 17. + * These are registered in _PG_init when shared_preload_libraries is used. + * On PG 17+, DSM is used instead and these functions are not called. + */ +#if PG_VERSION_NUM < 170000 +void age_graph_version_shmem_request(void) +{ + RequestAddinShmemSpace(MAXALIGN(sizeof(GraphVersionState))); +} + +void age_graph_version_shmem_startup(void) +{ + bool found; + + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + + shmem_version_state = + (GraphVersionState *) ShmemInitStruct("AGE Graph Version State", + sizeof(GraphVersionState), + &found); + if (!found) + { + LWLockInitialize(&shmem_version_state->lock, + LWLockNewTrancheId()); + LWLockRegisterTranche(shmem_version_state->lock.tranche, + "age_graph_version"); + shmem_version_state->num_entries = 0; + memset(shmem_version_state->entries, 0, + sizeof(shmem_version_state->entries)); + } + + LWLockRelease(AddinShmemInitLock); +} +#endif /* PG_VERSION_NUM < 170000 */ + +/* + * Detect which version mode to use. Called once per backend on first access. + * Emits a DEBUG1 log message indicating the chosen mode. + */ +static void detect_version_mode(void) +{ +#if PG_VERSION_NUM >= 170000 + version_mode = VERSION_MODE_DSM; + elog(DEBUG1, "AGE: VLE cache using DSM version counter"); +#else + if (shmem_version_state != NULL) + { + version_mode = VERSION_MODE_SHMEM; + elog(DEBUG1, "AGE: VLE cache using SHMEM version counter"); + } + else + { + version_mode = VERSION_MODE_SNAPSHOT; + elog(DEBUG1, "AGE: VLE cache using snapshot-based invalidation " + "(add AGE to shared_preload_libraries for better caching)"); + } +#endif +} + +/* + * Get a pointer to the GraphVersionState, regardless of mode. + * Returns NULL only in SNAPSHOT mode (no shared memory available). + */ +static GraphVersionState *get_version_state(void) +{ + if (version_mode == VERSION_MODE_UNKNOWN) + { + detect_version_mode(); + } + +#if PG_VERSION_NUM >= 170000 + if (version_mode == VERSION_MODE_DSM) + { + return get_version_state_dsm(); + } +#endif + + if (version_mode == VERSION_MODE_SHMEM) + { + return shmem_version_state; + } + + return NULL; +} + +/* + * Get the current version counter for a graph. + * Returns 0 if the graph has never been tracked or if shared memory + * is not available. Lock-free read via pg_atomic_read_u64. + */ +uint64 get_graph_version(Oid graph_oid) +{ + GraphVersionState *state = get_version_state(); + int i; + + if (state == NULL) + { + return 0; + } + + /* lock-free scan of the array */ + for (i = 0; i < state->num_entries; i++) + { + if (state->entries[i].graph_oid == graph_oid) + { + return pg_atomic_read_u64(&state->entries[i].version); + } + } + + return 0; +} + +/* + * Increment the version counter for a graph. + * Called after any graph mutation (Cypher or SQL trigger). + * Lock-free for existing entries; acquires LWLock only to allocate new slots. + */ +void increment_graph_version(Oid graph_oid) +{ + GraphVersionState *state = get_version_state(); + int i; + + if (state == NULL) + { + return; + } + + /* try to find existing entry (lock-free) */ + for (i = 0; i < state->num_entries; i++) + { + if (state->entries[i].graph_oid == graph_oid) + { + pg_atomic_fetch_add_u64(&state->entries[i].version, 1); + return; + } + } + + /* new graph — need lock to allocate slot */ + LWLockAcquire(&state->lock, LW_EXCLUSIVE); + + /* re-check after acquiring lock (another backend may have added it) */ + for (i = 0; i < state->num_entries; i++) + { + if (state->entries[i].graph_oid == graph_oid) + { + LWLockRelease(&state->lock); + pg_atomic_fetch_add_u64(&state->entries[i].version, 1); + return; + } + } + + /* add new entry */ + if (state->num_entries < AGE_MAX_GRAPHS) + { + int idx = state->num_entries; + + state->entries[idx].graph_oid = graph_oid; + pg_atomic_init_u64(&state->entries[idx].version, 1); + + /* + * Write barrier ensures the entry fields are fully visible to + * other backends before num_entries is incremented. This prevents + * readers on weak memory-ordering architectures (e.g., ARM) from + * seeing the incremented count before the entry is initialized. + */ + pg_write_barrier(); + state->num_entries++; + } + else + { + elog(WARNING, "AGE: graph version counter table full (%d graphs)", + AGE_MAX_GRAPHS); + } + + LWLockRelease(&state->lock); +} + +/* + * Helper function to look up the graph OID for a given label table OID. + * Uses AGE's label relation cache for fast lookup. + * Returns InvalidOid if the table is not a graph label table. + */ +Oid get_graph_oid_for_table(Oid table_oid) +{ + label_cache_data *lcd = NULL; + + lcd = search_label_relation_cache(table_oid); + + if (lcd != NULL) + { + return lcd->graph; + } + + return InvalidOid; +} + +/* + * SQL-callable trigger function for VLE cache invalidation. + * Installed on graph label tables (AFTER INSERT/UPDATE/DELETE FOR EACH STATEMENT). + * Looks up which graph the triggering table belongs to and increments + * that graph's version counter. + */ +PG_FUNCTION_INFO_V1(age_invalidate_graph_cache); + +Datum age_invalidate_graph_cache(PG_FUNCTION_ARGS) +{ + TriggerData *trigdata; + Oid table_oid; + Oid graph_oid; + + /* verify called as trigger */ + if (!CALLED_AS_TRIGGER(fcinfo)) + { + ereport(ERROR, + (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), + errmsg("age_invalidate_graph_cache: not called as trigger"))); + } + + trigdata = (TriggerData *) fcinfo->context; + table_oid = RelationGetRelid(trigdata->tg_relation); + + /* look up which graph this label table belongs to */ + graph_oid = get_graph_oid_for_table(table_oid); + + if (OidIsValid(graph_oid)) + { + increment_graph_version(graph_oid); + } + + /* + * Trigger protocol: return a null pointer without setting fcinfo->isnull. + * PG_RETURN_NULL() sets isnull=true, which violates the trigger protocol + * and causes "trigger function returned null value" errors during COPY. + */ + PG_RETURN_POINTER(NULL); +} diff --git a/src/backend/utils/adt/age_graphid_ds.c b/src/backend/utils/adt/age_graphid_ds.c index 625a6947c..cee53bd83 100644 --- a/src/backend/utils/adt/age_graphid_ds.c +++ b/src/backend/utils/adt/age_graphid_ds.c @@ -258,3 +258,94 @@ graphid pop_graphid_stack(ListGraphId *stack) /* return the id */ return id; } + +/* + * ============================================================================ + * GraphIdStack — Array-based stack for VLE DFS traversal + * + * Uses a flat graphid array with size/capacity tracking. Push appends to + * the end (doubling capacity when full), pop decrements size. No per-element + * allocation — just sequential array access. + * + * This is intentionally separate from ListGraphId to avoid the lifetime + * and complexity issues that arise from modifying linked-list behavior. + * ============================================================================ + */ + +#define GID_STACK_INITIAL_CAPACITY 64 + +GraphIdStack *new_gid_stack(void) +{ + GraphIdStack *s = palloc(sizeof(GraphIdStack)); + + s->array = palloc(sizeof(graphid) * GID_STACK_INITIAL_CAPACITY); + s->size = 0; + s->capacity = GID_STACK_INITIAL_CAPACITY; + return s; +} + +void free_gid_stack(GraphIdStack *stack) +{ + if (stack == NULL) + { + return; + } + + if (stack->array != NULL) + { + pfree(stack->array); + } + + pfree(stack); +} + +void gid_stack_push(GraphIdStack *s, graphid id) +{ + Assert(s != NULL); + + /* double capacity if full */ + if (s->size >= s->capacity) + { + s->capacity *= 2; + s->array = repalloc(s->array, sizeof(graphid) * s->capacity); + } + + s->array[s->size] = id; + s->size++; +} + +graphid gid_stack_pop(GraphIdStack *s) +{ + Assert(s != NULL); + Assert(s->size > 0); + + s->size--; + return s->array[s->size]; +} + +graphid gid_stack_peek(GraphIdStack *s) +{ + Assert(s != NULL); + Assert(s->size > 0); + + return s->array[s->size - 1]; +} + +bool gid_stack_is_empty(GraphIdStack *s) +{ + return s->size == 0; +} + +int64 gid_stack_size(GraphIdStack *s) +{ + return s->size; +} + +/* Access element by index — 0 is bottom, size-1 is top */ +graphid gid_stack_get(GraphIdStack *s, int64 i) +{ + Assert(s != NULL); + Assert(i >= 0 && i < s->size); + + return s->array[i]; +} diff --git a/src/backend/utils/adt/age_vle.c b/src/backend/utils/adt/age_vle.c index 9224ed612..9aeeadf9b 100644 --- a/src/backend/utils/adt/age_vle.c +++ b/src/backend/utils/adt/age_vle.c @@ -79,9 +79,9 @@ typedef struct VLE_local_context bool uidx_infinite; /* flag if the upper bound is omitted */ cypher_rel_dir edge_direction; /* the direction of the edge */ HTAB *edge_state_hashtable; /* local state hashtable for our edges */ - ListGraphId *dfs_vertex_stack; /* dfs stack for vertices */ - ListGraphId *dfs_edge_stack; /* dfs stack for edges */ - ListGraphId *dfs_path_stack; /* dfs stack containing the path */ + GraphIdStack *dfs_vertex_stack; /* dfs stack for vertices (array-based) */ + GraphIdStack *dfs_edge_stack; /* dfs stack for edges (array-based) */ + GraphIdStack *dfs_path_stack; /* dfs stack containing the path (array-based) */ VLE_path_function path_function; /* which path function to use */ GraphIdNode *next_vertex; /* for VLE_FUNCTION_PATHS_TO */ int64 vle_grammar_node_id; /* the unique VLE grammar assigned node id */ @@ -364,54 +364,70 @@ static bool is_an_edge_match(VLE_local_context *vlelctx, edge_entry *ee) return false; } - /* get our edge's properties */ - edge_property = DATUM_GET_AGTYPE_P(get_edge_entry_properties(ee)); - /* get the containers */ - agtc_edge_property_constraint = &vlelctx->edge_property_constraint->root; - agtc_edge_property = &edge_property->root; - /* get the number of properties in the edge to be matched */ - num_edge_properties = AGTYPE_CONTAINER_SIZE(agtc_edge_property); - /* - * Check to see if the edge_properties object has AT LEAST as many pairs - * to compare as the edge_property_constraint object has pairs. If not, it - * can't possibly match. + * Fast path: if the label matched (or wasn't constrained) and there + * are no property constraints, the edge is a match. This avoids + * accessing edge properties entirely for label-only VLE patterns + * like [:KNOWS*1..3] which are the common case. */ - if (num_edge_property_constraints > num_edge_properties) + if (num_edge_property_constraints == 0) { - return false; + return true; } /* - * If the number of constraints are the same as the number of properties, - * then the datums would be the same if they match. + * Fetch edge properties once and cache locally. With thin entries, + * get_edge_entry_properties() does a heap_fetch, so we avoid calling + * it multiple times for the same edge. */ - if (num_edge_property_constraints == num_edge_properties) { - Datum edge_props = get_edge_entry_properties(ee); - uint32 edge_props_hash = datum_image_hash(edge_props, false, -1); + Datum edge_props_datum = get_edge_entry_properties(ee); + + edge_property = DATUM_GET_AGTYPE_P(edge_props_datum); + agtc_edge_property_constraint = &vlelctx->edge_property_constraint->root; + agtc_edge_property = &edge_property->root; + num_edge_properties = AGTYPE_CONTAINER_SIZE(agtc_edge_property); - /* check the hash first */ - if (vlelctx->edge_property_constraint_hash == edge_props_hash) + /* + * Check to see if the edge_properties object has AT LEAST as many + * pairs to compare as the edge_property_constraint object has pairs. + * If not, it can't possibly match. + */ + if (num_edge_property_constraints > num_edge_properties) { - /* if the hashes match, check the datum images */ - if (datum_image_eq(vlelctx->edge_property_constraint_datum, - edge_props, false, -1)) + return false; + } + + /* + * If the number of constraints are the same as the number of + * properties, then the datums would be the same if they match. + */ + if (num_edge_property_constraints == num_edge_properties) + { + uint32 edge_props_hash = datum_image_hash(edge_props_datum, + false, -1); + /* check the hash first */ + if (vlelctx->edge_property_constraint_hash == edge_props_hash) { - return true; + /* if the hashes match, check the datum images */ + if (datum_image_eq(vlelctx->edge_property_constraint_datum, + edge_props_datum, false, -1)) + { + return true; + } } - } - /* if we got here they aren't the same */ - return false; - } + /* if we got here they aren't the same */ + return false; + } - /* get the iterators */ - constraint_it = agtype_iterator_init(agtc_edge_property_constraint); - property_it = agtype_iterator_init(agtc_edge_property); + /* get the iterators */ + constraint_it = agtype_iterator_init(agtc_edge_property_constraint); + property_it = agtype_iterator_init(agtc_edge_property); - /* return the value of deep contains */ - return agtype_deep_contains(&property_it, &constraint_it, false); + /* return the value of deep contains */ + return agtype_deep_contains(&property_it, &constraint_it, false); + } } /* @@ -449,22 +465,23 @@ static void free_VLE_local_context(VLE_local_context *vlelctx) vlelctx->edge_state_hashtable = NULL; /* - * We need to free the contents of our stacks if the context is not dirty. - * These stacks are created in a more volatile memory context. If the - * process was interrupted, they will be garbage collected by PG. The only - * time we will ever clean them here is if the cache isn't being used. + * Free the DFS stacks. When is_dirty is false, the stacks are in the + * current context and need explicit cleanup. When is_dirty is true + * (cached context), only free the containers — the contents were + * allocated in a volatile SRF context that was already cleaned up. */ - if (vlelctx->is_dirty == false) + if (vlelctx->dfs_vertex_stack != NULL) { - free_graphid_stack(vlelctx->dfs_vertex_stack); - free_graphid_stack(vlelctx->dfs_edge_stack); - free_graphid_stack(vlelctx->dfs_path_stack); + free_gid_stack(vlelctx->dfs_vertex_stack); + } + if (vlelctx->dfs_edge_stack != NULL) + { + free_gid_stack(vlelctx->dfs_edge_stack); + } + if (vlelctx->dfs_path_stack != NULL) + { + free_gid_stack(vlelctx->dfs_path_stack); } - - /* free the containers */ - pfree_if_not_null(vlelctx->dfs_vertex_stack); - pfree_if_not_null(vlelctx->dfs_edge_stack); - pfree_if_not_null(vlelctx->dfs_path_stack); vlelctx->dfs_vertex_stack = NULL; vlelctx->dfs_edge_stack = NULL; vlelctx->dfs_path_stack = NULL; @@ -812,9 +829,9 @@ static VLE_local_context *build_local_vle_context(FunctionCallInfo fcinfo, create_VLE_local_state_hashtable(vlelctx); /* initialize the dfs stacks */ - vlelctx->dfs_vertex_stack = new_graphid_stack(); - vlelctx->dfs_edge_stack = new_graphid_stack(); - vlelctx->dfs_path_stack = new_graphid_stack(); + vlelctx->dfs_vertex_stack = new_gid_stack(); + vlelctx->dfs_edge_stack = new_gid_stack(); + vlelctx->dfs_path_stack = new_gid_stack(); /* load in the starting edge(s) */ load_initial_dfs_stacks(vlelctx); @@ -885,7 +902,7 @@ static graphid get_next_vertex(VLE_local_context *vlelctx, edge_entry *ee) case CYPHER_REL_DIR_NONE: { - ListGraphId *vertex_stack = NULL; + GraphIdStack *vertex_stack = NULL; graphid parent_vertex_id; vertex_stack = vlelctx->dfs_vertex_stack; @@ -894,7 +911,7 @@ static graphid get_next_vertex(VLE_local_context *vlelctx, edge_entry *ee) * as un-directional, where we go to next depends on where we came * from. This is because we can go against an edge. */ - parent_vertex_id = PEEK_GRAPHID_STACK(vertex_stack); + parent_vertex_id = gid_stack_peek(vertex_stack); /* find the terminal vertex */ if (get_edge_entry_start_vertex_id(ee) == parent_vertex_id) { @@ -933,9 +950,9 @@ static graphid get_next_vertex(VLE_local_context *vlelctx, edge_entry *ee) */ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) { - ListGraphId *vertex_stack = NULL; - ListGraphId *edge_stack = NULL; - ListGraphId *path_stack = NULL; + GraphIdStack *vertex_stack = NULL; + GraphIdStack *edge_stack = NULL; + GraphIdStack *path_stack = NULL; graphid end_vertex_id; Assert(vlelctx != NULL); @@ -947,7 +964,7 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) end_vertex_id = vlelctx->veid; /* while we have edges to process */ - while (!(IS_GRAPHID_STACK_EMPTY(edge_stack))) + while (!(gid_stack_is_empty(edge_stack))) { graphid edge_id; graphid next_vertex_id; @@ -956,7 +973,7 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) bool found = false; /* get an edge, but leave it on the stack for now */ - edge_id = PEEK_GRAPHID_STACK(edge_stack); + edge_id = gid_stack_peek(edge_stack); /* get the edge's state */ ese = get_edge_state(vlelctx, edge_id); /* @@ -972,18 +989,18 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) graphid path_edge_id; /* get the edge id on the top of the path stack (last edge) */ - path_edge_id = PEEK_GRAPHID_STACK(path_stack); + path_edge_id = gid_stack_peek(path_stack); /* * If the ids are the same, we're backing up. So, remove it from the * path stack and reset used_in_path. */ if (edge_id == path_edge_id) { - pop_graphid_stack(path_stack); + gid_stack_pop(path_stack); ese->used_in_path = false; } /* now remove it from the edge stack */ - pop_graphid_stack(edge_stack); + gid_stack_pop(edge_stack); /* * Remove its source vertex, if we are looking at edges as * un-directional. We only maintain the vertex stack when the @@ -992,7 +1009,7 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) */ if (vlelctx->edge_direction == CYPHER_REL_DIR_NONE) { - pop_graphid_stack(vertex_stack); + gid_stack_pop(vertex_stack); } /* move to the next edge */ continue; @@ -1003,7 +1020,7 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) * the edge stack as it is already there. */ ese->used_in_path = true; - push_graphid_stack(path_stack, edge_id); + gid_stack_push(path_stack, edge_id); /* now get the edge entry so we can get the next vertex to move to */ ee = get_edge_entry(vlelctx->ggctx, edge_id); @@ -1014,9 +1031,9 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) * within the bounds specified? */ if (next_vertex_id == end_vertex_id && - get_stack_size(path_stack) >= vlelctx->lidx && + gid_stack_size(path_stack) >= vlelctx->lidx && (vlelctx->uidx_infinite || - get_stack_size(path_stack) <= vlelctx->uidx)) + gid_stack_size(path_stack) <= vlelctx->uidx)) { /* we found one */ found = true; @@ -1028,14 +1045,14 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) */ if (next_vertex_id == end_vertex_id && !vlelctx->uidx_infinite && - get_stack_size(path_stack) > vlelctx->uidx) + gid_stack_size(path_stack) > vlelctx->uidx) { continue; } /* add in the edges for the next vertex if we won't exceed the bounds */ if (vlelctx->uidx_infinite || - get_stack_size(path_stack) < vlelctx->uidx) + gid_stack_size(path_stack) < vlelctx->uidx) { add_valid_vertex_edges(vlelctx, next_vertex_id); } @@ -1063,9 +1080,9 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) */ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) { - ListGraphId *vertex_stack = NULL; - ListGraphId *edge_stack = NULL; - ListGraphId *path_stack = NULL; + GraphIdStack *vertex_stack = NULL; + GraphIdStack *edge_stack = NULL; + GraphIdStack *path_stack = NULL; Assert(vlelctx != NULL); @@ -1075,7 +1092,7 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) path_stack = vlelctx->dfs_path_stack; /* while we have edges to process */ - while (!(IS_GRAPHID_STACK_EMPTY(edge_stack))) + while (!(gid_stack_is_empty(edge_stack))) { graphid edge_id; graphid next_vertex_id; @@ -1084,7 +1101,7 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) bool found = false; /* get an edge, but leave it on the stack for now */ - edge_id = PEEK_GRAPHID_STACK(edge_stack); + edge_id = gid_stack_peek(edge_stack); /* get the edge's state */ ese = get_edge_state(vlelctx, edge_id); /* @@ -1100,18 +1117,18 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) graphid path_edge_id; /* get the edge id on the top of the path stack (last edge) */ - path_edge_id = PEEK_GRAPHID_STACK(path_stack); + path_edge_id = gid_stack_peek(path_stack); /* * If the ids are the same, we're backing up. So, remove it from the * path stack and reset used_in_path. */ if (edge_id == path_edge_id) { - pop_graphid_stack(path_stack); + gid_stack_pop(path_stack); ese->used_in_path = false; } /* now remove it from the edge stack */ - pop_graphid_stack(edge_stack); + gid_stack_pop(edge_stack); /* * Remove its source vertex, if we are looking at edges as * un-directional. We only maintain the vertex stack when the @@ -1120,7 +1137,7 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) */ if (vlelctx->edge_direction == CYPHER_REL_DIR_NONE) { - pop_graphid_stack(vertex_stack); + gid_stack_pop(vertex_stack); } /* move to the next edge */ continue; @@ -1131,7 +1148,7 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) * the edge stack as it is already there. */ ese->used_in_path = true; - push_graphid_stack(path_stack, edge_id); + gid_stack_push(path_stack, edge_id); /* now get the edge entry so we can get the next vertex to move to */ ee = get_edge_entry(vlelctx->ggctx, edge_id); @@ -1141,9 +1158,9 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) * Is this a path that meets our requirements? Is its length within the * bounds specified? */ - if (get_stack_size(path_stack) >= vlelctx->lidx && + if (gid_stack_size(path_stack) >= vlelctx->lidx && (vlelctx->uidx_infinite || - get_stack_size(path_stack) <= vlelctx->uidx)) + gid_stack_size(path_stack) <= vlelctx->uidx)) { /* we found one */ found = true; @@ -1151,7 +1168,7 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) /* add in the edges for the next vertex if we won't exceed the bounds */ if (vlelctx->uidx_infinite || - get_stack_size(path_stack) < vlelctx->uidx) + gid_stack_size(path_stack) < vlelctx->uidx) { add_valid_vertex_edges(vlelctx, next_vertex_id); } @@ -1173,20 +1190,16 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) */ static bool is_edge_in_path(VLE_local_context *vlelctx, graphid edge_id) { - GraphIdNode *edge = NULL; + GraphIdStack *stack = vlelctx->dfs_path_stack; + int64 i; - /* start at the top of the stack */ - edge = peek_stack_head(vlelctx->dfs_path_stack); - - /* go through the path stack, return true if we find the edge */ - while (edge != NULL) + /* scan the array-based path stack */ + for (i = 0; i < gid_stack_size(stack); i++) { - if (get_graphid(edge) == edge_id) + if (gid_stack_get(stack, i) == edge_id) { return true; } - /* get the next stack element */ - edge = next_GraphIdNode(edge); } /* we didn't find it if we get here */ return false; @@ -1205,8 +1218,8 @@ static bool is_edge_in_path(VLE_local_context *vlelctx, graphid edge_id) static void add_valid_vertex_edges(VLE_local_context *vlelctx, graphid vertex_id) { - ListGraphId *vertex_stack = NULL; - ListGraphId *edge_stack = NULL; + GraphIdStack *vertex_stack = NULL; + GraphIdStack *edge_stack = NULL; ListGraphId *edges = NULL; vertex_entry *ve = NULL; GraphIdNode *edge_in = NULL; @@ -1267,7 +1280,7 @@ static void add_valid_vertex_edges(VLE_local_context *vlelctx, * This is a fast existence check, relative to the hash search, for when * the path stack is small. If the edge is in the path, we skip it. */ - if (get_stack_size(vlelctx->dfs_path_stack) < 10 && + if (gid_stack_size(vlelctx->dfs_path_stack) < 10 && is_edge_in_path(vlelctx, edge_id)) { /* set to the next available edge */ @@ -1325,9 +1338,9 @@ static void add_valid_vertex_edges(VLE_local_context *vlelctx, */ if (vlelctx->edge_direction == CYPHER_REL_DIR_NONE) { - push_graphid_stack(vertex_stack, get_vertex_entry_id(ve)); + gid_stack_push(vertex_stack, get_vertex_entry_id(ve)); } - push_graphid_stack(edge_stack, edge_id); + gid_stack_push(edge_stack, edge_id); } } /* get the next working edge */ @@ -1412,13 +1425,13 @@ static VLE_path_container *create_VLE_path_container(int64 path_size) static VLE_path_container *build_VLE_path_container(VLE_local_context *vlelctx) { - ListGraphId *stack = vlelctx->dfs_path_stack; + GraphIdStack *stack = vlelctx->dfs_path_stack; VLE_path_container *vpc = NULL; graphid *graphid_array = NULL; - GraphIdNode *edge = NULL; graphid vid = 0; int index = 0; int ssize = 0; + int j = 0; if (stack == NULL) { @@ -1426,7 +1439,7 @@ static VLE_path_container *build_VLE_path_container(VLE_local_context *vlelctx) } /* allocate the graphid array */ - ssize = get_stack_size(stack); + ssize = gid_stack_size(stack); /* * Create the container. Note that the path size will always be 2 times the @@ -1444,25 +1457,21 @@ static VLE_path_container *build_VLE_path_container(VLE_local_context *vlelctx) vid = vlelctx->vsid; graphid_array[0] = vid; - /* get the head of the stack */ - edge = peek_stack_head(stack); - /* - * We need to fill in the array from the back to the front. This is due - * to the order of the path stack - last in first out. Remember that the - * last entry is a vertex. + * Fill in edge entries from the back to the front. The path stack + * is array-based with index 0 = bottom (first pushed) and + * index size-1 = top (last pushed). We iterate from top to bottom + * to fill the graphid_array from back to front. */ index = vpc->graphid_array_size - 2; - /* copy while we have an edge to copy */ - while (edge != NULL) + for (j = ssize - 1; j >= 0; j--) { /* 0 is the vsid, we should never get here */ Assert(index > 0); - /* store and set to the next edge */ - graphid_array[index] = get_graphid(edge); - edge = next_GraphIdNode(edge); + /* store the edge from stack position j */ + graphid_array[index] = gid_stack_get(stack, j); /* we need to skip over the interior vertices */ index -= 2; @@ -1487,13 +1496,13 @@ static VLE_path_container *build_VLE_path_container(VLE_local_context *vlelctx) /* helper function to build a VPC for just the start vertex */ static VLE_path_container *build_VLE_zero_container(VLE_local_context *vlelctx) { - ListGraphId *stack = vlelctx->dfs_path_stack; + GraphIdStack *stack = vlelctx->dfs_path_stack; VLE_path_container *vpc = NULL; graphid *graphid_array = NULL; graphid vid = 0; /* we should have an empty stack */ - if (get_stack_size(stack) != 0) + if (gid_stack_size(stack) != 0) { ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), @@ -2011,10 +2020,6 @@ PG_FUNCTION_INFO_V1(age_match_vle_edge_to_id_qual); Datum age_match_vle_edge_to_id_qual(PG_FUNCTION_ARGS) { - int nargs = 0; - Datum *args = NULL; - bool *nulls = NULL; - Oid *types = NULL; agtype *agt_arg_vpc = NULL; agtype *edge_id = NULL; agtype *pos_agt = NULL; @@ -2023,11 +2028,10 @@ Datum age_match_vle_edge_to_id_qual(PG_FUNCTION_ARGS) graphid *array = NULL; bool vle_is_on_left = false; graphid gid = 0; + Oid type1; - /* extract argument values */ - nargs = extract_variadic_args(fcinfo, 0, true, &args, &types, &nulls); - - if (nargs != 3) + /* check argument count */ + if (PG_NARGS() != 3) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("age_match_vle_edge_to_id_qual() invalid number of arguments"))); @@ -2038,13 +2042,13 @@ Datum age_match_vle_edge_to_id_qual(PG_FUNCTION_ARGS) * OPTIONAL MATCH (LEFT JOIN) contexts where a preceding clause * produced no results. */ - if (nulls[0] || nulls[1] || nulls[2]) + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) { PG_RETURN_BOOL(false); } /* get the VLE_path_container argument */ - agt_arg_vpc = DATUM_GET_AGTYPE_P(args[0]); + agt_arg_vpc = DATUM_GET_AGTYPE_P(PG_GETARG_DATUM(0)); if (!AGT_ROOT_IS_BINARY(agt_arg_vpc) || AGT_ROOT_BINARY_FLAGS(agt_arg_vpc) != AGT_FBINARY_TYPE_VLE_PATH) @@ -2058,11 +2062,23 @@ Datum age_match_vle_edge_to_id_qual(PG_FUNCTION_ARGS) vle_path = (VLE_path_container *)agt_arg_vpc; array = GET_GRAPHID_ARRAY_FROM_CONTAINER(vle_path); - if (types[1] == AGTYPEOID) + /* + * Get arg type for argument 1 — cache in fn_extra to avoid + * repeated expression type resolution. + */ + if (fcinfo->flinfo->fn_extra == NULL) + { + Oid *cached_type = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt, + sizeof(Oid)); + *cached_type = get_fn_expr_argtype(fcinfo->flinfo, 1); + fcinfo->flinfo->fn_extra = cached_type; + } + type1 = *(Oid *)fcinfo->flinfo->fn_extra; + + if (type1 == AGTYPEOID) { /* Get the edge id we are checking the end of the list too */ edge_id = AG_GET_ARG_AGTYPE_P(1); - if (!AGT_ROOT_IS_SCALAR(edge_id)) { ereport(ERROR, @@ -2081,11 +2097,9 @@ Datum age_match_vle_edge_to_id_qual(PG_FUNCTION_ARGS) gid = id->val.int_value; } - else if (types[1] == GRAPHIDOID) + else if (type1 == GRAPHIDOID) { - - gid = DATUM_GET_GRAPHID(args[1]); - + gid = DATUM_GET_GRAPHID(PG_GETARG_DATUM(1)); } else { @@ -2240,10 +2254,6 @@ PG_FUNCTION_INFO_V1(age_match_vle_terminal_edge); Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) { - int nargs = 0; - Datum *args = NULL; - bool *nulls = NULL; - Oid *types = NULL; VLE_path_container *vpc = NULL; agtype *agt_arg_vsid = NULL; agtype *agt_arg_veid = NULL; @@ -2253,11 +2263,10 @@ Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) graphid veid = 0; graphid *gida = NULL; int gidasize = 0; + Oid type0, type1; - /* extract argument values */ - nargs = extract_variadic_args(fcinfo, 0, true, &args, &types, &nulls); - - if (nargs != 3) + /* check argument count */ + if (PG_NARGS() != 3) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("age_match_vle_terminal_edge() invalid number of arguments"))); @@ -2269,13 +2278,13 @@ Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) * where a preceding OPTIONAL MATCH produced no results. Returning * FALSE allows PostgreSQL to produce the correct NULL-extended rows. */ - if (nulls[0] || nulls[1] || nulls[2]) + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) { PG_RETURN_BOOL(false); } /* get the vpc */ - agt_arg_path = DATUM_GET_AGTYPE_P(args[2]); + agt_arg_path = DATUM_GET_AGTYPE_P(PG_GETARG_DATUM(2)); /* if the vpc is an agtype NULL, return FALSE */ if (is_agtype_null(agt_arg_path)) @@ -2302,14 +2311,29 @@ Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) /* verify the minimum size is 3 or 1 */ Assert(gidasize >= 3 || gidasize == 1); + /* + * Get argument types directly instead of using extract_variadic_args. + * This avoids the expensive exprType/get_call_expr_argtype overhead + * on every call. Cache the types in fn_extra on first invocation. + */ + if (fcinfo->flinfo->fn_extra == NULL) + { + Oid *cached_types = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt, + 2 * sizeof(Oid)); + cached_types[0] = get_fn_expr_argtype(fcinfo->flinfo, 0); + cached_types[1] = get_fn_expr_argtype(fcinfo->flinfo, 1); + fcinfo->flinfo->fn_extra = cached_types; + } + type0 = ((Oid *)fcinfo->flinfo->fn_extra)[0]; + type1 = ((Oid *)fcinfo->flinfo->fn_extra)[1]; + /* get the vsid */ - if (types[0] == AGTYPEOID) + if (type0 == AGTYPEOID) { - agt_arg_vsid = DATUM_GET_AGTYPE_P(args[0]); + agt_arg_vsid = DATUM_GET_AGTYPE_P(PG_GETARG_DATUM(0)); if (!is_agtype_null(agt_arg_vsid)) { - agtv_temp = get_ith_agtype_value_from_container(&agt_arg_vsid->root, 0); @@ -2321,9 +2345,9 @@ Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) PG_RETURN_BOOL(false); } } - else if (types[0] == GRAPHIDOID) + else if (type0 == GRAPHIDOID) { - vsid = DATUM_GET_GRAPHID(args[0]); + vsid = DATUM_GET_GRAPHID(PG_GETARG_DATUM(0)); } else { @@ -2333,9 +2357,9 @@ Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) } /* get the veid */ - if (types[1] == AGTYPEOID) + if (type1 == AGTYPEOID) { - agt_arg_veid = DATUM_GET_AGTYPE_P(args[1]); + agt_arg_veid = DATUM_GET_AGTYPE_P(PG_GETARG_DATUM(1)); if (!is_agtype_null(agt_arg_veid)) { @@ -2349,9 +2373,9 @@ Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) PG_RETURN_BOOL(false); } } - else if (types[1] == GRAPHIDOID) + else if (type1 == GRAPHIDOID) { - veid = DATUM_GET_GRAPHID(args[1]); + veid = DATUM_GET_GRAPHID(PG_GETARG_DATUM(1)); } else { diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index 8fad00bdf..b887305fa 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -4102,6 +4102,115 @@ Datum agtype_access_operator(PG_FUNCTION_ARGS) agtype *result = NULL; int i = 0; + /* + * Fast path for the common 2-argument case (object.property or + * array[index]). Avoids extract_variadic_args overhead which + * includes exprType, get_call_expr_argtype, and memory allocation + * on every call. + */ + if (PG_NARGS() == 2) + { + agtype *key = NULL; + + /* check for NULLs */ + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) + { + PG_RETURN_NULL(); + } + + /* get the container argument */ + container = DATUM_GET_AGTYPE_P(PG_GETARG_DATUM(0)); + + /* handle binary container (VLE vpc) */ + if (AGT_ROOT_IS_BINARY(container)) + { + if (AGT_ROOT_BINARY_FLAGS(container) == AGT_FBINARY_TYPE_VLE_PATH) + { + container_value = agtv_materialize_vle_edges(container); + container = NULL; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("binary container must be a VLE vpc"))); + } + } + /* handle scalar (vertex or edge) */ + else if (AGT_ROOT_IS_SCALAR(container)) + { + container_value = get_ith_agtype_value_from_container( + &container->root, 0); + if (container_value->type != AGTV_EDGE && + container_value->type != AGTV_VERTEX) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("scalar object must be a vertex or edge"))); + } + container = NULL; + } + + /* get the key */ + key = DATUM_GET_AGTYPE_P(PG_GETARG_DATUM(1)); + + if (!(AGT_ROOT_IS_SCALAR(key))) + { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("key must resolve to a scalar value"))); + } + + /* extract properties from vertex/edge */ + if (container_value != NULL && + (container_value->type == AGTV_EDGE || + container_value->type == AGTV_VERTEX)) + { + container_value = (container_value->type == AGTV_EDGE) + ? &container_value->val.object.pairs[4].value + : &container_value->val.object.pairs[2].value; + } + + /* map access */ + if ((container_value != NULL && + (container_value->type == AGTV_OBJECT || + (container_value->type == AGTV_BINARY && + AGTYPE_CONTAINER_IS_OBJECT(container_value->val.binary.data)))) || + (container != NULL && AGT_ROOT_IS_OBJECT(container))) + { + container_value = execute_map_access_operator(container, + container_value, key); + } + /* array access */ + else if ((container_value != NULL && + (container_value->type == AGTV_ARRAY || + (container_value->type == AGTV_BINARY && + AGTYPE_CONTAINER_IS_ARRAY(container_value->val.binary.data)))) || + (container != NULL && AGT_ROOT_IS_ARRAY(container))) + { + container_value = execute_array_access_operator(container, + container_value, + key); + } + else + { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("container must be an array or object"))); + } + + if (container_value == NULL || container_value->type == AGTV_NULL) + { + PG_RETURN_NULL(); + } + + result = agtype_value_to_agtype(container_value); + return AGTYPE_P_GET_DATUM(result); + } + + /* + * Standard variadic path for 3+ arguments (chained access like a.b.c) + * or edge cases. + */ + /* extract our args, we need at least 2 */ nargs = extract_variadic_args_min(fcinfo, 0, true, &args, &types, &nulls, 2); @@ -6653,24 +6762,52 @@ Datum age_tointeger(PG_FUNCTION_ARGS) Oid type; int64 result; - /* extract argument values */ - nargs = extract_variadic_args(fcinfo, 0, true, &args, &types, &nulls); + /* + * Fast path: toInteger() always takes exactly 1 argument. + * Avoid extract_variadic_args overhead by accessing the arg directly + * and caching the type via fn_extra. + */ + if (PG_NARGS() == 1) + { + if (PG_ARGISNULL(0)) + { + PG_RETURN_NULL(); + } - /* check number of args */ - if (nargs > 1) - ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("toInteger() only supports one argument"))); + arg = PG_GETARG_DATUM(0); - /* check for null */ - if (nargs < 0 || nulls[0]) - PG_RETURN_NULL(); + /* cache the arg type on first call */ + if (fcinfo->flinfo->fn_extra == NULL) + { + Oid *cached = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt, + sizeof(Oid)); + *cached = get_fn_expr_argtype(fcinfo->flinfo, 0); + fcinfo->flinfo->fn_extra = cached; + } + type = *(Oid *)fcinfo->flinfo->fn_extra; + nargs = 1; + } + else + { + /* fallback variadic path */ + nargs = extract_variadic_args(fcinfo, 0, true, &args, &types, &nulls); - /* - * toInteger() supports integer, float, numeric, text, cstring, or the - * agtype integer, float, numeric, and string input - */ - arg = args[0]; - type = types[0]; + /* check number of args */ + if (nargs > 1) + { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("toInteger() only supports one argument"))); + } + + /* check for null */ + if (nargs < 0 || nulls[0]) + { + PG_RETURN_NULL(); + } + + arg = args[0]; + type = types[0]; + } if (type != AGTYPEOID) { diff --git a/src/include/utils/age_global_graph.h b/src/include/utils/age_global_graph.h index 2b336a411..92044fc7e 100644 --- a/src/include/utils/age_global_graph.h +++ b/src/include/utils/age_global_graph.h @@ -59,4 +59,16 @@ Oid get_edge_entry_label_table_oid(edge_entry *ee); Datum get_edge_entry_properties(edge_entry *ee); graphid get_edge_entry_start_vertex_id(edge_entry *ee); graphid get_edge_entry_end_vertex_id(edge_entry *ee); + +/* Graph version counter functions — shared memory (DSM or shmem) */ +uint64 get_graph_version(Oid graph_oid); +void increment_graph_version(Oid graph_oid); +Oid get_graph_oid_for_table(Oid table_oid); + +/* Shared memory initialization for PG < 17 (shmem_request_hook path) */ +#if PG_VERSION_NUM < 170000 +void age_graph_version_shmem_request(void); +void age_graph_version_shmem_startup(void); +#endif + #endif diff --git a/src/include/utils/age_graphid_ds.h b/src/include/utils/age_graphid_ds.h index a5bb5273f..ebd1290d0 100644 --- a/src/include/utils/age_graphid_ds.h +++ b/src/include/utils/age_graphid_ds.h @@ -40,6 +40,21 @@ typedef struct GraphIdNode GraphIdNode; /* declare the ListGraphId container */ typedef struct ListGraphId ListGraphId; +/* + * Array-based stack for graphid values. Used by VLE DFS traversal + * for the vertex, edge, and path stacks. Provides O(1) push/pop + * with sequential memory access (no linked-list pointer chasing). + * + * This is intentionally separate from ListGraphId, which is used + * for vertex adjacency lists in the global graph context. + */ +typedef struct GraphIdStack +{ + graphid *array; /* contiguous graphid array */ + int64 size; /* current number of elements */ + int64 capacity; /* allocated capacity */ +} GraphIdStack; + /* GraphIdNode access functions */ GraphIdNode *next_GraphIdNode(GraphIdNode *node); graphid get_graphid(GraphIdNode *node); @@ -73,4 +88,14 @@ GraphIdNode *get_list_head(ListGraphId *list); /* get the size of the passed list */ int64 get_list_size(ListGraphId *list); +/* GraphIdStack functions — array-based stack for VLE DFS */ +GraphIdStack *new_gid_stack(void); +void free_gid_stack(GraphIdStack *stack); +void gid_stack_push(GraphIdStack *s, graphid id); +graphid gid_stack_pop(GraphIdStack *s); +graphid gid_stack_peek(GraphIdStack *s); +bool gid_stack_is_empty(GraphIdStack *s); +int64 gid_stack_size(GraphIdStack *s); +graphid gid_stack_get(GraphIdStack *s, int64 i); + #endif From 6f520fedda7885efcfc104b925338cdfa933f935 Mon Sep 17 00:00:00 2001 From: Sai Asish Y Date: Tue, 21 Apr 2026 02:49:59 -0700 Subject: [PATCH 056/100] Propagate null in agtype_access_slice bounds (#2400) * agtype: propagate null through list slice bounds in agtype_access_slice instead of treating AGTV_NULL as omitted agtype_access_slice() decides what to do with each bound in two steps: 1. If the SQL argument is NULL (`PG_ARGISNULL(1|2)`), the caller did not supply that bound at all (e.g. `list[..2]`) - treat it as "no bound" and fall back to 0 / array_size. This is correct. 2. Otherwise the argument is non-NULL agtype. If *the agtype value* inside happens to be AGTV_NULL (e.g. `list[null..2]`), the existing code also treated it as "no bound" and returned the full slice. This is wrong. Under Cypher null-propagation semantics `list[a..b]` is null whenever either bound is null; Neo4j and Memgraph both return null, and differential testing against both flagged the divergence. Change step 2 to PG_RETURN_NULL() when the supplied bound is AGTV_NULL. Step 1 is untouched, so `[..n]` / `[n..]` still work as before. Regression tests are updated accordingly: * The two pre-existing agtype_access_slice() calls with an explicit 'null'::agtype argument now expect the null result. - The patch only fixes agtype_access_slice function, not direct list slices in cypher query. Co-developed with an AI coding assistant (Claude Code). Signed-off-by: SAY-5 --- regress/expected/expr.out | 4 ++-- src/backend/utils/adt/agtype.c | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/regress/expected/expr.out b/regress/expected/expr.out index e95fe14e3..4a332a335 100644 --- a/regress/expected/expr.out +++ b/regress/expected/expr.out @@ -489,13 +489,13 @@ $$RETURN [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10][-1..10]$$) AS r(c agtype); SELECT agtype_access_slice('[0]'::agtype, 'null'::agtype, '1'::agtype); agtype_access_slice --------------------- - [0] + (1 row) SELECT agtype_access_slice('[0]'::agtype, '0'::agtype, 'null'::agtype); agtype_access_slice --------------------- - [0] + (1 row) -- should error - ERROR: slice must access a list diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index b887305fa..6700be3f3 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -4433,11 +4433,14 @@ Datum agtype_access_slice(PG_FUNCTION_ARGS) { agt_lidx = AG_GET_ARG_AGTYPE_P(1); lidx_value = get_ith_agtype_value_from_container(&agt_lidx->root, 0); - /* adjust for AGTV_NULL */ + /* + * Under Cypher null-propagation semantics, list[a..b] is null when + * either bound is null. Return null directly instead of silently + * treating AGTV_NULL as an omitted bound. + */ if (lidx_value->type == AGTV_NULL) { - lower_index = 0; - lidx_value = NULL; + PG_RETURN_NULL(); } } @@ -4450,11 +4453,10 @@ Datum agtype_access_slice(PG_FUNCTION_ARGS) { agt_uidx = AG_GET_ARG_AGTYPE_P(2); uidx_value = get_ith_agtype_value_from_container(&agt_uidx->root, 0); - /* adjust for AGTV_NULL */ + /* Symmetric to the lower bound: null propagates to a null result. */ if (uidx_value->type == AGTV_NULL) { - upper_index = array_size; - uidx_value = NULL; + PG_RETURN_NULL(); } } From 84e29542f0d54c9d848a3d5ebbc7a8f4d5a67437 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Wed, 22 Apr 2026 05:06:59 -0400 Subject: [PATCH 057/100] Add MERGE ON CREATE SET / ON MATCH SET support (#2347) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add MERGE ON CREATE SET / ON MATCH SET support Implements the openCypher-standard ON CREATE SET and ON MATCH SET clauses for the MERGE statement. This allows conditional property updates depending on whether MERGE created a new path or matched an existing one: MERGE (n:Person {name: 'Alice'}) ON CREATE SET n.created = timestamp() ON MATCH SET n.updated = timestamp() Implementation spans parser, planner, and executor: - Grammar: new merge_actions_opt/merge_actions/merge_action rules in cypher_gram.y, with ON keyword added to cypher_kwlist.h - Nodes: on_match/on_create lists on cypher_merge, corresponding on_match_set_info/on_create_set_info on cypher_merge_information, and prop_expr on cypher_update_item (all serialized through copy/out/read funcs) - Transform: cypher_clause.c transforms ON SET items and stores prop_expr for direct expression evaluation - Executor: cypher_set.c extracts apply_update_list() from process_update_list(); cypher_merge.c calls it at all merge decision points (simple merge, terminal, non-terminal with eager buffering, and first-clause-with-followers paths) Key design choice: prop_expr stores the Expr* directly in cypher_update_item rather than using prop_position into the scan tuple. The planner strips target list entries for SET expressions that CustomScan doesn't need, making prop_position references dangling. By storing the expression directly (only for MERGE ON SET items), we evaluate it with ExecInitExpr/ExecEvalExpr independent of the scan tuple layout. Includes regression tests covering: basic ON CREATE SET, basic ON MATCH SET, combined ON CREATE + ON MATCH, multiple SET items, expression evaluation, interaction with WITH clause, and edge property updates. - Move ExecInitExpr for ON CREATE/MATCH SET items from per-row execution in apply_update_list() to plan initialization in begin_cypher_merge(). Follows the established pattern used by cypher_target_node (id_expr_state, prop_expr_state). - Add prop_expr_state field to cypher_update_item with serialization support in outfuncs/readfuncs/copyfuncs. - apply_update_list() uses pre-initialized state when available, falls back to per-row init for plain SET callers. - Fix misleading comment: "ON MATCH SET" → "ON CREATE SET" for Case 1 first-run test. - Add Case 1 second-run test that triggers ON MATCH SET with a predecessor clause (MATCH ... MERGE ... ON MATCH SET). - Add ON to safe_keywords in cypher_gram.y so that property keys and labels named 'on' still work (e.g., n.on, MATCH (n:on)). All other keywords added as tokens are also in safe_keywords. - Add chained (non-terminal) MERGE regression tests exercising the eager-buffering code path with ON CREATE SET and ON MATCH SET. First run creates both nodes (ON CREATE SET fires), second run matches both (ON MATCH SET fires). - Move ExecStoreVirtualTuple before apply_update_list unconditionally in Case 1 non-terminal and terminal MERGE paths, matching the pattern at Case 3 (line 994). Ensures tts_nvalid is set for downstream ExecProject even when ON CREATE SET is absent. - Add resolve_merge_set_exprs() helper to deduplicate the prop_expr resolution loops for ON MATCH SET and ON CREATE SET. Includes ereport when target entry is missing (internal error, should never happen). - Add regression test for ON keyword as label name, confirming backward compatibility via safe_keywords grammar path. - The four ExecStoreVirtualTuple calls in exec_cypher_merge were triggering an Assert failure under --enable-cassert: TRAP: failed Assert("TTS_EMPTY(slot)"), File: execTuples.c, Line: 1748 ExecStoreVirtualTuple (execTuples.c:1748) asserts that its target slot is in the TTS_EMPTY state. In our MERGE executor, process_path writes directly into the subquery's scan tuple slot -- which already holds the subquery's output tuple and therefore is NOT empty. On a release build the assertion compiles out and ExecStoreVirtualTuple just clears the flag and sets tts_nvalid; on an --enable-cassert build the backend aborts and takes down the regression run. We only need the bookkeeping half of ExecStoreVirtualTuple (clear TTS_FLAG_EMPTY and set tts_nvalid = natts) -- not the "store semantics" that motivate the assertion. Add a small static helper mark_scan_slot_valid() that does exactly the bookkeeping, and replace the four call sites. Release-build behavior is byte-identical since Assert() compiles to nothing; cassert-build behavior now matches release. - Fix MERGE ON CREATE/MATCH SET crash when RHS references a bound variable When MERGE has a previous clause (e.g. MATCH, UNWIND), transform_cypher_merge takes the lateral-left-join path via transform_merge_make_lateral_join. That helper called addRangeTableEntryForJoin with nscolumns=NULL, leaving the join ParseNamespaceItem's p_nscolumns unset. For queries that did not subsequently resolve a column reference against that nsitem (e.g. RETURN, which runs in a fresh namespace built by handle_prev_clause), the NULL was harmless. Our ON CREATE / ON MATCH SET transform runs in-line, before the MERGE query becomes a subquery, so transform_cypher_set_item_list consulted the join's nsitem directly. colNameToVar -> scanNSItemForColumn then dereferenced p_nscolumns[attnum-1] = NULL[0] and the backend segfaulted on any ON SET whose RHS referenced a bound variable. Populate the join's p_nscolumns from res_colvars. The Var we end up producing for a bound entity lives inside prop_expr, which is opaque to the planner, so it is not rewritten to match the plan's output slots. At ExecEvalScalarVar time only varattno is consulted, and scantuple's layout mirrors the join's eref->colnames (via make_target_list_from_join). Use the join rtindex and 1-based eref position so scantuple[varattno - 1] resolves to the correct entity column at runtime; without this, Vars for a (varno=l_rte) and b (varno=r_rte) with varattno=1 both hit scantuple[0] and b.id evaluated to a.id. Also initialise apply_update_list's new_property_value at its declaration. All control paths reach the single alter_property_value call with the variable set, but -Wmaybe-uninitialized fires at -O2 because the compiler cannot prove remove_property == isnull when prop_expr is non-NULL. --------- Co-authored-by: Claude Opus 4.6 --- .gitignore | 1 + regress/expected/cypher_merge.out | 249 +++++++++++++++++++++++++++ regress/sql/cypher_merge.sql | 157 +++++++++++++++++ src/backend/executor/cypher_merge.c | 133 +++++++++++++- src/backend/executor/cypher_set.c | 67 +++++-- src/backend/nodes/cypher_copyfuncs.c | 4 + src/backend/nodes/cypher_outfuncs.c | 8 +- src/backend/nodes/cypher_readfuncs.c | 4 + src/backend/parser/cypher_clause.c | 114 +++++++++++- src/backend/parser/cypher_gram.y | 65 ++++++- src/include/executor/cypher_utils.h | 6 + src/include/nodes/cypher_nodes.h | 7 + src/include/parser/cypher_kwlist.h | 1 + 13 files changed, 788 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 03923b03e..1e2f8f674 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ __pycache__ **/apache_age_python.egg-info drivers/python/build +*.bc diff --git a/regress/expected/cypher_merge.out b/regress/expected/cypher_merge.out index 4242f2f59..ac08a971a 100644 --- a/regress/expected/cypher_merge.out +++ b/regress/expected/cypher_merge.out @@ -2001,9 +2001,258 @@ SELECT * FROM cypher('issue_1954', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype --- (0 rows) +-- +-- ON CREATE SET / ON MATCH SET tests (issue #1619) +-- +SELECT create_graph('merge_actions'); +NOTICE: graph "merge_actions" has been created + create_graph +-------------- + +(1 row) + +-- Basic ON CREATE SET: first run creates the node +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Alice'}) + ON CREATE SET n.created = true + RETURN n.name, n.created +$$) AS (name agtype, created agtype); + name | created +---------+--------- + "Alice" | true +(1 row) + +-- ON MATCH SET: second run matches the existing node +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Alice'}) + ON MATCH SET n.found = true + RETURN n.name, n.created, n.found +$$) AS (name agtype, created agtype, found agtype); + name | created | found +---------+---------+------- + "Alice" | true | true +(1 row) + +-- Both ON CREATE SET and ON MATCH SET (first run = create) +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Bob'}) + ON CREATE SET n.created = true + ON MATCH SET n.matched = true + RETURN n.name, n.created, n.matched +$$) AS (name agtype, created agtype, matched agtype); + name | created | matched +-------+---------+--------- + "Bob" | true | +(1 row) + +-- Both ON CREATE SET and ON MATCH SET (second run = match) +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Bob'}) + ON CREATE SET n.created = true + ON MATCH SET n.matched = true + RETURN n.name, n.created, n.matched +$$) AS (name agtype, created agtype, matched agtype); + name | created | matched +-------+---------+--------- + "Bob" | true | true +(1 row) + +-- ON CREATE SET with MERGE after MATCH (Case 1: has predecessor, first run = create) +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Alice'}) + MERGE (a)-[:KNOWS]->(b:Person {name: 'Charlie'}) + ON CREATE SET b.source = 'merge_create' + RETURN a.name, b.name, b.source +$$) AS (a agtype, b agtype, source agtype); + a | b | source +---------+-----------+---------------- + "Alice" | "Charlie" | "merge_create" +(1 row) + +-- ON MATCH SET with MERGE after MATCH (Case 1: has predecessor, second run = match) +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Alice'}) + MERGE (a)-[:KNOWS]->(b:Person {name: 'Charlie'}) + ON MATCH SET b.visited = true + RETURN a.name, b.name, b.visited +$$) AS (a agtype, b agtype, visited agtype); + a | b | visited +---------+-----------+--------- + "Alice" | "Charlie" | true +(1 row) + +-- Multiple SET items in a single ON CREATE SET +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Dave'}) + ON CREATE SET n.a = 1, n.b = 2 + RETURN n.name, n.a, n.b +$$) AS (name agtype, a agtype, b agtype); + name | a | b +--------+---+--- + "Dave" | 1 | 2 +(1 row) + +-- Reverse order: ON MATCH before ON CREATE should work +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Eve'}) + ON MATCH SET n.seen = true + ON CREATE SET n.new = true + RETURN n.name, n.new +$$) AS (name agtype, new agtype); + name | new +-------+------ + "Eve" | true +(1 row) + +-- Error: ON CREATE SET specified more than once +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Bad'}) + ON CREATE SET n.a = 1 + ON CREATE SET n.b = 2 + RETURN n +$$) AS (n agtype); +ERROR: ON CREATE SET specified more than once +LINE 1: SELECT * FROM cypher('merge_actions', $$ + ^ +-- Error: ON MATCH SET specified more than once +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Bad'}) + ON MATCH SET n.a = 1 + ON MATCH SET n.b = 2 + RETURN n +$$) AS (n agtype); +ERROR: ON MATCH SET specified more than once +LINE 1: SELECT * FROM cypher('merge_actions', $$ + ^ +-- Chained (non-terminal) MERGE with ON CREATE SET (eager-buffering path) +SELECT * FROM cypher('merge_actions', $$ + MERGE (a:Person {name: 'Frank'}) + ON CREATE SET a.created = true + MERGE (a)-[:KNOWS]->(b:Person {name: 'Grace'}) + ON CREATE SET b.created = true + RETURN a.name, a.created, b.name, b.created +$$) AS (a_name agtype, a_created agtype, b_name agtype, b_created agtype); + a_name | a_created | b_name | b_created +---------+-----------+---------+----------- + "Frank" | true | "Grace" | true +(1 row) + +-- Chained (non-terminal) MERGE with ON MATCH SET (second run = match) +SELECT * FROM cypher('merge_actions', $$ + MERGE (a:Person {name: 'Frank'}) + ON MATCH SET a.matched = true + MERGE (a)-[:KNOWS]->(b:Person {name: 'Grace'}) + ON MATCH SET b.matched = true + RETURN a.name, a.matched, b.name, b.matched +$$) AS (a_name agtype, a_matched agtype, b_name agtype, b_matched agtype); + a_name | a_matched | b_name | b_matched +---------+-----------+---------+----------- + "Frank" | true | "Grace" | true +(1 row) + +-- ON keyword as label name (backward compat via safe_keywords) +SELECT * FROM cypher('merge_actions', $$ + CREATE (n:on {name: 'test'}) + RETURN n.name +$$) AS (name agtype); + name +-------- + "test" +(1 row) + +-- Issue #2347: RHS of ON CREATE / ON MATCH SET referencing a bound +-- variable crashed the backend when MERGE had a previous clause, because +-- the lateral-join's ParseNamespaceItem had p_nscolumns=NULL. +-- ON CREATE SET with RHS referencing the outer MATCH's variable +SELECT * FROM cypher('merge_actions', $$ CREATE (:Person {name:'Anchor'}) $$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Anchor'}) + MERGE (b:Person {name: 'FromOuter'}) + ON CREATE SET b.source_name = a.name + RETURN a.name, b.name, b.source_name +$$) AS (a_name agtype, b_name agtype, b_source agtype); + a_name | b_name | b_source +----------+-------------+---------- + "Anchor" | "FromOuter" | "Anchor" +(1 row) + +-- ON CREATE SET with RHS referencing the MERGE-bound variable itself +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Anchor'}) + MERGE (b:Person {name: 'SelfRef'}) + ON CREATE SET b.echo_name = b.name + RETURN b.name, b.echo_name +$$) AS (b_name agtype, b_echo agtype); + b_name | b_echo +-----------+----------- + "SelfRef" | "SelfRef" +(1 row) + +-- ON CREATE SET driven by UNWIND with self-reference on the RHS +-- (Muhammad's second reproducer) +SELECT * FROM cypher('merge_actions', $$ + UNWIND ['U1', 'U2'] AS nm + MERGE (n:Person {name: nm}) + ON CREATE SET n.copy_name = n.name + RETURN n.name, n.copy_name +$$) AS (n_name agtype, n_copy agtype); + n_name | n_copy +--------+-------- + "U1" | "U1" + "U2" | "U2" +(2 rows) + +-- Multiple SET items mixing outer-ref, self-ref, and literal RHS +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Anchor'}) + MERGE (b:Person {name: 'MultiItem'}) + ON CREATE SET b.from_a = a.name, b.self = b.name, b.lit = 'literal' + RETURN b.from_a, b.self, b.lit +$$) AS (fa agtype, sf agtype, lit agtype); + fa | sf | lit +----------+-------------+----------- + "Anchor" | "MultiItem" | "literal" +(1 row) + +-- ON MATCH SET with variable RHS (second run on existing node) +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Anchor'}) + MERGE (b:Person {name: 'FromOuter'}) + ON CREATE SET b.source_name = a.name + ON MATCH SET b.last_seen_by = a.name + RETURN b.source_name, b.last_seen_by +$$) AS (src agtype, last agtype); + src | last +----------+---------- + "Anchor" | "Anchor" +(1 row) + +-- cleanup +SELECT * FROM cypher('merge_actions', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); + a +--- +(0 rows) + -- -- delete graphs -- +SELECT drop_graph('merge_actions', true); +NOTICE: drop cascades to 5 other objects +DETAIL: drop cascades to table merge_actions._ag_label_vertex +drop cascades to table merge_actions._ag_label_edge +drop cascades to table merge_actions."Person" +drop cascades to table merge_actions."KNOWS" +drop cascades to table merge_actions."on" +NOTICE: graph "merge_actions" has been dropped + drop_graph +------------ + +(1 row) + SELECT drop_graph('issue_1907', true); NOTICE: drop cascades to 4 other objects DETAIL: drop cascades to table issue_1907._ag_label_vertex diff --git a/regress/sql/cypher_merge.sql b/regress/sql/cypher_merge.sql index 5939c42a8..86b3e0235 100644 --- a/regress/sql/cypher_merge.sql +++ b/regress/sql/cypher_merge.sql @@ -932,9 +932,166 @@ SELECT * FROM cypher('issue_1709', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype SELECT * FROM cypher('issue_1446', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); SELECT * FROM cypher('issue_1954', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); +-- +-- ON CREATE SET / ON MATCH SET tests (issue #1619) +-- +SELECT create_graph('merge_actions'); + +-- Basic ON CREATE SET: first run creates the node +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Alice'}) + ON CREATE SET n.created = true + RETURN n.name, n.created +$$) AS (name agtype, created agtype); + +-- ON MATCH SET: second run matches the existing node +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Alice'}) + ON MATCH SET n.found = true + RETURN n.name, n.created, n.found +$$) AS (name agtype, created agtype, found agtype); + +-- Both ON CREATE SET and ON MATCH SET (first run = create) +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Bob'}) + ON CREATE SET n.created = true + ON MATCH SET n.matched = true + RETURN n.name, n.created, n.matched +$$) AS (name agtype, created agtype, matched agtype); + +-- Both ON CREATE SET and ON MATCH SET (second run = match) +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Bob'}) + ON CREATE SET n.created = true + ON MATCH SET n.matched = true + RETURN n.name, n.created, n.matched +$$) AS (name agtype, created agtype, matched agtype); + +-- ON CREATE SET with MERGE after MATCH (Case 1: has predecessor, first run = create) +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Alice'}) + MERGE (a)-[:KNOWS]->(b:Person {name: 'Charlie'}) + ON CREATE SET b.source = 'merge_create' + RETURN a.name, b.name, b.source +$$) AS (a agtype, b agtype, source agtype); + +-- ON MATCH SET with MERGE after MATCH (Case 1: has predecessor, second run = match) +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Alice'}) + MERGE (a)-[:KNOWS]->(b:Person {name: 'Charlie'}) + ON MATCH SET b.visited = true + RETURN a.name, b.name, b.visited +$$) AS (a agtype, b agtype, visited agtype); + +-- Multiple SET items in a single ON CREATE SET +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Dave'}) + ON CREATE SET n.a = 1, n.b = 2 + RETURN n.name, n.a, n.b +$$) AS (name agtype, a agtype, b agtype); + +-- Reverse order: ON MATCH before ON CREATE should work +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Eve'}) + ON MATCH SET n.seen = true + ON CREATE SET n.new = true + RETURN n.name, n.new +$$) AS (name agtype, new agtype); + +-- Error: ON CREATE SET specified more than once +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Bad'}) + ON CREATE SET n.a = 1 + ON CREATE SET n.b = 2 + RETURN n +$$) AS (n agtype); + +-- Error: ON MATCH SET specified more than once +SELECT * FROM cypher('merge_actions', $$ + MERGE (n:Person {name: 'Bad'}) + ON MATCH SET n.a = 1 + ON MATCH SET n.b = 2 + RETURN n +$$) AS (n agtype); + +-- Chained (non-terminal) MERGE with ON CREATE SET (eager-buffering path) +SELECT * FROM cypher('merge_actions', $$ + MERGE (a:Person {name: 'Frank'}) + ON CREATE SET a.created = true + MERGE (a)-[:KNOWS]->(b:Person {name: 'Grace'}) + ON CREATE SET b.created = true + RETURN a.name, a.created, b.name, b.created +$$) AS (a_name agtype, a_created agtype, b_name agtype, b_created agtype); + +-- Chained (non-terminal) MERGE with ON MATCH SET (second run = match) +SELECT * FROM cypher('merge_actions', $$ + MERGE (a:Person {name: 'Frank'}) + ON MATCH SET a.matched = true + MERGE (a)-[:KNOWS]->(b:Person {name: 'Grace'}) + ON MATCH SET b.matched = true + RETURN a.name, a.matched, b.name, b.matched +$$) AS (a_name agtype, a_matched agtype, b_name agtype, b_matched agtype); + +-- ON keyword as label name (backward compat via safe_keywords) +SELECT * FROM cypher('merge_actions', $$ + CREATE (n:on {name: 'test'}) + RETURN n.name +$$) AS (name agtype); + +-- Issue #2347: RHS of ON CREATE / ON MATCH SET referencing a bound +-- variable crashed the backend when MERGE had a previous clause, because +-- the lateral-join's ParseNamespaceItem had p_nscolumns=NULL. + +-- ON CREATE SET with RHS referencing the outer MATCH's variable +SELECT * FROM cypher('merge_actions', $$ CREATE (:Person {name:'Anchor'}) $$) AS (a agtype); +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Anchor'}) + MERGE (b:Person {name: 'FromOuter'}) + ON CREATE SET b.source_name = a.name + RETURN a.name, b.name, b.source_name +$$) AS (a_name agtype, b_name agtype, b_source agtype); + +-- ON CREATE SET with RHS referencing the MERGE-bound variable itself +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Anchor'}) + MERGE (b:Person {name: 'SelfRef'}) + ON CREATE SET b.echo_name = b.name + RETURN b.name, b.echo_name +$$) AS (b_name agtype, b_echo agtype); + +-- ON CREATE SET driven by UNWIND with self-reference on the RHS +-- (Muhammad's second reproducer) +SELECT * FROM cypher('merge_actions', $$ + UNWIND ['U1', 'U2'] AS nm + MERGE (n:Person {name: nm}) + ON CREATE SET n.copy_name = n.name + RETURN n.name, n.copy_name +$$) AS (n_name agtype, n_copy agtype); + +-- Multiple SET items mixing outer-ref, self-ref, and literal RHS +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Anchor'}) + MERGE (b:Person {name: 'MultiItem'}) + ON CREATE SET b.from_a = a.name, b.self = b.name, b.lit = 'literal' + RETURN b.from_a, b.self, b.lit +$$) AS (fa agtype, sf agtype, lit agtype); + +-- ON MATCH SET with variable RHS (second run on existing node) +SELECT * FROM cypher('merge_actions', $$ + MATCH (a:Person {name: 'Anchor'}) + MERGE (b:Person {name: 'FromOuter'}) + ON CREATE SET b.source_name = a.name + ON MATCH SET b.last_seen_by = a.name + RETURN b.source_name, b.last_seen_by +$$) AS (src agtype, last agtype); + +-- cleanup +SELECT * FROM cypher('merge_actions', $$ MATCH (n) DETACH DELETE n $$) AS (a agtype); + -- -- delete graphs -- +SELECT drop_graph('merge_actions', true); SELECT drop_graph('issue_1907', true); SELECT drop_graph('cypher_merge', true); SELECT drop_graph('issue_1630', true); diff --git a/src/backend/executor/cypher_merge.c b/src/backend/executor/cypher_merge.c index c2db878b7..2b3d1f7dd 100644 --- a/src/backend/executor/cypher_merge.c +++ b/src/backend/executor/cypher_merge.c @@ -81,6 +81,7 @@ static bool check_path(cypher_merge_custom_scan_state *css, static void process_path(cypher_merge_custom_scan_state *css, path_entry **path_array, bool should_insert); static void mark_tts_isnull(TupleTableSlot *slot); +static void mark_scan_slot_valid(TupleTableSlot *slot); const CustomExecMethods cypher_merge_exec_methods = {MERGE_SCAN_STATE_NAME, begin_cypher_merge, @@ -192,6 +193,35 @@ static void begin_cypher_merge(CustomScanState *node, EState *estate, } } + /* + * Pre-initialize ExprStates for ON CREATE SET / ON MATCH SET items. + * This must happen once at plan init time, not per-row. + */ + if (css->on_create_set_info != NULL) + { + foreach(lc, css->on_create_set_info->set_items) + { + cypher_update_item *item = (cypher_update_item *)lfirst(lc); + if (item->prop_expr != NULL) + { + item->prop_expr_state = ExecInitExpr( + (Expr *)item->prop_expr, (PlanState *)node); + } + } + } + if (css->on_match_set_info != NULL) + { + foreach(lc, css->on_match_set_info->set_items) + { + cypher_update_item *item = (cypher_update_item *)lfirst(lc); + if (item->prop_expr != NULL) + { + item->prop_expr_state = ExecInitExpr( + (Expr *)item->prop_expr, (PlanState *)node); + } + } + } + /* * Postgres does not assign the es_output_cid in queries that do * not write to disk, ie: SELECT commands. We need the command id @@ -322,11 +352,49 @@ static void process_simple_merge(CustomScanState *node) /* setup the scantuple that the process_path needs */ econtext->ecxt_scantuple = sss->ss.ss_ScanTupleSlot; + mark_tts_isnull(econtext->ecxt_scantuple); process_path(css, NULL, true); + + /* ON CREATE SET: path was just created */ + if (css->on_create_set_info) + { + mark_scan_slot_valid(econtext->ecxt_scantuple); + apply_update_list(&css->css, css->on_create_set_info); + } + } + else + { + /* ON MATCH SET: path already exists */ + if (css->on_match_set_info) + { + ExprContext *econtext = node->ss.ps.ps_ExprContext; + + econtext->ecxt_scantuple = + node->ss.ps.lefttree->ps_ProjInfo->pi_exprContext->ecxt_scantuple; + + apply_update_list(&css->css, css->on_match_set_info); + } } } +/* + * mark_scan_slot_valid - mark a scan slot as populated after direct writes + * to tts_values[] by process_path. + * + * This does the same bookkeeping as ExecStoreVirtualTuple (clear TTS_EMPTY, + * set tts_nvalid = natts) but without the TTS_EMPTY precondition assertion. + * We cannot use ExecStoreVirtualTuple here because process_path writes into + * a scan slot that already holds the subquery's output tuple -- the slot is + * NOT empty, and asserting it is would fire under --enable-cassert while + * silently clearing the flag on release builds. + */ +static void mark_scan_slot_valid(TupleTableSlot *slot) +{ + slot->tts_flags &= ~TTS_FLAG_EMPTY; + slot->tts_nvalid = slot->tts_tupleDescriptor->natts; +} + /* * Iterate through the TupleTableSlot's tts_values and marks the isnull field * with true. @@ -658,6 +726,11 @@ static TupleTableSlot *exec_cypher_merge(CustomScanState *node) free_path_entry_array(prebuilt_path_array, path_length); process_path(css, found_path_array, false); + + /* ON MATCH SET: path was found as duplicate */ + if (css->on_match_set_info) + apply_update_list(&css->css, + css->on_match_set_info); } else { @@ -669,8 +742,20 @@ static TupleTableSlot *exec_cypher_merge(CustomScanState *node) css->created_paths_list = new_path; process_path(css, prebuilt_path_array, true); + mark_scan_slot_valid(econtext->ecxt_scantuple); + + /* ON CREATE SET: path was just created */ + if (css->on_create_set_info) + apply_update_list(&css->css, + css->on_create_set_info); } } + else + { + /* ON MATCH SET: path already existed from lateral join */ + if (css->on_match_set_info) + apply_update_list(&css->css, css->on_match_set_info); + } /* Project the result and save a copy */ econtext->ecxt_scantuple = @@ -743,6 +828,10 @@ static TupleTableSlot *exec_cypher_merge(CustomScanState *node) { free_path_entry_array(prebuilt_path_array, path_length); process_path(css, found_path_array, false); + + /* ON MATCH SET: path was found as duplicate */ + if (css->on_match_set_info) + apply_update_list(&css->css, css->on_match_set_info); } else { @@ -753,8 +842,19 @@ static TupleTableSlot *exec_cypher_merge(CustomScanState *node) css->created_paths_list = new_path; process_path(css, prebuilt_path_array, true); + mark_scan_slot_valid(econtext->ecxt_scantuple); + + /* ON CREATE SET: path was just created */ + if (css->on_create_set_info) + apply_update_list(&css->css, css->on_create_set_info); } } + else + { + /* ON MATCH SET: path already existed from lateral join */ + if (css->on_match_set_info) + apply_update_list(&css->css, css->on_match_set_info); + } } while (true); @@ -827,6 +927,14 @@ static TupleTableSlot *exec_cypher_merge(CustomScanState *node) */ css->found_a_path = true; + /* ON MATCH SET: path already exists */ + if (css->on_match_set_info) + { + econtext->ecxt_scantuple = + node->ss.ps.lefttree->ps_ProjInfo->pi_exprContext->ecxt_scantuple; + apply_update_list(&css->css, css->on_match_set_info); + } + econtext->ecxt_scantuple = ExecProject(node->ss.ps.lefttree->ps_ProjInfo); return ExecProject(node->ss.ps.ps_ProjInfo); } @@ -887,20 +995,25 @@ static TupleTableSlot *exec_cypher_merge(CustomScanState *node) /* setup the scantuple that the process_path needs */ econtext->ecxt_scantuple = sss->ss.ss_ScanTupleSlot; + /* + * Initialize the scan tuple slot as all-null before process_path + * populates it with the created entities. This ensures the slot + * is properly set up for apply_update_list. + */ + mark_tts_isnull(econtext->ecxt_scantuple); + /* create the path */ process_path(css, NULL, true); - /* mark the create_new_path flag to true. */ - css->created_new_path = true; + /* mark the slot as valid so tts_nvalid reflects natts */ + mark_scan_slot_valid(econtext->ecxt_scantuple); - /* - * find the tts_values that process_path did not populate and - * mark as null. - */ - mark_tts_isnull(econtext->ecxt_scantuple); + /* ON CREATE SET: path was just created */ + if (css->on_create_set_info) + apply_update_list(&css->css, css->on_create_set_info); - /* store the heap tuble */ - ExecStoreVirtualTuple(econtext->ecxt_scantuple); + /* mark the create_new_path flag to true. */ + css->created_new_path = true; /* * make the subquery's projection scan slot be the tuple table we @@ -1036,6 +1149,8 @@ Node *create_cypher_merge_plan_state(CustomScan *cscan) cypher_css->created_new_path = false; cypher_css->found_a_path = false; cypher_css->graph_oid = merge_information->graph_oid; + cypher_css->on_match_set_info = merge_information->on_match_set_info; + cypher_css->on_create_set_info = merge_information->on_create_set_info; cypher_css->css.ss.ps.type = T_CustomScanState; cypher_css->css.methods = &cypher_merge_exec_methods; diff --git a/src/backend/executor/cypher_set.c b/src/backend/executor/cypher_set.c index 8dfa63268..a6e64ba56 100644 --- a/src/backend/executor/cypher_set.c +++ b/src/backend/executor/cypher_set.c @@ -327,8 +327,7 @@ static agtype_value *replace_entity_in_path(agtype_value *path, static void update_all_paths(CustomScanState *node, graphid id, agtype *updated_entity) { - cypher_set_custom_scan_state *css = (cypher_set_custom_scan_state *)node; - ExprContext *econtext = css->css.ss.ps.ps_ExprContext; + ExprContext *econtext = node->ss.ps.ps_ExprContext; TupleTableSlot *scanTupleSlot = econtext->ecxt_scantuple; int i; @@ -374,13 +373,18 @@ static void update_all_paths(CustomScanState *node, graphid id, } } -static void process_update_list(CustomScanState *node) +/* + * Core SET logic that can be called from any executor (SET, MERGE, etc.). + * Takes the CustomScanState for expression context and a + * cypher_update_information describing which properties to set. + */ +void apply_update_list(CustomScanState *node, + cypher_update_information *set_info) { - cypher_set_custom_scan_state *css = (cypher_set_custom_scan_state *)node; - ExprContext *econtext = css->css.ss.ps.ps_ExprContext; + ExprContext *econtext = node->ss.ps.ps_ExprContext; TupleTableSlot *scanTupleSlot = econtext->ecxt_scantuple; ListCell *lc; - EState *estate = css->css.ss.ps.state; + EState *estate = node->ss.ps.state; int *luindex = NULL; int lidx = 0; HTAB *qual_cache = NULL; @@ -415,7 +419,7 @@ static void process_update_list(CustomScanState *node) * to correctly update an 'entity' after all other previous updates to that * 'entity' have been done. */ - foreach (lc, css->set_list->set_items) + foreach (lc, set_info->set_items) { cypher_update_item *update_item = NULL; @@ -430,7 +434,7 @@ static void process_update_list(CustomScanState *node) lidx = 0; /* iterate through SET set items */ - foreach (lc, css->set_list->set_items) + foreach (lc, set_info->set_items) { agtype_value *altered_properties; agtype_value *original_entity_value; @@ -438,7 +442,7 @@ static void process_update_list(CustomScanState *node) agtype_value *id; agtype_value *label; agtype *original_entity; - agtype *new_property_value; + agtype *new_property_value = NULL; TupleTableSlot *slot; ResultRelInfo *resultRelInfo; ScanKeyData scan_keys[1]; @@ -448,7 +452,7 @@ static void process_update_list(CustomScanState *node) cypher_update_item *update_item; Datum new_entity; HeapTuple heap_tuple; - char *clause_name = css->set_list->clause_name; + char *clause_name = set_info->clause_name; int cid; Oid index_oid = InvalidOid; Relation rel; @@ -501,11 +505,43 @@ static void process_update_list(CustomScanState *node) * this is a REMOVE clause or the variable references a variable that is * NULL. It will be possible for a variable to be NULL when OPTIONAL * MATCH is implemented. + * + * If prop_expr is set (used by MERGE ON CREATE/MATCH SET), evaluate + * the expression directly rather than reading from the scan tuple. + * The planner may have stripped the target entry at prop_position. */ if (update_item->remove_item) { remove_property = true; } + else if (update_item->prop_expr != NULL) + { + ExprState *expr_state; + Datum val; + bool isnull; + + /* + * Use the pre-initialized ExprState if available (set during + * plan init in begin_cypher_merge). Fall back to per-row init + * for callers that haven't pre-initialized (e.g. plain SET). + */ + if (update_item->prop_expr_state != NULL) + { + expr_state = update_item->prop_expr_state; + } + else + { + expr_state = ExecInitExpr((Expr *)update_item->prop_expr, + (PlanState *)node); + } + val = ExecEvalExpr(expr_state, econtext, &isnull); + remove_property = isnull; + + if (!isnull) + { + new_property_value = DATUM_GET_AGTYPE_P(val); + } + } else { remove_property = scanTupleSlot->tts_isnull[update_item->prop_position - 1]; @@ -519,7 +555,7 @@ static void process_update_list(CustomScanState *node) { new_property_value = NULL; } - else + else if (update_item->prop_expr == NULL) { new_property_value = DATUM_GET_AGTYPE_P(scanTupleSlot->tts_values[update_item->prop_position - 1]); } @@ -552,7 +588,7 @@ static void process_update_list(CustomScanState *node) } resultRelInfo = create_entity_result_rel_info( - estate, css->set_list->graph_name, label_name); + estate, set_info->graph_name, label_name); rel = resultRelInfo->ri_RelationDesc; relid = RelationGetRelid(rel); @@ -799,6 +835,13 @@ static void process_update_list(CustomScanState *node) pfree_if_not_null(luindex); } +static void process_update_list(CustomScanState *node) +{ + cypher_set_custom_scan_state *css = (cypher_set_custom_scan_state *)node; + + apply_update_list(node, css->set_list); +} + static TupleTableSlot *exec_cypher_set(CustomScanState *node) { cypher_set_custom_scan_state *css = (cypher_set_custom_scan_state *)node; diff --git a/src/backend/nodes/cypher_copyfuncs.c b/src/backend/nodes/cypher_copyfuncs.c index 420ab1d22..283096ca7 100644 --- a/src/backend/nodes/cypher_copyfuncs.c +++ b/src/backend/nodes/cypher_copyfuncs.c @@ -136,6 +136,8 @@ void copy_cypher_update_item(ExtensibleNode *newnode, const ExtensibleNode *from COPY_NODE_FIELD(qualified_name); COPY_SCALAR_FIELD(remove_item); COPY_SCALAR_FIELD(is_add); + COPY_NODE_FIELD(prop_expr); + COPY_NODE_FIELD(prop_expr_state); } /* copy function for cypher_delete_information */ @@ -168,6 +170,8 @@ void copy_cypher_merge_information(ExtensibleNode *newnode, const ExtensibleNode COPY_SCALAR_FIELD(graph_oid); COPY_SCALAR_FIELD(merge_function_attr); COPY_NODE_FIELD(path); + COPY_NODE_FIELD(on_match_set_info); + COPY_NODE_FIELD(on_create_set_info); } /* copy function for cypher_predicate_function */ diff --git a/src/backend/nodes/cypher_outfuncs.c b/src/backend/nodes/cypher_outfuncs.c index cf8a400fc..84d32a8f8 100644 --- a/src/backend/nodes/cypher_outfuncs.c +++ b/src/backend/nodes/cypher_outfuncs.c @@ -200,12 +200,14 @@ void out_cypher_predicate_function(StringInfo str, const ExtensibleNode *node) WRITE_NODE_FIELD(where); } -/* serialization function for the cypher_delete ExtensibleNode. */ +/* serialization function for the cypher_merge ExtensibleNode. */ void out_cypher_merge(StringInfo str, const ExtensibleNode *node) { DEFINE_AG_NODE(cypher_merge); WRITE_NODE_FIELD(path); + WRITE_NODE_FIELD(on_match); + WRITE_NODE_FIELD(on_create); } /* serialization function for the cypher_path ExtensibleNode. */ @@ -438,6 +440,8 @@ void out_cypher_update_item(StringInfo str, const ExtensibleNode *node) WRITE_NODE_FIELD(qualified_name); WRITE_BOOL_FIELD(remove_item); WRITE_BOOL_FIELD(is_add); + WRITE_NODE_FIELD(prop_expr); + WRITE_NODE_FIELD(prop_expr_state); } /* serialization function for the cypher_delete_information ExtensibleNode. */ @@ -470,6 +474,8 @@ void out_cypher_merge_information(StringInfo str, const ExtensibleNode *node) WRITE_INT32_FIELD(graph_oid); WRITE_INT32_FIELD(merge_function_attr); WRITE_NODE_FIELD(path); + WRITE_NODE_FIELD(on_match_set_info); + WRITE_NODE_FIELD(on_create_set_info); } /* diff --git a/src/backend/nodes/cypher_readfuncs.c b/src/backend/nodes/cypher_readfuncs.c index 14b553dbb..1e7e0ef82 100644 --- a/src/backend/nodes/cypher_readfuncs.c +++ b/src/backend/nodes/cypher_readfuncs.c @@ -269,6 +269,8 @@ void read_cypher_update_item(struct ExtensibleNode *node) READ_NODE_FIELD(qualified_name); READ_BOOL_FIELD(remove_item); READ_BOOL_FIELD(is_add); + READ_NODE_FIELD(prop_expr); + READ_NODE_FIELD(prop_expr_state); } /* @@ -310,6 +312,8 @@ void read_cypher_merge_information(struct ExtensibleNode *node) READ_UINT_FIELD(graph_oid); READ_INT_FIELD(merge_function_attr); READ_NODE_FIELD(path); + READ_NODE_FIELD(on_match_set_info); + READ_NODE_FIELD(on_create_set_info); } /* diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 0ab574fe6..3083c52e1 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -7269,6 +7269,33 @@ Query *cypher_parse_sub_analyze(Node *parseTree, * similar to OPTIONAL MATCH, however with the added feature of creating the * path if not there, rather than just emitting NULL. */ +/* + * Resolve prop_expr for each SET item by looking up its target entry. + * The planner may strip SET expression target entries from the plan, + * so we embed the Expr in the update item for direct evaluation. + */ +static void +resolve_merge_set_exprs(List *set_items, List *targetList, + const char *clause_name) +{ + ListCell *lc; + + foreach(lc, set_items) + { + cypher_update_item *item = lfirst(lc); + TargetEntry *set_tle = get_tle_by_resno(targetList, + item->prop_position); + if (set_tle == NULL) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("%s target entry not found at position %d", + clause_name, item->prop_position))); + } + item->prop_expr = (Node *)set_tle->expr; + } +} + static Query *transform_cypher_merge(cypher_parsestate *cpstate, cypher_clause *clause) { @@ -7341,6 +7368,32 @@ static Query *transform_cypher_merge(cypher_parsestate *cpstate, merge_information->graph_oid = cpstate->graph_oid; merge_information->path = merge_path; + /* Transform ON MATCH SET items, if any */ + if (self->on_match != NIL) + { + merge_information->on_match_set_info = + transform_cypher_set_item_list(cpstate, self->on_match, query); + merge_information->on_match_set_info->clause_name = "MERGE ON MATCH SET"; + merge_information->on_match_set_info->graph_name = cpstate->graph_name; + + resolve_merge_set_exprs( + merge_information->on_match_set_info->set_items, + query->targetList, "ON MATCH SET"); + } + + /* Transform ON CREATE SET items, if any */ + if (self->on_create != NIL) + { + merge_information->on_create_set_info = + transform_cypher_set_item_list(cpstate, self->on_create, query); + merge_information->on_create_set_info->clause_name = "MERGE ON CREATE SET"; + merge_information->on_create_set_info->graph_name = cpstate->graph_name; + + resolve_merge_set_exprs( + merge_information->on_create_set_info->set_items, + query->targetList, "ON CREATE SET"); + } + if (!clause->next) { merge_information->flags |= CYPHER_CLAUSE_FLAG_TERMINAL; @@ -7456,10 +7509,63 @@ transform_merge_make_lateral_join(cypher_parsestate *cpstate, Query *query, */ get_res_cols(pstate, l_nsitem, r_nsitem, &res_colnames, &res_colvars); - /* make the RTE for the join */ - jnsitem = addRangeTableEntryForJoin(pstate, res_colnames, NULL, j->jointype, - 0, res_colvars, NIL, NIL, j->alias, - NULL, true); + /* + * Build a ParseNamespaceColumn array for the join RTE so that + * subsequent name lookups (e.g. transform_cypher_set_item_list for an + * ON CREATE SET / ON MATCH SET expression) can resolve references to + * variables bound in the prev clause or the MERGE's path via + * colNameToVar → scanNSItemForColumn, which dereferences + * nsitem->p_nscolumns. Passing NULL here left p_nscolumns unset and + * caused a segfault whenever an ON SET item's RHS referenced a bound + * variable (issue #2347). + * + * Each column's nscolumn references the join RTE (via its rtindex) with + * p_varattno = position in res_colnames. This matches the scantuple + * layout that apply_update_list sees at execution time: the join's + * target list (built by make_target_list_from_join below) iterates + * eref->colnames in order, so scantuple[i-1] corresponds to the i-th + * entry in eref->colnames. Using the underlying RTE's varno/varattno + * would be semantically equivalent for planner-rewritten Vars in the + * query tree, but the Vars we produce here end up inside prop_expr -- + * opaque metadata the planner does not walk -- so they stay un-remapped + * and must index the scantuple layout directly. + * + * addRangeTableEntryForJoin appends the new RTE to pstate->p_rtable, so + * its rtindex is list_length(p_rtable) + 1 at this point. + */ + { + int colcount = list_length(res_colvars); + int join_rtindex = list_length(pstate->p_rtable) + 1; + ParseNamespaceColumn *nscolumns; + ListCell *lvar; + int col_idx = 0; + + nscolumns = (ParseNamespaceColumn *) + palloc0(colcount * sizeof(ParseNamespaceColumn)); + + foreach (lvar, res_colvars) + { + Var *v = (Var *) lfirst(lvar); + + /* res_colvars is populated by get_res_cols via expandRTE */ + Assert(IsA(v, Var)); + + nscolumns[col_idx].p_varno = join_rtindex; + nscolumns[col_idx].p_varattno = col_idx + 1; + nscolumns[col_idx].p_vartype = v->vartype; + nscolumns[col_idx].p_vartypmod = v->vartypmod; + nscolumns[col_idx].p_varcollid = v->varcollid; + nscolumns[col_idx].p_varnosyn = join_rtindex; + nscolumns[col_idx].p_varattnosyn = col_idx + 1; + col_idx++; + } + + /* make the RTE for the join */ + jnsitem = addRangeTableEntryForJoin(pstate, res_colnames, nscolumns, + j->jointype, 0, res_colvars, + NIL, NIL, j->alias, NULL, true); + Assert(jnsitem->p_rtindex == join_rtindex); + } j->rtindex = jnsitem->p_rtindex; diff --git a/src/backend/parser/cypher_gram.y b/src/backend/parser/cypher_gram.y index e14c6b481..c857724dd 100644 --- a/src/backend/parser/cypher_gram.y +++ b/src/backend/parser/cypher_gram.y @@ -64,6 +64,10 @@ bool boolean; Node *node; List *list; + struct { + List *on_match; + List *on_create; + } merge_actions; } %token INTEGER @@ -89,7 +93,7 @@ LIMIT MATCH MERGE NONE NOT NULL_P - OPERATOR OPTIONAL OR ORDER + ON OPERATOR OPTIONAL OR ORDER REMOVE RETURN SET SINGLE SKIP STARTS THEN TRUE_P @@ -139,6 +143,7 @@ /* MERGE clause */ %type merge +%type merge_actions_opt merge_actions merge_action /* CALL ... YIELD clause */ %type call_stmt yield_item @@ -1139,17 +1144,72 @@ detach_opt: * MERGE clause */ merge: - MERGE path + MERGE path merge_actions_opt { cypher_merge *n; n = make_ag_node(cypher_merge); n->path = $2; + n->on_match = $3.on_match; + n->on_create = $3.on_create; $$ = (Node *)n; } ; +merge_actions_opt: + /* empty */ + { + $$.on_match = NIL; + $$.on_create = NIL; + } + | merge_actions + { + $$ = $1; + } + ; + +merge_actions: + merge_action + { + $$ = $1; + } + | merge_actions merge_action + { + if ($2.on_match != NIL) + { + if ($1.on_match != NIL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ON MATCH SET specified more than once"))); + $$.on_match = $2.on_match; + $$.on_create = $1.on_create; + } + else + { + if ($1.on_create != NIL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ON CREATE SET specified more than once"))); + $$.on_create = $2.on_create; + $$.on_match = $1.on_match; + } + } + ; + +merge_action: + ON MATCH SET set_item_list + { + $$.on_match = $4; + $$.on_create = NIL; + } + | ON CREATE SET set_item_list + { + $$.on_match = NIL; + $$.on_create = $4; + } + ; + /* * common */ @@ -2423,6 +2483,7 @@ safe_keywords: | MERGE { $$ = KEYWORD_STRDUP($1); } | NONE { $$ = KEYWORD_STRDUP($1); } | NOT { $$ = KEYWORD_STRDUP($1); } + | ON { $$ = KEYWORD_STRDUP($1); } | OPERATOR { $$ = KEYWORD_STRDUP($1); } | OPTIONAL { $$ = KEYWORD_STRDUP($1); } | OR { $$ = KEYWORD_STRDUP($1); } diff --git a/src/include/executor/cypher_utils.h b/src/include/executor/cypher_utils.h index cdd8fa33e..ac4b5ea5a 100644 --- a/src/include/executor/cypher_utils.h +++ b/src/include/executor/cypher_utils.h @@ -111,8 +111,14 @@ typedef struct cypher_merge_custom_scan_state List *eager_tuples; int eager_tuples_index; bool eager_buffer_filled; + cypher_update_information *on_match_set_info; /* NULL if not specified */ + cypher_update_information *on_create_set_info; /* NULL if not specified */ } cypher_merge_custom_scan_state; +/* Reusable SET logic callable from MERGE executor */ +void apply_update_list(CustomScanState *node, + cypher_update_information *set_info); + TupleTableSlot *populate_vertex_tts(TupleTableSlot *elemTupleSlot, agtype_value *id, agtype_value *properties); TupleTableSlot *populate_edge_tts( diff --git a/src/include/nodes/cypher_nodes.h b/src/include/nodes/cypher_nodes.h index 93bbe01de..3433bebb0 100644 --- a/src/include/nodes/cypher_nodes.h +++ b/src/include/nodes/cypher_nodes.h @@ -124,6 +124,8 @@ typedef struct cypher_merge { ExtensibleNode extensible; Node *path; + List *on_match; /* List of cypher_set_item, or NIL */ + List *on_create; /* List of cypher_set_item, or NIL */ } cypher_merge; /* @@ -472,6 +474,9 @@ typedef struct cypher_update_item List *qualified_name; bool remove_item; bool is_add; + Node *prop_expr; /* SET value expression, used by MERGE ON CREATE/MATCH SET + * where the expression is not in the plan's target list */ + ExprState *prop_expr_state; /* initialized at plan init, not per-row */ } cypher_update_item; typedef struct cypher_delete_information @@ -498,6 +503,8 @@ typedef struct cypher_merge_information uint32 graph_oid; AttrNumber merge_function_attr; cypher_create_path *path; + cypher_update_information *on_match_set_info; /* NULL if no ON MATCH SET */ + cypher_update_information *on_create_set_info; /* NULL if no ON CREATE SET */ } cypher_merge_information; /* grammar node for typecasts */ diff --git a/src/include/parser/cypher_kwlist.h b/src/include/parser/cypher_kwlist.h index 0de294979..44ac09452 100644 --- a/src/include/parser/cypher_kwlist.h +++ b/src/include/parser/cypher_kwlist.h @@ -31,6 +31,7 @@ PG_KEYWORD("merge", MERGE, RESERVED_KEYWORD) PG_KEYWORD("none", NONE, RESERVED_KEYWORD) PG_KEYWORD("not", NOT, RESERVED_KEYWORD) PG_KEYWORD("null", NULL_P, RESERVED_KEYWORD) +PG_KEYWORD("on", ON, RESERVED_KEYWORD) PG_KEYWORD("operator", OPERATOR, RESERVED_KEYWORD) PG_KEYWORD("optional", OPTIONAL, RESERVED_KEYWORD) PG_KEYWORD("or", OR, RESERVED_KEYWORD) From 5a74048725ac6ddbe7bc15377c3a42cb0e31f22b Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Sat, 25 Apr 2026 05:10:31 +0500 Subject: [PATCH 058/100] Fix property access on list comprehension / predicate loop variables (#2402) transform_column_ref_for_indirection errored with "could not find properties for " when the referenced RTE had no "properties" column (as is the case for the unnest() RTE used by list comprehension and the any/all/none/single predicate functions), breaking queries like [x IN list | x.name] and any(x IN list WHERE x.n > 1). Return NULL instead so the caller continues transforming the ColumnRef as an agtype value and applies the indirection via agtype_access_operator. Co-authored-by: Claude Opus 4.7 --- regress/expected/list_comprehension.out | 25 ++++ regress/expected/predicate_functions.out | 148 +++++++++++++++++++++++ regress/sql/list_comprehension.sql | 6 + regress/sql/predicate_functions.sql | 84 +++++++++++++ src/backend/parser/cypher_expr.c | 15 +-- 5 files changed, 267 insertions(+), 11 deletions(-) diff --git a/regress/expected/list_comprehension.out b/regress/expected/list_comprehension.out index e12ad621d..e431c0d9c 100644 --- a/regress/expected/list_comprehension.out +++ b/regress/expected/list_comprehension.out @@ -503,6 +503,31 @@ SELECT * FROM cypher('list_comprehension', $$ CREATE n=()-[:edge]->() RETURN [n [{"id": 281474976710664, "label": "", "properties": {}}::vertex, {"id": 281474976710665, "label": "", "properties": {}}::vertex] (1 row) +-- Property access on the list-comprehension loop variable +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [{name:'a'}, {name:'b'}] | x.name] $$) AS (result agtype); + result +------------ + ["a", "b"] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [{n:1}, {n:2}, {n:3}] WHERE x.n > 1 | x.n] $$) AS (result agtype); + result +-------- + [2, 3] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ MATCH p=()-[:edge]->() RETURN [n IN nodes(p) | n.name] $$) AS (result agtype); + result +-------------- + [null, null] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ MATCH (u:csm_match) WITH collect(u) AS ns RETURN [x IN ns | x.list] $$) AS (result agtype); + result +------------------------- + [["abc", "def", "ghi"]] +(1 row) + -- Multiple list comprehensions in RETURN and WITH clause SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [1,2,3]], [u IN [1,2,3]] $$) AS (result agtype, result2 agtype); result | result2 diff --git a/regress/expected/predicate_functions.out b/regress/expected/predicate_functions.out index 47226453d..9ee673a4e 100644 --- a/regress/expected/predicate_functions.out +++ b/regress/expected/predicate_functions.out @@ -351,6 +351,154 @@ $$) AS (result agtype); "even" (1 row) +-- +-- Property access on the loop variable +-- +-- any: true (2 > 1) / false (none > 5) +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [{n: 1}, {n: 2}] WHERE x.n > 1) +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [{n: 1}, {n: 2}] WHERE x.n > 5) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- all: true (both > 0) / false (not all > 1) +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [{n: 1}, {n: 2}] WHERE x.n > 0) +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [{n: 1}, {n: 2}] WHERE x.n > 1) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- none: true (neither > 2) / false (one matches) +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [{n: 1}, {n: 2}] WHERE x.n > 2) +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [{n: 1}, {n: 2}] WHERE x.n = 1) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- single: true (exactly one) / false (both match) +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [{n: 1}, {n: 2}] WHERE x.n = 1) +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [{n: 1}, {n: 2}] WHERE x.n > 0) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- Property access on vertex loop variables over a collected node list +-- any: true ('even' exists) / false (no 'missing') +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN any(x IN ns WHERE x.name = 'even') +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN any(x IN ns WHERE x.name = 'missing') +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- all: true (all have non-empty vals) / false (not all named 'even') +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN all(x IN ns WHERE size(x.vals) > 0) +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN all(x IN ns WHERE x.name = 'even') +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- none: true (none 'missing') / false ('even' matches) +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN none(x IN ns WHERE x.name = 'missing') +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN none(x IN ns WHERE x.name = 'even') +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- single: true (only one 'odd') / false (all have non-empty vals) +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN single(x IN ns WHERE x.name = 'odd') +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN single(x IN ns WHERE size(x.vals) > 0) +$$) AS (result agtype); + result +-------- + false +(1 row) + -- -- Predicate functions in boolean expressions -- diff --git a/regress/sql/list_comprehension.sql b/regress/sql/list_comprehension.sql index 572b2e6bb..0ef07c974 100644 --- a/regress/sql/list_comprehension.sql +++ b/regress/sql/list_comprehension.sql @@ -123,6 +123,12 @@ SELECT * FROM cypher('list_comprehension', $$ WITH 1 AS m RETURN [m IN [1, 2, 3] SELECT * FROM cypher('list_comprehension', $$ WITH [m IN [1,2,3]] AS m RETURN [m IN [1, 2, 3]], m $$) AS (result agtype, result2 agtype); SELECT * FROM cypher('list_comprehension', $$ CREATE n=()-[:edge]->() RETURN [n IN nodes(n)] $$) AS (u agtype); +-- Property access on the list-comprehension loop variable +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [{name:'a'}, {name:'b'}] | x.name] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [{n:1}, {n:2}, {n:3}] WHERE x.n > 1 | x.n] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ MATCH p=()-[:edge]->() RETURN [n IN nodes(p) | n.name] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ MATCH (u:csm_match) WITH collect(u) AS ns RETURN [x IN ns | x.list] $$) AS (result agtype); + -- Multiple list comprehensions in RETURN and WITH clause SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [1,2,3]], [u IN [1,2,3]] $$) AS (result agtype, result2 agtype); SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [1,2,3] WHERE u>1], [u IN [1,2,3] WHERE u>2] $$) AS (result agtype, result2 agtype); diff --git a/regress/sql/predicate_functions.sql b/regress/sql/predicate_functions.sql index 7466cc2a4..82933f87a 100644 --- a/regress/sql/predicate_functions.sql +++ b/regress/sql/predicate_functions.sql @@ -217,6 +217,90 @@ SELECT * FROM cypher('predicate_functions', $$ ORDER BY u.name $$) AS (result agtype); +-- +-- Property access on the loop variable +-- +-- any: true (2 > 1) / false (none > 5) +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [{n: 1}, {n: 2}] WHERE x.n > 1) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [{n: 1}, {n: 2}] WHERE x.n > 5) +$$) AS (result agtype); + +-- all: true (both > 0) / false (not all > 1) +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [{n: 1}, {n: 2}] WHERE x.n > 0) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [{n: 1}, {n: 2}] WHERE x.n > 1) +$$) AS (result agtype); + +-- none: true (neither > 2) / false (one matches) +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [{n: 1}, {n: 2}] WHERE x.n > 2) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [{n: 1}, {n: 2}] WHERE x.n = 1) +$$) AS (result agtype); + +-- single: true (exactly one) / false (both match) +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [{n: 1}, {n: 2}] WHERE x.n = 1) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [{n: 1}, {n: 2}] WHERE x.n > 0) +$$) AS (result agtype); + +-- Property access on vertex loop variables over a collected node list +-- any: true ('even' exists) / false (no 'missing') +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN any(x IN ns WHERE x.name = 'even') +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN any(x IN ns WHERE x.name = 'missing') +$$) AS (result agtype); + +-- all: true (all have non-empty vals) / false (not all named 'even') +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN all(x IN ns WHERE size(x.vals) > 0) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN all(x IN ns WHERE x.name = 'even') +$$) AS (result agtype); + +-- none: true (none 'missing') / false ('even' matches) +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN none(x IN ns WHERE x.name = 'missing') +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN none(x IN ns WHERE x.name = 'even') +$$) AS (result agtype); + +-- single: true (only one 'odd') / false (all have non-empty vals) +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN single(x IN ns WHERE x.name = 'odd') +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + MATCH (u) WITH collect(u) AS ns + RETURN single(x IN ns WHERE size(x.vals) > 0) +$$) AS (result agtype); + -- -- Predicate functions in boolean expressions -- diff --git a/src/backend/parser/cypher_expr.c b/src/backend/parser/cypher_expr.c index fc0335def..1ed777486 100644 --- a/src/backend/parser/cypher_expr.c +++ b/src/backend/parser/cypher_expr.c @@ -1326,21 +1326,14 @@ static Node *transform_column_ref_for_indirection(cypher_parsestate *cpstate, } /* find the properties column of the NSI and return a var for it */ - node = scanNSItemForColumn(pstate, pnsi, levels_up, "properties", + node = scanNSItemForColumn(pstate, pnsi, levels_up, "properties", cr->location); /* - * Error out if we couldn't find it. - * - * TODO: Should we error out or return NULL for further processing? - * For now, just error out. + * If there's no "properties" column, continue transforming the + * ColumnRef as an agtype value and try to apply the indirection via + * agtype_access_operator. */ - if (!node) - { - ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("could not find properties for %s", relname))); - } - return node; } From 9f9d0f3b18b231d229b17b3994a907aba7519695 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Fri, 24 Apr 2026 20:15:43 -0400 Subject: [PATCH 059/100] Add VLE semantics and cost model design note (#2349) (#2413) Adds a block comment at the top of age_vle.c documenting (a) the edge-isomorphism semantics that openCypher mandates for variable-length relationship matching, (b) the cost model distinction between bounded and unbounded patterns, and (c) a pointer to the enforcement site (edge_state_entry.used_in_path, dfs_find_a_path_*, is_edge_in_path). Motivation: issue #2349 misdiagnosed the VLE as lacking cycle detection and proposed a visited-node filter as a fix. Cycle prevention already exists via edge-uniqueness, and swapping to vertex-isomorphism would silently drop spec-valid paths (e.g. triangle traversals where the endpoint coincides with the start vertex). Capturing the semantics and cost model in-source prevents future readers from repeating the same misanalysis. No behavior change. Cassert installcheck: 33/33. --- src/backend/utils/adt/age_vle.c | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/backend/utils/adt/age_vle.c b/src/backend/utils/adt/age_vle.c index 9aeeadf9b..22c268cdf 100644 --- a/src/backend/utils/adt/age_vle.c +++ b/src/backend/utils/adt/age_vle.c @@ -17,6 +17,50 @@ * under the License. */ +/* + * VLE (Variable-Length Edge) semantics and cost model + * --------------------------------------------------- + * + * This file implements variable-length relationship matching for Cypher + * patterns of the form (a)-[*min..max]->(b). The semantics and cost model + * are often misunderstood; this note exists to prevent future + * misdiagnoses (see issue #2349). + * + * Semantics: edge-isomorphism (openCypher-mandated) + * + * A path is valid iff no edge appears in it more than once. Vertices MAY + * recur. This is "edge-isomorphism" (a.k.a. relationship-uniqueness) per + * the openCypher specification; it is NOT vertex-isomorphism. + * + * Example: in the triangle A-[e1]->B-[e2]->C-[e3]->A, the query + * MATCH (a)-[*3]->(b) WHERE id(a) = id(A) + * MUST return the path (A, e1, B, e2, C, e3, A) with b = A. Switching + * to vertex-isomorphism would silently drop this path and violate the + * spec. Any "optimization" that tracks visited vertices as a filter + * rather than visited edges is therefore incorrect, not merely faster. + * + * Cost model + * + * With E total edges in the traversal-reachable subgraph and a bounded + * pattern [*min..max], the number of enumerated paths is bounded by + * sum_{k=min..max} P(E, k) where P(E, k) = E! / (E - k)! -- polynomial + * in E for fixed max, but factorial in the depth bound. + * + * Unbounded patterns ([*], [*1..]) have no termination guarantee other + * than edge-uniqueness depletion. On a cycle-rich graph the worst case + * is O(E!). This is inherent to edge-isomorphic path enumeration and + * cannot be reduced by algorithm change without changing semantics. + * Users who want reachability (not full enumeration) should bound the + * upper length or use a dedicated function such as shortestPath(). + * + * Implementation pointer + * + * Cycle prevention is enforced by edge_state_entry.used_in_path, set + * and cleared during DFS traversal in dfs_find_a_path_between() and + * dfs_find_a_path_from(). The helper is_edge_in_path() inspects the + * current path stack. See those functions for the enforcement site. + */ + #include "postgres.h" #include "common/hashfn.h" From 22f4c94fb821376d9b4de5b31dc3240225c187fa Mon Sep 17 00:00:00 2001 From: Prashant Chinnam <5108573+crprashant@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:32:27 -0700 Subject: [PATCH 060/100] Update README.md (#2414) docs: Clarify transaction/commit semantics with non-autocommit clients Adds a "Using AGE with Non-Autocommit Clients" section explaining PostgreSQL transaction visibility rules as they apply to AGE DDL-like functions (create_graph, create_vlabel, etc.), with broken/fixed psycopg v3 examples and a JDBC note. Refs apache/age#2195 --- README.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/README.md b/README.md index dad7f1032..819d9dcde 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,105 @@ LOAD 'age'; SET search_path = ag_catalog, "$user", public; ``` +

  Using AGE with Non-Autocommit Clients (psycopg, JDBC, etc.)

+If you are using AGE from a database client that does **not** default to autocommit — most commonly `psycopg` v3 or JDBC — you must understand how PostgreSQL's transaction semantics apply to AGE's setup and DDL-like functions. Otherwise, you may see graphs or labels that appear to be created successfully, but are not visible from new connections. + +This is **not** a bug in AGE — it is standard PostgreSQL behavior. AGE's DDL-like functions write to the catalog, and catalog writes only become visible to other sessions after the enclosing transaction is committed. + +### What is and isn't transactional + +| Statement | Scope | Needs commit to be visible elsewhere? | +|---|---|---| +| `LOAD 'age'` | Session-local (loads the .so into the current backend) | No | +| `SET search_path = ag_catalog, "$user", public` | Session-local | No | +| `SELECT create_graph('g')` | **Writes** to `ag_graph` and creates a schema | **Yes** | +| `SELECT create_vlabel('g', 'L')` / `create_elabel(...)` | **Writes** to `ag_label` and creates a table | **Yes** | +| `SELECT drop_graph('g', true)` / `drop_label(...)` | **Writes** to catalog | **Yes** | +| `SELECT load_labels_from_file(...)` / `load_edges_from_file(...)` | **Writes** to catalog + data | **Yes** | +| `cypher('g', $$ CREATE (:L {...}) $$)` | **Writes** data | **Yes** | + +In a client that defaults to autocommit (e.g. `psql`), every statement commits automatically, so this is never noticed. In a non-autocommit client, the first statement you run implicitly opens a transaction that stays open until you call `commit()`, `rollback()`, or close the connection. + +### psycopg v3 — the "savepoint gotcha" + +The common pitfall is that `with connection.transaction():` in psycopg does **not** start a new top-level transaction when one is already open — it creates a **savepoint** inside the existing outer transaction. Releasing a savepoint is not a commit, so your `create_graph` write stays invisible to other sessions until the outer transaction is explicitly committed. + +#### ❌ Broken: graph is not visible from a new connection + +```python +import psycopg + +params = {"host": "localhost", "port": 5432, "user": "postgres", + "password": "pw", "dbname": "mydb"} + +# --- First connection --- +conn = psycopg.connect(**params) +conn.execute("LOAD 'age'") # implicitly opens a txn +conn.execute("SET search_path = ag_catalog, '$user', public") + +with conn.transaction(), conn.cursor() as cur: # <-- SAVEPOINT, not a real txn + cur.execute("SELECT * FROM create_graph('my_graph')") +# outer transaction is STILL OPEN here + +conn.close() # outer transaction is rolled back on close → my_graph is gone + +# --- New connection --- +conn = psycopg.connect(**params) +conn.execute("LOAD 'age'") +conn.execute("SET search_path = ag_catalog, '$user', public") +with conn.cursor() as cur: + cur.execute("SELECT name FROM ag_graph;") + # 'my_graph' is NOT in the results +``` + +#### ✅ Fix 1: explicit `commit()` after setup + +```python +conn = psycopg.connect(**params) +conn.execute("LOAD 'age'") +conn.execute("SET search_path = ag_catalog, '$user', public") +conn.commit() # <-- closes the implicit outer txn + +with conn.transaction(), conn.cursor() as cur: + cur.execute("SELECT * FROM create_graph('my_graph')") +# this transaction block is now top-level and commits on exit +conn.close() +``` + +#### ✅ Fix 2: enable autocommit on the connection + +```python +conn = psycopg.connect(**params, autocommit=True) +conn.execute("LOAD 'age'") +conn.execute("SET search_path = ag_catalog, '$user', public") +conn.execute("SELECT * FROM create_graph('my_graph')") # commits immediately +conn.close() +``` + +You can also toggle autocommit at runtime with `conn.set_autocommit(True)`. + +### JDBC + +JDBC connections also default to autocommit **true** per the JDBC spec, but many frameworks (Spring, etc.) flip it off. If you are running AGE DDL-like calls from JDBC, either: + +```java +connection.setAutoCommit(true); +// ... LOAD 'age'; SET search_path ...; SELECT create_graph(...); +``` + +or keep autocommit off and explicitly commit after DDL-like calls: + +```java +stmt.execute("LOAD 'age'"); +stmt.execute("SET search_path = ag_catalog, \"$user\", public;"); +stmt.execute("SELECT create_graph('my_graph');"); +connection.commit(); // make the graph visible to other sessions +``` + +### Rule of thumb + +> If an AGE call creates, drops, or modifies a graph, label, vertex, edge, or property, it is a **transactional write**. In a non-autocommit client, it will not be visible to other sessions until you explicitly `commit()`.

  Quick Start

From 774e781b43878fadf829e185be202dd51eb6def8 Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Mon, 27 Apr 2026 23:35:00 +0500 Subject: [PATCH 061/100] Propagate null through agtype arithmetic operators (#2405) The scalar branch of agtype_add lacked an AGTV_NULL case and fell through to the scalar-scalar concat path, so `null + 1` produced the two-element list `[null, 1]` instead of null. This surfaced most visibly inside a list comprehension projection (e.g. `[x IN [1, null, 2] | x + 1]` yielded `[2, [null, 1], 3]`) because there both operands are typed agtype and bypass the agtype_any_add wrapper that already short-circuits on null. The sibling operators (-, *, /, %, ^, unary -) had the same gap but surfaced as "Invalid input parameter types" errors rather than concats. Short-circuit all seven functions to SQL NULL when any operand is AGTV_NULL. SQL NULL (not a wrapped AGTV_NULL scalar) matches AGE's existing convention: `RETURN null AS v` already yields SQL NULL at the row level, and agtype_any_add already returns PG NULL for AGTV_NULL input, so the two operator-resolution paths stay indistinguishable. List aggregation re-packs SQL NULL entries as AGTV_NULL, so `[x IN [1, null, 2] | x + 1]` now renders as `[2, null, 3]`. Co-authored-by: Claude Opus 4.7 --- regress/expected/agtype.out | 44 ++++++++++++++++- regress/expected/list_comprehension.out | 63 +++++++++++++++++++++++++ regress/sql/agtype.sql | 8 +++- regress/sql/list_comprehension.sql | 14 ++++++ src/backend/utils/adt/agtype_ops.c | 42 +++++++++++++++++ 5 files changed, 168 insertions(+), 3 deletions(-) diff --git a/regress/expected/agtype.out b/regress/expected/agtype.out index 065f357f1..cb97b3bfa 100644 --- a/regress/expected/agtype.out +++ b/regress/expected/agtype.out @@ -704,6 +704,48 @@ SELECT '3.14::numeric'::agtype + '3.14::numeric'::agtype; 6.28::numeric (1 row) +SELECT 'null'::agtype - '1'; + ?column? +---------- + +(1 row) + +SELECT 'null'::agtype + '1'; + ?column? +---------- + +(1 row) + +SELECT 'null'::agtype * '1'; + ?column? +---------- + +(1 row) + +SELECT 'null'::agtype / '1'; + ?column? +---------- + +(1 row) + +SELECT 'null'::agtype % '1'; + ?column? +---------- + +(1 row) + +SELECT 'null'::agtype ^ '1'; + ?column? +---------- + +(1 row) + +SELECT -'null'::agtype; + ?column? +---------- + +(1 row) + -- -- Test operator - for extended functionality -- @@ -1065,8 +1107,6 @@ SELECT '{"a":1 , "b":2, "c":3}'::agtype - 'null'; ERROR: expected agtype string, not agtype NULL SELECT '{"a":1 , "b":2, "c":3}'::agtype - '["c","b"]' - '[1]' - '["a"]'; ERROR: expected agtype string, not agtype integer -SELECT 'null'::agtype - '1'; -ERROR: Invalid input parameter types for agtype_sub SELECT 'null'::agtype - '[1]'; ERROR: must be object or array, not a scalar value SELECT '{"id": 1125899906842625, "label": "Vertex", "properties": {"a": "xyz", "b": true, "c": -19.888, "e": {"f": "abcdef", "g": {}, "h": [[], {}]}, "i": {"j": 199, "k": {"l": "mnopq"}}}}::vertex'::agtype - '"a"'; diff --git a/regress/expected/list_comprehension.out b/regress/expected/list_comprehension.out index e431c0d9c..e47b2d569 100644 --- a/regress/expected/list_comprehension.out +++ b/regress/expected/list_comprehension.out @@ -746,6 +746,69 @@ SELECT * FROM cypher('list_comprehension', $$ MATCH (u {list: [0, 2, 4, 6, 8, 10 {"id": 281474976710668, "label": "", "properties": {"b": [0, 1, 2, 3, 4, 5], "c": [0, 2, 4, 6, 8, 10, 12], "list": [0, 2, 4, 6, 8, 10, 12]}}::vertex (2 rows) +-- Issue 2394 - projection over a null element should propagate null, +-- not concatenate the null and the projection result into a sublist and +-- not error out for operators other than '+'. +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [null] | x + 1] $$) AS (result agtype); + result +-------- + [null] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x + 1] $$) AS (result agtype); + result +-------------- + [2, null, 3] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | 1 + x] $$) AS (result agtype); + result +-------------- + [2, null, 3] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x] $$) AS (result agtype); + result +-------------- + [1, null, 2] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x - 1] $$) AS (result agtype); + result +-------------- + [0, null, 1] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x * 2] $$) AS (result agtype); + result +-------------- + [2, null, 4] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x / 1] $$) AS (result agtype); + result +-------------- + [1, null, 2] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x % 2] $$) AS (result agtype); + result +-------------- + [1, null, 0] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x ^ 2] $$) AS (result agtype); + result +------------------ + [1.0, null, 4.0] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | -x] $$) AS (result agtype); + result +---------------- + [-1, null, -2] +(1 row) + -- Clean up SELECT * FROM drop_graph('list_comprehension', true); NOTICE: drop cascades to 4 other objects diff --git a/regress/sql/agtype.sql b/regress/sql/agtype.sql index 6dab6bc30..505097761 100644 --- a/regress/sql/agtype.sql +++ b/regress/sql/agtype.sql @@ -229,6 +229,13 @@ SELECT '3'::agtype + '3.14'::agtype; SELECT '3'::agtype + '3.14::numeric'::agtype; SELECT '3.14'::agtype + '3.14::numeric'::agtype; SELECT '3.14::numeric'::agtype + '3.14::numeric'::agtype; +SELECT 'null'::agtype - '1'; +SELECT 'null'::agtype + '1'; +SELECT 'null'::agtype * '1'; +SELECT 'null'::agtype / '1'; +SELECT 'null'::agtype % '1'; +SELECT 'null'::agtype ^ '1'; +SELECT -'null'::agtype; -- -- Test operator - for extended functionality @@ -305,7 +312,6 @@ SELECT '{"a":1 , "b":2, "c":3}'::agtype - '[null]'; SELECT '{"a":1 , "b":2, "c":3}'::agtype - '1'; SELECT '{"a":1 , "b":2, "c":3}'::agtype - 'null'; SELECT '{"a":1 , "b":2, "c":3}'::agtype - '["c","b"]' - '[1]' - '["a"]'; -SELECT 'null'::agtype - '1'; SELECT 'null'::agtype - '[1]'; SELECT '{"id": 1125899906842625, "label": "Vertex", "properties": {"a": "xyz", "b": true, "c": -19.888, "e": {"f": "abcdef", "g": {}, "h": [[], {}]}, "i": {"j": 199, "k": {"l": "mnopq"}}}}::vertex'::agtype - '"a"'; SELECT '{"id": 1125899906842625, "label": "Vertex", "properties": {"a": "xyz", "b": true, "c": -19.888, "e": {"f": "abcdef", "g": {}, "h": [[], {}]}, "i": {"j": 199, "k": {"l": "mnopq"}}}}::vertex'::agtype - '["a"]'; diff --git a/regress/sql/list_comprehension.sql b/regress/sql/list_comprehension.sql index 0ef07c974..57b43221b 100644 --- a/regress/sql/list_comprehension.sql +++ b/regress/sql/list_comprehension.sql @@ -180,5 +180,19 @@ SELECT * FROM cypher('list_comprehension', $$ MATCH (u {list: [0, 2, 4, 6, 8, 10 SELECT * FROM cypher('list_comprehension', $$ MATCH (u {list: [0, 2, 4, 6, 8, 10, 12]}) WHERE u.list = [u IN [1, u]] RETURN u $$) AS (u agtype); SELECT * FROM cypher('list_comprehension', $$ MATCH (u {list: [0, 2, 4, 6, 8, 10, 12]}) WHERE u.list IN [u IN [1, u.list]] RETURN u $$) AS (u agtype); +-- Issue 2394 - projection over a null element should propagate null, +-- not concatenate the null and the projection result into a sublist and +-- not error out for operators other than '+'. +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [null] | x + 1] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x + 1] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | 1 + x] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x - 1] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x * 2] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x / 1] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x % 2] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x ^ 2] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | -x] $$) AS (result agtype); + -- Clean up SELECT * FROM drop_graph('list_comprehension', true); \ No newline at end of file diff --git a/src/backend/utils/adt/agtype_ops.c b/src/backend/utils/adt/agtype_ops.c index d831447b0..f7f45b467 100644 --- a/src/backend/utils/adt/agtype_ops.c +++ b/src/backend/utils/adt/agtype_ops.c @@ -165,6 +165,12 @@ Datum agtype_add(PG_FUNCTION_ARGS) agtv_lhs = get_ith_agtype_value_from_container(&lhs->root, 0); agtv_rhs = get_ith_agtype_value_from_container(&rhs->root, 0); + /* openCypher: arithmetic over null yields null. */ + if (agtv_lhs->type == AGTV_NULL || agtv_rhs->type == AGTV_NULL) + { + PG_RETURN_NULL(); + } + /* * One or both values is a string OR one is a string and the other is * either an integer, float, or numeric. If so, concatenate them. @@ -525,6 +531,12 @@ Datum agtype_sub(PG_FUNCTION_ARGS) agtv_lhs = get_ith_agtype_value_from_container(&lhs->root, 0); agtv_rhs = get_ith_agtype_value_from_container(&rhs->root, 0); + /* openCypher: arithmetic over null yields null. */ + if (agtv_lhs->type == AGTV_NULL || agtv_rhs->type == AGTV_NULL) + { + PG_RETURN_NULL(); + } + if (agtv_lhs->type == AGTV_INTEGER && agtv_rhs->type == AGTV_INTEGER) { agtv_result.type = AGTV_INTEGER; @@ -615,6 +627,12 @@ Datum agtype_neg(PG_FUNCTION_ARGS) agtv_value = get_ith_agtype_value_from_container(&v->root, 0); + /* openCypher: arithmetic over null yields null. */ + if (agtv_value->type == AGTV_NULL) + { + PG_RETURN_NULL(); + } + if (agtv_value->type == AGTV_INTEGER) { agtv_result.type = AGTV_INTEGER; @@ -666,6 +684,12 @@ Datum agtype_mul(PG_FUNCTION_ARGS) agtv_lhs = get_ith_agtype_value_from_container(&lhs->root, 0); agtv_rhs = get_ith_agtype_value_from_container(&rhs->root, 0); + /* openCypher: arithmetic over null yields null. */ + if (agtv_lhs->type == AGTV_NULL || agtv_rhs->type == AGTV_NULL) + { + PG_RETURN_NULL(); + } + if (agtv_lhs->type == AGTV_INTEGER && agtv_rhs->type == AGTV_INTEGER) { agtv_result.type = AGTV_INTEGER; @@ -756,6 +780,12 @@ Datum agtype_div(PG_FUNCTION_ARGS) agtv_lhs = get_ith_agtype_value_from_container(&lhs->root, 0); agtv_rhs = get_ith_agtype_value_from_container(&rhs->root, 0); + /* openCypher: arithmetic over null yields null. */ + if (agtv_lhs->type == AGTV_NULL || agtv_rhs->type == AGTV_NULL) + { + PG_RETURN_NULL(); + } + if (agtv_lhs->type == AGTV_INTEGER && agtv_rhs->type == AGTV_INTEGER) { if (agtv_rhs->val.int_value == 0) @@ -874,6 +904,12 @@ Datum agtype_mod(PG_FUNCTION_ARGS) agtv_lhs = get_ith_agtype_value_from_container(&lhs->root, 0); agtv_rhs = get_ith_agtype_value_from_container(&rhs->root, 0); + /* openCypher: arithmetic over null yields null. */ + if (agtv_lhs->type == AGTV_NULL || agtv_rhs->type == AGTV_NULL) + { + PG_RETURN_NULL(); + } + if (agtv_lhs->type == AGTV_INTEGER && agtv_rhs->type == AGTV_INTEGER) { agtv_result.type = AGTV_INTEGER; @@ -964,6 +1000,12 @@ Datum agtype_pow(PG_FUNCTION_ARGS) agtv_lhs = get_ith_agtype_value_from_container(&lhs->root, 0); agtv_rhs = get_ith_agtype_value_from_container(&rhs->root, 0); + /* openCypher: arithmetic over null yields null. */ + if (agtv_lhs->type == AGTV_NULL || agtv_rhs->type == AGTV_NULL) + { + PG_RETURN_NULL(); + } + if (agtv_lhs->type == AGTV_INTEGER && agtv_rhs->type == AGTV_INTEGER) { agtv_result.type = AGTV_FLOAT; From ff47828b9f1ba474647923bdb0b73299a96a3a79 Mon Sep 17 00:00:00 2001 From: Prashant Chinnam <5108573+crprashant@users.noreply.github.com> Date: Wed, 29 Apr 2026 15:13:45 -0700 Subject: [PATCH 062/100] Allow safe Cypher reserved keywords in alias positions (#2355) (#2415) Cypher productions in `cypher_gram.y` that bind an alias via the AS keyword (RETURN/WITH/YIELD ... AS x and UNWIND ... AS x) only accepted plain identifiers. As a result, completely valid Cypher such as SELECT * FROM cypher('g', $$ RETURN 1 AS count $$) AS (a agtype); failed with `syntax error at or near "count"`, even though `count` is already accepted in other identifier positions (it appears in the existing `safe_keywords` list and is permitted in `func_name`; `schema_name` accepts the broader `reserved_keyword` set). This patch introduces a dedicated `var_name_alias` non-terminal used only in the three alias-binding sites (yield_item, return_item, unwind). It accepts everything `var_name` accepts, plus the entire `safe_keywords` set, so the 49 non-conflicting reserved keywords (count, exists, coalesce, match, return, where, order, limit, distinct, optional, detach, contains, starts, ends, in, is, not, ...) are now usable as aliases. The change is intentionally scoped to alias positions: * `var_name` itself (used by pattern-variable bindings like `(x:Label)`, edge bindings, named paths, and `expr_var` references) is unchanged. Allowing safe_keywords there triggers 156 shift/reduce conflicts because keyword tokens collide with their roles inside expressions and patterns. * `conflicted_keywords` (END, NULL, TRUE, FALSE) remain rejected in every position; they are genuinely ambiguous with literal/CASE productions. Reading a keyword-named alias back through `expr_var` still fails (e.g. `WITH 1 AS count RETURN count`) because `expr_var` reads through `var_name`. That asymmetry is captured as a known limitation in the regression suite and tracked separately in #2416. Regression coverage lives in `regress/sql/reserved_keyword_alias.sql` and `regress/expected/reserved_keyword_alias.out`, exercising: * the original repro, * representative safe_keywords across RETURN/WITH/UNWIND, * multiple keyword aliases in one projection, * a backtick-quoted alias positive case, * the known read-back limitation as a negative test, and * explicit negatives proving END/NULL/TRUE/FALSE and pattern-position keywords still error out. Closes #2355. --- Makefile | 3 +- regress/expected/reserved_keyword_alias.out | 248 ++++++++++++++++++++ regress/sql/reserved_keyword_alias.sql | 102 ++++++++ src/backend/parser/cypher_gram.y | 44 +++- 4 files changed, 392 insertions(+), 5 deletions(-) create mode 100644 regress/expected/reserved_keyword_alias.out create mode 100755 regress/sql/reserved_keyword_alias.sql diff --git a/Makefile b/Makefile index 5405665d8..cb33e1047 100644 --- a/Makefile +++ b/Makefile @@ -177,7 +177,8 @@ REGRESS = scan \ predicate_functions \ map_projection \ direct_field_access \ - security + security \ + reserved_keyword_alias ifneq ($(EXTRA_TESTS),) REGRESS += $(EXTRA_TESTS) diff --git a/regress/expected/reserved_keyword_alias.out b/regress/expected/reserved_keyword_alias.out new file mode 100644 index 000000000..0127dbb0e --- /dev/null +++ b/regress/expected/reserved_keyword_alias.out @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Regression coverage for issue #2355: + * Cypher reserved keywords from the `safe_keywords` set must be accepted + * in alias positions (RETURN/WITH/YIELD ... AS , UNWIND ... AS ). + * Conflicting tokens (END / NULL / TRUE / FALSE) must remain rejected. + * Pattern variable bindings are intentionally still restricted to plain + * identifiers. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +SELECT * FROM create_graph('issue_2355'); +NOTICE: graph "issue_2355" has been created + create_graph +-------------- + +(1 row) + +-- The exact reproducer from the issue (previously failed with +-- "syntax error at or near "count""). +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS count $$) AS (a agtype); + a +--- + 1 +(1 row) + +-- Representative coverage across the safe_keywords set as RETURN aliases. +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS exists $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS coalesce $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS match $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS return $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS where $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS order $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS limit $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS distinct $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS optional $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS detach $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS contains $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS starts $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS ends $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS in $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS is $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS not $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS yield $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS call $$) AS (a agtype); + a +--- + 1 +(1 row) + +-- Multiple keyword aliases in one projection. +SELECT * FROM cypher('issue_2355', + $$ RETURN 1 AS count, 2 AS exists, 3 AS where $$ +) AS (count agtype, ex agtype, w agtype); + count | ex | w +-------+----+--- + 1 | 2 | 3 +(1 row) + +-- WITH ... AS : alias binding works. +SELECT * FROM cypher('issue_2355', + $$ WITH 1 AS count RETURN 1 AS x $$ +) AS (a agtype); + a +--- + 1 +(1 row) + +-- UNWIND ... AS : alias binding works. +SELECT * FROM cypher('issue_2355', + $$ UNWIND [1, 2, 3] AS row RETURN 1 AS x $$ +) AS (a agtype); + a +--- + 1 + 1 + 1 +(3 rows) + +-- conflicted_keywords (END, NULL, TRUE, FALSE) MUST still be rejected +-- because they introduce real grammar ambiguity with literal/expression +-- productions. +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS null $$) AS (a agtype); +ERROR: syntax error at or near "null" +LINE 1: SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS null $$) ... + ^ +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS true $$) AS (a agtype); +ERROR: syntax error at or near "true" +LINE 1: SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS true $$) ... + ^ +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS false $$) AS (a agtype); +ERROR: syntax error at or near "false" +LINE 1: SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS false $$) ... + ^ +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS end $$) AS (a agtype); +ERROR: syntax error at or near "end" +LINE 1: SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS end $$) ... + ^ +-- Pattern-variable positions are intentionally NOT broadened (would +-- create shift/reduce conflicts). Confirm they still error. +SELECT * FROM cypher('issue_2355', + $$ MATCH (count) RETURN 1 AS x $$ +) AS (a agtype); +ERROR: syntax error at or near "count" +LINE 2: $$ MATCH (count) RETURN 1 AS x $$ + ^ +-- Plain identifiers naturally remain unaffected. +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS my_alias $$) AS (a agtype); + a +--- + 1 +(1 row) + +-- Backtick-quoted alias positive case: forces the IDENTIFIER token +-- path, so future grammar refactors don't accidentally regress quoted +-- identifiers when the unquoted form is also a keyword. +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS `count` $$) AS (a agtype); + a +--- + 1 +(1 row) + +SELECT * FROM cypher('issue_2355', $$ WITH 1 AS `count` RETURN `count` $$) AS (a agtype); + a +--- + 1 +(1 row) + +-- Known limitation: reading a keyword-named alias back fails because +-- expr_var reads through var_name (which is unchanged here). Tracked +-- in issue #2416. Captured in the expected output so the next +-- contributor who fixes expr_var has a precise file to update. +SELECT * FROM cypher('issue_2355', $$ WITH 1 AS count RETURN count $$) AS (a agtype); +ERROR: syntax error at end of input +LINE 1: ...her('issue_2355', $$ WITH 1 AS count RETURN count $$) AS (a ... + ^ +SELECT * FROM drop_graph('issue_2355', true); +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table issue_2355._ag_label_vertex +drop cascades to table issue_2355._ag_label_edge +NOTICE: graph "issue_2355" has been dropped + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/reserved_keyword_alias.sql b/regress/sql/reserved_keyword_alias.sql new file mode 100755 index 000000000..eab27ee37 --- /dev/null +++ b/regress/sql/reserved_keyword_alias.sql @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Regression coverage for issue #2355: + * Cypher reserved keywords from the `safe_keywords` set must be accepted + * in alias positions (RETURN/WITH/YIELD ... AS , UNWIND ... AS ). + * Conflicting tokens (END / NULL / TRUE / FALSE) must remain rejected. + * Pattern variable bindings are intentionally still restricted to plain + * identifiers. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +SELECT * FROM create_graph('issue_2355'); + +-- The exact reproducer from the issue (previously failed with +-- "syntax error at or near "count""). +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS count $$) AS (a agtype); + +-- Representative coverage across the safe_keywords set as RETURN aliases. +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS exists $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS coalesce $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS match $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS return $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS where $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS order $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS limit $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS distinct $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS optional $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS detach $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS contains $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS starts $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS ends $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS in $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS is $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS not $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS yield $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS call $$) AS (a agtype); + +-- Multiple keyword aliases in one projection. +SELECT * FROM cypher('issue_2355', + $$ RETURN 1 AS count, 2 AS exists, 3 AS where $$ +) AS (count agtype, ex agtype, w agtype); + +-- WITH ... AS : alias binding works. +SELECT * FROM cypher('issue_2355', + $$ WITH 1 AS count RETURN 1 AS x $$ +) AS (a agtype); + +-- UNWIND ... AS : alias binding works. +SELECT * FROM cypher('issue_2355', + $$ UNWIND [1, 2, 3] AS row RETURN 1 AS x $$ +) AS (a agtype); + +-- conflicted_keywords (END, NULL, TRUE, FALSE) MUST still be rejected +-- because they introduce real grammar ambiguity with literal/expression +-- productions. +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS null $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS true $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS false $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS end $$) AS (a agtype); + +-- Pattern-variable positions are intentionally NOT broadened (would +-- create shift/reduce conflicts). Confirm they still error. +SELECT * FROM cypher('issue_2355', + $$ MATCH (count) RETURN 1 AS x $$ +) AS (a agtype); + +-- Plain identifiers naturally remain unaffected. +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS my_alias $$) AS (a agtype); + +-- Backtick-quoted alias positive case: forces the IDENTIFIER token +-- path, so future grammar refactors don't accidentally regress quoted +-- identifiers when the unquoted form is also a keyword. +SELECT * FROM cypher('issue_2355', $$ RETURN 1 AS `count` $$) AS (a agtype); +SELECT * FROM cypher('issue_2355', $$ WITH 1 AS `count` RETURN `count` $$) AS (a agtype); + +-- Known limitation: reading a keyword-named alias back fails because +-- expr_var reads through var_name (which is unchanged here). Tracked +-- in issue #2416. Captured in the expected output so the next +-- contributor who fixes expr_var has a precise file to update. +SELECT * FROM cypher('issue_2355', $$ WITH 1 AS count RETURN count $$) AS (a agtype); + +SELECT * FROM drop_graph('issue_2355', true); diff --git a/src/backend/parser/cypher_gram.y b/src/backend/parser/cypher_gram.y index c857724dd..c614e1dbe 100644 --- a/src/backend/parser/cypher_gram.y +++ b/src/backend/parser/cypher_gram.y @@ -175,7 +175,7 @@ %type expr_subquery /* names */ -%type property_key_name var_name var_name_opt label_name +%type property_key_name var_name var_name_alias var_name_opt label_name %type symbolic_name schema_name type_name %type reserved_keyword safe_keywords conflicted_keywords %type func_name @@ -480,7 +480,7 @@ yield_item_list: ; yield_item: - expr AS var_name + expr AS var_name_alias { ResTarget *n; @@ -791,7 +791,7 @@ return_item_list: ; return_item: - expr AS var_name + expr AS var_name_alias { ResTarget *n; @@ -985,7 +985,7 @@ optional_opt: unwind: - UNWIND expr AS var_name + UNWIND expr AS var_name_alias { ResTarget *res; cypher_unwind *n; @@ -2337,6 +2337,42 @@ var_name: } ; +/* + * var_name_alias is used in alias positions (RETURN/WITH/YIELD ... AS x, + * UNWIND ... AS x) where the AS keyword removes any lookahead ambiguity. + * Beyond plain identifiers, it permits the same set of non-conflicting + * reserved keywords accepted by safe_keywords (already accepted in + * func_name; schema_name accepts the broader reserved_keyword), so that + * legitimate Cypher such as + * RETURN 1 AS count + * RETURN n AS exists + * UNWIND [1, 2] AS row + * is parsed correctly. Truly conflicting tokens (END, NULL, TRUE, FALSE) + * are listed under conflicted_keywords (not safe_keywords) and remain + * rejected here. See issue #2355. + * + * It is intentionally NOT used in pattern variable positions + * ((x:Label), [r:REL]) or named-path bindings (p = ...), because + * allowing reserved keywords there introduces shift/reduce ambiguity. + * + * NOTE: Reading a keyword-named alias back (e.g. WITH 1 AS count + * RETURN count) is intentionally still rejected -- broadening expr_var + * (which reads through var_name) to accept safe_keywords reintroduces + * ~156 shift/reduce conflicts in bison. That asymmetry (writable but + * not readable) is tracked in issue #2416. + */ +var_name_alias: + var_name + | safe_keywords + { + /* safe_keywords already returns a pnstrdup-allocated copy via + * KEYWORD_STRDUP, so no further pstrdup is needed. Mirrors the + * established pattern used by schema_name's reserved_keyword + * branch. */ + $$ = (char *) $1; + } + ; + var_name_opt: /* empty */ { From 54e19fac9f257ecd173e3dcf647c990e74a28323 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Thu, 30 Apr 2026 00:16:31 -0700 Subject: [PATCH 063/100] Fix upgrade test: replace data-integrity checks with catalog comparison (#2403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The age_upgrade regression test (added in #2364, improved in #2377, #2397) was designed to validate the upgrade template (age----y.y.y.sql) by creating graph data before the upgrade and verifying it survived afterward. This approach had two fundamental problems: 1. It did not detect incomplete upgrade templates. The test verified that graph data (vertices, edges, checksums, GIN indexes) survived ALTER EXTENSION UPDATE, but never checked whether new SQL objects (functions, views, relations, indexes, types, operators, casts, constraints) were actually created by the template. A developer could add a new function to sql/ and sql_files, forget to add it to the upgrade template, and all tests would pass — the function existed via the fresh CREATE EXTENSION install that ran before the upgrade test, but would be missing for users who upgraded via ALTER EXTENSION UPDATE. 2. The data-integrity checks relied on cypher queries (MATCH/RETURN) within the same backend session after DROP EXTENSION + CREATE EXTENSION. This caused intermittent failures on some PostgreSQL versions where AGE's internal type cache (agtype OID) was not properly refreshed after the extension was dropped and recreated, resulting in 'type with OID 0 does not exist' errors. The data-integrity aspect was also redundant — ALTER EXTENSION UPDATE runs DDL statements and does not touch heap data, so data survival is guaranteed by PostgreSQL and not a meaningful test. The fix replaces the entire test with a comprehensive catalog comparison: 1. Snapshot the ag_catalog schema from the fresh install across seven PostgreSQL system catalogs: - pg_proc: functions, aggregates, procedures (name, args, and properties: volatility, strictness, kind, return type, setof) - pg_class: tables, views, sequences, indexes (name, kind) - pg_type: types (name, type category) - pg_operator: operators (name, left/right operand types) - pg_cast: casts involving AGE types (source, target, context) - pg_opclass: operator classes (name, access method) - pg_constraint: constraints (name, type, table, referenced table) 2. DROP EXTENSION, CREATE EXTENSION at the synthetic initial version, then ALTER EXTENSION UPDATE to the current version via the stamped upgrade template. 3. Snapshot the catalog again after upgrade. 4. Compare: any object present in the fresh snapshot but missing after upgrade means the template is incomplete. Any object present after upgrade but not in the fresh snapshot means the template creates something unexpected. Function properties (volatility, strictness, prokind, return type) are also compared for functions that exist in both — catching cases where a CREATE OR REPLACE in the template changes a function's signature or behavior. Additional improvements from code review feedback: - Graph cleanup in Step 1 uses a DO block with PERFORM and suppressed NOTICEs to produce deterministic output regardless of prior test state. - The pg_class snapshot includes indexes (relkind 'i') in addition to tables, views, and sequences. - Diagnostic output includes relkind/typtype suffixes for actionable diffs. - Summary uses boolean equality checks (funcs_match, rels_match, etc.) instead of absolute counts, so the expected output does not need updating when new objects are added to AGE. Developers who correctly add objects to both sql/ and the template will never need to modify this test or its expected output. This approach: - Catches the actual failure mode: incomplete upgrade templates. - Covers all SQL object categories: functions (including aggregates), relations, types, operators, casts, operator classes, and constraints. - Detects property changes on existing functions (volatility, strictness, kind, return type changes). - Uses only plain SQL catalog queries — no cypher, no .so cache issues. - Works reliably across all PostgreSQL versions. - Reports the exact missing/extra/changed object in the diff output. - Is maintenance-free: no expected output changes needed when AGE grows. Makefile: updated step 5 comment to reflect catalog comparison approach. All 33 regression tests pass. Co-authored-by: Claude modified: Makefile modified: regress/expected/age_upgrade.out modified: regress/sql/age_upgrade.sql --- Makefile | 8 +- regress/expected/age_upgrade.out | 694 +++++++++++++++---------------- regress/sql/age_upgrade.sql | 552 +++++++++++++----------- 3 files changed, 645 insertions(+), 609 deletions(-) diff --git a/Makefile b/Makefile index cb33e1047..3c5a28c29 100644 --- a/Makefile +++ b/Makefile @@ -33,15 +33,17 @@ age_sql = age--1.7.0.sql # 4. Temporarily installs the synthetic files into the PG extension directory # so that CREATE EXTENSION age VERSION '' and ALTER EXTENSION # age UPDATE TO '' can find them. -# 5. The age_upgrade regression test exercises the full upgrade path: install -# at INIT, create data, ALTER EXTENSION UPDATE to CURR, verify data. +# 5. The age_upgrade regression test snapshots the ag_catalog schema from +# a fresh install, then installs at INIT, upgrades to CURR, and compares +# the catalog across seven system catalogs to detect missing, extra, +# or changed objects. # 6. The test SQL cleans up the synthetic files via a generated shell script. # # This forces developers to keep the upgrade template in sync: any SQL object # added after the version-bump commit must also appear in the template, or the # upgrade test will fail (the object will be missing after ALTER EXTENSION UPDATE). # -# Because the default install SQL comes from HEAD, all 31 non-upgrade tests +# Because the default install SQL comes from HEAD, all non-upgrade tests # run with every SQL function registered — no functions are missing. # # Graceful degradation — the upgrade test is silently skipped when: diff --git a/regress/expected/age_upgrade.out b/regress/expected/age_upgrade.out index 7c3da0150..edf8e6022 100644 --- a/regress/expected/age_upgrade.out +++ b/regress/expected/age_upgrade.out @@ -17,36 +17,105 @@ * under the License. */ -- --- Extension upgrade regression test +-- Extension upgrade template regression test -- --- This test validates the upgrade template (age----y.y.y.sql) by: --- 1. Dropping AGE and reinstalling at the synthetic "initial" version --- (built from the version-bump commit's SQL — the "day-one" state) --- 2. Creating three graphs with multiple labels, edges, GIN indexes, --- and numeric properties that serve as integrity checksums --- 3. Upgrading to the current (default) version via the stamped template --- 4. Verifying all data, structure, and checksums survived the upgrade +-- Validates the upgrade template (age----y.y.y.sql) by comparing +-- the pg_catalog entries of a fresh install against an upgraded install. +-- If any object exists in the fresh install but is missing after upgrade, +-- the template is incomplete. This catches the case where a developer adds +-- a new SQL object to sql/ and sql_files but forgets to add it to the +-- upgrade template. -- --- The Makefile builds: --- age--.sql from current HEAD's sql/sql_files (default) --- age--_initial.sql from the version-bump commit (synthetic) --- age--_initial--.sql stamped from the upgrade template +-- Compared catalogs: +-- pg_proc — functions, aggregates, procedures (name, args, properties) +-- pg_class — tables, views, sequences, indexes (name, kind) +-- pg_type — types (name, type category) +-- pg_operator — operators (name, left/right types) +-- pg_cast — casts involving AGE types (source, target, context) +-- pg_opclass — operator classes (name, access method) +-- pg_constraint — constraints (name, type, table, referenced table) -- --- All version discovery is dynamic — no hardcoded versions anywhere. --- This test is version-agnostic and works on any branch for any version. +-- All comparison queries should return 0 rows. -- LOAD 'age'; SET search_path TO ag_catalog; --- Step 1: Clean up any state left by prior tests, then drop AGE entirely. --- The --load-extension=age flag installed AGE at the current (default) version. --- We need to remove it so we can reinstall at the synthetic initial version. -SELECT drop_graph(name, true) FROM ag_graph ORDER BY name; - drop_graph ------------- -(0 rows) +-- Step 1: Clean up any graphs left by prior tests (deterministic, no output). +DO $$ +DECLARE + graph_name ag_graph.name%TYPE; +BEGIN + PERFORM set_config('client_min_messages', 'warning', true); + FOR graph_name IN + SELECT name + FROM ag_graph + ORDER BY name + LOOP + PERFORM drop_graph(graph_name, true); + END LOOP; +END +$$; +-- ===================================================================== +-- FRESH INSTALL SNAPSHOTS (Steps 2-7) +-- Capture the catalog state from the default CREATE EXTENSION install. +-- ===================================================================== +-- Step 2: Snapshot functions (includes aggregates via prokind). +CREATE TEMP TABLE _fresh_funcs AS +SELECT proname::text, + pg_get_function_identity_arguments(oid) AS args, + provolatile::text, proisstrict::text, prokind::text, + prorettype::regtype::text AS rettype, proretset::text +FROM pg_proc +WHERE pronamespace = 'ag_catalog'::regnamespace +ORDER BY proname, args; +-- Step 3: Snapshot relations (tables, views, sequences, indexes). +CREATE TEMP TABLE _fresh_rels AS +SELECT relname::text, relkind::text +FROM pg_class +WHERE relnamespace = 'ag_catalog'::regnamespace + AND relkind IN ('r', 'v', 'S', 'i') +ORDER BY relname; +-- Step 4: Snapshot types. +CREATE TEMP TABLE _fresh_types AS +SELECT typname::text, typtype::text +FROM pg_type +WHERE typnamespace = 'ag_catalog'::regnamespace + AND typname NOT LIKE 'pg_toast%' +ORDER BY typname; +-- Step 5: Snapshot operators. +CREATE TEMP TABLE _fresh_ops AS +SELECT oprname::text, + oprleft::regtype::text AS lefttype, + oprright::regtype::text AS righttype +FROM pg_operator +WHERE oprnamespace = 'ag_catalog'::regnamespace +ORDER BY oprname, lefttype, righttype; +-- Step 6: Snapshot casts involving AGE types, and operator classes. +CREATE TEMP TABLE _fresh_casts AS +SELECT castsource::regtype::text AS source_type, + casttarget::regtype::text AS target_type, + castcontext::text +FROM pg_cast +WHERE castsource IN (SELECT oid FROM pg_type WHERE typnamespace = 'ag_catalog'::regnamespace) + OR casttarget IN (SELECT oid FROM pg_type WHERE typnamespace = 'ag_catalog'::regnamespace) +ORDER BY source_type, target_type; +CREATE TEMP TABLE _fresh_opclass AS +SELECT opcname::text, + (SELECT amname FROM pg_am WHERE oid = opcmethod)::text AS amname +FROM pg_opclass +WHERE opcnamespace = 'ag_catalog'::regnamespace +ORDER BY opcname; +-- Step 7: Snapshot constraints. +CREATE TEMP TABLE _fresh_constraints AS +SELECT conname::text, contype::text, + conrelid::regclass::text AS table_name, + confrelid::regclass::text AS ref_table +FROM pg_constraint +WHERE connamespace = 'ag_catalog'::regnamespace +ORDER BY conname; +-- Step 8: Drop AGE entirely. DROP EXTENSION age; --- Step 2: Verify we have multiple installable versions. +-- Step 9: Verify we have an upgrade path available. SELECT count(*) > 1 AS has_upgrade_path FROM pg_available_extension_versions WHERE name = 'age'; has_upgrade_path @@ -54,7 +123,7 @@ FROM pg_available_extension_versions WHERE name = 'age'; t (1 row) --- Step 3: Install AGE at the synthetic initial version (pre-upgrade state). +-- Step 10: Install AGE at the synthetic initial version. DO $$ DECLARE init_ver text; BEGIN @@ -66,204 +135,10 @@ BEGIN IF init_ver IS NULL THEN RAISE EXCEPTION 'No initial version available for upgrade test'; END IF; - EXECUTE format('CREATE EXTENSION age VERSION %L', init_ver); END; $$; -SELECT extversion IS NOT NULL AS version_installed FROM pg_extension WHERE extname = 'age'; - version_installed -------------------- - t -(1 row) - --- Step 4: Create three test graphs with diverse labels, edges, and data. -LOAD 'age'; -SET search_path TO ag_catalog, "$user", public; --- --- Graph 1: "company" — organization hierarchy with numeric checksums. --- Labels: Employee, Department, Project --- Edges: WORKS_IN, MANAGES, ASSIGNED_TO --- Each vertex has a "val" property (float) for checksum validation. --- -SELECT create_graph('company'); -NOTICE: graph "company" has been created - create_graph --------------- - -(1 row) - -SELECT * FROM cypher('company', $$ - CREATE (e1:Employee {name: 'Alice', role: 'VP', val: 3.14159}) - CREATE (e2:Employee {name: 'Bob', role: 'Manager', val: 2.71828}) - CREATE (e3:Employee {name: 'Charlie', role: 'Engineer', val: 1.41421}) - CREATE (e4:Employee {name: 'Diana', role: 'Engineer', val: 1.73205}) - CREATE (d1:Department {name: 'Engineering', budget: 500000, val: 42.0}) - CREATE (d2:Department {name: 'Research', budget: 300000, val: 17.5}) - CREATE (p1:Project {name: 'Atlas', priority: 1, val: 99.99}) - CREATE (p2:Project {name: 'Beacon', priority: 2, val: 88.88}) - CREATE (p3:Project {name: 'Cipher', priority: 3, val: 77.77}) - CREATE (e1)-[:WORKS_IN {since: 2019}]->(d1) - CREATE (e2)-[:WORKS_IN {since: 2020}]->(d1) - CREATE (e3)-[:WORKS_IN {since: 2021}]->(d1) - CREATE (e4)-[:WORKS_IN {since: 2022}]->(d2) - CREATE (e1)-[:MANAGES {level: 1}]->(e2) - CREATE (e2)-[:MANAGES {level: 2}]->(e3) - CREATE (e3)-[:ASSIGNED_TO {hours: 40}]->(p1) - CREATE (e3)-[:ASSIGNED_TO {hours: 20}]->(p2) - CREATE (e4)-[:ASSIGNED_TO {hours: 30}]->(p2) - CREATE (e4)-[:ASSIGNED_TO {hours: 10}]->(p3) - RETURN 'company graph created' -$$) AS (result agtype); - result -------------------------- - "company graph created" -(1 row) - --- GIN index on Employee properties in company graph -CREATE INDEX company_employee_gin ON company."Employee" USING GIN (properties); --- --- Graph 2: "network" — social network with weighted edges. --- Labels: User, Post --- Edges: FOLLOWS, AUTHORED, LIKES --- -SELECT create_graph('network'); -NOTICE: graph "network" has been created - create_graph --------------- - -(1 row) - -SELECT * FROM cypher('network', $$ - CREATE (u1:User {handle: '@alpha', score: 1000.01}) - CREATE (u2:User {handle: '@beta', score: 2000.02}) - CREATE (u3:User {handle: '@gamma', score: 3000.03}) - CREATE (u4:User {handle: '@delta', score: 4000.04}) - CREATE (u5:User {handle: '@epsilon', score: 5000.05}) - CREATE (p1:Post {title: 'Hello World', views: 150}) - CREATE (p2:Post {title: 'Graph Databases 101', views: 890}) - CREATE (p3:Post {title: 'AGE is awesome', views: 2200}) - CREATE (u1)-[:FOLLOWS {weight: 0.9}]->(u2) - CREATE (u2)-[:FOLLOWS {weight: 0.8}]->(u3) - CREATE (u3)-[:FOLLOWS {weight: 0.7}]->(u4) - CREATE (u4)-[:FOLLOWS {weight: 0.6}]->(u5) - CREATE (u5)-[:FOLLOWS {weight: 0.5}]->(u1) - CREATE (u1)-[:AUTHORED]->(p1) - CREATE (u2)-[:AUTHORED]->(p2) - CREATE (u3)-[:AUTHORED]->(p3) - CREATE (u4)-[:LIKES]->(p1) - CREATE (u5)-[:LIKES]->(p2) - CREATE (u1)-[:LIKES]->(p3) - CREATE (u2)-[:LIKES]->(p3) - RETURN 'network graph created' -$$) AS (result agtype); - result -------------------------- - "network graph created" -(1 row) - --- GIN indexes on network graph -CREATE INDEX network_user_gin ON network."User" USING GIN (properties); -CREATE INDEX network_post_gin ON network."Post" USING GIN (properties); --- --- Graph 3: "routes" — geographic routing with precise coordinates. --- Labels: City, Airport --- Edges: ROAD, FLIGHT --- Coordinates use precise decimals that are easy to checksum. --- -SELECT create_graph('routes'); -NOTICE: graph "routes" has been created - create_graph --------------- - -(1 row) - -SELECT * FROM cypher('routes', $$ - CREATE (c1:City {name: 'Portland', lat: 45.5152, lon: -122.6784, pop: 652503}) - CREATE (c2:City {name: 'Seattle', lat: 47.6062, lon: -122.3321, pop: 749256}) - CREATE (c3:City {name: 'Vancouver', lat: 49.2827, lon: -123.1207, pop: 631486}) - CREATE (a1:Airport {code: 'PDX', elev: 30.5}) - CREATE (a2:Airport {code: 'SEA', elev: 131.7}) - CREATE (a3:Airport {code: 'YVR', elev: 4.3}) - CREATE (c1)-[:ROAD {distance_km: 279.5, toll: 0.0}]->(c2) - CREATE (c2)-[:ROAD {distance_km: 225.3, toll: 5.0}]->(c3) - CREATE (c1)-[:ROAD {distance_km: 502.1, toll: 5.0}]->(c3) - CREATE (a1)-[:FLIGHT {distance_km: 229.0, duration_min: 55}]->(a2) - CREATE (a2)-[:FLIGHT {distance_km: 198.0, duration_min: 50}]->(a3) - CREATE (a1)-[:FLIGHT {distance_km: 426.0, duration_min: 75}]->(a3) - RETURN 'routes graph created' -$$) AS (result agtype); - result ------------------------- - "routes graph created" -(1 row) - --- GIN index on routes graph -CREATE INDEX routes_city_gin ON routes."City" USING GIN (properties); --- Step 5: Record pre-upgrade integrity checksums. --- These sums use the "val" / "score" / coordinate properties as fingerprints. --- company: sum of all val properties (should be a precise known value) -SELECT * FROM cypher('company', $$ - MATCH (n) WHERE n.val IS NOT NULL RETURN sum(n.val) -$$) AS (company_val_sum_before agtype); - company_val_sum_before ------------------------- - 335.14612999999997 -(1 row) - --- network: sum of all score properties -SELECT * FROM cypher('network', $$ - MATCH (n:User) RETURN sum(n.score) -$$) AS (network_score_sum_before agtype); - network_score_sum_before --------------------------- - 15000.149999999998 -(1 row) - --- routes: sum of all latitude values -SELECT * FROM cypher('routes', $$ - MATCH (c:City) RETURN sum(c.lat) -$$) AS (routes_lat_sum_before agtype); - routes_lat_sum_before ------------------------ - 142.4041 -(1 row) - --- Total vertex and edge counts across all three graphs -SELECT sum(cnt)::int AS total_vertices_before FROM ( - SELECT count(*) AS cnt FROM cypher('company', $$ MATCH (n) RETURN n $$) AS (n agtype) - UNION ALL - SELECT count(*) FROM cypher('network', $$ MATCH (n) RETURN n $$) AS (n agtype) - UNION ALL - SELECT count(*) FROM cypher('routes', $$ MATCH (n) RETURN n $$) AS (n agtype) -) sub; - total_vertices_before ------------------------ - 23 -(1 row) - -SELECT sum(cnt)::int AS total_edges_before FROM ( - SELECT count(*) AS cnt FROM cypher('company', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) - UNION ALL - SELECT count(*) FROM cypher('network', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) - UNION ALL - SELECT count(*) FROM cypher('routes', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) -) sub; - total_edges_before --------------------- - 28 -(1 row) - --- Count of distinct labels (ag_label entries) across all graphs -SELECT count(*)::int AS total_labels_before -FROM ag_label al JOIN ag_graph ag ON al.graph = ag.graphid -WHERE ag.name <> '_ag_catalog'; - total_labels_before ---------------------- - 21 -(1 row) - --- Step 6: Upgrade AGE from the initial version to the current (default) version --- via the stamped upgrade template. +-- Step 11: Upgrade to the current (default) version via the stamped template. DO $$ DECLARE curr_ver text; BEGIN @@ -273,11 +148,10 @@ BEGIN IF curr_ver IS NULL THEN RAISE EXCEPTION 'No default version found for upgrade test'; END IF; - EXECUTE format('ALTER EXTENSION age UPDATE TO %L', curr_ver); END; $$; --- Step 7: Confirm version is now the default (current HEAD) version. +-- Step 12: Confirm the upgrade succeeded. SELECT installed_version = default_version AS upgraded_to_current FROM pg_available_extensions WHERE name = 'age'; upgraded_to_current @@ -285,169 +159,253 @@ FROM pg_available_extensions WHERE name = 'age'; t (1 row) --- Step 8: Verify all data survived — reload and recheck. -LOAD 'age'; -SET search_path TO ag_catalog, "$user", public; --- Repeat integrity checksums — must match pre-upgrade values exactly. -SELECT * FROM cypher('company', $$ - MATCH (n) WHERE n.val IS NOT NULL RETURN sum(n.val) -$$) AS (company_val_sum_after agtype); - company_val_sum_after ------------------------ - 335.14612999999997 -(1 row) +-- ===================================================================== +-- UPGRADED INSTALL SNAPSHOTS (Steps 13-18) +-- Capture the catalog state after upgrade from initial to current. +-- ===================================================================== +-- Step 13: Snapshot functions. +CREATE TEMP TABLE _upgraded_funcs AS +SELECT proname::text, + pg_get_function_identity_arguments(oid) AS args, + provolatile::text, proisstrict::text, prokind::text, + prorettype::regtype::text AS rettype, proretset::text +FROM pg_proc +WHERE pronamespace = 'ag_catalog'::regnamespace +ORDER BY proname, args; +-- Step 14: Snapshot relations. +CREATE TEMP TABLE _upgraded_rels AS +SELECT relname::text, relkind::text +FROM pg_class +WHERE relnamespace = 'ag_catalog'::regnamespace + AND relkind IN ('r', 'v', 'S', 'i') +ORDER BY relname; +-- Step 15: Snapshot types. +CREATE TEMP TABLE _upgraded_types AS +SELECT typname::text, typtype::text +FROM pg_type +WHERE typnamespace = 'ag_catalog'::regnamespace + AND typname NOT LIKE 'pg_toast%' +ORDER BY typname; +-- Step 16: Snapshot operators. +CREATE TEMP TABLE _upgraded_ops AS +SELECT oprname::text, + oprleft::regtype::text AS lefttype, + oprright::regtype::text AS righttype +FROM pg_operator +WHERE oprnamespace = 'ag_catalog'::regnamespace +ORDER BY oprname, lefttype, righttype; +-- Step 17: Snapshot casts and operator classes. +CREATE TEMP TABLE _upgraded_casts AS +SELECT castsource::regtype::text AS source_type, + casttarget::regtype::text AS target_type, + castcontext::text +FROM pg_cast +WHERE castsource IN (SELECT oid FROM pg_type WHERE typnamespace = 'ag_catalog'::regnamespace) + OR casttarget IN (SELECT oid FROM pg_type WHERE typnamespace = 'ag_catalog'::regnamespace) +ORDER BY source_type, target_type; +CREATE TEMP TABLE _upgraded_opclass AS +SELECT opcname::text, + (SELECT amname FROM pg_am WHERE oid = opcmethod)::text AS amname +FROM pg_opclass +WHERE opcnamespace = 'ag_catalog'::regnamespace +ORDER BY opcname; +-- Step 18: Snapshot constraints. +CREATE TEMP TABLE _upgraded_constraints AS +SELECT conname::text, contype::text, + conrelid::regclass::text AS table_name, + confrelid::regclass::text AS ref_table +FROM pg_constraint +WHERE connamespace = 'ag_catalog'::regnamespace +ORDER BY conname; +-- ===================================================================== +-- COMPARISON: Missing or extra objects (Steps 19-33) +-- Any rows returned indicate a template deficiency. +-- ===================================================================== +-- Step 19: Functions MISSING after upgrade. +SELECT f.proname || '(' || f.args || ')' AS missing_function +FROM _fresh_funcs f +LEFT JOIN _upgraded_funcs u USING (proname, args) +WHERE u.proname IS NULL +ORDER BY 1; + missing_function +------------------ +(0 rows) -SELECT * FROM cypher('network', $$ - MATCH (n:User) RETURN sum(n.score) -$$) AS (network_score_sum_after agtype); - network_score_sum_after -------------------------- - 15000.149999999998 -(1 row) +-- Step 20: Functions EXTRA after upgrade. +SELECT u.proname || '(' || u.args || ')' AS extra_function +FROM _upgraded_funcs u +LEFT JOIN _fresh_funcs f USING (proname, args) +WHERE f.proname IS NULL +ORDER BY 1; + extra_function +---------------- +(0 rows) -SELECT * FROM cypher('routes', $$ - MATCH (c:City) RETURN sum(c.lat) -$$) AS (routes_lat_sum_after agtype); - routes_lat_sum_after ----------------------- - 142.4041 -(1 row) +-- Step 21: Function PROPERTY changes (volatility, strictness, kind, return type). +SELECT f.proname || '(' || f.args || ')' AS function_name, + CASE WHEN f.prokind <> u.prokind THEN 'prokind: ' || f.prokind || '->' || u.prokind END AS kind_change, + CASE WHEN f.provolatile<> u.provolatile THEN 'volatile: ' || f.provolatile|| '->' || u.provolatile END AS volatility_change, + CASE WHEN f.proisstrict<> u.proisstrict THEN 'strict: ' || f.proisstrict|| '->' || u.proisstrict END AS strict_change, + CASE WHEN f.rettype <> u.rettype THEN 'rettype: ' || f.rettype || '->' || u.rettype END AS rettype_change, + CASE WHEN f.proretset <> u.proretset THEN 'retset: ' || f.proretset || '->' || u.proretset END AS retset_change +FROM _fresh_funcs f +JOIN _upgraded_funcs u USING (proname, args) +WHERE f.provolatile <> u.provolatile + OR f.proisstrict <> u.proisstrict + OR f.prokind <> u.prokind + OR f.rettype <> u.rettype + OR f.proretset <> u.proretset +ORDER BY 1; + function_name | kind_change | volatility_change | strict_change | rettype_change | retset_change +---------------+-------------+-------------------+---------------+----------------+--------------- +(0 rows) -SELECT sum(cnt)::int AS total_vertices_after FROM ( - SELECT count(*) AS cnt FROM cypher('company', $$ MATCH (n) RETURN n $$) AS (n agtype) - UNION ALL - SELECT count(*) FROM cypher('network', $$ MATCH (n) RETURN n $$) AS (n agtype) - UNION ALL - SELECT count(*) FROM cypher('routes', $$ MATCH (n) RETURN n $$) AS (n agtype) -) sub; - total_vertices_after ----------------------- - 23 -(1 row) +-- Step 22: Relations MISSING after upgrade. +SELECT f.relname || ' (' || f.relkind || ')' AS missing_relation +FROM _fresh_rels f +LEFT JOIN _upgraded_rels u USING (relname, relkind) +WHERE u.relname IS NULL +ORDER BY 1; + missing_relation +------------------ +(0 rows) -SELECT sum(cnt)::int AS total_edges_after FROM ( - SELECT count(*) AS cnt FROM cypher('company', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) - UNION ALL - SELECT count(*) FROM cypher('network', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) - UNION ALL - SELECT count(*) FROM cypher('routes', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) -) sub; - total_edges_after -------------------- - 28 -(1 row) +-- Step 23: Relations EXTRA after upgrade. +SELECT u.relname || ' (' || u.relkind || ')' AS extra_relation +FROM _upgraded_rels u +LEFT JOIN _fresh_rels f USING (relname, relkind) +WHERE f.relname IS NULL +ORDER BY 1; + extra_relation +---------------- +(0 rows) -SELECT count(*)::int AS total_labels_after -FROM ag_label al JOIN ag_graph ag ON al.graph = ag.graphid -WHERE ag.name <> '_ag_catalog'; - total_labels_after --------------------- - 21 -(1 row) +-- Step 24: Types MISSING after upgrade. +SELECT f.typname || ' (' || f.typtype || ')' AS missing_type +FROM _fresh_types f +LEFT JOIN _upgraded_types u USING (typname, typtype) +WHERE u.typname IS NULL +ORDER BY 1; + missing_type +-------------- +(0 rows) --- Step 9: Verify specific structural queries across all three graphs. --- company: management chain -SELECT * FROM cypher('company', $$ - MATCH (boss:Employee)-[:MANAGES*]->(report:Employee) - RETURN boss.name, report.name - ORDER BY boss.name, report.name -$$) AS (boss agtype, report agtype); - boss | report ----------+----------- - "Alice" | "Bob" - "Alice" | "Charlie" - "Bob" | "Charlie" -(3 rows) +-- Step 25: Types EXTRA after upgrade. +SELECT u.typname || ' (' || u.typtype || ')' AS extra_type +FROM _upgraded_types u +LEFT JOIN _fresh_types f USING (typname, typtype) +WHERE f.typname IS NULL +ORDER BY 1; + extra_type +------------ +(0 rows) --- network: circular follow chain (proves full cycle survived) -SELECT * FROM cypher('network', $$ - MATCH (a:User)-[:FOLLOWS]->(b:User) - RETURN a.handle, b.handle - ORDER BY a.handle -$$) AS (follower agtype, followed agtype); - follower | followed -------------+------------ - "@alpha" | "@beta" - "@beta" | "@gamma" - "@delta" | "@epsilon" - "@epsilon" | "@alpha" - "@gamma" | "@delta" -(5 rows) +-- Step 26: Operators MISSING after upgrade. +SELECT f.oprname || ' (' || f.lefttype || ', ' || f.righttype || ')' AS missing_operator +FROM _fresh_ops f +LEFT JOIN _upgraded_ops u USING (oprname, lefttype, righttype) +WHERE u.oprname IS NULL +ORDER BY 1; + missing_operator +------------------ +(0 rows) --- routes: all flights with distances (proves edge properties intact) -SELECT * FROM cypher('routes', $$ - MATCH (a:Airport)-[f:FLIGHT]->(b:Airport) - RETURN a.code, b.code, f.distance_km - ORDER BY a.code, b.code -$$) AS (origin agtype, dest agtype, dist agtype); - origin | dest | dist ---------+-------+------- - "PDX" | "SEA" | 229.0 - "PDX" | "YVR" | 426.0 - "SEA" | "YVR" | 198.0 -(3 rows) +-- Step 27: Operators EXTRA after upgrade. +SELECT u.oprname || ' (' || u.lefttype || ', ' || u.righttype || ')' AS extra_operator +FROM _upgraded_ops u +LEFT JOIN _fresh_ops f USING (oprname, lefttype, righttype) +WHERE f.oprname IS NULL +ORDER BY 1; + extra_operator +---------------- +(0 rows) --- Step 10: Verify GIN indexes still exist after upgrade. -SELECT indexname FROM pg_indexes -WHERE schemaname IN ('company', 'network', 'routes') - AND tablename IN ('Employee', 'User', 'Post', 'City') - AND indexdef LIKE '%gin%' -ORDER BY indexname; - indexname ----------------------- - company_employee_gin - network_post_gin - network_user_gin - routes_city_gin -(4 rows) +-- Step 28: Casts MISSING after upgrade. +SELECT f.source_type || ' -> ' || f.target_type || ' (' || f.castcontext || ')' AS missing_cast +FROM _fresh_casts f +LEFT JOIN _upgraded_casts u USING (source_type, target_type, castcontext) +WHERE u.source_type IS NULL +ORDER BY 1; + missing_cast +-------------- +(0 rows) --- Step 11: Cleanup and restore AGE at the default version for subsequent tests. -SELECT drop_graph('routes', true); -NOTICE: drop cascades to 6 other objects -DETAIL: drop cascades to table routes._ag_label_vertex -drop cascades to table routes._ag_label_edge -drop cascades to table routes."City" -drop cascades to table routes."Airport" -drop cascades to table routes."ROAD" -drop cascades to table routes."FLIGHT" -NOTICE: graph "routes" has been dropped - drop_graph +-- Step 29: Casts EXTRA after upgrade. +SELECT u.source_type || ' -> ' || u.target_type || ' (' || u.castcontext || ')' AS extra_cast +FROM _upgraded_casts u +LEFT JOIN _fresh_casts f USING (source_type, target_type, castcontext) +WHERE f.source_type IS NULL +ORDER BY 1; + extra_cast ------------ - -(1 row) +(0 rows) -SELECT drop_graph('network', true); -NOTICE: drop cascades to 7 other objects -DETAIL: drop cascades to table network._ag_label_vertex -drop cascades to table network._ag_label_edge -drop cascades to table network."User" -drop cascades to table network."Post" -drop cascades to table network."FOLLOWS" -drop cascades to table network."AUTHORED" -drop cascades to table network."LIKES" -NOTICE: graph "network" has been dropped - drop_graph ------------- - -(1 row) +-- Step 30: Operator classes MISSING after upgrade. +SELECT f.opcname || ' (' || f.amname || ')' AS missing_opclass +FROM _fresh_opclass f +LEFT JOIN _upgraded_opclass u USING (opcname, amname) +WHERE u.opcname IS NULL +ORDER BY 1; + missing_opclass +----------------- +(0 rows) -SELECT drop_graph('company', true); -NOTICE: drop cascades to 8 other objects -DETAIL: drop cascades to table company._ag_label_vertex -drop cascades to table company._ag_label_edge -drop cascades to table company."Employee" -drop cascades to table company."Department" -drop cascades to table company."Project" -drop cascades to table company."WORKS_IN" -drop cascades to table company."MANAGES" -drop cascades to table company."ASSIGNED_TO" -NOTICE: graph "company" has been dropped - drop_graph ------------- - +-- Step 31: Operator classes EXTRA after upgrade. +SELECT u.opcname || ' (' || u.amname || ')' AS extra_opclass +FROM _upgraded_opclass u +LEFT JOIN _fresh_opclass f USING (opcname, amname) +WHERE f.opcname IS NULL +ORDER BY 1; + extra_opclass +--------------- +(0 rows) + +-- Step 32: Constraints MISSING after upgrade. +SELECT f.conname || ' (' || f.contype || ' on ' || f.table_name || ')' AS missing_constraint +FROM _fresh_constraints f +LEFT JOIN _upgraded_constraints u USING (conname, contype, table_name) +WHERE u.conname IS NULL +ORDER BY 1; + missing_constraint +-------------------- +(0 rows) + +-- Step 33: Constraints EXTRA after upgrade. +SELECT u.conname || ' (' || u.contype || ' on ' || u.table_name || ')' AS extra_constraint +FROM _upgraded_constraints u +LEFT JOIN _fresh_constraints f USING (conname, contype, table_name) +WHERE f.conname IS NULL +ORDER BY 1; + extra_constraint +------------------ +(0 rows) + +-- ===================================================================== +-- SUMMARY (Step 34) +-- ===================================================================== +-- Step 34: Verify all counts match (result: single row, all true). +SELECT + (SELECT count(*) FROM _fresh_funcs) = (SELECT count(*) FROM _upgraded_funcs) AS funcs_match, + (SELECT count(*) FROM _fresh_rels) = (SELECT count(*) FROM _upgraded_rels) AS rels_match, + (SELECT count(*) FROM _fresh_types) = (SELECT count(*) FROM _upgraded_types) AS types_match, + (SELECT count(*) FROM _fresh_ops) = (SELECT count(*) FROM _upgraded_ops) AS ops_match, + (SELECT count(*) FROM _fresh_casts) = (SELECT count(*) FROM _upgraded_casts) AS casts_match, + (SELECT count(*) FROM _fresh_opclass) = (SELECT count(*) FROM _upgraded_opclass) AS opclass_match, + (SELECT count(*) FROM _fresh_constraints) = (SELECT count(*) FROM _upgraded_constraints) AS constraints_match; + funcs_match | rels_match | types_match | ops_match | casts_match | opclass_match | constraints_match +-------------+------------+-------------+-----------+-------------+---------------+------------------- + t | t | t | t | t | t | t (1 row) +-- ===================================================================== +-- CLEANUP (Steps 35-36) +-- ===================================================================== +-- Step 35: Drop temp tables, restore AGE at default version. +DROP TABLE _fresh_funcs, _upgraded_funcs, _fresh_rels, _upgraded_rels, + _fresh_types, _upgraded_types, _fresh_ops, _upgraded_ops, + _fresh_casts, _upgraded_casts, _fresh_opclass, _upgraded_opclass, + _fresh_constraints, _upgraded_constraints; DROP EXTENSION age; CREATE EXTENSION age; --- Step 12: Remove synthetic upgrade test files from the extension directory. +-- Step 36: Remove synthetic upgrade test files from the extension directory. \! sh ./regress/age_upgrade_cleanup.sh diff --git a/regress/sql/age_upgrade.sql b/regress/sql/age_upgrade.sql index 70d45064a..f56f7ca93 100644 --- a/regress/sql/age_upgrade.sql +++ b/regress/sql/age_upgrade.sql @@ -18,39 +18,121 @@ */ -- --- Extension upgrade regression test +-- Extension upgrade template regression test -- --- This test validates the upgrade template (age----y.y.y.sql) by: --- 1. Dropping AGE and reinstalling at the synthetic "initial" version --- (built from the version-bump commit's SQL — the "day-one" state) --- 2. Creating three graphs with multiple labels, edges, GIN indexes, --- and numeric properties that serve as integrity checksums --- 3. Upgrading to the current (default) version via the stamped template --- 4. Verifying all data, structure, and checksums survived the upgrade +-- Validates the upgrade template (age----y.y.y.sql) by comparing +-- the pg_catalog entries of a fresh install against an upgraded install. +-- If any object exists in the fresh install but is missing after upgrade, +-- the template is incomplete. This catches the case where a developer adds +-- a new SQL object to sql/ and sql_files but forgets to add it to the +-- upgrade template. -- --- The Makefile builds: --- age--.sql from current HEAD's sql/sql_files (default) --- age--_initial.sql from the version-bump commit (synthetic) --- age--_initial--.sql stamped from the upgrade template +-- Compared catalogs: +-- pg_proc — functions, aggregates, procedures (name, args, properties) +-- pg_class — tables, views, sequences, indexes (name, kind) +-- pg_type — types (name, type category) +-- pg_operator — operators (name, left/right types) +-- pg_cast — casts involving AGE types (source, target, context) +-- pg_opclass — operator classes (name, access method) +-- pg_constraint — constraints (name, type, table, referenced table) -- --- All version discovery is dynamic — no hardcoded versions anywhere. --- This test is version-agnostic and works on any branch for any version. +-- All comparison queries should return 0 rows. -- LOAD 'age'; SET search_path TO ag_catalog; --- Step 1: Clean up any state left by prior tests, then drop AGE entirely. --- The --load-extension=age flag installed AGE at the current (default) version. --- We need to remove it so we can reinstall at the synthetic initial version. -SELECT drop_graph(name, true) FROM ag_graph ORDER BY name; +-- Step 1: Clean up any graphs left by prior tests (deterministic, no output). +DO $$ +DECLARE + graph_name ag_graph.name%TYPE; +BEGIN + PERFORM set_config('client_min_messages', 'warning', true); + + FOR graph_name IN + SELECT name + FROM ag_graph + ORDER BY name + LOOP + PERFORM drop_graph(graph_name, true); + END LOOP; +END +$$; + +-- ===================================================================== +-- FRESH INSTALL SNAPSHOTS (Steps 2-7) +-- Capture the catalog state from the default CREATE EXTENSION install. +-- ===================================================================== + +-- Step 2: Snapshot functions (includes aggregates via prokind). +CREATE TEMP TABLE _fresh_funcs AS +SELECT proname::text, + pg_get_function_identity_arguments(oid) AS args, + provolatile::text, proisstrict::text, prokind::text, + prorettype::regtype::text AS rettype, proretset::text +FROM pg_proc +WHERE pronamespace = 'ag_catalog'::regnamespace +ORDER BY proname, args; + +-- Step 3: Snapshot relations (tables, views, sequences, indexes). +CREATE TEMP TABLE _fresh_rels AS +SELECT relname::text, relkind::text +FROM pg_class +WHERE relnamespace = 'ag_catalog'::regnamespace + AND relkind IN ('r', 'v', 'S', 'i') +ORDER BY relname; + +-- Step 4: Snapshot types. +CREATE TEMP TABLE _fresh_types AS +SELECT typname::text, typtype::text +FROM pg_type +WHERE typnamespace = 'ag_catalog'::regnamespace + AND typname NOT LIKE 'pg_toast%' +ORDER BY typname; + +-- Step 5: Snapshot operators. +CREATE TEMP TABLE _fresh_ops AS +SELECT oprname::text, + oprleft::regtype::text AS lefttype, + oprright::regtype::text AS righttype +FROM pg_operator +WHERE oprnamespace = 'ag_catalog'::regnamespace +ORDER BY oprname, lefttype, righttype; + +-- Step 6: Snapshot casts involving AGE types, and operator classes. +CREATE TEMP TABLE _fresh_casts AS +SELECT castsource::regtype::text AS source_type, + casttarget::regtype::text AS target_type, + castcontext::text +FROM pg_cast +WHERE castsource IN (SELECT oid FROM pg_type WHERE typnamespace = 'ag_catalog'::regnamespace) + OR casttarget IN (SELECT oid FROM pg_type WHERE typnamespace = 'ag_catalog'::regnamespace) +ORDER BY source_type, target_type; + +CREATE TEMP TABLE _fresh_opclass AS +SELECT opcname::text, + (SELECT amname FROM pg_am WHERE oid = opcmethod)::text AS amname +FROM pg_opclass +WHERE opcnamespace = 'ag_catalog'::regnamespace +ORDER BY opcname; + +-- Step 7: Snapshot constraints. +CREATE TEMP TABLE _fresh_constraints AS +SELECT conname::text, contype::text, + conrelid::regclass::text AS table_name, + confrelid::regclass::text AS ref_table +FROM pg_constraint +WHERE connamespace = 'ag_catalog'::regnamespace +ORDER BY conname; + +-- Step 8: Drop AGE entirely. DROP EXTENSION age; --- Step 2: Verify we have multiple installable versions. +-- Step 9: Verify we have an upgrade path available. SELECT count(*) > 1 AS has_upgrade_path FROM pg_available_extension_versions WHERE name = 'age'; --- Step 3: Install AGE at the synthetic initial version (pre-upgrade state). +-- Step 10: Install AGE at the synthetic initial version. DO $$ DECLARE init_ver text; BEGIN @@ -62,154 +144,11 @@ BEGIN IF init_ver IS NULL THEN RAISE EXCEPTION 'No initial version available for upgrade test'; END IF; - EXECUTE format('CREATE EXTENSION age VERSION %L', init_ver); END; $$; -SELECT extversion IS NOT NULL AS version_installed FROM pg_extension WHERE extname = 'age'; --- Step 4: Create three test graphs with diverse labels, edges, and data. -LOAD 'age'; -SET search_path TO ag_catalog, "$user", public; - --- --- Graph 1: "company" — organization hierarchy with numeric checksums. --- Labels: Employee, Department, Project --- Edges: WORKS_IN, MANAGES, ASSIGNED_TO --- Each vertex has a "val" property (float) for checksum validation. --- -SELECT create_graph('company'); - -SELECT * FROM cypher('company', $$ - CREATE (e1:Employee {name: 'Alice', role: 'VP', val: 3.14159}) - CREATE (e2:Employee {name: 'Bob', role: 'Manager', val: 2.71828}) - CREATE (e3:Employee {name: 'Charlie', role: 'Engineer', val: 1.41421}) - CREATE (e4:Employee {name: 'Diana', role: 'Engineer', val: 1.73205}) - CREATE (d1:Department {name: 'Engineering', budget: 500000, val: 42.0}) - CREATE (d2:Department {name: 'Research', budget: 300000, val: 17.5}) - CREATE (p1:Project {name: 'Atlas', priority: 1, val: 99.99}) - CREATE (p2:Project {name: 'Beacon', priority: 2, val: 88.88}) - CREATE (p3:Project {name: 'Cipher', priority: 3, val: 77.77}) - CREATE (e1)-[:WORKS_IN {since: 2019}]->(d1) - CREATE (e2)-[:WORKS_IN {since: 2020}]->(d1) - CREATE (e3)-[:WORKS_IN {since: 2021}]->(d1) - CREATE (e4)-[:WORKS_IN {since: 2022}]->(d2) - CREATE (e1)-[:MANAGES {level: 1}]->(e2) - CREATE (e2)-[:MANAGES {level: 2}]->(e3) - CREATE (e3)-[:ASSIGNED_TO {hours: 40}]->(p1) - CREATE (e3)-[:ASSIGNED_TO {hours: 20}]->(p2) - CREATE (e4)-[:ASSIGNED_TO {hours: 30}]->(p2) - CREATE (e4)-[:ASSIGNED_TO {hours: 10}]->(p3) - RETURN 'company graph created' -$$) AS (result agtype); - --- GIN index on Employee properties in company graph -CREATE INDEX company_employee_gin ON company."Employee" USING GIN (properties); - --- --- Graph 2: "network" — social network with weighted edges. --- Labels: User, Post --- Edges: FOLLOWS, AUTHORED, LIKES --- -SELECT create_graph('network'); - -SELECT * FROM cypher('network', $$ - CREATE (u1:User {handle: '@alpha', score: 1000.01}) - CREATE (u2:User {handle: '@beta', score: 2000.02}) - CREATE (u3:User {handle: '@gamma', score: 3000.03}) - CREATE (u4:User {handle: '@delta', score: 4000.04}) - CREATE (u5:User {handle: '@epsilon', score: 5000.05}) - CREATE (p1:Post {title: 'Hello World', views: 150}) - CREATE (p2:Post {title: 'Graph Databases 101', views: 890}) - CREATE (p3:Post {title: 'AGE is awesome', views: 2200}) - CREATE (u1)-[:FOLLOWS {weight: 0.9}]->(u2) - CREATE (u2)-[:FOLLOWS {weight: 0.8}]->(u3) - CREATE (u3)-[:FOLLOWS {weight: 0.7}]->(u4) - CREATE (u4)-[:FOLLOWS {weight: 0.6}]->(u5) - CREATE (u5)-[:FOLLOWS {weight: 0.5}]->(u1) - CREATE (u1)-[:AUTHORED]->(p1) - CREATE (u2)-[:AUTHORED]->(p2) - CREATE (u3)-[:AUTHORED]->(p3) - CREATE (u4)-[:LIKES]->(p1) - CREATE (u5)-[:LIKES]->(p2) - CREATE (u1)-[:LIKES]->(p3) - CREATE (u2)-[:LIKES]->(p3) - RETURN 'network graph created' -$$) AS (result agtype); - --- GIN indexes on network graph -CREATE INDEX network_user_gin ON network."User" USING GIN (properties); -CREATE INDEX network_post_gin ON network."Post" USING GIN (properties); - --- --- Graph 3: "routes" — geographic routing with precise coordinates. --- Labels: City, Airport --- Edges: ROAD, FLIGHT --- Coordinates use precise decimals that are easy to checksum. --- -SELECT create_graph('routes'); - -SELECT * FROM cypher('routes', $$ - CREATE (c1:City {name: 'Portland', lat: 45.5152, lon: -122.6784, pop: 652503}) - CREATE (c2:City {name: 'Seattle', lat: 47.6062, lon: -122.3321, pop: 749256}) - CREATE (c3:City {name: 'Vancouver', lat: 49.2827, lon: -123.1207, pop: 631486}) - CREATE (a1:Airport {code: 'PDX', elev: 30.5}) - CREATE (a2:Airport {code: 'SEA', elev: 131.7}) - CREATE (a3:Airport {code: 'YVR', elev: 4.3}) - CREATE (c1)-[:ROAD {distance_km: 279.5, toll: 0.0}]->(c2) - CREATE (c2)-[:ROAD {distance_km: 225.3, toll: 5.0}]->(c3) - CREATE (c1)-[:ROAD {distance_km: 502.1, toll: 5.0}]->(c3) - CREATE (a1)-[:FLIGHT {distance_km: 229.0, duration_min: 55}]->(a2) - CREATE (a2)-[:FLIGHT {distance_km: 198.0, duration_min: 50}]->(a3) - CREATE (a1)-[:FLIGHT {distance_km: 426.0, duration_min: 75}]->(a3) - RETURN 'routes graph created' -$$) AS (result agtype); - --- GIN index on routes graph -CREATE INDEX routes_city_gin ON routes."City" USING GIN (properties); - --- Step 5: Record pre-upgrade integrity checksums. --- These sums use the "val" / "score" / coordinate properties as fingerprints. - --- company: sum of all val properties (should be a precise known value) -SELECT * FROM cypher('company', $$ - MATCH (n) WHERE n.val IS NOT NULL RETURN sum(n.val) -$$) AS (company_val_sum_before agtype); - --- network: sum of all score properties -SELECT * FROM cypher('network', $$ - MATCH (n:User) RETURN sum(n.score) -$$) AS (network_score_sum_before agtype); - --- routes: sum of all latitude values -SELECT * FROM cypher('routes', $$ - MATCH (c:City) RETURN sum(c.lat) -$$) AS (routes_lat_sum_before agtype); - --- Total vertex and edge counts across all three graphs -SELECT sum(cnt)::int AS total_vertices_before FROM ( - SELECT count(*) AS cnt FROM cypher('company', $$ MATCH (n) RETURN n $$) AS (n agtype) - UNION ALL - SELECT count(*) FROM cypher('network', $$ MATCH (n) RETURN n $$) AS (n agtype) - UNION ALL - SELECT count(*) FROM cypher('routes', $$ MATCH (n) RETURN n $$) AS (n agtype) -) sub; - -SELECT sum(cnt)::int AS total_edges_before FROM ( - SELECT count(*) AS cnt FROM cypher('company', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) - UNION ALL - SELECT count(*) FROM cypher('network', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) - UNION ALL - SELECT count(*) FROM cypher('routes', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) -) sub; - --- Count of distinct labels (ag_label entries) across all graphs -SELECT count(*)::int AS total_labels_before -FROM ag_label al JOIN ag_graph ag ON al.graph = ag.graphid -WHERE ag.name <> '_ag_catalog'; - --- Step 6: Upgrade AGE from the initial version to the current (default) version --- via the stamped upgrade template. +-- Step 11: Upgrade to the current (default) version via the stamped template. DO $$ DECLARE curr_ver text; BEGIN @@ -219,88 +158,225 @@ BEGIN IF curr_ver IS NULL THEN RAISE EXCEPTION 'No default version found for upgrade test'; END IF; - EXECUTE format('ALTER EXTENSION age UPDATE TO %L', curr_ver); END; $$; --- Step 7: Confirm version is now the default (current HEAD) version. +-- Step 12: Confirm the upgrade succeeded. SELECT installed_version = default_version AS upgraded_to_current FROM pg_available_extensions WHERE name = 'age'; --- Step 8: Verify all data survived — reload and recheck. -LOAD 'age'; -SET search_path TO ag_catalog, "$user", public; - --- Repeat integrity checksums — must match pre-upgrade values exactly. -SELECT * FROM cypher('company', $$ - MATCH (n) WHERE n.val IS NOT NULL RETURN sum(n.val) -$$) AS (company_val_sum_after agtype); - -SELECT * FROM cypher('network', $$ - MATCH (n:User) RETURN sum(n.score) -$$) AS (network_score_sum_after agtype); - -SELECT * FROM cypher('routes', $$ - MATCH (c:City) RETURN sum(c.lat) -$$) AS (routes_lat_sum_after agtype); - -SELECT sum(cnt)::int AS total_vertices_after FROM ( - SELECT count(*) AS cnt FROM cypher('company', $$ MATCH (n) RETURN n $$) AS (n agtype) - UNION ALL - SELECT count(*) FROM cypher('network', $$ MATCH (n) RETURN n $$) AS (n agtype) - UNION ALL - SELECT count(*) FROM cypher('routes', $$ MATCH (n) RETURN n $$) AS (n agtype) -) sub; - -SELECT sum(cnt)::int AS total_edges_after FROM ( - SELECT count(*) AS cnt FROM cypher('company', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) - UNION ALL - SELECT count(*) FROM cypher('network', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) - UNION ALL - SELECT count(*) FROM cypher('routes', $$ MATCH ()-[e]->() RETURN e $$) AS (e agtype) -) sub; - -SELECT count(*)::int AS total_labels_after -FROM ag_label al JOIN ag_graph ag ON al.graph = ag.graphid -WHERE ag.name <> '_ag_catalog'; - --- Step 9: Verify specific structural queries across all three graphs. - --- company: management chain -SELECT * FROM cypher('company', $$ - MATCH (boss:Employee)-[:MANAGES*]->(report:Employee) - RETURN boss.name, report.name - ORDER BY boss.name, report.name -$$) AS (boss agtype, report agtype); - --- network: circular follow chain (proves full cycle survived) -SELECT * FROM cypher('network', $$ - MATCH (a:User)-[:FOLLOWS]->(b:User) - RETURN a.handle, b.handle - ORDER BY a.handle -$$) AS (follower agtype, followed agtype); - --- routes: all flights with distances (proves edge properties intact) -SELECT * FROM cypher('routes', $$ - MATCH (a:Airport)-[f:FLIGHT]->(b:Airport) - RETURN a.code, b.code, f.distance_km - ORDER BY a.code, b.code -$$) AS (origin agtype, dest agtype, dist agtype); - --- Step 10: Verify GIN indexes still exist after upgrade. -SELECT indexname FROM pg_indexes -WHERE schemaname IN ('company', 'network', 'routes') - AND tablename IN ('Employee', 'User', 'Post', 'City') - AND indexdef LIKE '%gin%' -ORDER BY indexname; - --- Step 11: Cleanup and restore AGE at the default version for subsequent tests. -SELECT drop_graph('routes', true); -SELECT drop_graph('network', true); -SELECT drop_graph('company', true); +-- ===================================================================== +-- UPGRADED INSTALL SNAPSHOTS (Steps 13-18) +-- Capture the catalog state after upgrade from initial to current. +-- ===================================================================== + +-- Step 13: Snapshot functions. +CREATE TEMP TABLE _upgraded_funcs AS +SELECT proname::text, + pg_get_function_identity_arguments(oid) AS args, + provolatile::text, proisstrict::text, prokind::text, + prorettype::regtype::text AS rettype, proretset::text +FROM pg_proc +WHERE pronamespace = 'ag_catalog'::regnamespace +ORDER BY proname, args; + +-- Step 14: Snapshot relations. +CREATE TEMP TABLE _upgraded_rels AS +SELECT relname::text, relkind::text +FROM pg_class +WHERE relnamespace = 'ag_catalog'::regnamespace + AND relkind IN ('r', 'v', 'S', 'i') +ORDER BY relname; + +-- Step 15: Snapshot types. +CREATE TEMP TABLE _upgraded_types AS +SELECT typname::text, typtype::text +FROM pg_type +WHERE typnamespace = 'ag_catalog'::regnamespace + AND typname NOT LIKE 'pg_toast%' +ORDER BY typname; + +-- Step 16: Snapshot operators. +CREATE TEMP TABLE _upgraded_ops AS +SELECT oprname::text, + oprleft::regtype::text AS lefttype, + oprright::regtype::text AS righttype +FROM pg_operator +WHERE oprnamespace = 'ag_catalog'::regnamespace +ORDER BY oprname, lefttype, righttype; + +-- Step 17: Snapshot casts and operator classes. +CREATE TEMP TABLE _upgraded_casts AS +SELECT castsource::regtype::text AS source_type, + casttarget::regtype::text AS target_type, + castcontext::text +FROM pg_cast +WHERE castsource IN (SELECT oid FROM pg_type WHERE typnamespace = 'ag_catalog'::regnamespace) + OR casttarget IN (SELECT oid FROM pg_type WHERE typnamespace = 'ag_catalog'::regnamespace) +ORDER BY source_type, target_type; + +CREATE TEMP TABLE _upgraded_opclass AS +SELECT opcname::text, + (SELECT amname FROM pg_am WHERE oid = opcmethod)::text AS amname +FROM pg_opclass +WHERE opcnamespace = 'ag_catalog'::regnamespace +ORDER BY opcname; + + +-- Step 18: Snapshot constraints. +CREATE TEMP TABLE _upgraded_constraints AS +SELECT conname::text, contype::text, + conrelid::regclass::text AS table_name, + confrelid::regclass::text AS ref_table +FROM pg_constraint +WHERE connamespace = 'ag_catalog'::regnamespace +ORDER BY conname; + +-- ===================================================================== +-- COMPARISON: Missing or extra objects (Steps 19-33) +-- Any rows returned indicate a template deficiency. +-- ===================================================================== + +-- Step 19: Functions MISSING after upgrade. +SELECT f.proname || '(' || f.args || ')' AS missing_function +FROM _fresh_funcs f +LEFT JOIN _upgraded_funcs u USING (proname, args) +WHERE u.proname IS NULL +ORDER BY 1; + +-- Step 20: Functions EXTRA after upgrade. +SELECT u.proname || '(' || u.args || ')' AS extra_function +FROM _upgraded_funcs u +LEFT JOIN _fresh_funcs f USING (proname, args) +WHERE f.proname IS NULL +ORDER BY 1; + +-- Step 21: Function PROPERTY changes (volatility, strictness, kind, return type). +SELECT f.proname || '(' || f.args || ')' AS function_name, + CASE WHEN f.prokind <> u.prokind THEN 'prokind: ' || f.prokind || '->' || u.prokind END AS kind_change, + CASE WHEN f.provolatile<> u.provolatile THEN 'volatile: ' || f.provolatile|| '->' || u.provolatile END AS volatility_change, + CASE WHEN f.proisstrict<> u.proisstrict THEN 'strict: ' || f.proisstrict|| '->' || u.proisstrict END AS strict_change, + CASE WHEN f.rettype <> u.rettype THEN 'rettype: ' || f.rettype || '->' || u.rettype END AS rettype_change, + CASE WHEN f.proretset <> u.proretset THEN 'retset: ' || f.proretset || '->' || u.proretset END AS retset_change +FROM _fresh_funcs f +JOIN _upgraded_funcs u USING (proname, args) +WHERE f.provolatile <> u.provolatile + OR f.proisstrict <> u.proisstrict + OR f.prokind <> u.prokind + OR f.rettype <> u.rettype + OR f.proretset <> u.proretset +ORDER BY 1; + +-- Step 22: Relations MISSING after upgrade. +SELECT f.relname || ' (' || f.relkind || ')' AS missing_relation +FROM _fresh_rels f +LEFT JOIN _upgraded_rels u USING (relname, relkind) +WHERE u.relname IS NULL +ORDER BY 1; + +-- Step 23: Relations EXTRA after upgrade. +SELECT u.relname || ' (' || u.relkind || ')' AS extra_relation +FROM _upgraded_rels u +LEFT JOIN _fresh_rels f USING (relname, relkind) +WHERE f.relname IS NULL +ORDER BY 1; + +-- Step 24: Types MISSING after upgrade. +SELECT f.typname || ' (' || f.typtype || ')' AS missing_type +FROM _fresh_types f +LEFT JOIN _upgraded_types u USING (typname, typtype) +WHERE u.typname IS NULL +ORDER BY 1; + +-- Step 25: Types EXTRA after upgrade. +SELECT u.typname || ' (' || u.typtype || ')' AS extra_type +FROM _upgraded_types u +LEFT JOIN _fresh_types f USING (typname, typtype) +WHERE f.typname IS NULL +ORDER BY 1; + +-- Step 26: Operators MISSING after upgrade. +SELECT f.oprname || ' (' || f.lefttype || ', ' || f.righttype || ')' AS missing_operator +FROM _fresh_ops f +LEFT JOIN _upgraded_ops u USING (oprname, lefttype, righttype) +WHERE u.oprname IS NULL +ORDER BY 1; + +-- Step 27: Operators EXTRA after upgrade. +SELECT u.oprname || ' (' || u.lefttype || ', ' || u.righttype || ')' AS extra_operator +FROM _upgraded_ops u +LEFT JOIN _fresh_ops f USING (oprname, lefttype, righttype) +WHERE f.oprname IS NULL +ORDER BY 1; + +-- Step 28: Casts MISSING after upgrade. +SELECT f.source_type || ' -> ' || f.target_type || ' (' || f.castcontext || ')' AS missing_cast +FROM _fresh_casts f +LEFT JOIN _upgraded_casts u USING (source_type, target_type, castcontext) +WHERE u.source_type IS NULL +ORDER BY 1; + +-- Step 29: Casts EXTRA after upgrade. +SELECT u.source_type || ' -> ' || u.target_type || ' (' || u.castcontext || ')' AS extra_cast +FROM _upgraded_casts u +LEFT JOIN _fresh_casts f USING (source_type, target_type, castcontext) +WHERE f.source_type IS NULL +ORDER BY 1; + +-- Step 30: Operator classes MISSING after upgrade. +SELECT f.opcname || ' (' || f.amname || ')' AS missing_opclass +FROM _fresh_opclass f +LEFT JOIN _upgraded_opclass u USING (opcname, amname) +WHERE u.opcname IS NULL +ORDER BY 1; + +-- Step 31: Operator classes EXTRA after upgrade. +SELECT u.opcname || ' (' || u.amname || ')' AS extra_opclass +FROM _upgraded_opclass u +LEFT JOIN _fresh_opclass f USING (opcname, amname) +WHERE f.opcname IS NULL +ORDER BY 1; + +-- Step 32: Constraints MISSING after upgrade. +SELECT f.conname || ' (' || f.contype || ' on ' || f.table_name || ')' AS missing_constraint +FROM _fresh_constraints f +LEFT JOIN _upgraded_constraints u USING (conname, contype, table_name) +WHERE u.conname IS NULL +ORDER BY 1; + +-- Step 33: Constraints EXTRA after upgrade. +SELECT u.conname || ' (' || u.contype || ' on ' || u.table_name || ')' AS extra_constraint +FROM _upgraded_constraints u +LEFT JOIN _fresh_constraints f USING (conname, contype, table_name) +WHERE f.conname IS NULL +ORDER BY 1; + +-- ===================================================================== +-- SUMMARY (Step 34) +-- ===================================================================== + +-- Step 34: Verify all counts match (result: single row, all true). +SELECT + (SELECT count(*) FROM _fresh_funcs) = (SELECT count(*) FROM _upgraded_funcs) AS funcs_match, + (SELECT count(*) FROM _fresh_rels) = (SELECT count(*) FROM _upgraded_rels) AS rels_match, + (SELECT count(*) FROM _fresh_types) = (SELECT count(*) FROM _upgraded_types) AS types_match, + (SELECT count(*) FROM _fresh_ops) = (SELECT count(*) FROM _upgraded_ops) AS ops_match, + (SELECT count(*) FROM _fresh_casts) = (SELECT count(*) FROM _upgraded_casts) AS casts_match, + (SELECT count(*) FROM _fresh_opclass) = (SELECT count(*) FROM _upgraded_opclass) AS opclass_match, + (SELECT count(*) FROM _fresh_constraints) = (SELECT count(*) FROM _upgraded_constraints) AS constraints_match; + +-- ===================================================================== +-- CLEANUP (Steps 35-36) +-- ===================================================================== + +-- Step 35: Drop temp tables, restore AGE at default version. +DROP TABLE _fresh_funcs, _upgraded_funcs, _fresh_rels, _upgraded_rels, + _fresh_types, _upgraded_types, _fresh_ops, _upgraded_ops, + _fresh_casts, _upgraded_casts, _fresh_opclass, _upgraded_opclass, + _fresh_constraints, _upgraded_constraints; DROP EXTENSION age; CREATE EXTENSION age; --- Step 12: Remove synthetic upgrade test files from the extension directory. +-- Step 36: Remove synthetic upgrade test files from the extension directory. \! sh ./regress/age_upgrade_cleanup.sh From e9ef30b177040662a79dcb55307ca3b1a5cf7582 Mon Sep 17 00:00:00 2001 From: Hari Krishna Sunder Date: Mon, 4 May 2026 11:23:25 -0700 Subject: [PATCH 064/100] Zero-initialize parent_cpstate in analyze_cypher (#2423) cypher_parsestate parent_cpstate is declared on the stack in analyze_cypher() and only pstate is explicitly set before it is passed to make_cypher_parsestate(). The latter reads parent_cpstate->subquery_where_flag (and other fields) in cypher_parse_node.c, which leaves them with indeterminate values. UBSan flagged the garbage bool (value 8) and aborted the backend. Use MemSet to zero the struct before populating pstate so all remaining members start with a defined value. Co-authored-by: Hari Krishna Sunder <12418230+hari90@users.noreply.github.com> --- src/backend/parser/cypher_analyze.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/parser/cypher_analyze.c b/src/backend/parser/cypher_analyze.c index 7844af2f0..b2c9256ce 100644 --- a/src/backend/parser/cypher_analyze.c +++ b/src/backend/parser/cypher_analyze.c @@ -961,9 +961,8 @@ static Query *analyze_cypher(List *stmt, ParseState *parent_pstate, * convert ParseState into cypher_parsestate temporarily to pass it to * make_cypher_parsestate() */ + MemSet(&parent_cpstate, 0, sizeof(parent_cpstate)); parent_cpstate.pstate = *parent_pstate; - parent_cpstate.graph_name = NULL; - parent_cpstate.params = NULL; cpstate = make_cypher_parsestate(&parent_cpstate); From a1b749af383dfdf0958ba6c1531b99aeab9207f5 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Tue, 5 May 2026 14:02:19 -0700 Subject: [PATCH 065/100] Fix upgrade test: allow function removal (#2422) Fix upgrade test: allow function removal and detect more deficiencies. The age_upgrade regression test (added in #2364, refined in #2377, #2397, install and a synthetic-initial -> current upgrade. Three gaps surfaced in practice: 1. Function removal forced permanent C stubs. The synthetic '_initial' install is built from a fixed historical commit. CREATE EXTENSION resolves every CREATE FUNCTION ... AS '$libdir/age', '' via dlsym at install time when check_function_bodies is on (the default). If a developer retires a C entry point in HEAD's age.so, step 10 aborts with "could not find function ... in file age.so" -- even though the immediately-following ALTER EXTENSION UPDATE would DROP that SQL declaration. The only way to keep the test green was to leave a permanent error-raising stub in age.so, and to remember to add a DROP to the upgrade template. 2. Modifications were under-detected. The function-property-change query did not compare probin or prosrc, so a C function whose symbol was renamed in the upgrade template, or a SQL/plpgsql function whose body changed in either path, slipped through. 3. Extension membership was not checked. A template that CREATEs an object but never ALTER EXTENSION ADDs it leaves a row in pg_proc/pg_class but no pg_depend deptype='e' link. pg_dump --extension would diverge, but the existing per-catalog diff queries all returned 0 rows. Changes (regress/sql/age_upgrade.sql + regress/expected/age_upgrade.out): * Step 10 wraps the synthetic CREATE EXTENSION in SET check_function_bodies = off; ... RESET check_function_bodies; Symbol resolution is deferred to call time. Step 11's ALTER EXTENSION UPDATE then DROPs any retired functions before any plan can call them. Step 35's fresh CREATE EXTENSION runs at the GUC default, so HEAD's sql/ <-> HEAD's age.so consistency is still enforced on the production install path. * Steps 2 and 13 add probin and prosrc to the function snapshot. Step 21 reports probin and prosrc divergences alongside the existing property-change columns. * Steps 7b and 18b add an extension-membership snapshot from pg_depend deptype='e' filtered to the AGE extension OID. Every member is labeled by stable identity (regprocedure, regtype, regoperator, opfname+strategy+types, etc.), never by raw OID, so OID drift between fresh and upgrade installs cannot produce false positives. Steps 33a and 33b report MISSING / EXTRA members. Step 34 adds extmembers_match to the summary row. * Section-header step ranges updated to include the new sub-steps. The change is fully self-contained: only regress/sql/age_upgrade.sql and regress/expected/age_upgrade.out are modified. No production C, SQL, build, or test files are touched. All 34 regression tests pass on a clean tree. Mutation-tested with 8 cases against the unmutated tree: baseline pass; remove-function-with-DROP pass (no stub needed); remove-function-forget- DROP fail; add-function-with-CREATE pass; add-function-forget-CREATE fail; volatility-change-no-template fail; volatility-change-with-CREATE- OR-REPLACE pass; C-symbol-rename-no-template fail. All eight expected outcomes observed. All 34 regression tests pass. Co-authored-by: Claude modified: regress/expected/age_upgrade.out modified: regress/sql/age_upgrade.sql --- regress/expected/age_upgrade.out | 208 ++++++++++++++++++++++++++++--- regress/sql/age_upgrade.sql | 194 +++++++++++++++++++++++++--- 2 files changed, 367 insertions(+), 35 deletions(-) diff --git a/regress/expected/age_upgrade.out b/regress/expected/age_upgrade.out index edf8e6022..446e0eb76 100644 --- a/regress/expected/age_upgrade.out +++ b/regress/expected/age_upgrade.out @@ -27,16 +27,30 @@ -- upgrade template. -- -- Compared catalogs: --- pg_proc — functions, aggregates, procedures (name, args, properties) +-- pg_proc — functions, aggregates, procedures (name, args, properties +-- including probin/prosrc to catch C-symbol renames and +-- SQL-body changes) -- pg_class — tables, views, sequences, indexes (name, kind) -- pg_type — types (name, type category) -- pg_operator — operators (name, left/right types) -- pg_cast — casts involving AGE types (source, target, context) -- pg_opclass — operator classes (name, access method) -- pg_constraint — constraints (name, type, table, referenced table) +-- pg_depend — extension membership (every AGE-owned object must be +-- linked back to the extension via deptype='e'; catches +-- a missing ALTER EXTENSION age ADD ... in the template) -- -- All comparison queries should return 0 rows. -- +-- Note on synthetic-initial install (step 10): the synthetic '*_initial' +-- snapshot is built from a fixed historical commit, so its CREATE FUNCTION +-- statements may reference C symbols that have since been removed from the +-- current age.so. Step 10 disables check_function_bodies so that dlsym is +-- deferred to call time; the immediately-following ALTER EXTENSION UPDATE +-- (step 11) DROPs any such retired functions before any plan can call them. +-- This lets developers cleanly remove deprecated C entry points without +-- needing to keep error-raising stubs in age.so. +-- LOAD 'age'; SET search_path TO ag_catalog; -- Step 1: Clean up any graphs left by prior tests (deterministic, no output). @@ -56,15 +70,23 @@ BEGIN END $$; -- ===================================================================== --- FRESH INSTALL SNAPSHOTS (Steps 2-7) +-- FRESH INSTALL SNAPSHOTS (Steps 2-7b) -- Capture the catalog state from the default CREATE EXTENSION install. -- ===================================================================== -- Step 2: Snapshot functions (includes aggregates via prokind). +-- probin/prosrc capture the binding to the implementation: +-- * LANGUAGE c : probin = '$libdir/age', prosrc = C symbol name +-- (a renamed/retargeted symbol shows up here) +-- * LANGUAGE sql/plpgsql: probin = NULL, prosrc = function body text +-- (a body change in the upgrade template shows up here) +-- * LANGUAGE internal : probin = NULL, prosrc = builtin name CREATE TEMP TABLE _fresh_funcs AS SELECT proname::text, pg_get_function_identity_arguments(oid) AS args, provolatile::text, proisstrict::text, prokind::text, - prorettype::regtype::text AS rettype, proretset::text + prorettype::regtype::text AS rettype, proretset::text, + COALESCE(probin, '') AS probin, + COALESCE(prosrc, '') AS prosrc FROM pg_proc WHERE pronamespace = 'ag_catalog'::regnamespace ORDER BY proname, args; @@ -113,6 +135,57 @@ SELECT conname::text, contype::text, FROM pg_constraint WHERE connamespace = 'ag_catalog'::regnamespace ORDER BY conname; +-- Step 7b: Snapshot extension membership (pg_depend deptype='e'). +-- Every object that CREATE EXTENSION owns has a row in pg_depend linking +-- it to the extension. The upgrade template must produce the same set: +-- if it CREATEs an object but forgets to ALTER EXTENSION ADD it, the +-- catalog row exists (so funcs_match/rels_match would pass) but the +-- pg_depend link is absent and pg_dump --extension would diverge. +CREATE TEMP TABLE _fresh_extmembers AS +SELECT + CASE d.classid + WHEN 'pg_proc'::regclass + THEN 'function: ' || d.objid::regprocedure::text + WHEN 'pg_type'::regclass + THEN 'type: ' || d.objid::regtype::text + WHEN 'pg_class'::regclass + THEN 'relation: ' || d.objid::regclass::text + || ' (' || (SELECT relkind::text FROM pg_class WHERE oid = d.objid) || ')' + WHEN 'pg_operator'::regclass + THEN 'operator: ' || d.objid::regoperator::text + WHEN 'pg_cast'::regclass + THEN 'cast: ' || (SELECT castsource::regtype::text || ' -> ' || casttarget::regtype::text + FROM pg_cast WHERE oid = d.objid) + WHEN 'pg_opclass'::regclass + THEN 'opclass: ' || (SELECT opcname || ' (' || (SELECT amname FROM pg_am WHERE oid = opcmethod) || ')' + FROM pg_opclass WHERE oid = d.objid) + WHEN 'pg_constraint'::regclass + THEN 'constraint: ' || (SELECT conname || ' on ' || conrelid::regclass::text + FROM pg_constraint WHERE oid = d.objid) + WHEN 'pg_opfamily'::regclass + THEN 'opfamily: ' || (SELECT opfname || ' (' || (SELECT amname FROM pg_am WHERE oid = opfmethod) || ')' + FROM pg_opfamily WHERE oid = d.objid) + WHEN 'pg_amop'::regclass + THEN 'amop: ' || (SELECT (SELECT opfname FROM pg_opfamily WHERE oid = a.amopfamily) + || ' [strategy ' || a.amopstrategy || '] ' + || a.amoplefttype::regtype::text || ',' + || a.amoprighttype::regtype::text || ' op ' + || a.amopopr::regoperator::text + FROM pg_amop a WHERE a.oid = d.objid) + WHEN 'pg_amproc'::regclass + THEN 'amproc: ' || (SELECT (SELECT opfname FROM pg_opfamily WHERE oid = p.amprocfamily) + || ' [proc ' || p.amprocnum || '] ' + || p.amproclefttype::regtype::text || ',' + || p.amprocrighttype::regtype::text || ' fn ' + || p.amproc::regprocedure::text + FROM pg_amproc p WHERE p.oid = d.objid) + ELSE 'unhandled[' || d.classid::regclass::text || ']' + END AS member +FROM pg_depend d +WHERE d.deptype = 'e' + AND d.refclassid = 'pg_extension'::regclass + AND d.refobjid = (SELECT oid FROM pg_extension WHERE extname = 'age') +ORDER BY 1; -- Step 8: Drop AGE entirely. DROP EXTENSION age; -- Step 9: Verify we have an upgrade path available. @@ -124,6 +197,19 @@ FROM pg_available_extension_versions WHERE name = 'age'; (1 row) -- Step 10: Install AGE at the synthetic initial version. +-- +-- Disable check_function_bodies for this CREATE only. The synthetic +-- '*_initial' SQL is pulled from a fixed historical commit and may +-- declare C functions whose symbols have since been removed from the +-- current age.so. With check_function_bodies=on, PostgreSQL would dlsym +-- each such symbol at CREATE FUNCTION time and abort. Deferring the +-- symbol probe to call time is safe because step 11 (ALTER EXTENSION +-- UPDATE) immediately runs the upgrade template, which DROPs any +-- removed-in-HEAD functions before the test (or any user) can call them. +-- The fresh CREATE EXTENSION at step 35 keeps the GUC at its default, +-- so any inconsistency between HEAD's SQL and HEAD's age.so is still +-- caught at install time on the production code path. +SET check_function_bodies = off; DO $$ DECLARE init_ver text; BEGIN @@ -138,6 +224,7 @@ BEGIN EXECUTE format('CREATE EXTENSION age VERSION %L', init_ver); END; $$; +RESET check_function_bodies; -- Step 11: Upgrade to the current (default) version via the stamped template. DO $$ DECLARE curr_ver text; @@ -160,15 +247,17 @@ FROM pg_available_extensions WHERE name = 'age'; (1 row) -- ===================================================================== --- UPGRADED INSTALL SNAPSHOTS (Steps 13-18) +-- UPGRADED INSTALL SNAPSHOTS (Steps 13-18b) -- Capture the catalog state after upgrade from initial to current. -- ===================================================================== --- Step 13: Snapshot functions. +-- Step 13: Snapshot functions (probin/prosrc included; see step 2). CREATE TEMP TABLE _upgraded_funcs AS SELECT proname::text, pg_get_function_identity_arguments(oid) AS args, provolatile::text, proisstrict::text, prokind::text, - prorettype::regtype::text AS rettype, proretset::text + prorettype::regtype::text AS rettype, proretset::text, + COALESCE(probin, '') AS probin, + COALESCE(prosrc, '') AS prosrc FROM pg_proc WHERE pronamespace = 'ag_catalog'::regnamespace ORDER BY proname, args; @@ -217,8 +306,54 @@ SELECT conname::text, contype::text, FROM pg_constraint WHERE connamespace = 'ag_catalog'::regnamespace ORDER BY conname; +-- Step 18b: Snapshot extension membership after upgrade (see step 7b). +CREATE TEMP TABLE _upgraded_extmembers AS +SELECT + CASE d.classid + WHEN 'pg_proc'::regclass + THEN 'function: ' || d.objid::regprocedure::text + WHEN 'pg_type'::regclass + THEN 'type: ' || d.objid::regtype::text + WHEN 'pg_class'::regclass + THEN 'relation: ' || d.objid::regclass::text + || ' (' || (SELECT relkind::text FROM pg_class WHERE oid = d.objid) || ')' + WHEN 'pg_operator'::regclass + THEN 'operator: ' || d.objid::regoperator::text + WHEN 'pg_cast'::regclass + THEN 'cast: ' || (SELECT castsource::regtype::text || ' -> ' || casttarget::regtype::text + FROM pg_cast WHERE oid = d.objid) + WHEN 'pg_opclass'::regclass + THEN 'opclass: ' || (SELECT opcname || ' (' || (SELECT amname FROM pg_am WHERE oid = opcmethod) || ')' + FROM pg_opclass WHERE oid = d.objid) + WHEN 'pg_constraint'::regclass + THEN 'constraint: ' || (SELECT conname || ' on ' || conrelid::regclass::text + FROM pg_constraint WHERE oid = d.objid) + WHEN 'pg_opfamily'::regclass + THEN 'opfamily: ' || (SELECT opfname || ' (' || (SELECT amname FROM pg_am WHERE oid = opfmethod) || ')' + FROM pg_opfamily WHERE oid = d.objid) + WHEN 'pg_amop'::regclass + THEN 'amop: ' || (SELECT (SELECT opfname FROM pg_opfamily WHERE oid = a.amopfamily) + || ' [strategy ' || a.amopstrategy || '] ' + || a.amoplefttype::regtype::text || ',' + || a.amoprighttype::regtype::text || ' op ' + || a.amopopr::regoperator::text + FROM pg_amop a WHERE a.oid = d.objid) + WHEN 'pg_amproc'::regclass + THEN 'amproc: ' || (SELECT (SELECT opfname FROM pg_opfamily WHERE oid = p.amprocfamily) + || ' [proc ' || p.amprocnum || '] ' + || p.amproclefttype::regtype::text || ',' + || p.amprocrighttype::regtype::text || ' fn ' + || p.amproc::regprocedure::text + FROM pg_amproc p WHERE p.oid = d.objid) + ELSE 'unhandled[' || d.classid::regclass::text || ']' + END AS member +FROM pg_depend d +WHERE d.deptype = 'e' + AND d.refclassid = 'pg_extension'::regclass + AND d.refobjid = (SELECT oid FROM pg_extension WHERE extname = 'age') +ORDER BY 1; -- ===================================================================== --- COMPARISON: Missing or extra objects (Steps 19-33) +-- COMPARISON: Missing or extra objects (Steps 19-33b) -- Any rows returned indicate a template deficiency. -- ===================================================================== -- Step 19: Functions MISSING after upgrade. @@ -241,13 +376,20 @@ ORDER BY 1; ---------------- (0 rows) --- Step 21: Function PROPERTY changes (volatility, strictness, kind, return type). +-- Step 21: Function PROPERTY changes +-- (kind, volatility, strictness, return type, return-set, binding, +-- body/symbol). The probin/prosrc check catches: +-- * a C function whose symbol was renamed in the upgrade template +-- * a SQL/plpgsql function whose body was changed in either path +-- * a language change between the fresh and upgrade installs. SELECT f.proname || '(' || f.args || ')' AS function_name, - CASE WHEN f.prokind <> u.prokind THEN 'prokind: ' || f.prokind || '->' || u.prokind END AS kind_change, - CASE WHEN f.provolatile<> u.provolatile THEN 'volatile: ' || f.provolatile|| '->' || u.provolatile END AS volatility_change, - CASE WHEN f.proisstrict<> u.proisstrict THEN 'strict: ' || f.proisstrict|| '->' || u.proisstrict END AS strict_change, - CASE WHEN f.rettype <> u.rettype THEN 'rettype: ' || f.rettype || '->' || u.rettype END AS rettype_change, - CASE WHEN f.proretset <> u.proretset THEN 'retset: ' || f.proretset || '->' || u.proretset END AS retset_change + CASE WHEN f.prokind <> u.prokind THEN 'prokind: ' || f.prokind || '->' || u.prokind END AS kind_change, + CASE WHEN f.provolatile <> u.provolatile THEN 'volatile: ' || f.provolatile || '->' || u.provolatile END AS volatility_change, + CASE WHEN f.proisstrict <> u.proisstrict THEN 'strict: ' || f.proisstrict || '->' || u.proisstrict END AS strict_change, + CASE WHEN f.rettype <> u.rettype THEN 'rettype: ' || f.rettype || '->' || u.rettype END AS rettype_change, + CASE WHEN f.proretset <> u.proretset THEN 'retset: ' || f.proretset || '->' || u.proretset END AS retset_change, + CASE WHEN f.probin <> u.probin THEN 'probin: ' || f.probin || '->' || u.probin END AS probin_change, + CASE WHEN f.prosrc <> u.prosrc THEN 'prosrc changed' END AS prosrc_change FROM _fresh_funcs f JOIN _upgraded_funcs u USING (proname, args) WHERE f.provolatile <> u.provolatile @@ -255,9 +397,11 @@ WHERE f.provolatile <> u.provolatile OR f.prokind <> u.prokind OR f.rettype <> u.rettype OR f.proretset <> u.proretset + OR f.probin <> u.probin + OR f.prosrc <> u.prosrc ORDER BY 1; - function_name | kind_change | volatility_change | strict_change | rettype_change | retset_change ----------------+-------------+-------------------+---------------+----------------+--------------- + function_name | kind_change | volatility_change | strict_change | rettype_change | retset_change | probin_change | prosrc_change +---------------+-------------+-------------------+---------------+----------------+---------------+---------------+--------------- (0 rows) -- Step 22: Relations MISSING after upgrade. @@ -380,6 +524,28 @@ ORDER BY 1; ------------------ (0 rows) +-- Step 33a: Extension members MISSING after upgrade +-- (object exists in pg_proc/pg_class/etc. but is not linked to the AGE +-- extension via pg_depend, i.e. ALTER EXTENSION age ADD ... was forgotten). +SELECT f.member AS missing_extension_member +FROM _fresh_extmembers f +LEFT JOIN _upgraded_extmembers u USING (member) +WHERE u.member IS NULL +ORDER BY 1; + missing_extension_member +-------------------------- +(0 rows) + +-- Step 33b: Extension members EXTRA after upgrade. +SELECT u.member AS extra_extension_member +FROM _upgraded_extmembers u +LEFT JOIN _fresh_extmembers f USING (member) +WHERE f.member IS NULL +ORDER BY 1; + extra_extension_member +------------------------ +(0 rows) + -- ===================================================================== -- SUMMARY (Step 34) -- ===================================================================== @@ -391,10 +557,11 @@ SELECT (SELECT count(*) FROM _fresh_ops) = (SELECT count(*) FROM _upgraded_ops) AS ops_match, (SELECT count(*) FROM _fresh_casts) = (SELECT count(*) FROM _upgraded_casts) AS casts_match, (SELECT count(*) FROM _fresh_opclass) = (SELECT count(*) FROM _upgraded_opclass) AS opclass_match, - (SELECT count(*) FROM _fresh_constraints) = (SELECT count(*) FROM _upgraded_constraints) AS constraints_match; - funcs_match | rels_match | types_match | ops_match | casts_match | opclass_match | constraints_match --------------+------------+-------------+-----------+-------------+---------------+------------------- - t | t | t | t | t | t | t + (SELECT count(*) FROM _fresh_constraints) = (SELECT count(*) FROM _upgraded_constraints) AS constraints_match, + (SELECT count(*) FROM _fresh_extmembers) = (SELECT count(*) FROM _upgraded_extmembers) AS extmembers_match; + funcs_match | rels_match | types_match | ops_match | casts_match | opclass_match | constraints_match | extmembers_match +-------------+------------+-------------+-----------+-------------+---------------+-------------------+------------------ + t | t | t | t | t | t | t | t (1 row) -- ===================================================================== @@ -404,7 +571,8 @@ SELECT DROP TABLE _fresh_funcs, _upgraded_funcs, _fresh_rels, _upgraded_rels, _fresh_types, _upgraded_types, _fresh_ops, _upgraded_ops, _fresh_casts, _upgraded_casts, _fresh_opclass, _upgraded_opclass, - _fresh_constraints, _upgraded_constraints; + _fresh_constraints, _upgraded_constraints, + _fresh_extmembers, _upgraded_extmembers; DROP EXTENSION age; CREATE EXTENSION age; -- Step 36: Remove synthetic upgrade test files from the extension directory. diff --git a/regress/sql/age_upgrade.sql b/regress/sql/age_upgrade.sql index f56f7ca93..98ce3e21e 100644 --- a/regress/sql/age_upgrade.sql +++ b/regress/sql/age_upgrade.sql @@ -28,16 +28,30 @@ -- upgrade template. -- -- Compared catalogs: --- pg_proc — functions, aggregates, procedures (name, args, properties) +-- pg_proc — functions, aggregates, procedures (name, args, properties +-- including probin/prosrc to catch C-symbol renames and +-- SQL-body changes) -- pg_class — tables, views, sequences, indexes (name, kind) -- pg_type — types (name, type category) -- pg_operator — operators (name, left/right types) -- pg_cast — casts involving AGE types (source, target, context) -- pg_opclass — operator classes (name, access method) -- pg_constraint — constraints (name, type, table, referenced table) +-- pg_depend — extension membership (every AGE-owned object must be +-- linked back to the extension via deptype='e'; catches +-- a missing ALTER EXTENSION age ADD ... in the template) -- -- All comparison queries should return 0 rows. -- +-- Note on synthetic-initial install (step 10): the synthetic '*_initial' +-- snapshot is built from a fixed historical commit, so its CREATE FUNCTION +-- statements may reference C symbols that have since been removed from the +-- current age.so. Step 10 disables check_function_bodies so that dlsym is +-- deferred to call time; the immediately-following ALTER EXTENSION UPDATE +-- (step 11) DROPs any such retired functions before any plan can call them. +-- This lets developers cleanly remove deprecated C entry points without +-- needing to keep error-raising stubs in age.so. +-- LOAD 'age'; SET search_path TO ag_catalog; @@ -60,16 +74,24 @@ END $$; -- ===================================================================== --- FRESH INSTALL SNAPSHOTS (Steps 2-7) +-- FRESH INSTALL SNAPSHOTS (Steps 2-7b) -- Capture the catalog state from the default CREATE EXTENSION install. -- ===================================================================== -- Step 2: Snapshot functions (includes aggregates via prokind). +-- probin/prosrc capture the binding to the implementation: +-- * LANGUAGE c : probin = '$libdir/age', prosrc = C symbol name +-- (a renamed/retargeted symbol shows up here) +-- * LANGUAGE sql/plpgsql: probin = NULL, prosrc = function body text +-- (a body change in the upgrade template shows up here) +-- * LANGUAGE internal : probin = NULL, prosrc = builtin name CREATE TEMP TABLE _fresh_funcs AS SELECT proname::text, pg_get_function_identity_arguments(oid) AS args, provolatile::text, proisstrict::text, prokind::text, - prorettype::regtype::text AS rettype, proretset::text + prorettype::regtype::text AS rettype, proretset::text, + COALESCE(probin, '') AS probin, + COALESCE(prosrc, '') AS prosrc FROM pg_proc WHERE pronamespace = 'ag_catalog'::regnamespace ORDER BY proname, args; @@ -125,6 +147,58 @@ FROM pg_constraint WHERE connamespace = 'ag_catalog'::regnamespace ORDER BY conname; +-- Step 7b: Snapshot extension membership (pg_depend deptype='e'). +-- Every object that CREATE EXTENSION owns has a row in pg_depend linking +-- it to the extension. The upgrade template must produce the same set: +-- if it CREATEs an object but forgets to ALTER EXTENSION ADD it, the +-- catalog row exists (so funcs_match/rels_match would pass) but the +-- pg_depend link is absent and pg_dump --extension would diverge. +CREATE TEMP TABLE _fresh_extmembers AS +SELECT + CASE d.classid + WHEN 'pg_proc'::regclass + THEN 'function: ' || d.objid::regprocedure::text + WHEN 'pg_type'::regclass + THEN 'type: ' || d.objid::regtype::text + WHEN 'pg_class'::regclass + THEN 'relation: ' || d.objid::regclass::text + || ' (' || (SELECT relkind::text FROM pg_class WHERE oid = d.objid) || ')' + WHEN 'pg_operator'::regclass + THEN 'operator: ' || d.objid::regoperator::text + WHEN 'pg_cast'::regclass + THEN 'cast: ' || (SELECT castsource::regtype::text || ' -> ' || casttarget::regtype::text + FROM pg_cast WHERE oid = d.objid) + WHEN 'pg_opclass'::regclass + THEN 'opclass: ' || (SELECT opcname || ' (' || (SELECT amname FROM pg_am WHERE oid = opcmethod) || ')' + FROM pg_opclass WHERE oid = d.objid) + WHEN 'pg_constraint'::regclass + THEN 'constraint: ' || (SELECT conname || ' on ' || conrelid::regclass::text + FROM pg_constraint WHERE oid = d.objid) + WHEN 'pg_opfamily'::regclass + THEN 'opfamily: ' || (SELECT opfname || ' (' || (SELECT amname FROM pg_am WHERE oid = opfmethod) || ')' + FROM pg_opfamily WHERE oid = d.objid) + WHEN 'pg_amop'::regclass + THEN 'amop: ' || (SELECT (SELECT opfname FROM pg_opfamily WHERE oid = a.amopfamily) + || ' [strategy ' || a.amopstrategy || '] ' + || a.amoplefttype::regtype::text || ',' + || a.amoprighttype::regtype::text || ' op ' + || a.amopopr::regoperator::text + FROM pg_amop a WHERE a.oid = d.objid) + WHEN 'pg_amproc'::regclass + THEN 'amproc: ' || (SELECT (SELECT opfname FROM pg_opfamily WHERE oid = p.amprocfamily) + || ' [proc ' || p.amprocnum || '] ' + || p.amproclefttype::regtype::text || ',' + || p.amprocrighttype::regtype::text || ' fn ' + || p.amproc::regprocedure::text + FROM pg_amproc p WHERE p.oid = d.objid) + ELSE 'unhandled[' || d.classid::regclass::text || ']' + END AS member +FROM pg_depend d +WHERE d.deptype = 'e' + AND d.refclassid = 'pg_extension'::regclass + AND d.refobjid = (SELECT oid FROM pg_extension WHERE extname = 'age') +ORDER BY 1; + -- Step 8: Drop AGE entirely. DROP EXTENSION age; @@ -133,6 +207,19 @@ SELECT count(*) > 1 AS has_upgrade_path FROM pg_available_extension_versions WHERE name = 'age'; -- Step 10: Install AGE at the synthetic initial version. +-- +-- Disable check_function_bodies for this CREATE only. The synthetic +-- '*_initial' SQL is pulled from a fixed historical commit and may +-- declare C functions whose symbols have since been removed from the +-- current age.so. With check_function_bodies=on, PostgreSQL would dlsym +-- each such symbol at CREATE FUNCTION time and abort. Deferring the +-- symbol probe to call time is safe because step 11 (ALTER EXTENSION +-- UPDATE) immediately runs the upgrade template, which DROPs any +-- removed-in-HEAD functions before the test (or any user) can call them. +-- The fresh CREATE EXTENSION at step 35 keeps the GUC at its default, +-- so any inconsistency between HEAD's SQL and HEAD's age.so is still +-- caught at install time on the production code path. +SET check_function_bodies = off; DO $$ DECLARE init_ver text; BEGIN @@ -147,6 +234,7 @@ BEGIN EXECUTE format('CREATE EXTENSION age VERSION %L', init_ver); END; $$; +RESET check_function_bodies; -- Step 11: Upgrade to the current (default) version via the stamped template. DO $$ @@ -167,16 +255,18 @@ SELECT installed_version = default_version AS upgraded_to_current FROM pg_available_extensions WHERE name = 'age'; -- ===================================================================== --- UPGRADED INSTALL SNAPSHOTS (Steps 13-18) +-- UPGRADED INSTALL SNAPSHOTS (Steps 13-18b) -- Capture the catalog state after upgrade from initial to current. -- ===================================================================== --- Step 13: Snapshot functions. +-- Step 13: Snapshot functions (probin/prosrc included; see step 2). CREATE TEMP TABLE _upgraded_funcs AS SELECT proname::text, pg_get_function_identity_arguments(oid) AS args, provolatile::text, proisstrict::text, prokind::text, - prorettype::regtype::text AS rettype, proretset::text + prorettype::regtype::text AS rettype, proretset::text, + COALESCE(probin, '') AS probin, + COALESCE(prosrc, '') AS prosrc FROM pg_proc WHERE pronamespace = 'ag_catalog'::regnamespace ORDER BY proname, args; @@ -233,8 +323,55 @@ FROM pg_constraint WHERE connamespace = 'ag_catalog'::regnamespace ORDER BY conname; +-- Step 18b: Snapshot extension membership after upgrade (see step 7b). +CREATE TEMP TABLE _upgraded_extmembers AS +SELECT + CASE d.classid + WHEN 'pg_proc'::regclass + THEN 'function: ' || d.objid::regprocedure::text + WHEN 'pg_type'::regclass + THEN 'type: ' || d.objid::regtype::text + WHEN 'pg_class'::regclass + THEN 'relation: ' || d.objid::regclass::text + || ' (' || (SELECT relkind::text FROM pg_class WHERE oid = d.objid) || ')' + WHEN 'pg_operator'::regclass + THEN 'operator: ' || d.objid::regoperator::text + WHEN 'pg_cast'::regclass + THEN 'cast: ' || (SELECT castsource::regtype::text || ' -> ' || casttarget::regtype::text + FROM pg_cast WHERE oid = d.objid) + WHEN 'pg_opclass'::regclass + THEN 'opclass: ' || (SELECT opcname || ' (' || (SELECT amname FROM pg_am WHERE oid = opcmethod) || ')' + FROM pg_opclass WHERE oid = d.objid) + WHEN 'pg_constraint'::regclass + THEN 'constraint: ' || (SELECT conname || ' on ' || conrelid::regclass::text + FROM pg_constraint WHERE oid = d.objid) + WHEN 'pg_opfamily'::regclass + THEN 'opfamily: ' || (SELECT opfname || ' (' || (SELECT amname FROM pg_am WHERE oid = opfmethod) || ')' + FROM pg_opfamily WHERE oid = d.objid) + WHEN 'pg_amop'::regclass + THEN 'amop: ' || (SELECT (SELECT opfname FROM pg_opfamily WHERE oid = a.amopfamily) + || ' [strategy ' || a.amopstrategy || '] ' + || a.amoplefttype::regtype::text || ',' + || a.amoprighttype::regtype::text || ' op ' + || a.amopopr::regoperator::text + FROM pg_amop a WHERE a.oid = d.objid) + WHEN 'pg_amproc'::regclass + THEN 'amproc: ' || (SELECT (SELECT opfname FROM pg_opfamily WHERE oid = p.amprocfamily) + || ' [proc ' || p.amprocnum || '] ' + || p.amproclefttype::regtype::text || ',' + || p.amprocrighttype::regtype::text || ' fn ' + || p.amproc::regprocedure::text + FROM pg_amproc p WHERE p.oid = d.objid) + ELSE 'unhandled[' || d.classid::regclass::text || ']' + END AS member +FROM pg_depend d +WHERE d.deptype = 'e' + AND d.refclassid = 'pg_extension'::regclass + AND d.refobjid = (SELECT oid FROM pg_extension WHERE extname = 'age') +ORDER BY 1; + -- ===================================================================== --- COMPARISON: Missing or extra objects (Steps 19-33) +-- COMPARISON: Missing or extra objects (Steps 19-33b) -- Any rows returned indicate a template deficiency. -- ===================================================================== @@ -252,13 +389,20 @@ LEFT JOIN _fresh_funcs f USING (proname, args) WHERE f.proname IS NULL ORDER BY 1; --- Step 21: Function PROPERTY changes (volatility, strictness, kind, return type). +-- Step 21: Function PROPERTY changes +-- (kind, volatility, strictness, return type, return-set, binding, +-- body/symbol). The probin/prosrc check catches: +-- * a C function whose symbol was renamed in the upgrade template +-- * a SQL/plpgsql function whose body was changed in either path +-- * a language change between the fresh and upgrade installs. SELECT f.proname || '(' || f.args || ')' AS function_name, - CASE WHEN f.prokind <> u.prokind THEN 'prokind: ' || f.prokind || '->' || u.prokind END AS kind_change, - CASE WHEN f.provolatile<> u.provolatile THEN 'volatile: ' || f.provolatile|| '->' || u.provolatile END AS volatility_change, - CASE WHEN f.proisstrict<> u.proisstrict THEN 'strict: ' || f.proisstrict|| '->' || u.proisstrict END AS strict_change, - CASE WHEN f.rettype <> u.rettype THEN 'rettype: ' || f.rettype || '->' || u.rettype END AS rettype_change, - CASE WHEN f.proretset <> u.proretset THEN 'retset: ' || f.proretset || '->' || u.proretset END AS retset_change + CASE WHEN f.prokind <> u.prokind THEN 'prokind: ' || f.prokind || '->' || u.prokind END AS kind_change, + CASE WHEN f.provolatile <> u.provolatile THEN 'volatile: ' || f.provolatile || '->' || u.provolatile END AS volatility_change, + CASE WHEN f.proisstrict <> u.proisstrict THEN 'strict: ' || f.proisstrict || '->' || u.proisstrict END AS strict_change, + CASE WHEN f.rettype <> u.rettype THEN 'rettype: ' || f.rettype || '->' || u.rettype END AS rettype_change, + CASE WHEN f.proretset <> u.proretset THEN 'retset: ' || f.proretset || '->' || u.proretset END AS retset_change, + CASE WHEN f.probin <> u.probin THEN 'probin: ' || f.probin || '->' || u.probin END AS probin_change, + CASE WHEN f.prosrc <> u.prosrc THEN 'prosrc changed' END AS prosrc_change FROM _fresh_funcs f JOIN _upgraded_funcs u USING (proname, args) WHERE f.provolatile <> u.provolatile @@ -266,6 +410,8 @@ WHERE f.provolatile <> u.provolatile OR f.prokind <> u.prokind OR f.rettype <> u.rettype OR f.proretset <> u.proretset + OR f.probin <> u.probin + OR f.prosrc <> u.prosrc ORDER BY 1; -- Step 22: Relations MISSING after upgrade. @@ -352,6 +498,22 @@ LEFT JOIN _fresh_constraints f USING (conname, contype, table_name) WHERE f.conname IS NULL ORDER BY 1; +-- Step 33a: Extension members MISSING after upgrade +-- (object exists in pg_proc/pg_class/etc. but is not linked to the AGE +-- extension via pg_depend, i.e. ALTER EXTENSION age ADD ... was forgotten). +SELECT f.member AS missing_extension_member +FROM _fresh_extmembers f +LEFT JOIN _upgraded_extmembers u USING (member) +WHERE u.member IS NULL +ORDER BY 1; + +-- Step 33b: Extension members EXTRA after upgrade. +SELECT u.member AS extra_extension_member +FROM _upgraded_extmembers u +LEFT JOIN _fresh_extmembers f USING (member) +WHERE f.member IS NULL +ORDER BY 1; + -- ===================================================================== -- SUMMARY (Step 34) -- ===================================================================== @@ -364,7 +526,8 @@ SELECT (SELECT count(*) FROM _fresh_ops) = (SELECT count(*) FROM _upgraded_ops) AS ops_match, (SELECT count(*) FROM _fresh_casts) = (SELECT count(*) FROM _upgraded_casts) AS casts_match, (SELECT count(*) FROM _fresh_opclass) = (SELECT count(*) FROM _upgraded_opclass) AS opclass_match, - (SELECT count(*) FROM _fresh_constraints) = (SELECT count(*) FROM _upgraded_constraints) AS constraints_match; + (SELECT count(*) FROM _fresh_constraints) = (SELECT count(*) FROM _upgraded_constraints) AS constraints_match, + (SELECT count(*) FROM _fresh_extmembers) = (SELECT count(*) FROM _upgraded_extmembers) AS extmembers_match; -- ===================================================================== -- CLEANUP (Steps 35-36) @@ -374,7 +537,8 @@ SELECT DROP TABLE _fresh_funcs, _upgraded_funcs, _fresh_rels, _upgraded_rels, _fresh_types, _upgraded_types, _fresh_ops, _upgraded_ops, _fresh_casts, _upgraded_casts, _fresh_opclass, _upgraded_opclass, - _fresh_constraints, _upgraded_constraints; + _fresh_constraints, _upgraded_constraints, + _fresh_extmembers, _upgraded_extmembers; DROP EXTENSION age; CREATE EXTENSION age; From 01ee9414f9b483782cbf50cfe0744860bcf375ae Mon Sep 17 00:00:00 2001 From: Ueslei Lima Date: Wed, 13 May 2026 12:16:14 -0300 Subject: [PATCH 066/100] feat(python-driver): add public API for connection pooling and model dict conversion (#2374) * feat(python-driver): add public API for connection pooling and model dict conversion Add two enhancements to the Python driver's public API: 1. Add configure_connection() function that registers AGE agtype adapters on an existing psycopg connection without creating a new one. This enables use with external connection pools (e.g. psycopg_pool) and managed PostgreSQL services where LOAD 'age' may be restricted. Also explicitly export AgeLoader and ClientCursor as public symbols in age/__init__.py. (#2369) 2. Add to_dict() methods to Vertex, Edge, and Path model classes for conversion to plain Python dicts. This enables direct JSON serialization with json.dumps() without requiring custom conversion logic. (#2371) - Vertex.to_dict() returns {id, label, properties} - Edge.to_dict() returns {id, label, start_id, end_id, properties} - Path.to_dict() returns a list of to_dict() results Closes #2369 Closes #2371 * Fix configure_connection: correct parameter semantics, add load_from_plugins - Replace confusing `skip_load` (double-negative) with `load` (positive boolean, default False). The default now correctly matches the intent: no LOAD by default for connection pool / managed PostgreSQL use cases. - Add `load_from_plugins` parameter for parity with setUpAge(). - Fix docstring to accurately describe parameter behavior. - Add 6 unit tests for configure_connection covering: default no-load, explicit load, load_from_plugins, search_path always set, adapter registration, and graph_name check delegation. Made-with: Cursor * Address review feedback for configure_connection and to_dict - Move TypeInfo.fetch() inside cursor block so search_path change is visible regardless of transaction isolation mode - Raise ValueError when load_from_plugins=True but load=False - Add type annotations to configure_connection signature - Document shallow-copy semantics in Vertex/Edge to_dict() - Path.to_dict() uses str() fallback for non-AGObj entities to guarantee JSON-serializable output - Add test for AgeNotSet when TypeInfo.fetch returns None - Add test for load_from_plugins=True without load=True - Replace fragile string assertions with assert_called_with/assert_any_call Made-with: Cursor * Fix Path.to_dict() to preserve JSON-native types, add tests to suite - Path.to_dict(): leave dict/list/str/int/float/bool/None unchanged instead of converting to str(); handle entities=None safely - Add TestModelToDict, TestPublicImports, TestConfigureConnection to the __main__ suite so they run via direct script execution Made-with: Cursor * fix(python-driver): Python 3.9-safe hints and correct $user in search_path - Use Optional[str] for configure_connection graph_name (PEP 604 unions are invalid on Python 3.9). - Import Any/Optional from typing for annotations. - Quote $user in SET search_path; align unit test expectations. Made-with: Cursor * test(python-driver): add configure_connection + to_dict integration test Existing tests for the new public API are unit-only: - TestConfigureConnection mocks the psycopg connection, so it never proves that AgeLoader actually registers against real agtype OIDs. - TestModelToDict hand-constructs Vertex/Edge/Path via kwargs, so it never serialises objects produced by the ANTLR parser. Add a single TestAgeBasic.testConfigureConnection that: - opens a raw psycopg connection (bypassing age.connect()), - calls configure_connection(..., load=True) on it, - runs a Cypher CREATE/RETURN through the configured connection, - asserts the returned values are real Vertex/Edge instances and that their to_dict() output is JSON-serialisable with the expected label/start_id/end_id/properties shape, - repeats the round-trip for a Path returned by MATCH. This is the smallest test that proves the configure_connection + to_dict pipeline works end-to-end against a live AGE database. Made-with: Cursor --- drivers/python/age/__init__.py | 1 + drivers/python/age/age.py | 67 ++++++++++ drivers/python/age/models.py | 40 ++++++ drivers/python/test_age_py.py | 234 ++++++++++++++++++++++++++++++++- 4 files changed, 341 insertions(+), 1 deletion(-) diff --git a/drivers/python/age/__init__.py b/drivers/python/age/__init__.py index caee6a43c..2da5ef102 100644 --- a/drivers/python/age/__init__.py +++ b/drivers/python/age/__init__.py @@ -16,6 +16,7 @@ import psycopg.conninfo as conninfo from . import age from .age import * +from .age import AgeLoader, ClientCursor, configure_connection from .models import * from .builder import ResultHandler, DummyResultHandler, parseAgeValue, newResultHandler from . import VERSION diff --git a/drivers/python/age/age.py b/drivers/python/age/age.py index ae76bcf50..a580936a3 100644 --- a/drivers/python/age/age.py +++ b/drivers/python/age/age.py @@ -14,6 +14,8 @@ # under the License. import re +from typing import Any, Optional + import psycopg from psycopg.types import TypeInfo from psycopg import sql @@ -170,6 +172,71 @@ def setUpAge(conn:psycopg.connection, graphName:str, load_from_plugins:bool=Fals if graphName != None: checkGraphCreated(conn, graphName) + +def configure_connection( + conn: psycopg.connection, + graph_name: Optional[str] = None, + load: bool = False, + load_from_plugins: bool = False, +) -> None: + """Register AGE agtype adapters on an existing connection. + + This enables use of AGE with externally-managed connections, such as + those from psycopg_pool.ConnectionPool. By default the function does + **not** execute ``LOAD 'age'``, making it safe for managed PostgreSQL + services (Azure, AWS RDS) where the extension is pre-loaded via + ``shared_preload_libraries``. + + Performs: + - ``SET search_path`` to include ``ag_catalog`` + - Fetches agtype OIDs and registers ``AgeLoader`` + - Optionally loads the AGE extension (``load=True``) + - Optionally checks/creates the graph + + Args: + conn: An existing psycopg connection. + graph_name: Optional graph name to check/create. + load: If True, execute ``LOAD 'age'`` (or the plugins path). + Default False — suitable for environments where AGE is + already loaded. + load_from_plugins: If True (and ``load=True``), use + ``LOAD '$libdir/plugins/age'`` instead of ``LOAD 'age'``. + + Raises: + ValueError: If ``load_from_plugins=True`` but ``load=False``. + AgeNotSet: If the agtype type is not found in the database. + """ + if load_from_plugins and not load: + raise ValueError( + "load_from_plugins=True requires load=True. " + "Set load=True to enable extension loading." + ) + + with conn.cursor() as cursor: + if load: + if load_from_plugins: + cursor.execute("LOAD '$libdir/plugins/age';") + else: + cursor.execute("LOAD 'age';") + + cursor.execute('SET search_path = ag_catalog, "$user", public;') + + ag_info = TypeInfo.fetch(conn, 'agtype') + + if not ag_info: + raise AgeNotSet( + "AGE agtype type not found. Ensure the AGE extension is " + "installed and loaded in the current database. " + "Run CREATE EXTENSION age; first." + ) + + conn.adapters.register_loader(ag_info.oid, AgeLoader) + conn.adapters.register_loader(ag_info.array_oid, AgeLoader) + + if graph_name is not None: + checkGraphCreated(conn, graph_name) + + # Create the graph, if it does not exist def checkGraphCreated(conn:psycopg.connection, graphName:str): validate_graph_name(graphName) diff --git a/drivers/python/age/models.py b/drivers/python/age/models.py index 6d9095485..62215c160 100644 --- a/drivers/python/age/models.py +++ b/drivers/python/age/models.py @@ -118,6 +118,20 @@ def toJson(self) -> str: return buf.getvalue() + def to_dict(self) -> list: + # AGObj elements are recursively converted; JSON-native types + # (dict, list, str, int, float, bool, None) pass through unchanged. + # Non-serializable objects fall back to str() as a safety net. + result = [] + for e in (self.entities or []): + if isinstance(e, AGObj): + result.append(e.to_dict()) + elif isinstance(e, (dict, list, str, int, float, bool, type(None))): + result.append(e) + else: + result.append(str(e)) + return result + @@ -146,6 +160,18 @@ def __str__(self) -> str: def __repr__(self) -> str: return self.toString() + def to_dict(self) -> dict: + """Return a plain dict suitable for JSON serialization. + + Properties are shallow-copied; nested mutable values will share + references with the original Vertex. + """ + return { + "id": self.id, + "label": self.label, + "properties": dict(self.properties) if self.properties else {}, + } + def toString(self) -> str: return nodeToString(self) @@ -186,6 +212,20 @@ def __str__(self) -> str: def __repr__(self) -> str: return self.toString() + def to_dict(self) -> dict: + """Return a plain dict suitable for JSON serialization. + + Properties are shallow-copied; nested mutable values will share + references with the original Edge. + """ + return { + "id": self.id, + "label": self.label, + "start_id": self.start_id, + "end_id": self.end_id, + "properties": dict(self.properties) if self.properties else {}, + } + def extraStrFormat(node, buf): if node.start_id != None: buf.write(", start_id:") diff --git a/drivers/python/test_age_py.py b/drivers/python/test_age_py.py index 12dd7bd55..bb92f7d51 100644 --- a/drivers/python/test_age_py.py +++ b/drivers/python/test_age_py.py @@ -15,7 +15,7 @@ import json import re -from age.models import Vertex +from age.models import Vertex, Edge, Path import unittest import unittest.mock import decimal @@ -171,6 +171,160 @@ def test_validate_column_quoting(self): self.assertEqual(_validate_column("my_col"), '"my_col" agtype') +class TestModelToDict(unittest.TestCase): + """Unit tests for Vertex/Edge/Path to_dict() — no DB required.""" + + def test_vertex_to_dict(self): + v = Vertex(id=123, label="Person", properties={"name": "Alice", "age": 30}) + d = v.to_dict() + self.assertEqual(d["id"], 123) + self.assertEqual(d["label"], "Person") + self.assertEqual(d["properties"], {"name": "Alice", "age": 30}) + # Verify it's a plain dict (JSON-serializable) + json_str = json.dumps(d) + self.assertIn("Alice", json_str) + + def test_vertex_to_dict_empty_properties(self): + v = Vertex(id=1, label="Empty", properties=None) + d = v.to_dict() + self.assertEqual(d["properties"], {}) + + def test_edge_to_dict(self): + e = Edge(id=456, label="KNOWS", properties={"since": 2020}) + e.start_id = 123 + e.end_id = 789 + d = e.to_dict() + self.assertEqual(d["id"], 456) + self.assertEqual(d["label"], "KNOWS") + self.assertEqual(d["start_id"], 123) + self.assertEqual(d["end_id"], 789) + self.assertEqual(d["properties"], {"since": 2020}) + json_str = json.dumps(d) + self.assertIn("KNOWS", json_str) + + def test_path_to_dict(self): + v1 = Vertex(id=1, label="A", properties={"name": "start"}) + e = Edge(id=10, label="r", properties={"w": 1}) + e.start_id = 1 + e.end_id = 2 + v2 = Vertex(id=2, label="B", properties={"name": "end"}) + p = Path([v1, e, v2]) + d = p.to_dict() + self.assertEqual(len(d), 3) + self.assertEqual(d[0]["label"], "A") + self.assertEqual(d[1]["label"], "r") + self.assertEqual(d[1]["start_id"], 1) + self.assertEqual(d[2]["label"], "B") + # Verify the whole path is JSON-serializable + json_str = json.dumps(d) + self.assertIn("start", json_str) + + def test_vertex_to_dict_is_plain_dict(self): + """to_dict() returns standard dict, not a model object.""" + v = Vertex(id=1, label="X", properties={"k": "v"}) + d = v.to_dict() + self.assertIsInstance(d, dict) + self.assertIsInstance(d["properties"], dict) + + +class TestPublicImports(unittest.TestCase): + """Verify that public API symbols are importable without type: ignore.""" + + def test_import_configure_connection(self): + from age import configure_connection + self.assertTrue(callable(configure_connection)) + + def test_import_age_loader(self): + from age import AgeLoader + self.assertIsNotNone(AgeLoader) + + def test_import_client_cursor(self): + from age import ClientCursor + self.assertIsNotNone(ClientCursor) + + +class TestConfigureConnection(unittest.TestCase): + """Unit tests for configure_connection() — no DB required.""" + + def _make_mock_conn(self): + mock_conn = unittest.mock.MagicMock() + mock_cursor = unittest.mock.MagicMock() + mock_conn.cursor.return_value.__enter__ = unittest.mock.Mock(return_value=mock_cursor) + mock_conn.cursor.return_value.__exit__ = unittest.mock.Mock(return_value=False) + mock_conn.adapters = unittest.mock.MagicMock() + mock_type_info = unittest.mock.MagicMock() + mock_type_info.oid = 1 + mock_type_info.array_oid = 2 + return mock_conn, mock_cursor, mock_type_info + + def test_default_does_not_load(self): + """By default, configure_connection should NOT execute LOAD.""" + mock_conn, mock_cursor, mock_type_info = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=mock_type_info), \ + unittest.mock.patch("age.age.checkGraphCreated"): + age.age.configure_connection(mock_conn) + mock_cursor.execute.assert_called_once_with( + 'SET search_path = ag_catalog, "$user", public;' + ) + + def test_load_true_executes_load(self): + """When load=True, LOAD 'age' must be executed.""" + mock_conn, mock_cursor, mock_type_info = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=mock_type_info), \ + unittest.mock.patch("age.age.checkGraphCreated"): + age.age.configure_connection(mock_conn, load=True) + mock_cursor.execute.assert_any_call("LOAD 'age';") + + def test_load_from_plugins(self): + """When load=True and load_from_plugins=True, use plugins path.""" + mock_conn, mock_cursor, mock_type_info = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=mock_type_info), \ + unittest.mock.patch("age.age.checkGraphCreated"): + age.age.configure_connection(mock_conn, load=True, load_from_plugins=True) + mock_cursor.execute.assert_any_call("LOAD '$libdir/plugins/age';") + + def test_load_from_plugins_without_load_raises(self): + """load_from_plugins=True without load=True must raise ValueError.""" + mock_conn, _, _ = self._make_mock_conn() + with self.assertRaises(ValueError): + age.age.configure_connection(mock_conn, load_from_plugins=True) + + def test_always_sets_search_path(self): + """search_path must always be set regardless of load parameter.""" + mock_conn, mock_cursor, mock_type_info = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=mock_type_info), \ + unittest.mock.patch("age.age.checkGraphCreated"): + age.age.configure_connection(mock_conn) + mock_cursor.execute.assert_any_call( + 'SET search_path = ag_catalog, "$user", public;' + ) + + def test_registers_agtype_adapters(self): + """AgeLoader must be registered for agtype OIDs.""" + mock_conn, mock_cursor, mock_type_info = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=mock_type_info), \ + unittest.mock.patch("age.age.checkGraphCreated"): + age.age.configure_connection(mock_conn) + mock_conn.adapters.register_loader.assert_any_call(1, age.age.AgeLoader) + mock_conn.adapters.register_loader.assert_any_call(2, age.age.AgeLoader) + + def test_graph_name_triggers_check(self): + """When graph_name is provided, checkGraphCreated must be called.""" + mock_conn, mock_cursor, mock_type_info = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=mock_type_info), \ + unittest.mock.patch("age.age.checkGraphCreated") as mock_check: + age.age.configure_connection(mock_conn, graph_name="my_graph") + mock_check.assert_called_once_with(mock_conn, "my_graph") + + def test_age_not_set_when_type_info_is_none(self): + """AgeNotSet must be raised when TypeInfo.fetch returns None.""" + from age.exceptions import AgeNotSet + mock_conn, _, _ = self._make_mock_conn() + with unittest.mock.patch("age.age.TypeInfo.fetch", return_value=None): + with self.assertRaises(AgeNotSet): + age.age.configure_connection(mock_conn) + + class TestAgeBasic(unittest.TestCase): ag = None args: argparse.Namespace = argparse.Namespace( @@ -591,6 +745,78 @@ def testSerialization(self): self.assertFalse(as_str.endswith(", }}::VERTEX")) print("Vertex.toString() 'properties' field is formatted properly.") + def testConfigureConnection(self): + """Integration: configure_connection() on an externally-opened + connection must register agtype adapters so cypher queries return + real Vertex/Edge/Path objects, and to_dict() must round-trip those + parser-produced objects through json.dumps().""" + print("\n-------------------------------------------------------") + print("Test 8: configure_connection + to_dict end-to-end.....") + print("-------------------------------------------------------\n") + + import psycopg + + from age import configure_connection + + dsn = "host={host} port={port} dbname={database} user={user} password={password}".format( + **vars(self.args) + ) + # Deliberately bypass age.connect(): the whole point of + # configure_connection() is to enable AGE on a caller-managed + # connection (e.g. one obtained from psycopg_pool.ConnectionPool). + raw = psycopg.connect(dsn) + try: + configure_connection(raw, graph_name=self.args.graphName, load=True) + + graph = self.args.graphName + with raw.cursor() as cur: + cur.execute( + f"SELECT * FROM cypher('{graph}', $$ " + "CREATE (a:Person {name: 'Alice'})-[r:KNOWS {since: 2020}]->(b:Person {name: 'Bob'}) " + "RETURN a, r, b " + "$$) AS (a agtype, r agtype, b agtype);" + ) + row = cur.fetchone() + raw.commit() + + v_a, e, v_b = row + self.assertIsInstance(v_a, Vertex) + self.assertIsInstance(e, Edge) + self.assertIsInstance(v_b, Vertex) + self.assertEqual(v_a["name"], "Alice") + self.assertEqual(v_b["name"], "Bob") + self.assertEqual(e["since"], 2020) + + payload = { + "start": v_a.to_dict(), + "edge": e.to_dict(), + "end": v_b.to_dict(), + } + serialised = json.loads(json.dumps(payload)) + self.assertEqual(serialised["start"]["label"], "Person") + self.assertEqual(serialised["edge"]["label"], "KNOWS") + self.assertEqual(serialised["edge"]["start_id"], v_a.id) + self.assertEqual(serialised["edge"]["end_id"], v_b.id) + self.assertEqual(serialised["start"]["properties"]["name"], "Alice") + + with raw.cursor() as cur: + cur.execute( + f"SELECT * FROM cypher('{graph}', $$ " + "MATCH p=(:Person)-[:KNOWS]->(:Person) RETURN p " + "$$) AS (p agtype);" + ) + path = cur.fetchone()[0] + + self.assertIsInstance(path, Path) + path_dict = json.loads(json.dumps(path.to_dict())) + self.assertEqual(len(path_dict), 3) + self.assertEqual(path_dict[0]["label"], "Person") + self.assertEqual(path_dict[1]["label"], "KNOWS") + self.assertEqual(path_dict[2]["label"], "Person") + print("\nTest 8 Successful....") + finally: + raw.close() + if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -628,9 +854,14 @@ def testSerialization(self): args = parser.parse_args() suite = unittest.TestSuite() + # Unit tests (no DB required) loader = unittest.TestLoader() suite.addTests(loader.loadTestsFromTestCase(TestSetUpAge)) suite.addTests(loader.loadTestsFromTestCase(TestBuildCypher)) + suite.addTests(loader.loadTestsFromTestCase(TestModelToDict)) + suite.addTests(loader.loadTestsFromTestCase(TestPublicImports)) + suite.addTests(loader.loadTestsFromTestCase(TestConfigureConnection)) + # Integration tests (require DB) suite.addTest(TestAgeBasic("testExec")) suite.addTest(TestAgeBasic("testQuery")) suite.addTest(TestAgeBasic("testChangeData")) @@ -638,5 +869,6 @@ def testSerialization(self): suite.addTest(TestAgeBasic("testMultipleEdges")) suite.addTest(TestAgeBasic("testCollect")) suite.addTest(TestAgeBasic("testSerialization")) + suite.addTest(TestAgeBasic("testConfigureConnection")) TestAgeBasic.args = args unittest.TextTestRunner().run(suite) From 9960e9c6034ed301b90e6e6777c80b7c95b45cf0 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Wed, 13 May 2026 20:02:14 -0400 Subject: [PATCH 067/100] Add agtype <-> jsonb bidirectional casts (#350) (#2361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add agtype <-> jsonb bidirectional casts (issue #350) Register explicit casts between agtype and jsonb, enabling: SELECT properties(n)::jsonb FROM cypher(...) AS (n agtype); SELECT '{"key": "value"}'::jsonb::agtype; -- Use jsonb operators on graph data: SELECT (props::jsonb)->>'name' FROM cypher(...) AS (props agtype); Implementation uses SQL language functions that go through proven text-intermediate paths: agtype -> jsonb: agtype_to_json() -> json::jsonb jsonb -> agtype: jsonb::text -> text::agtype This approach is safe because agtype extends jsonb's binary format with types (AGTV_INTEGER, AGTV_FLOAT, AGTV_VERTEX, AGTV_EDGE, AGTV_PATH) that jsonb does not recognize, making direct binary conversion unreliable. The text roundtrip handles all value types correctly including graph types (vertex/edge properties are extracted as JSON objects). All 31 regression tests pass. Co-Authored-By: Claude Opus 4.6 (1M context) * Address Copilot review: fix comment, add regression tests for jsonb casts - Fix comment in agtype_coercions.sql: document json-intermediate path (agtype_to_json -> json::jsonb) instead of incorrect "text intermediate" - Add agtype_jsonb_cast regression test covering: string/null/array/object agtype->jsonb, all jsonb scalar types->agtype, roundtrips, vertex/edge ->jsonb with structural key checks, NULL handling - Register agtype_jsonb_cast in Makefile REGRESS list Co-Authored-By: Claude Opus 4.6 (1M context) * Register agtype <-> jsonb casts in upgrade template The age_upgrade regression test (added after this PR was originally opened) verifies that every SQL object in the install SQL also appears in the version upgrade template (age----.sql). Surfaced during rebase onto current master: agtype_to_jsonb / jsonb_to_agtype and their casts were missing from the template, causing 4 entries in the "missing_function" / "missing_cast" check. Adds the two functions and two casts to the end of the template, following the file's own "add all additions to the end of this file" convention. Cassert installcheck now 34/34 AGE tests green (pgvector skipped — env-only). --------- Co-authored-by: Claude Opus 4.6 (1M context) --- Makefile | 3 +- age--1.7.0--y.y.y.sql | 31 +++ regress/expected/agtype_jsonb_cast.out | 261 +++++++++++++++++++++++++ regress/sql/agtype_jsonb_cast.sql | 137 +++++++++++++ sql/agtype_coercions.sql | 27 +++ 5 files changed, 458 insertions(+), 1 deletion(-) create mode 100644 regress/expected/agtype_jsonb_cast.out create mode 100644 regress/sql/agtype_jsonb_cast.sql diff --git a/Makefile b/Makefile index 3c5a28c29..7e5fb3d2d 100644 --- a/Makefile +++ b/Makefile @@ -180,7 +180,8 @@ REGRESS = scan \ map_projection \ direct_field_access \ security \ - reserved_keyword_alias + reserved_keyword_alias \ + agtype_jsonb_cast ifneq ($(EXTRA_TESTS),) REGRESS += $(EXTRA_TESTS) diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index a3cdea279..520447770 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -459,3 +459,34 @@ BEGIN END LOOP; END; $$; + +-- +-- agtype <-> jsonb bidirectional casts +-- + +-- agtype -> jsonb (explicit) +-- Uses json intermediate (agtype_to_json -> json::jsonb) because agtype +-- extends jsonb's binary format with types (AGTV_INTEGER, AGTV_FLOAT, +-- AGTV_VERTEX, AGTV_EDGE, AGTV_PATH) that jsonb does not recognize. +CREATE FUNCTION ag_catalog.agtype_to_jsonb(agtype) + RETURNS jsonb + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'SELECT ag_catalog.agtype_to_json($1)::jsonb'; + +CREATE CAST (agtype AS jsonb) + WITH FUNCTION ag_catalog.agtype_to_jsonb(agtype); + +-- jsonb -> agtype (explicit) +CREATE FUNCTION ag_catalog.jsonb_to_agtype(jsonb) + RETURNS agtype + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'SELECT $1::text::agtype'; + +CREATE CAST (jsonb AS agtype) + WITH FUNCTION ag_catalog.jsonb_to_agtype(jsonb); diff --git a/regress/expected/agtype_jsonb_cast.out b/regress/expected/agtype_jsonb_cast.out new file mode 100644 index 000000000..49bf1f887 --- /dev/null +++ b/regress/expected/agtype_jsonb_cast.out @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +-- +-- agtype -> jsonb casts +-- +-- String scalar +SELECT '"hello"'::agtype::jsonb; + jsonb +--------- + "hello" +(1 row) + +-- Null +SELECT 'null'::agtype::jsonb; + jsonb +------- + null +(1 row) + +-- Array +SELECT '[1, "two", null]'::agtype::jsonb; + jsonb +------------------ + [1, "two", null] +(1 row) + +-- Nested array +SELECT '[[1, 2], [3, 4]]'::agtype::jsonb; + jsonb +------------------ + [[1, 2], [3, 4]] +(1 row) + +-- Empty array +SELECT '[]'::agtype::jsonb; + jsonb +------- + [] +(1 row) + +-- Object +SELECT '{"name": "Alice", "age": 30}'::agtype::jsonb; + jsonb +------------------------------ + {"age": 30, "name": "Alice"} +(1 row) + +-- Nested object +SELECT '{"a": {"b": {"c": 1}}}'::agtype::jsonb; + jsonb +------------------------ + {"a": {"b": {"c": 1}}} +(1 row) + +-- Object with array values +SELECT '{"tags": ["a", "b"], "count": 2}'::agtype::jsonb; + jsonb +---------------------------------- + {"tags": ["a", "b"], "count": 2} +(1 row) + +-- Empty object +SELECT '{}'::agtype::jsonb; + jsonb +------- + {} +(1 row) + +-- +-- jsonb -> agtype casts +-- +-- String scalar +SELECT '"hello"'::jsonb::agtype; + agtype +--------- + "hello" +(1 row) + +-- Numeric scalar +SELECT '42'::jsonb::agtype; + agtype +-------- + 42 +(1 row) + +-- Float scalar +SELECT '3.14'::jsonb::agtype; + agtype +-------- + 3.14 +(1 row) + +-- Boolean +SELECT 'true'::jsonb::agtype; + agtype +-------- + true +(1 row) + +-- Null +SELECT 'null'::jsonb::agtype; + agtype +-------- + null +(1 row) + +-- Array +SELECT '[1, "two", null]'::jsonb::agtype; + agtype +------------------ + [1, "two", null] +(1 row) + +-- Nested array +SELECT '[[1, 2], [3, 4]]'::jsonb::agtype; + agtype +------------------ + [[1, 2], [3, 4]] +(1 row) + +-- Empty array +SELECT '[]'::jsonb::agtype; + agtype +-------- + [] +(1 row) + +-- Object +SELECT '{"name": "Alice", "age": 30}'::jsonb::agtype; + agtype +------------------------------ + {"age": 30, "name": "Alice"} +(1 row) + +-- Nested object +SELECT '{"a": {"b": {"c": 1}}}'::jsonb::agtype; + agtype +------------------------ + {"a": {"b": {"c": 1}}} +(1 row) + +-- Empty object +SELECT '{}'::jsonb::agtype; + agtype +-------- + {} +(1 row) + +-- +-- Roundtrip: jsonb -> agtype -> jsonb +-- +SELECT ('{"key": "value"}'::jsonb::agtype)::jsonb; + jsonb +------------------ + {"key": "value"} +(1 row) + +SELECT ('[1, 2, 3]'::jsonb::agtype)::jsonb; + jsonb +----------- + [1, 2, 3] +(1 row) + +SELECT ('null'::jsonb::agtype)::jsonb; + jsonb +------- + null +(1 row) + +-- +-- Graph data -> jsonb (vertex and edge) +-- +SELECT create_graph('agtype_jsonb_test'); +NOTICE: graph "agtype_jsonb_test" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('agtype_jsonb_test', $$ + CREATE (a:Person {name: 'Alice', age: 30})-[:KNOWS {since: 2020}]->(b:Person {name: 'Bob', age: 25}) + RETURN a, b +$$) AS (a agtype, b agtype); + a | b +------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "Person", "properties": {"age": 30, "name": "Alice"}}::vertex | {"id": 844424930131970, "label": "Person", "properties": {"age": 25, "name": "Bob"}}::vertex +(1 row) + +-- Vertex to jsonb: check structure +SELECT v::jsonb ? 'label' AS has_label, + v::jsonb ? 'properties' AS has_properties, + (v::jsonb -> 'properties' ->> 'name') AS name +FROM cypher('agtype_jsonb_test', $$ + MATCH (n:Person) RETURN n ORDER BY n.name +$$) AS (v agtype); + has_label | has_properties | name +-----------+----------------+------- + t | t | Alice + t | t | Bob +(2 rows) + +-- Edge to jsonb: check structure +SELECT e::jsonb ? 'label' AS has_label, + (e::jsonb ->> 'label') AS label, + (e::jsonb -> 'properties' ->> 'since') AS since +FROM cypher('agtype_jsonb_test', $$ + MATCH ()-[r:KNOWS]->() RETURN r +$$) AS (e agtype); + has_label | label | since +-----------+-------+------- + t | KNOWS | 2020 +(1 row) + +-- +-- NULL handling +-- +SELECT NULL::agtype::jsonb; + jsonb +------- + +(1 row) + +SELECT NULL::jsonb::agtype; + agtype +-------- + +(1 row) + +-- +-- Cleanup +-- +SELECT drop_graph('agtype_jsonb_test', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table agtype_jsonb_test._ag_label_vertex +drop cascades to table agtype_jsonb_test._ag_label_edge +drop cascades to table agtype_jsonb_test."Person" +drop cascades to table agtype_jsonb_test."KNOWS" +NOTICE: graph "agtype_jsonb_test" has been dropped + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/agtype_jsonb_cast.sql b/regress/sql/agtype_jsonb_cast.sql new file mode 100644 index 000000000..c3a6a8e27 --- /dev/null +++ b/regress/sql/agtype_jsonb_cast.sql @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +-- +-- agtype -> jsonb casts +-- + +-- String scalar +SELECT '"hello"'::agtype::jsonb; + +-- Null +SELECT 'null'::agtype::jsonb; + +-- Array +SELECT '[1, "two", null]'::agtype::jsonb; + +-- Nested array +SELECT '[[1, 2], [3, 4]]'::agtype::jsonb; + +-- Empty array +SELECT '[]'::agtype::jsonb; + +-- Object +SELECT '{"name": "Alice", "age": 30}'::agtype::jsonb; + +-- Nested object +SELECT '{"a": {"b": {"c": 1}}}'::agtype::jsonb; + +-- Object with array values +SELECT '{"tags": ["a", "b"], "count": 2}'::agtype::jsonb; + +-- Empty object +SELECT '{}'::agtype::jsonb; + +-- +-- jsonb -> agtype casts +-- + +-- String scalar +SELECT '"hello"'::jsonb::agtype; + +-- Numeric scalar +SELECT '42'::jsonb::agtype; + +-- Float scalar +SELECT '3.14'::jsonb::agtype; + +-- Boolean +SELECT 'true'::jsonb::agtype; + +-- Null +SELECT 'null'::jsonb::agtype; + +-- Array +SELECT '[1, "two", null]'::jsonb::agtype; + +-- Nested array +SELECT '[[1, 2], [3, 4]]'::jsonb::agtype; + +-- Empty array +SELECT '[]'::jsonb::agtype; + +-- Object +SELECT '{"name": "Alice", "age": 30}'::jsonb::agtype; + +-- Nested object +SELECT '{"a": {"b": {"c": 1}}}'::jsonb::agtype; + +-- Empty object +SELECT '{}'::jsonb::agtype; + +-- +-- Roundtrip: jsonb -> agtype -> jsonb +-- + +SELECT ('{"key": "value"}'::jsonb::agtype)::jsonb; +SELECT ('[1, 2, 3]'::jsonb::agtype)::jsonb; +SELECT ('null'::jsonb::agtype)::jsonb; + +-- +-- Graph data -> jsonb (vertex and edge) +-- + +SELECT create_graph('agtype_jsonb_test'); + +SELECT * FROM cypher('agtype_jsonb_test', $$ + CREATE (a:Person {name: 'Alice', age: 30})-[:KNOWS {since: 2020}]->(b:Person {name: 'Bob', age: 25}) + RETURN a, b +$$) AS (a agtype, b agtype); + +-- Vertex to jsonb: check structure +SELECT v::jsonb ? 'label' AS has_label, + v::jsonb ? 'properties' AS has_properties, + (v::jsonb -> 'properties' ->> 'name') AS name +FROM cypher('agtype_jsonb_test', $$ + MATCH (n:Person) RETURN n ORDER BY n.name +$$) AS (v agtype); + +-- Edge to jsonb: check structure +SELECT e::jsonb ? 'label' AS has_label, + (e::jsonb ->> 'label') AS label, + (e::jsonb -> 'properties' ->> 'since') AS since +FROM cypher('agtype_jsonb_test', $$ + MATCH ()-[r:KNOWS]->() RETURN r +$$) AS (e agtype); + +-- +-- NULL handling +-- + +SELECT NULL::agtype::jsonb; +SELECT NULL::jsonb::agtype; + +-- +-- Cleanup +-- + +SELECT drop_graph('agtype_jsonb_test', true); diff --git a/sql/agtype_coercions.sql b/sql/agtype_coercions.sql index 933375fc1..7763360f2 100644 --- a/sql/agtype_coercions.sql +++ b/sql/agtype_coercions.sql @@ -174,6 +174,33 @@ AS 'MODULE_PATHNAME'; CREATE CAST (agtype AS json) WITH FUNCTION ag_catalog.agtype_to_json(agtype); +-- agtype -> jsonb (explicit) +-- Uses json intermediate (agtype_to_json -> json::jsonb) because agtype +-- extends jsonb's binary format with types (AGTV_INTEGER, AGTV_FLOAT, +-- AGTV_VERTEX, AGTV_EDGE, AGTV_PATH) that jsonb does not recognize. +CREATE FUNCTION ag_catalog.agtype_to_jsonb(agtype) + RETURNS jsonb + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'SELECT ag_catalog.agtype_to_json($1)::jsonb'; + +CREATE CAST (agtype AS jsonb) + WITH FUNCTION ag_catalog.agtype_to_jsonb(agtype); + +-- jsonb -> agtype (explicit) +CREATE FUNCTION ag_catalog.jsonb_to_agtype(jsonb) + RETURNS agtype + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'SELECT $1::text::agtype'; + +CREATE CAST (jsonb AS agtype) + WITH FUNCTION ag_catalog.jsonb_to_agtype(jsonb); + CREATE FUNCTION ag_catalog.agtype_array_to_agtype(agtype[]) RETURNS agtype LANGUAGE c From f02eda0e0504bf3e941dd4a05e3060d2f3bb4f04 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Mon, 1 Jun 2026 09:05:49 -0700 Subject: [PATCH 068/100] perf: VLE terminal-qual rewrite (#2420) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit perf: VLE terminal-qual rewrite — emit endpoint equalities instead of SRF qual functions. Removes the per-row age_match_vle_terminal_edge and age_match_two_vle_edges qual functions from VLE query plans. The cypher transformer now emits the endpoint match as a plain graphid/int8 equality on new SRF output columns, evaluated by the planner like any other join clause — no detoasting, no per-row C function dispatch. Stages land as one commit: S1 Inline start_vid/end_vid in VLE_path_container header S2 Read VLE qual endpoints from header-only TOAST slice S4 Emit start_id/end_id as scalar SRF output columns (age_vle now RETURNS SETOF record with edges/start_id/end_id) S5 Cypher transformer rewrites terminal-edge match quals as integer equalities (drops age_match_vle_terminal_edge call) S6 Cypher transformer emits graphid equality for two-VLE-edge joins (drops age_match_two_vle_edges call) Performance (SF3 LDBC SNB, 5 runs/3 warmup, vs clean master baseline_v2): IC sum 198,958 → 109,322 ms −45.05 % (1.82× end-to-end speedup) IC1 8,625 → 4,600 ms −46.67 % IC3 21,239 → 9,784 ms −53.93 % IC5 21,051 → 5,696 ms −72.94 % IC6 15,916 → 4,447 ms −72.06 % IC9 44,839 → 21,161 ms −52.81 % IC10 13,104 → 2,432 ms −81.44 % IC11 11,676 → 241 ms −97.93 % (48× speedup) IC2/4/7/8/12: parity (within ±3.3 %; IC4 is −2.47 %, no regression) IS sum: 1,009 → 1,004 ms −0.51 % (no VLE traffic) IU sum: 77 → 71 ms −8.38 % (IU1 −16.09 %; incidental) Memory: header-only TOAST slice for VLE qual evaluation avoids detoasting full path containers on every row; reduces per-call palloc/pfree churn in long DFS paths. No measured RSS change. Dead-code removal: - Bodies of age_match_vle_terminal_edge and age_match_two_vle_edges are gone from age_vle.c (~225 lines). C entry points remain as error-raising stubs solely so the upgrade-test snapshot loader (which sources an older 1.7.0_initial SQL against the current age.so) can resolve the symbols before the immediate ALTER EXTENSION UPDATE drops them. No regress test references either function. - SQL CREATE FUNCTION declarations removed from fresh install (sql/agtype_typecast.sql). - DROP FUNCTION IF EXISTS for both qual functions added to the upgrade script (age--1.7.0--y.y.y.sql). API change: ag_catalog.age_vle(...) now RETURNS SETOF record with output columns (edges agtype, start_id graphid, end_id graphid) instead of RETURNS SETOF agtype. Both 7-arg and 8-arg overloads are updated in fresh-install (sql/agtype_typecast.sql) and upgrade (age--1.7.0--y.y.y.sql) paths. age_match_vle_terminal_edge and age_match_two_vle_edges are dropped on upgrade and absent from fresh installs. Internal AGE callers are unaffected; external SQL that called any of these directly must adapt. Hardening (in response to PR #2420 review feedback): - cypher_clause.c: terminal-edge and two-VLE-edge join-qual emission paths now bracket the existing Assert(vle_alias != NULL) with a runtime ereport(ERROR, ...) so a missing alias produces a clean error in production builds (where Asserts compile out) instead of a NULL-deref crash inside makeString(). - age_vle.c: VLE_path_container struct comment now documents that the layout is transient (consumed within the producing query and never persisted), so the new start_vid/end_vid fields do not require an AGT_FBINARY_TYPE_VLE_PATH version bump or backward- compatible reader. The note also flags the constraint a future change would need to honor if the container ever became persistable. Tested on PostgreSQL 18.3 (REL_18_STABLE): all 34 regression tests pass (installcheck), warning-free build. modified: age--1.7.0--y.y.y.sql modified: regress/expected/cypher_match.out modified: regress/expected/cypher_vle.out modified: regress/expected/expr.out modified: sql/agtype_typecast.sql modified: src/backend/parser/cypher_clause.c modified: src/backend/parser/cypher_transform_entity.c modified: src/backend/utils/adt/age_vle.c modified: src/include/parser/cypher_transform_entity.h --- age--1.7.0--y.y.y.sql | 46 +++ regress/expected/cypher_match.out | 6 +- regress/expected/cypher_vle.out | 28 +- regress/expected/expr.out | 64 ++-- sql/agtype_typecast.sql | 31 +- src/backend/parser/cypher_clause.c | 117 +++++-- src/backend/parser/cypher_transform_entity.c | 1 + src/backend/utils/adt/age_vle.c | 310 +++++++------------ src/include/parser/cypher_transform_entity.h | 11 + 9 files changed, 316 insertions(+), 298 deletions(-) diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index 520447770..f7ef80662 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -490,3 +490,49 @@ AS 'SELECT $1::text::agtype'; CREATE CAST (jsonb AS agtype) WITH FUNCTION ag_catalog.jsonb_to_agtype(jsonb); + +-- +-- S4: VLE SRF signature change +-- +-- The age_vle SRF now emits start_id and end_id as scalar graphid columns +-- alongside the existing `edges` column. This allows the cypher transformer +-- to rewrite terminal-edge match quals as plain integer equalities, +-- removing the per-row age_match_vle_terminal_edge and age_match_two_vle_edges +-- function calls from VLE query plans. Both qual functions are dropped. +-- +-- BREAKING CHANGE for any external SQL that called age_vle(...) directly +-- and relied on `RETURNS SETOF agtype`, or called age_match_vle_terminal_edge +-- / age_match_two_vle_edges directly. Internal AGE callers (the cypher +-- transformer) are not affected. +-- +DROP FUNCTION IF EXISTS ag_catalog.age_match_vle_terminal_edge(variadic "any"); +DROP FUNCTION IF EXISTS ag_catalog.age_match_two_vle_edges(agtype, agtype); + +DROP FUNCTION IF EXISTS ag_catalog.age_vle(agtype, agtype, agtype, agtype, + agtype, agtype, agtype); +DROP FUNCTION IF EXISTS ag_catalog.age_vle(agtype, agtype, agtype, agtype, + agtype, agtype, agtype, agtype); + +CREATE FUNCTION ag_catalog.age_vle(IN agtype, IN agtype, IN agtype, IN agtype, + IN agtype, IN agtype, IN agtype, + OUT edges agtype, + OUT start_id graphid, + OUT end_id graphid) + RETURNS SETOF record +LANGUAGE C +STABLE +CALLED ON NULL INPUT +PARALLEL UNSAFE +AS 'MODULE_PATHNAME'; + +CREATE FUNCTION ag_catalog.age_vle(IN agtype, IN agtype, IN agtype, IN agtype, + IN agtype, IN agtype, IN agtype, IN agtype, + OUT edges agtype, + OUT start_id graphid, + OUT end_id graphid) + RETURNS SETOF record +LANGUAGE C +STABLE +CALLED ON NULL INPUT +PARALLEL UNSAFE +AS 'MODULE_PATHNAME'; diff --git a/regress/expected/cypher_match.out b/regress/expected/cypher_match.out index e55ed23c3..e99460e5b 100644 --- a/regress/expected/cypher_match.out +++ b/regress/expected/cypher_match.out @@ -2784,13 +2784,13 @@ SELECT * FROM cypher('cypher_match', $$ MATCH p=()-[*]->() RETURN length(p) $$) length -------- 1 - 2 1 + 2 1 1 2 - 1 2 + 1 (8 rows) SELECT * FROM cypher('cypher_match', $$ MATCH p=()-[*]->() WHERE length(p) > 1 RETURN length(p) $$) as (length agtype); @@ -2812,8 +2812,8 @@ SELECT * FROM cypher('cypher_match', $$ MATCH p=()-[*]->() WHERE size(nodes(p)) SELECT * FROM cypher('cypher_match', $$ MATCH (n {name:'Dave'}) MATCH p=()-[*]->() WHERE nodes(p)[0] = n RETURN length(p) $$) as (length agtype); length -------- - 1 2 + 1 (2 rows) SELECT * FROM cypher('cypher_match', $$ MATCH p1=(n {name:'Dave'})-[]->() MATCH p2=()-[*]->() WHERE p2=p1 RETURN p2=p1 $$) as (path agtype); diff --git a/regress/expected/cypher_vle.out b/regress/expected/cypher_vle.out index 6574e0608..27e3bb822 100644 --- a/regress/expected/cypher_vle.out +++ b/regress/expected/cypher_vle.out @@ -508,37 +508,37 @@ SELECT * FROM cypher('cypher_vle', $$MATCH p=(u)-[e*0..0]->(v) RETURN id(u), p, SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[*0..0]->()-[]->() RETURN p $$) AS (p agtype); p ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1970324836974593, "label": "self_loop", "end_id": 1407374883553281, "start_id": 1407374883553281, "properties": {"name": "self loop", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1970324836974593, "label": "self_loop", "end_id": 1407374883553281, "start_id": 1407374883553281, "properties": {"name": "self loop", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path (13 rows) SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[]->()-[*0..0]->() RETURN p $$) AS (p agtype); p ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1970324836974593, "label": "self_loop", "end_id": 1407374883553281, "start_id": 1407374883553281, "properties": {"name": "self loop", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path - [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path (13 rows) -- @@ -598,9 +598,9 @@ SELECT * FROM cypher('mygraph', $$ $$) AS (g5 agtype); g5 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 844424930131969, "label": "Node", "properties": {"name": "a"}}::vertex, {"id": 1125899906842627, "label": "Edge", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex]::path [{"id": 844424930131969, "label": "Node", "properties": {"name": "a"}}::vertex, {"id": 1125899906842627, "label": "Edge", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex, {"id": 1125899906842626, "label": "Edge", "end_id": 844424930131970, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131970, "label": "Node", "properties": {"name": "c"}}::vertex]::path [{"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex, {"id": 1125899906842626, "label": "Edge", "end_id": 844424930131970, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131970, "label": "Node", "properties": {"name": "c"}}::vertex]::path + [{"id": 844424930131969, "label": "Node", "properties": {"name": "a"}}::vertex, {"id": 1125899906842627, "label": "Edge", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex]::path (3 rows) SELECT drop_graph('mygraph', true); @@ -1027,9 +1027,9 @@ SELECT * FROM cypher('issue_1043', $$ CREATE (n)-[:KNOWS {n:'hello'}]->({n:'hell SELECT * FROM cypher('issue_1043', $$ MATCH (x)<-[y *]-(),({n:y[0].n}) RETURN x $$) as (a agtype); a ---------------------------------------------------------------------------- - {"id": 281474976710658, "label": "", "properties": {"n": "hello"}}::vertex {"id": 281474976710658, "label": "", "properties": {"n": "hello"}}::vertex {"id": 281474976710660, "label": "", "properties": {"n": "hello"}}::vertex + {"id": 281474976710658, "label": "", "properties": {"n": "hello"}}::vertex {"id": 281474976710660, "label": "", "properties": {"n": "hello"}}::vertex (4 rows) diff --git a/regress/expected/expr.out b/regress/expected/expr.out index 4a332a335..23a3af28a 100644 --- a/regress/expected/expr.out +++ b/regress/expected/expr.out @@ -8354,11 +8354,11 @@ $$) AS (vle_array agtype); vle_array -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] - [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] (6 rows) SELECT * FROM cypher('expr', $$ @@ -8368,8 +8368,8 @@ SELECT * FROM cypher('expr', $$ $$) AS (vle_array agtype); vle_array -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] (2 rows) SELECT * FROM cypher('expr', $$ @@ -8388,12 +8388,12 @@ SELECT * FROM cypher('expr', $$ $$) AS (vle_array agtype); vle_array -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] - [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] (6 rows) -- head @@ -8403,11 +8403,11 @@ SELECT * FROM cypher('expr', $$ $$) AS (head agtype); head --------------------------------------------------------------------------------------------------------------------------- - {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge - {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge (6 rows) @@ -8418,12 +8418,12 @@ SELECT * FROM cypher('expr', $$ $$) AS (head agtype); head -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] - [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] (6 rows) SELECT * FROM cypher('expr', $$ @@ -8433,12 +8433,12 @@ SELECT * FROM cypher('expr', $$ $$) AS (head agtype); head -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] - [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] (6 rows) SELECT * FROM cypher('expr', $$ @@ -8459,11 +8459,11 @@ $$) AS (head agtype); head --------------------------------------------------------------------------------------------------------------------------- {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge - {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge + {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge - {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge + {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge (6 rows) SELECT * FROM cypher('expr', $$ @@ -8473,9 +8473,9 @@ SELECT * FROM cypher('expr', $$ $$) AS (head agtype); head ----------------------------------------------------------------------------------------------------------------------------- - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] (4 rows) @@ -8486,12 +8486,12 @@ SELECT * FROM cypher('expr', $$ $$) AS (head agtype); head -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] - [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] (6 rows) SELECT * FROM cypher('expr', $$ @@ -8522,11 +8522,11 @@ $$) AS (head agtype); head -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] - [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] (6 rows) SELECT * FROM cypher('expr', $$ @@ -8537,11 +8537,11 @@ $$) AS (head agtype); head -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] - [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] (6 rows) SELECT * FROM cypher('expr', $$ @@ -8552,11 +8552,11 @@ $$) AS (head agtype); head -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] - [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] (6 rows) -- reverse @@ -8567,11 +8567,11 @@ $$) as (u agtype); u -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] (6 rows) SELECT * FROM cypher('expr', $$ @@ -8581,12 +8581,12 @@ SELECT * FROM cypher('expr', $$ $$) as (u agtype); u -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge, {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] (6 rows) -- IN operator @@ -8611,8 +8611,8 @@ $$) AS (a agtype); a -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge] - [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] [{"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] + [{"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge, {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge] (3 rows) SELECT * FROM cypher('expr', $$ diff --git a/sql/agtype_typecast.sql b/sql/agtype_typecast.sql index c29c0a657..abca5e518 100644 --- a/sql/agtype_typecast.sql +++ b/sql/agtype_typecast.sql @@ -70,10 +70,14 @@ PARALLEL SAFE AS 'MODULE_PATHNAME'; -- original VLE function definition +-- S4: emit start_id/end_id as scalar columns to enable transformer rewrite +-- of terminal-edge quals as integer equalities (see PERF_VLE_TERMINAL_QUAL_PLAN). CREATE FUNCTION ag_catalog.age_vle(IN agtype, IN agtype, IN agtype, IN agtype, IN agtype, IN agtype, IN agtype, - OUT edges agtype) - RETURNS SETOF agtype + OUT edges agtype, + OUT start_id graphid, + OUT end_id graphid) + RETURNS SETOF record LANGUAGE C STABLE CALLED ON NULL INPUT @@ -84,8 +88,10 @@ AS 'MODULE_PATHNAME'; -- caching mechanism to coexist with the previous VLE version. CREATE FUNCTION ag_catalog.age_vle(IN agtype, IN agtype, IN agtype, IN agtype, IN agtype, IN agtype, IN agtype, IN agtype, - OUT edges agtype) - RETURNS SETOF agtype + OUT edges agtype, + OUT start_id graphid, + OUT end_id graphid) + RETURNS SETOF record LANGUAGE C STABLE CALLED ON NULL INPUT @@ -100,15 +106,6 @@ CREATE FUNCTION ag_catalog.age_build_vle_match_edge(agtype, agtype) PARALLEL SAFE AS 'MODULE_PATHNAME'; --- function to match a terminal vle edge -CREATE FUNCTION ag_catalog.age_match_vle_terminal_edge(variadic "any") - RETURNS boolean - LANGUAGE C - STABLE -CALLED ON NULL INPUT -PARALLEL SAFE -AS 'MODULE_PATHNAME'; - -- function to create an AGTV_PATH from a VLE_path_container CREATE FUNCTION ag_catalog.age_materialize_vle_path(agtype) RETURNS agtype @@ -135,14 +132,6 @@ RETURNS NULL ON NULL INPUT PARALLEL SAFE AS 'MODULE_PATHNAME'; -CREATE FUNCTION ag_catalog.age_match_two_vle_edges(agtype, agtype) - RETURNS boolean - LANGUAGE C - STABLE -RETURNS NULL ON NULL INPUT -PARALLEL SAFE -AS 'MODULE_PATHNAME'; - -- list functions CREATE FUNCTION ag_catalog.age_keys(agtype) RETURNS agtype diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 3083c52e1..015c0719e 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -3982,10 +3982,6 @@ static List *make_join_condition_for_edge(cypher_parsestate *cpstate, { Node *left_id = NULL; Node *right_id = NULL; - String *ag_catalog = makeString("ag_catalog"); - String *func_name; - List *qualified_func_name; - List *args = NIL; List *quals = NIL; /* @@ -3998,56 +3994,107 @@ static List *make_join_condition_for_edge(cypher_parsestate *cpstate, } /* - * If the previous node and the next node are in the join tree, we need - * to create the age_match_vle_terminal_edge to compare the vle returned - * results against the two nodes. + * S5: if the previous and next nodes are both in the join tree, + * emit two graphid equality A_Exprs: + * .start_id = prev_node.id + * .end_id = next_node.id + * This replaces the historical per-row + * age_match_vle_terminal_edge(prev.id, next.id, edges) + * function call with plain integer (int8) equality quals on the + * SRF's S4 output columns. The planner can now drive the join + * directly on these keys (HashJoin hash keys, NestLoop index + * conditions where indexed). */ if (prev_node->in_join_tree) { - func_name = makeString("age_match_vle_terminal_edge"); - qualified_func_name = list_make2(ag_catalog, func_name); + ColumnRef *cr_start; + ColumnRef *cr_end; + A_Expr *eq_start; + A_Expr *eq_end; /* - * Get the vertex's id and pass to the function. Pass in NULL - * otherwise. + * Production-build runtime guard. Asserts compile out in + * non-debug builds, so a NULL vle_alias would otherwise reach + * makeString() and crash the backend during ColumnRef build. */ - left_id = (Node *)make_qual(cpstate, prev_node, "id"); + if (entity->vle_alias == NULL) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("VLE edge entity is missing its alias; cannot emit terminal-edge join qual"))); + } + Assert(entity->vle_alias != NULL); + + cr_start = makeNode(ColumnRef); + cr_start->fields = list_make2(makeString(entity->vle_alias), + makeString("start_id")); + cr_start->location = -1; + + cr_end = makeNode(ColumnRef); + cr_end->fields = list_make2(makeString(entity->vle_alias), + makeString("end_id")); + cr_end->location = -1; + + left_id = (Node *)make_qual(cpstate, prev_node, "id"); right_id = (Node *)make_qual(cpstate, next_node, "id"); - /* create the argument list */ - args = list_make3(left_id, right_id, entity->expr); + eq_start = makeSimpleA_Expr(AEXPR_OP, "=", + (Node *)cr_start, left_id, -1); + eq_end = makeSimpleA_Expr(AEXPR_OP, "=", + (Node *)cr_end, right_id, -1); - /* add to quals */ - quals = lappend(quals, makeFuncCall(qualified_func_name, args, - COERCE_EXPLICIT_CALL, -1)); + quals = lappend(quals, eq_start); + quals = lappend(quals, eq_end); } /* - * When the previous node is not in the join tree, but there is a vle - * edge before that join, then we need to compare this vle's start node - * against the previous vle's end node. No need to check the next edge, - * because that would be redundant. + * S6: when the previous node is not in the join tree but there is + * a vle edge before that join, emit a single graphid equality + * connecting the two VLE SRFs: + * + * prev_vle.end_id = this_vle.start_id + * + * This replaces the per-row age_match_two_vle_edges(prev, this) + * function call with a plain int8 equality on the S4 scalar + * output columns of both age_vle SRFs. No detoasting of either + * VLE_path_container is needed. */ if (!prev_node->in_join_tree && prev_edge != NULL && prev_edge->type == ENT_VLE_EDGE) { - List *qualified_name; - String *match_qual; - FuncCall *fc; + ColumnRef *cr_prev_end; + ColumnRef *cr_this_start; + A_Expr *eq_chain; - match_qual = makeString("age_match_two_vle_edges"); + /* + * Production-build runtime guard for both VLE aliases; see + * note above on entity->vle_alias. + */ + if (prev_edge->vle_alias == NULL || entity->vle_alias == NULL) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("VLE edge entity is missing its alias; cannot emit two-VLE-edge join qual"))); + } + Assert(prev_edge->vle_alias != NULL); + Assert(entity->vle_alias != NULL); - /* make the qualified function name */ - qualified_name = list_make2(ag_catalog, match_qual); + cr_prev_end = makeNode(ColumnRef); + cr_prev_end->fields = list_make2(makeString(prev_edge->vle_alias), + makeString("end_id")); + cr_prev_end->location = -1; - /* make the args */ - args = list_make2(prev_edge->expr, entity->expr); + cr_this_start = makeNode(ColumnRef); + cr_this_start->fields = list_make2(makeString(entity->vle_alias), + makeString("start_id")); + cr_this_start->location = -1; - /* create the function call */ - fc = makeFuncCall(qualified_name, args, COERCE_EXPLICIT_CALL, -1); + eq_chain = makeSimpleA_Expr(AEXPR_OP, "=", + (Node *)cr_prev_end, + (Node *)cr_this_start, -1); - quals = lappend(quals, fc); + quals = lappend(quals, eq_chain); } return quals; @@ -4898,6 +4945,12 @@ static transform_entity *transform_VLE_edge_entity(cypher_parsestate *cpstate, vle_entity = make_transform_entity(cpstate, ENT_VLE_EDGE, (Node *)rel, (Expr *)var); + /* + * S5: stash the auto-generated alias name so make_join_condition_for_edge + * can build ColumnRefs for the SRF's start_id/end_id output columns. + */ + vle_entity->vle_alias = alias->aliasname; + /* return the vle entity */ return vle_entity; } diff --git a/src/backend/parser/cypher_transform_entity.c b/src/backend/parser/cypher_transform_entity.c index 1b2ab0edd..13f733608 100644 --- a/src/backend/parser/cypher_transform_entity.c +++ b/src/backend/parser/cypher_transform_entity.c @@ -52,6 +52,7 @@ transform_entity *make_transform_entity(cypher_parsestate *cpstate, entity->declared_in_current_clause = true; entity->expr = expr; entity->in_join_tree = expr != NULL; + entity->vle_alias = NULL; return entity; } diff --git a/src/backend/utils/adt/age_vle.c b/src/backend/utils/adt/age_vle.c index 22c268cdf..c312a62ee 100644 --- a/src/backend/utils/adt/age_vle.c +++ b/src/backend/utils/adt/age_vle.c @@ -140,6 +140,36 @@ typedef struct VLE_local_context * structure is set up to contains a BINARY container that can be accessed by * functions that need to process the path. */ +/* + * Layout (offsets, with int64 alignment): + * + * 0: vl_len_[4] varlena length header (int32 + pad) + * 4: header AGT_FBINARY | AGT_FBINARY_TYPE_VLE_PATH + * 8: graph_oid source graph oid + * 12: (4 bytes pad) int64 alignment + * 16: graphid_array_size number of graphids in the path + * 24: container_size_bytes total bytes of this container + * 32: start_vid redundant cache of graphid_array[0] + * 40: end_vid redundant cache of + * graphid_array[graphid_array_size - 1] + * 48: graphid_array_data flexible array start + * + * start_vid / end_vid are populated whenever the container is built and let + * downstream consumers (the age_vle SRF's start_id/end_id output columns) + * read the join endpoints without traversing the (potentially toasted) + * variadic payload. + * + * Persistence note: VLE_path_container is a transient SRF output. It is + * consumed within the same query that produces it (by the planner-emitted + * endpoint equalities and by age_materialize_vle_path / _vle_edges) and is + * never written back to disk. Because no on-disk instance of this layout + * can exist, adding fields to the struct does not require a binary + * version bump or a backward-compatible decoder. If a future change ever + * makes a VLE container persistable (e.g. by allowing it to be returned + * directly as agtype and stored in a column), the AGT_FBINARY_TYPE_VLE_PATH + * tag must be versioned and the readers (GET_GRAPHID_ARRAY_FROM_CONTAINER + * etc.) must branch on the version. + */ typedef struct VLE_path_container { char vl_len_[4]; /* Do not touch this field! */ @@ -147,6 +177,8 @@ typedef struct VLE_path_container uint32 graph_oid; int64 graphid_array_size; int64 container_size_bytes; + graphid start_vid; + graphid end_vid; graphid graphid_array_data; } VLE_path_container; @@ -1421,9 +1453,11 @@ static VLE_path_container *create_VLE_path_container(int64 path_size) * One for both the header and graph oid (they are both 32 bits). * One for the size of the graphid_array_size. * One for the container_size_bytes. + * One for start_vid (Stage 1: inline endpoint cache). + * One for end_vid (Stage 1: inline endpoint cache). * */ - container_size_bytes = sizeof(graphid) * (path_size + 4); + container_size_bytes = sizeof(graphid) * (path_size + 6); /* allocate the container */ vpc = palloc0(container_size_bytes); @@ -1533,6 +1567,13 @@ static VLE_path_container *build_VLE_path_container(VLE_local_context *vlelctx) graphid_array[index+1] = vid; } + /* + * Stage 1: cache endpoints in the fixed header so the join qual can read + * them without touching the (possibly toasted) graphid array. + */ + vpc->start_vid = graphid_array[0]; + vpc->end_vid = graphid_array[vpc->graphid_array_size - 1]; + /* return the container */ return vpc; } @@ -1569,6 +1610,12 @@ static VLE_path_container *build_VLE_zero_container(VLE_local_context *vlelctx) vid = vlelctx->vsid; graphid_array[0] = vid; + /* + * Stage 1: zero-edge container; start and end are both the start vertex. + */ + vpc->start_vid = vid; + vpc->end_vid = vid; + return vpc; } @@ -1788,6 +1835,26 @@ Datum age_vle(PG_FUNCTION_ARGS) /* create a function context for cross-call persistence */ funcctx = SRF_FIRSTCALL_INIT(); + /* + * S4: capture the result tuple descriptor. age_vle now emits a + * composite (edges, start_id, end_id) row, so we need a blessed + * TupleDesc that survives across SRF calls. + */ + { + TupleDesc tupdesc; + MemoryContext tdesc_oldctx; + + tdesc_oldctx = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("age_vle: function returning record called in context that cannot accept type record"))); + } + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + MemoryContextSwitchTo(tdesc_oldctx); + } + /* build the local vle context */ vlelctx = build_local_vle_context(fcinfo, funcctx); @@ -1920,8 +1987,24 @@ Datum age_vle(PG_FUNCTION_ARGS) vpc = build_VLE_zero_container(vlelctx); } - /* return the result and signal that the function is not yet done */ - SRF_RETURN_NEXT(funcctx, PointerGetDatum(vpc)); + /* + * S4: emit a composite (edges, start_id, end_id) row. The + * scalar endpoint columns let the cypher transformer (S5) + * rewrite terminal-edge quals as integer equalities, removing + * the per-row age_match_vle_terminal_edge function call. + */ + { + Datum values[3]; + bool nulls[3] = {false, false, false}; + HeapTuple tuple; + + values[0] = PointerGetDatum(vpc); + values[1] = GRAPHID_GET_DATUM(vpc->start_vid); + values[2] = GRAPHID_GET_DATUM(vpc->end_vid); + + tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); + } } /* otherwise, we are done and we need to cleanup and signal done */ else @@ -1992,66 +2075,31 @@ agtype_value *agtv_materialize_vle_path(agtype *agt_arg_vpc) return agtv_path; } -/* PG function to match 2 VLE edges */ +/* + * age_match_two_vle_edges and age_match_vle_terminal_edge are retained as + * stub C symbols only. The cypher transformer no longer emits calls to + * either function: terminal-edge match quals are now plain graphid + * equalities on the age_vle SRF's start_id/end_id output columns + * (Stages S4/S5/S6 of the VLE terminal-qual rewrite). + * + * The corresponding SQL declarations have been removed from fresh + * installs (sql/agtype_typecast.sql) and are DROP'd by the upgrade + * script (age--1.7.0--y.y.y.sql). These C entry points exist solely so + * the upgrade-test machinery, which loads an older "1.7.0_initial" SQL + * snapshot against the current age.so, can resolve the symbols before + * the immediate ALTER EXTENSION UPDATE drops them. They should never + * be reachable from any committed SQL path. + */ PG_FUNCTION_INFO_V1(age_match_two_vle_edges); Datum age_match_two_vle_edges(PG_FUNCTION_ARGS) { - agtype *agt_arg_vpc = NULL; - VLE_path_container *left_path = NULL, *right_path = NULL; - graphid *left_array, *right_array; - int left_array_size; - - /* - * If either argument is NULL, return FALSE. This can occur in - * OPTIONAL MATCH (LEFT JOIN) contexts where a preceding clause - * produced no results. - */ - if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) - { - PG_RETURN_BOOL(false); - } - - /* get the VLE_path_container argument */ - agt_arg_vpc = AG_GET_ARG_AGTYPE_P(0); - - if (!AGT_ROOT_IS_BINARY(agt_arg_vpc) || - AGT_ROOT_BINARY_FLAGS(agt_arg_vpc) != AGT_FBINARY_TYPE_VLE_PATH) - { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("argument 1 of age_match_two_vle_edges must be a VLE_Path_Container"))); - } - - /* cast argument as a VLE_Path_Container and extract graphid array */ - left_path = (VLE_path_container *)agt_arg_vpc; - left_array_size = left_path->graphid_array_size; - left_array = GET_GRAPHID_ARRAY_FROM_CONTAINER(left_path); - - PG_FREE_IF_COPY(agt_arg_vpc, 0); - - agt_arg_vpc = AG_GET_ARG_AGTYPE_P(1); - - if (!AGT_ROOT_IS_BINARY(agt_arg_vpc) || - AGT_ROOT_BINARY_FLAGS(agt_arg_vpc) != AGT_FBINARY_TYPE_VLE_PATH) - { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("argument 2 of age_match_two_vle_edges must be a VLE_Path_Container"))); - } - - /* cast argument as a VLE_Path_Container and extract graphid array */ - right_path = (VLE_path_container *)agt_arg_vpc; - right_array = GET_GRAPHID_ARRAY_FROM_CONTAINER(right_path); - - PG_FREE_IF_COPY(agt_arg_vpc, 1); - - if (left_array[left_array_size - 1] != right_array[0]) - { - PG_RETURN_BOOL(false); - } - - PG_RETURN_BOOL(true); + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("age_match_two_vle_edges() is removed; " + "VLE endpoint matching is now handled by the planner via " + "the age_vle SRF's start_id/end_id output columns"))); + PG_RETURN_BOOL(false); } /* @@ -2289,147 +2337,17 @@ Datum age_materialize_vle_path(PG_FUNCTION_ARGS) PG_RETURN_POINTER(agt_materialize_vle_path(agt_arg_vpc)); } -/* - * PG function to take a VLE_path_container and return whether the supplied end - * vertex (target/veid) matches against the last edge in the VLE path. The VLE - * path is encoded in a BINARY container. - */ +/* Stub: see comment on age_match_two_vle_edges above. */ PG_FUNCTION_INFO_V1(age_match_vle_terminal_edge); Datum age_match_vle_terminal_edge(PG_FUNCTION_ARGS) { - VLE_path_container *vpc = NULL; - agtype *agt_arg_vsid = NULL; - agtype *agt_arg_veid = NULL; - agtype *agt_arg_path = NULL; - agtype_value *agtv_temp = NULL; - graphid vsid = 0; - graphid veid = 0; - graphid *gida = NULL; - int gidasize = 0; - Oid type0, type1; - - /* check argument count */ - if (PG_NARGS() != 3) - { - ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("age_match_vle_terminal_edge() invalid number of arguments"))); - } - - /* - * If any argument is NULL, return FALSE. This can occur when this - * function is used as a join qual in an OPTIONAL MATCH (LEFT JOIN) - * where a preceding OPTIONAL MATCH produced no results. Returning - * FALSE allows PostgreSQL to produce the correct NULL-extended rows. - */ - if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) - { - PG_RETURN_BOOL(false); - } - - /* get the vpc */ - agt_arg_path = DATUM_GET_AGTYPE_P(PG_GETARG_DATUM(2)); - - /* if the vpc is an agtype NULL, return FALSE */ - if (is_agtype_null(agt_arg_path)) - { - PG_RETURN_BOOL(false); - } - - /* - * The vpc (path) must be a binary container and the type of the object in - * the container must be an AGT_FBINARY_TYPE_VLE_PATH. - */ - Assert(AGT_ROOT_IS_BINARY(agt_arg_path)); - Assert(AGT_ROOT_BINARY_FLAGS(agt_arg_path) == AGT_FBINARY_TYPE_VLE_PATH); - - /* get the container */ - vpc = (VLE_path_container *)agt_arg_path; - - /* get the graphid array from the container */ - gida = GET_GRAPHID_ARRAY_FROM_CONTAINER(vpc); - - /* get the gida array size */ - gidasize = vpc->graphid_array_size; - - /* verify the minimum size is 3 or 1 */ - Assert(gidasize >= 3 || gidasize == 1); - - /* - * Get argument types directly instead of using extract_variadic_args. - * This avoids the expensive exprType/get_call_expr_argtype overhead - * on every call. Cache the types in fn_extra on first invocation. - */ - if (fcinfo->flinfo->fn_extra == NULL) - { - Oid *cached_types = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt, - 2 * sizeof(Oid)); - cached_types[0] = get_fn_expr_argtype(fcinfo->flinfo, 0); - cached_types[1] = get_fn_expr_argtype(fcinfo->flinfo, 1); - fcinfo->flinfo->fn_extra = cached_types; - } - type0 = ((Oid *)fcinfo->flinfo->fn_extra)[0]; - type1 = ((Oid *)fcinfo->flinfo->fn_extra)[1]; - - /* get the vsid */ - if (type0 == AGTYPEOID) - { - agt_arg_vsid = DATUM_GET_AGTYPE_P(PG_GETARG_DATUM(0)); - - if (!is_agtype_null(agt_arg_vsid)) - { - agtv_temp = - get_ith_agtype_value_from_container(&agt_arg_vsid->root, 0); - - Assert(agtv_temp->type == AGTV_INTEGER); - vsid = agtv_temp->val.int_value; - } - else - { - PG_RETURN_BOOL(false); - } - } - else if (type0 == GRAPHIDOID) - { - vsid = DATUM_GET_GRAPHID(PG_GETARG_DATUM(0)); - } - else - { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("match_vle_terminal_edge() argument 1 must be an agtype integer or a graphid"))); - } - - /* get the veid */ - if (type1 == AGTYPEOID) - { - agt_arg_veid = DATUM_GET_AGTYPE_P(PG_GETARG_DATUM(1)); - - if (!is_agtype_null(agt_arg_veid)) - { - agtv_temp = get_ith_agtype_value_from_container(&agt_arg_veid->root, - 0); - Assert(agtv_temp->type == AGTV_INTEGER); - veid = agtv_temp->val.int_value; - } - else - { - PG_RETURN_BOOL(false); - } - } - else if (type1 == GRAPHIDOID) - { - veid = DATUM_GET_GRAPHID(PG_GETARG_DATUM(1)); - } - else - { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("match_vle_terminal_edge() argument 2 must be an agtype integer or a graphid"))); - } - - /* compare the path beginning or end points */ - PG_RETURN_BOOL(gida[0] == vsid && veid == gida[gidasize - 1]); + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("age_match_vle_terminal_edge() is removed; " + "VLE endpoint matching is now handled by the planner via " + "the age_vle SRF's start_id/end_id output columns"))); + PG_RETURN_BOOL(false); } /* PG helper function to build an agtype (Datum) edge for matching */ diff --git a/src/include/parser/cypher_transform_entity.h b/src/include/parser/cypher_transform_entity.h index d24d4c372..d0492210f 100644 --- a/src/include/parser/cypher_transform_entity.h +++ b/src/include/parser/cypher_transform_entity.h @@ -75,6 +75,17 @@ typedef struct * declared by itself or a previous clause. */ bool declared_in_current_clause; + + /* + * S5: for ENT_VLE_EDGE entities, the auto-generated alias of the + * RangeFunction added to the FROM clause for the age_vle SRF. + * Used to build ColumnRefs for the SRF's start_id/end_id output + * columns when emitting the terminal-edge join quals as plain + * graphid equality A_Exprs (replacing the per-row + * age_match_vle_terminal_edge function call). NULL for non-VLE + * entities. + */ + char *vle_alias; /* The parse data structure that we transformed */ union { From 5ef7d6d55cbf73b359604ff816a8f86580f94b7f Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Mon, 1 Jun 2026 23:02:07 +0500 Subject: [PATCH 069/100] Add vertex and edge composite types with direct field access optimization (#2303) - Introduce vertex and edge as pg composite types vertex: (id, label, properties) edge: (id, label, start_id, end_id, properties) - end_id precedes start_id intentionally, it matches the agtype edge's canonical key order (keys sorted by length, end_id=6 < start_id=8), keeping the composite positionally consistent with the agtype representation. - Property access (a.name) now directly uses a.properties for agtype_access_operator instead of rebuilding via _agtype_build_vertex/edge - Optimize accessor functions (id, properties, label, type, start_id, end_id) to use direct FieldSelect on composite types instead of agtype functions - Add casts: vertex/edge to agtype, vertex/edge to json/jsonb - Add eq and not eq ops for vertex/edge composite types. - Fix label_name specific routine to use cache instead of ag_label scan - Write/update clauses have executors strictly tied to agtype, due to which the variables after any write/update clause are carried forward as agtype. - Allows users to completely skip agtype build functions and return vertex/edge for pure read queries. - Change _label_name to return agtype since record comparisons are not allowed with cstring. Consequently, _agtype_build_vertex/edge now accept agtype as label. - Fix MERGE clause type mismatch when accessing properties from previous MATCH clauses by wrapping columns with agtype_volatile_wrapper before namespace lookup. - Update expression index in pgvector.sql, since now it uses raw properties column instead of _agtype_build_vertex/edge. - Add regression tests Assisted-by AI --- age--1.7.0--y.y.y.sql | 217 +++++ regress/expected/agtype.out | 92 +-- regress/expected/cypher_match.out | 6 +- regress/expected/cypher_with.out | 312 +++++++ regress/expected/expr.out | 1142 +++++++++++++++++++++++++- regress/expected/pgvector.out | 4 +- regress/sql/agtype.sql | 92 +-- regress/sql/cypher_with.sql | 143 ++++ regress/sql/expr.sql | 400 +++++++++ regress/sql/pgvector.sql | 4 +- sql/age_main.sql | 7 - sql/age_scalar.sql | 9 + sql/agtype_graphid.sql | 180 +++- src/backend/catalog/ag_catalog.c | 1 + src/backend/catalog/ag_label.c | 33 +- src/backend/executor/cypher_create.c | 4 +- src/backend/executor/cypher_merge.c | 4 +- src/backend/executor/cypher_set.c | 5 +- src/backend/parser/cypher_clause.c | 389 ++++++--- src/backend/parser/cypher_expr.c | 390 ++++++++- src/backend/utils/adt/agtype.c | 562 +++++++++++-- src/include/catalog/ag_label.h | 2 +- src/include/parser/cypher_clause.h | 9 + src/include/parser/cypher_expr.h | 5 + src/include/utils/agtype.h | 7 + 25 files changed, 3643 insertions(+), 376 deletions(-) diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index f7ef80662..4e718d426 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -536,3 +536,220 @@ STABLE CALLED ON NULL INPUT PARALLEL UNSAFE AS 'MODULE_PATHNAME'; + +-- +-- Composite types for vertex and edge +-- +CREATE TYPE ag_catalog.vertex AS ( + id graphid, + label agtype, + properties agtype +); + +CREATE TYPE ag_catalog.edge AS ( + id graphid, + label agtype, + end_id graphid, + start_id graphid, + properties agtype +); + +-- +-- vertex/edge to agtype cast functions +-- +CREATE FUNCTION ag_catalog.vertex_to_agtype(vertex) + RETURNS agtype + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +CREATE FUNCTION ag_catalog.edge_to_agtype(edge) + RETURNS agtype + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +-- +-- Implicit casts from vertex/edge to agtype +-- +CREATE CAST (vertex AS agtype) + WITH FUNCTION ag_catalog.vertex_to_agtype(vertex) +AS IMPLICIT; + +CREATE CAST (edge AS agtype) + WITH FUNCTION ag_catalog.edge_to_agtype(edge) +AS IMPLICIT; + +CREATE FUNCTION ag_catalog.vertex_to_json(vertex) + RETURNS json + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +CREATE FUNCTION ag_catalog.edge_to_json(edge) + RETURNS json + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +CREATE CAST (vertex AS json) + WITH FUNCTION ag_catalog.vertex_to_json(vertex); + +CREATE CAST (edge AS json) + WITH FUNCTION ag_catalog.edge_to_json(edge); + +CREATE FUNCTION ag_catalog.vertex_to_jsonb(vertex) + RETURNS jsonb + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +CREATE FUNCTION ag_catalog.edge_to_jsonb(edge) + RETURNS jsonb + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +CREATE CAST (vertex AS jsonb) + WITH FUNCTION ag_catalog.vertex_to_jsonb(vertex); + +CREATE CAST (edge AS jsonb) + WITH FUNCTION ag_catalog.edge_to_jsonb(edge); + +-- +-- Equality operators for vertex and edge (compare by id) +-- +CREATE FUNCTION ag_catalog.vertex_eq(vertex, vertex) + RETURNS boolean + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS $$ SELECT $1.id = $2.id $$; + +CREATE OPERATOR = ( + FUNCTION = ag_catalog.vertex_eq, + LEFTARG = vertex, + RIGHTARG = vertex, + COMMUTATOR = =, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel +); + +CREATE FUNCTION ag_catalog.vertex_ne(vertex, vertex) + RETURNS boolean + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS $$ SELECT $1.id <> $2.id $$; + +CREATE OPERATOR <> ( + FUNCTION = ag_catalog.vertex_ne, + LEFTARG = vertex, + RIGHTARG = vertex, + COMMUTATOR = <>, + NEGATOR = =, + RESTRICT = neqsel, + JOIN = neqjoinsel +); + +CREATE FUNCTION ag_catalog.edge_eq(edge, edge) + RETURNS boolean + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS $$ SELECT $1.id = $2.id $$; + +CREATE OPERATOR = ( + FUNCTION = ag_catalog.edge_eq, + LEFTARG = edge, + RIGHTARG = edge, + COMMUTATOR = =, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel +); + +CREATE FUNCTION ag_catalog.edge_ne(edge, edge) + RETURNS boolean + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS $$ SELECT $1.id <> $2.id $$; + +CREATE OPERATOR <> ( + FUNCTION = ag_catalog.edge_ne, + LEFTARG = edge, + RIGHTARG = edge, + COMMUTATOR = <>, + NEGATOR = =, + RESTRICT = neqsel, + JOIN = neqjoinsel +); + +-- +-- Drop and recreate _label_name with new return type (cstring -> agtype) +-- +DROP FUNCTION IF EXISTS ag_catalog._label_name(oid, graphid); + +CREATE FUNCTION ag_catalog._label_name(graph_oid oid, graphid) + RETURNS agtype + LANGUAGE c + IMMUTABLE +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +-- +-- Drop and recreate _agtype_build_vertex with new signature (cstring -> agtype for label) +-- +DROP FUNCTION IF EXISTS ag_catalog._agtype_build_vertex(graphid, cstring, agtype); + +CREATE FUNCTION ag_catalog._agtype_build_vertex(graphid, agtype, agtype) + RETURNS agtype + LANGUAGE c + IMMUTABLE +CALLED ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +-- +-- Drop and recreate _agtype_build_edge with new signature (cstring -> agtype for label) +-- +DROP FUNCTION IF EXISTS ag_catalog._agtype_build_edge(graphid, graphid, graphid, cstring, agtype); + +CREATE FUNCTION ag_catalog._agtype_build_edge(graphid, graphid, graphid, + agtype, agtype) + RETURNS agtype + LANGUAGE c + IMMUTABLE +CALLED ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +-- +-- Helper function for optimized startNode/endNode +-- +CREATE FUNCTION ag_catalog._get_vertex_by_graphid(text, graphid) + RETURNS agtype + LANGUAGE c + STABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + diff --git a/regress/expected/agtype.out b/regress/expected/agtype.out index cb97b3bfa..b2f3b83e1 100644 --- a/regress/expected/agtype.out +++ b/regress/expected/agtype.out @@ -3291,39 +3291,39 @@ NOTICE: graph "agtype_null_duplicate_test" has been dropped -- Vertex -- --Basic Vertex Creation -SELECT _agtype_build_vertex('1'::graphid, $$label_name$$, agtype_build_map()); +SELECT _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map()); _agtype_build_vertex ------------------------------------------------------------ {"id": 1, "label": "label_name", "properties": {}}::vertex (1 row) -SELECT _agtype_build_vertex('1'::graphid, $$label$$, agtype_build_map('id', 2)); +SELECT _agtype_build_vertex('1'::graphid, '"label"', agtype_build_map('id', 2)); _agtype_build_vertex -------------------------------------------------------------- {"id": 1, "label": "label", "properties": {"id": 2}}::vertex (1 row) --Null properties -SELECT _agtype_build_vertex('1'::graphid, $$label_name$$, NULL); +SELECT _agtype_build_vertex('1'::graphid, '"label_name"', NULL); _agtype_build_vertex ------------------------------------------------------------ {"id": 1, "label": "label_name", "properties": {}}::vertex (1 row) --Test access operator -SELECT agtype_access_operator(_agtype_build_vertex('1'::graphid, $$label$$, +SELECT agtype_access_operator(_agtype_build_vertex('1'::graphid, '"label"', agtype_build_map('id', 2)), '"id"'); agtype_access_operator ------------------------ 2 (1 row) -SELECT _agtype_build_vertex('1'::graphid, $$label$$, agtype_build_list()); +SELECT _agtype_build_vertex('1'::graphid, '"label"', agtype_build_list()); ERROR: _agtype_build_vertex() properties argument must be an object --Vertex in a map SELECT agtype_build_map( 'vertex', - _agtype_build_vertex('1'::graphid, $$label_name$$, agtype_build_map())); + _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map())); agtype_build_map ------------------------------------------------------------------------ {"vertex": {"id": 1, "label": "label_name", "properties": {}}::vertex} @@ -3331,9 +3331,9 @@ SELECT agtype_build_map( SELECT agtype_access_operator( agtype_build_map( - 'vertex', _agtype_build_vertex('1'::graphid, $$label_name$$, + 'vertex', _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map('key', 'value')), - 'other_vertex', _agtype_build_vertex('1'::graphid, $$label_name$$, + 'other_vertex', _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map('key', 'other_value'))), '"vertex"'); agtype_access_operator @@ -3343,8 +3343,8 @@ SELECT agtype_access_operator( --Vertex in a list SELECT agtype_build_list( - _agtype_build_vertex('1'::graphid, $$label_name$$, agtype_build_map()), - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map())); + _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map())); agtype_build_list -------------------------------------------------------------------------------------------------------------------------- [{"id": 1, "label": "label_name", "properties": {}}::vertex, {"id": 2, "label": "label_name", "properties": {}}::vertex] @@ -3352,9 +3352,9 @@ SELECT agtype_build_list( SELECT agtype_access_operator( agtype_build_list( - _agtype_build_vertex('1'::graphid, $$label_name$$, + _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map('id', 3)), - _agtype_build_vertex('2'::graphid, $$label_name$$, + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map('id', 4))), '0'); agtype_access_operator ------------------------------------------------------------------- @@ -3366,14 +3366,14 @@ SELECT agtype_access_operator( -- --Basic Edge Creation SELECT _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map()); + '"label_name"', agtype_build_map()); _agtype_build_edge -------------------------------------------------------------------------------------- {"id": 1, "label": "label_name", "end_id": 3, "start_id": 2, "properties": {}}::edge (1 row) SELECT _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)); + '"label"', agtype_build_map('id', 2)); _agtype_build_edge ---------------------------------------------------------------------------------------- {"id": 1, "label": "label", "end_id": 3, "start_id": 2, "properties": {"id": 2}}::edge @@ -3381,7 +3381,7 @@ SELECT _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, --Null properties SELECT _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, NULL); + '"label_name"', NULL); _agtype_build_edge -------------------------------------------------------------------------------------- {"id": 1, "label": "label_name", "end_id": 3, "start_id": 2, "properties": {}}::edge @@ -3389,7 +3389,7 @@ SELECT _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, --Test access operator SELECT agtype_access_operator(_agtype_build_edge('1'::graphid, '2'::graphid, - '3'::graphid, $$label$$, agtype_build_map('id', 2)),'"id"'); + '3'::graphid, '"label"', agtype_build_map('id', 2)),'"id"'); agtype_access_operator ------------------------ 2 @@ -3399,7 +3399,7 @@ SELECT agtype_access_operator(_agtype_build_edge('1'::graphid, '2'::graphid, SELECT agtype_build_map( 'edge', _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map())); + '"label_name"', agtype_build_map())); agtype_build_map ------------------------------------------------------------------------------------------------ {"edge": {"id": 1, "label": "label_name", "end_id": 3, "start_id": 2, "properties": {}}::edge} @@ -3408,9 +3408,9 @@ SELECT agtype_build_map( SELECT agtype_access_operator( agtype_build_map( 'edge', _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map('key', 'value')), + '"label_name"', agtype_build_map('key', 'value')), 'other_edge', _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map('key', 'other_value'))), + '"label_name"', agtype_build_map('key', 'other_value'))), '"edge"'); agtype_access_operator ---------------------------------------------------------------------------------------------------- @@ -3420,9 +3420,9 @@ SELECT agtype_access_operator( --Edge in a list SELECT agtype_build_list( _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map()), + '"label_name"', agtype_build_map()), _agtype_build_edge('2'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map())); + '"label_name"', agtype_build_map())); agtype_build_list ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [{"id": 1, "label": "label_name", "end_id": 3, "start_id": 2, "properties": {}}::edge, {"id": 2, "label": "label_name", "end_id": 3, "start_id": 2, "properties": {}}::edge] @@ -3430,9 +3430,9 @@ SELECT agtype_build_list( SELECT agtype_access_operator( agtype_build_list( - _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, $$label_name$$, + _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, '"label_name"', agtype_build_map('id', 3)), - _agtype_build_edge('2'::graphid, '2'::graphid, '3'::graphid, $$label_name$$, + _agtype_build_edge('2'::graphid, '2'::graphid, '3'::graphid, '"label_name"', agtype_build_map('id', 4))), '0'); agtype_access_operator --------------------------------------------------------------------------------------------- @@ -3441,10 +3441,10 @@ SELECT agtype_access_operator( -- Path SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), - _agtype_build_vertex('3'::graphid, $$label_name$$, agtype_build_map()) + '"label"', agtype_build_map('id', 2)), + _agtype_build_vertex('3'::graphid, '"label_name"', agtype_build_map()) ); _agtype_build_path ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ @@ -3453,78 +3453,78 @@ SELECT _agtype_build_path( --All these paths should produce Errors SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)) + '"label"', agtype_build_map('id', 2)) ); ERROR: a path is of the form: [vertex, (edge, vertex)*i] where i >= 0 SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), - _agtype_build_vertex('3'::graphid, $$label_name$$, agtype_build_map()), + '"label"', agtype_build_map('id', 2)), + _agtype_build_vertex('3'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '4'::graphid, '5'::graphid, - $$label$$, agtype_build_map('id', 2)) + '"label"', agtype_build_map('id', 2)) ); ERROR: a path is of the form: [vertex, (edge, vertex)*i] where i >= 0 SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), + '"label"', agtype_build_map('id', 2)), NULL ); ERROR: argument 3 must not be null SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), + '"label"', agtype_build_map('id', 2)), 1 ); ERROR: argument 3 must be an agtype SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), + '"label"', agtype_build_map('id', 2)), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)) + '"label"', agtype_build_map('id', 2)) ); ERROR: paths consist of alternating vertices and edges HINT: argument 3 must be an vertex -- -- id, startid, endid -- -SELECT age_id(_agtype_build_vertex('1'::graphid, $$label_name$$, agtype_build_map())); +SELECT age_id(_agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map())); age_id -------- 1 (1 row) SELECT age_id(_agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map('id', 2))); + '"label_name"', agtype_build_map('id', 2))); age_id -------- 1 (1 row) SELECT age_start_id(_agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map('id', 2))); + '"label_name"', agtype_build_map('id', 2))); age_start_id -------------- 2 (1 row) SELECT age_end_id(_agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map('id', 2))); + '"label_name"', agtype_build_map('id', 2))); age_end_id ------------ 3 (1 row) SELECT age_id(_agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), - _agtype_build_vertex('3'::graphid, $$label$$, agtype_build_map('id', 2)) + '"label"', agtype_build_map('id', 2)), + _agtype_build_vertex('3'::graphid, '"label"', agtype_build_map('id', 2)) )); ERROR: id() argument must be a vertex, an edge or null SELECT age_id(agtype_in('1')); diff --git a/regress/expected/cypher_match.out b/regress/expected/cypher_match.out index e99460e5b..33c728f2a 100644 --- a/regress/expected/cypher_match.out +++ b/regress/expected/cypher_match.out @@ -710,10 +710,10 @@ $$) AS (r0 agtype); --------------------------------------------------------------------------------------------------------------------------- {"id": 1407374883553282, "label": "e1", "end_id": 1125899906842626, "start_id": 1125899906842625, "properties": {}}::edge {"id": 1407374883553281, "label": "e1", "end_id": 1125899906842627, "start_id": 1125899906842626, "properties": {}}::edge - {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge {"id": 1970324836974593, "label": "e2", "end_id": 1688849860263939, "start_id": 1688849860263938, "properties": {}}::edge - {"id": 2533274790395905, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685251, "properties": {}}::edge + {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge {"id": 2533274790395906, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685249, "properties": {}}::edge + {"id": 2533274790395905, "label": "e3", "end_id": 2251799813685250, "start_id": 2251799813685251, "properties": {}}::edge (6 rows) SELECT * FROM cypher('cypher_match', $$ @@ -730,8 +730,8 @@ SELECT * FROM cypher('cypher_match', $$ $$) AS (r0 agtype); r0 --------------------------------------------------------------------------------------------------------------------------- - {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge {"id": 1970324836974593, "label": "e2", "end_id": 1688849860263939, "start_id": 1688849860263938, "properties": {}}::edge + {"id": 1970324836974594, "label": "e2", "end_id": 1688849860263937, "start_id": 1688849860263938, "properties": {}}::edge (2 rows) SELECT * FROM cypher('cypher_match', $$ diff --git a/regress/expected/cypher_with.out b/regress/expected/cypher_with.out index 99ea320a0..8864f026f 100644 --- a/regress/expected/cypher_with.out +++ b/regress/expected/cypher_with.out @@ -357,6 +357,318 @@ NOTICE: graph "graph" has been dropped (1 row) +-- +-- Test accessor optimizations in WITH clause +-- +SELECT * FROM create_graph('with_accessor_opt'); +NOTICE: graph "with_accessor_opt" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('with_accessor_opt', $$ + CREATE (a:Person {name: 'Alice', age: 30})-[r:KNOWS {since: 2020}]->(b:Person {name: 'Bob', age: 25}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- Single with +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH id(n) AS node_id + RETURN node_id +$$) AS (node_id agtype); + QUERY PLAN +------------------------------------------ + Seq Scan on with_accessor_opt."Person" n + Output: (n.id)::agtype +(2 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH properties(n) AS props + RETURN props +$$) AS (props agtype); + QUERY PLAN +------------------------------------------ + Seq Scan on with_accessor_opt."Person" n + Output: n.properties +(2 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + MATCH (n:Person) + WITH label(n) AS lbl + RETURN lbl +$$) AS (lbl agtype); + lbl +---------- + "Person" + "Person" +(2 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n.name AS name, n.age AS age + RETURN name, age +$$) AS (name agtype, age agtype); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------- + Seq Scan on with_accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]), agtype_access_operator(VARIADIC ARRAY[n.properties, '"age"'::agtype]) +(2 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + WITH id(r) AS rid + RETURN rid +$$) AS (rid agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------- + Merge Join + Output: (r.id)::agtype + Merge Cond: (_age_default_alias_1.id = r.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on with_accessor_opt._ag_label_vertex _age_default_alias_1_1 + Output: _age_default_alias_1_1.id + -> Index Only Scan using "Person_pkey" on with_accessor_opt."Person" _age_default_alias_1_2 + Output: _age_default_alias_1_2.id + -> Materialize + Output: r.id, r.end_id + -> Nested Loop + Output: r.id, r.end_id + -> Index Scan using "KNOWS_end_id_idx" on with_accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Append + -> Seq Scan on with_accessor_opt._ag_label_vertex _age_default_alias_0_1 + Output: _age_default_alias_0_1.id + Filter: (_age_default_alias_0_1.id = r.start_id) + -> Index Only Scan using "Person_pkey" on with_accessor_opt."Person" _age_default_alias_0_2 + Output: _age_default_alias_0_2.id + Index Cond: (_age_default_alias_0_2.id = r.start_id) +(22 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n, id(n) AS nid + WITH n.name AS name, nid + RETURN name, nid +$$) AS (name agtype, nid agtype); + QUERY PLAN +-------------------------------------------------------------------------------------------------- + Seq Scan on with_accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]), (n.id)::agtype +(2 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + MATCH (n:Person) + WITH n as m + RETURN m +$$) AS (n vertex); + n +--------------------------------------------------------------------- + (844424930131969,"""Person""","{""age"": 30, ""name"": ""Alice""}") + (844424930131970,"""Person""","{""age"": 25, ""name"": ""Bob""}") +(2 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n + RETURN id(n), n.name +$$) AS (id agtype, name agtype); + QUERY PLAN +-------------------------------------------------------------------------------------------------- + Seq Scan on with_accessor_opt."Person" n + Output: (n.id)::agtype, agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]) +(2 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + MATCH ()-[r:KNOWS]->() + WITH r + RETURN id(r), type(r) +$$) AS (id agtype, typ agtype); + id | typ +------------------+--------- + 1125899906842625 | "KNOWS" +(1 row) + +-- +-- Chained WITH tests +-- +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person)-[r:KNOWS]->(b:Person) + WITH a, b, id(a) AS aid + WITH a.name AS aname, b.name AS bname, aid + RETURN aname, bname, aid +$$) AS (aname agtype, bname agtype, aid agtype); + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Hash Join + Output: agtype_access_operator(VARIADIC ARRAY[a.properties, '"name"'::agtype]), agtype_access_operator(VARIADIC ARRAY[b.properties, '"name"'::agtype]), (a.id)::agtype + Inner Unique: true + Hash Cond: (r.end_id = b.id) + -> Hash Join + Output: a.properties, a.id, r.end_id + Inner Unique: true + Hash Cond: (r.start_id = a.id) + -> Seq Scan on with_accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Hash + Output: a.properties, a.id + -> Seq Scan on with_accessor_opt."Person" a + Output: a.properties, a.id + -> Hash + Output: b.properties, b.id + -> Seq Scan on with_accessor_opt."Person" b + Output: b.properties, b.id +(18 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n, n.age AS age + WHERE age > 20 + RETURN n.name +$$) AS (name agtype); + QUERY PLAN +-------------------------------------------------------------------------------------------------- + Seq Scan on with_accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]) + Filter: (agtype_access_operator(VARIADIC ARRAY[n.properties, '"age"'::agtype]) > '20'::agtype) +(3 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n.name as name, id(n) AS nid + ORDER BY nid + RETURN name +$$) AS (name agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------- + Subquery Scan on _age_default_alias_previous_cypher_clause + Output: _age_default_alias_previous_cypher_clause.name + -> Index Scan using "Person_pkey" on with_accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]), n.id +(4 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH id(n) AS id, count(*) AS cnt + RETURN id, cnt +$$) AS (id agtype, cnt agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Subquery Scan on _age_default_alias_previous_cypher_clause + Output: (_age_default_alias_previous_cypher_clause.id)::agtype, _age_default_alias_previous_cypher_clause.cnt + -> HashAggregate + Output: n.id, (count(*))::agtype + Group Key: n.id + -> Seq Scan on with_accessor_opt."Person" n + Output: n.id, n.properties +(7 rows) + +SELECT * FROM cypher('with_accessor_opt', $$ + MATCH (n:Person) + WITH DISTINCT label(n) AS lbl + RETURN lbl +$$) AS (lbl agtype); + lbl +---------- + "Person" +(1 row) + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n + WITH n, id(n) AS nid + WITH n.name AS name, nid + RETURN name, nid +$$) AS (name agtype, nid agtype); + QUERY PLAN +-------------------------------------------------------------------------------------------------- + Seq Scan on with_accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]), (n.id)::agtype +(2 rows) + +-- MATCH -> WITH accessors -> MATCH -> RETURN +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person) + WITH a, id(a) AS aid + MATCH (a)-[r:KNOWS]->(b) + RETURN aid, b.name +$$) AS (aid agtype, bname agtype); + QUERY PLAN +-------------------------------------------------------------------------------------------------- + Merge Join + Output: (a.id)::agtype, agtype_access_operator(VARIADIC ARRAY[b.properties, '"name"'::agtype]) + Merge Cond: (b.id = r.end_id) + -> Merge Append + Sort Key: b.id + -> Index Scan using _ag_label_vertex_pkey on with_accessor_opt._ag_label_vertex b_1 + Output: b_1.properties, b_1.id + -> Index Scan using "Person_pkey" on with_accessor_opt."Person" b_2 + Output: b_2.properties, b_2.id + -> Sort + Output: a.id, r.end_id + Sort Key: r.end_id + -> Hash Join + Output: a.id, r.end_id + Inner Unique: true + Hash Cond: (r.start_id = a.id) + -> Seq Scan on with_accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Hash + Output: a.id + -> Seq Scan on with_accessor_opt."Person" a + Output: a.id +(22 rows) + +-- WITH + UNWIND with accessors +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH collect(id(n)) AS ids + UNWIND ids AS nid + RETURN nid +$$) AS (nid agtype); + QUERY PLAN +--------------------------------------------------------------- + Subquery Scan on _age_default_alias_previous_cypher_clause + Output: _age_default_alias_previous_cypher_clause.nid + -> ProjectSet + Output: NULL::agtype, age_unnest((age_collect(n.id))) + -> Aggregate + Output: age_collect(n.id) + -> Seq Scan on with_accessor_opt."Person" n + Output: n.id, n.properties +(8 rows) + +-- Clean up +SELECT drop_graph('with_accessor_opt', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table with_accessor_opt._ag_label_vertex +drop cascades to table with_accessor_opt._ag_label_edge +drop cascades to table with_accessor_opt."Person" +drop cascades to table with_accessor_opt."KNOWS" +NOTICE: graph "with_accessor_opt" has been dropped + drop_graph +------------ + +(1 row) + -- -- End of test -- diff --git a/regress/expected/expr.out b/regress/expected/expr.out index 23a3af28a..3c80bcae5 100644 --- a/regress/expected/expr.out +++ b/regress/expected/expr.out @@ -1544,12 +1544,16 @@ SELECT * FROM cypher('type_coercion', $$ MATCH (v) RETURN v $$) AS (i bigint); -ERROR: cannot cast agtype vertex to type int +ERROR: cannot cast type vertex to bigint for column "i" +LINE 1: SELECT * FROM cypher('type_coercion', $$ + ^ SELECT * FROM cypher('type_coercion', $$ MATCH ()-[e]-() RETURN e $$) AS (i bigint); -ERROR: cannot cast agtype edge to type int +ERROR: cannot cast type edge to bigint for column "i" +LINE 1: SELECT * FROM cypher('type_coercion', $$ + ^ SELECT * FROM cypher('type_coercion', $$ MATCH p=()-[]-() RETURN p @@ -2762,6 +2766,8 @@ SELECT * FROM cypher('expr', $$ MATCH (v) RETURN start_id(v) $$) AS (start_id agtype); ERROR: start_id() argument must be an edge or null +LINE 2: MATCH (v) RETURN start_id(v) + ^ SELECT * FROM cypher('expr', $$ RETURN start_id() $$) AS (start_id agtype); @@ -2795,6 +2801,8 @@ SELECT * FROM cypher('expr', $$ MATCH (v) RETURN end_id(v) $$) AS (end_id agtype); ERROR: end_id() argument must be an edge or null +LINE 2: MATCH (v) RETURN end_id(v) + ^ SELECT * FROM cypher('expr', $$ RETURN end_id() $$) AS (end_id agtype); @@ -2828,6 +2836,8 @@ SELECT * FROM cypher('expr', $$ MATCH (v) RETURN startNode(v) $$) AS (startNode agtype); ERROR: startNode() argument must be an edge or null +LINE 2: MATCH (v) RETURN startNode(v) + ^ SELECT * FROM cypher('expr', $$ RETURN startNode() $$) AS (startNode agtype); @@ -2861,6 +2871,8 @@ SELECT * FROM cypher('expr', $$ MATCH (v) RETURN endNode(v) $$) AS (endNode agtype); ERROR: endNode() argument must be an edge or null +LINE 2: MATCH (v) RETURN endNode(v) + ^ SELECT * FROM cypher('expr', $$ RETURN endNode() $$) AS (endNode agtype); @@ -2894,6 +2906,8 @@ SELECT * FROM cypher('expr', $$ MATCH (v) RETURN type(v) $$) AS (type agtype); ERROR: type() argument must be an edge or null +LINE 2: MATCH (v) RETURN type(v) + ^ SELECT * FROM cypher('expr', $$ RETURN type() $$) AS (type agtype); @@ -4114,15 +4128,15 @@ $$) as (u agtype); SELECT * FROM cypher('expr', $$ RETURN reverse(true) $$) AS (results agtype); -ERROR: reverse() unsupported argument agtype 5 +ERROR: reverse() unsupported argument agtype SELECT * FROM age_reverse(true); -ERROR: reverse() unsupported argument type 16 +ERROR: reverse() unsupported argument type SELECT * FROM cypher('expr', $$ RETURN reverse(3.14) $$) AS (results agtype); -ERROR: reverse() unsupported argument agtype 4 +ERROR: reverse() unsupported argument agtype SELECT * FROM age_reverse(3.14); -ERROR: reverse() unsupported argument type 1700 +ERROR: reverse() unsupported argument type SELECT * FROM cypher('expr', $$ RETURN reverse() $$) AS (results agtype); @@ -4139,7 +4153,7 @@ SELECT * FROM cypher('expr', $$ MATCH (v) RETURN reverse(v) $$) AS (results agtype); -ERROR: reverse() unsupported argument agtype 6 +ERROR: reverse() unsupported argument type SELECT * FROM cypher('expr', $$ RETURN reverse({}) $$) AS (results agtype); @@ -7436,12 +7450,12 @@ SELECT * FROM cypher('case_statement', $$ $$ ) AS (n agtype, case_statement agtype); n | case_statement ------------------------------------------------------------------------------------------------+---------------- - {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710661, "label": "", "properties": {"i": [], "j": [0, 1, 2], "id": 5}}::vertex | "not count" {"id": 281474976710657, "label": "", "properties": {"i": 1, "id": 1}}::vertex | "not count" {"id": 281474976710659, "label": "", "properties": {"i": 0, "j": 1, "id": 3}}::vertex | 1 - {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" + {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710662, "label": "", "properties": {"i": {}, "j": {"i": 1}, "id": 6}}::vertex | "not count" + {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" (6 rows) --concatenated @@ -7454,12 +7468,12 @@ SELECT * FROM cypher('case_statement', $$ $$ ) AS (n agtype, case_statement agtype); n | case_statement ------------------------------------------------------------------------------------------------+---------------- - {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710661, "label": "", "properties": {"i": [], "j": [0, 1, 2], "id": 5}}::vertex | "not count" {"id": 281474976710657, "label": "", "properties": {"i": 1, "id": 1}}::vertex | "not count" {"id": 281474976710659, "label": "", "properties": {"i": 0, "j": 1, "id": 3}}::vertex | 6 - {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" + {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710662, "label": "", "properties": {"i": {}, "j": {"i": 1}, "id": 6}}::vertex | "not count" + {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" (6 rows) --count(n) @@ -7472,12 +7486,12 @@ SELECT * FROM cypher('case_statement', $$ $$ ) AS (n agtype, case_statement agtype); n | case_statement ------------------------------------------------------------------------------------------------+---------------- - {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710661, "label": "", "properties": {"i": [], "j": [0, 1, 2], "id": 5}}::vertex | "not count" {"id": 281474976710657, "label": "", "properties": {"i": 1, "id": 1}}::vertex | "not count" {"id": 281474976710659, "label": "", "properties": {"i": 0, "j": 1, "id": 3}}::vertex | 1 - {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" + {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710662, "label": "", "properties": {"i": {}, "j": {"i": 1}, "id": 6}}::vertex | "not count" + {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" (6 rows) --concatenated @@ -7490,12 +7504,12 @@ SELECT * FROM cypher('case_statement', $$ $$ ) AS (n agtype, case_statement agtype); n | case_statement ------------------------------------------------------------------------------------------------+---------------- - {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710661, "label": "", "properties": {"i": [], "j": [0, 1, 2], "id": 5}}::vertex | "not count" {"id": 281474976710657, "label": "", "properties": {"i": 1, "id": 1}}::vertex | "not count" {"id": 281474976710659, "label": "", "properties": {"i": 0, "j": 1, "id": 3}}::vertex | 6 - {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" + {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710662, "label": "", "properties": {"i": {}, "j": {"i": 1}, "id": 6}}::vertex | "not count" + {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" (6 rows) --count(1) @@ -7508,12 +7522,12 @@ SELECT * FROM cypher('case_statement', $$ $$ ) AS (n agtype, case_statement agtype); n | case_statement ------------------------------------------------------------------------------------------------+---------------- - {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710661, "label": "", "properties": {"i": [], "j": [0, 1, 2], "id": 5}}::vertex | "not count" {"id": 281474976710657, "label": "", "properties": {"i": 1, "id": 1}}::vertex | "not count" {"id": 281474976710659, "label": "", "properties": {"i": 0, "j": 1, "id": 3}}::vertex | 1 - {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" + {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710662, "label": "", "properties": {"i": {}, "j": {"i": 1}, "id": 6}}::vertex | "not count" + {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" (6 rows) --concatenated @@ -7526,12 +7540,12 @@ SELECT * FROM cypher('case_statement', $$ $$ ) AS (n agtype, case_statement agtype); n | case_statement ------------------------------------------------------------------------------------------------+---------------- - {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710661, "label": "", "properties": {"i": [], "j": [0, 1, 2], "id": 5}}::vertex | "not count" {"id": 281474976710657, "label": "", "properties": {"i": 1, "id": 1}}::vertex | "not count" {"id": 281474976710659, "label": "", "properties": {"i": 0, "j": 1, "id": 3}}::vertex | 6 - {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" + {"id": 281474976710658, "label": "", "properties": {"i": "a", "j": "b", "id": 2}}::vertex | "not count" {"id": 281474976710662, "label": "", "properties": {"i": {}, "j": {"i": 1}, "id": 6}}::vertex | "not count" + {"id": 281474976710660, "label": "", "properties": {"i": true, "j": false, "id": 4}}::vertex | "not count" (6 rows) --CASE with EXISTS() @@ -9217,9 +9231,1099 @@ SELECT * FROM cypher('issue_2289', $$ RETURN (1 IN []) AS v $$) AS (v agtype); false (1 row) +-- +-- Accessor functions and properties extraction +-- without _agtype_build.. functions +-- +SELECT * FROM create_graph('accessor_opt'); +NOTICE: graph "accessor_opt" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('accessor_opt', $$ + CREATE (a:Person {name: 'Alice', age: 30})-[r:KNOWS {since: 2020}]->(b:Person {name: 'Bob', age: 25}) +$$) AS (a agtype); + a +--- +(0 rows) + +-- +-- Vertex accessor tests +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN id(n) +$$) AS (plan agtype); + QUERY PLAN +------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: (n.id)::agtype +(2 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN properties(n) +$$) AS (plan agtype); + QUERY PLAN +------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: n.properties +(2 rows) + +SELECT * FROM cypher('accessor_opt', $$ + MATCH (n:Person) + RETURN label(n) +$$) AS (plan agtype); + plan +---------- + "Person" + "Person" +(2 rows) + +-- should use n.properties in _agtype_access_operator +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN n.name, n.age +$$) AS (a agtype, b agtype); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]), agtype_access_operator(VARIADIC ARRAY[n.properties, '"age"'::agtype]) +(2 rows) + +-- +-- Edge accessor tests +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN id(r) +$$) AS (plan agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Merge Join + Output: (r.id)::agtype + Merge Cond: (_age_default_alias_1.id = r.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on accessor_opt._ag_label_vertex _age_default_alias_1_1 + Output: _age_default_alias_1_1.id + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_1_2 + Output: _age_default_alias_1_2.id + -> Materialize + Output: r.id, r.end_id + -> Nested Loop + Output: r.id, r.end_id + -> Index Scan using "KNOWS_end_id_idx" on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex _age_default_alias_0_1 + Output: _age_default_alias_0_1.id + Filter: (_age_default_alias_0_1.id = r.start_id) + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_0_2 + Output: _age_default_alias_0_2.id + Index Cond: (_age_default_alias_0_2.id = r.start_id) +(22 rows) + +SELECT * FROM cypher('accessor_opt', $$ + MATCH ()-[r:KNOWS]->() + RETURN type(r) +$$) AS (plan agtype); + plan +--------- + "KNOWS" +(1 row) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN start_id(r) +$$) AS (plan agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Merge Join + Output: (r.start_id)::agtype + Merge Cond: (_age_default_alias_1.id = r.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on accessor_opt._ag_label_vertex _age_default_alias_1_1 + Output: _age_default_alias_1_1.id + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_1_2 + Output: _age_default_alias_1_2.id + -> Materialize + Output: r.start_id, r.end_id + -> Nested Loop + Output: r.start_id, r.end_id + -> Index Scan using "KNOWS_end_id_idx" on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex _age_default_alias_0_1 + Output: _age_default_alias_0_1.id + Filter: (_age_default_alias_0_1.id = r.start_id) + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_0_2 + Output: _age_default_alias_0_2.id + Index Cond: (_age_default_alias_0_2.id = r.start_id) +(22 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN end_id(r) +$$) AS (plan agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Merge Join + Output: (r.end_id)::agtype + Merge Cond: (_age_default_alias_1.id = r.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on accessor_opt._ag_label_vertex _age_default_alias_1_1 + Output: _age_default_alias_1_1.id + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_1_2 + Output: _age_default_alias_1_2.id + -> Materialize + Output: r.end_id + -> Nested Loop + Output: r.end_id + -> Index Scan using "KNOWS_end_id_idx" on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex _age_default_alias_0_1 + Output: _age_default_alias_0_1.id + Filter: (_age_default_alias_0_1.id = r.start_id) + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_0_2 + Output: _age_default_alias_0_2.id + Index Cond: (_age_default_alias_0_2.id = r.start_id) +(22 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN properties(r) +$$) AS (plan agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Merge Join + Output: r.properties + Merge Cond: (_age_default_alias_1.id = r.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on accessor_opt._ag_label_vertex _age_default_alias_1_1 + Output: _age_default_alias_1_1.id + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_1_2 + Output: _age_default_alias_1_2.id + -> Materialize + Output: r.properties, r.end_id + -> Nested Loop + Output: r.properties, r.end_id + -> Index Scan using "KNOWS_end_id_idx" on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex _age_default_alias_0_1 + Output: _age_default_alias_0_1.id + Filter: (_age_default_alias_0_1.id = r.start_id) + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_0_2 + Output: _age_default_alias_0_2.id + Index Cond: (_age_default_alias_0_2.id = r.start_id) +(22 rows) + +-- should use r.properties in _agtype_access_operator +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN r.since +$$) AS (plan agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Merge Join + Output: agtype_access_operator(VARIADIC ARRAY[r.properties, '"since"'::agtype]) + Merge Cond: (_age_default_alias_1.id = r.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on accessor_opt._ag_label_vertex _age_default_alias_1_1 + Output: _age_default_alias_1_1.id + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_1_2 + Output: _age_default_alias_1_2.id + -> Materialize + Output: r.properties, r.end_id + -> Nested Loop + Output: r.properties, r.end_id + -> Index Scan using "KNOWS_end_id_idx" on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex _age_default_alias_0_1 + Output: _age_default_alias_0_1.id + Filter: (_age_default_alias_0_1.id = r.start_id) + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_0_2 + Output: _age_default_alias_0_2.id + Index Cond: (_age_default_alias_0_2.id = r.start_id) +(22 rows) + +-- +-- Multiple accessors in same query +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN id(n), properties(n), n.name +$$) AS (a agtype, c agtype, d agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: (n.id)::agtype, n.properties, agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]) +(2 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN id(r), start_id(r), end_id(r), properties(r) +$$) AS (a agtype, c agtype, d agtype, e agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Merge Join + Output: (r.id)::agtype, (r.start_id)::agtype, (r.end_id)::agtype, r.properties + Merge Cond: (_age_default_alias_1.id = r.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on accessor_opt._ag_label_vertex _age_default_alias_1_1 + Output: _age_default_alias_1_1.id + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_1_2 + Output: _age_default_alias_1_2.id + -> Materialize + Output: r.id, r.start_id, r.end_id, r.properties + -> Nested Loop + Output: r.id, r.start_id, r.end_id, r.properties + -> Index Scan using "KNOWS_end_id_idx" on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex _age_default_alias_0_1 + Output: _age_default_alias_0_1.id + Filter: (_age_default_alias_0_1.id = r.start_id) + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_0_2 + Output: _age_default_alias_0_2.id + Index Cond: (_age_default_alias_0_2.id = r.start_id) +(22 rows) + +-- +-- Accessors in WHERE clause +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WHERE id(n) > 0 + RETURN n.name +$$) AS (plan agtype); + QUERY PLAN +---------------------------------------------------------------------------------- + Bitmap Heap Scan on accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]) + Recheck Cond: (n.id > '0'::graphid) + -> Bitmap Index Scan on "Person_pkey" + Index Cond: (n.id > '0'::graphid) +(5 rows) + +-- Compare two node ids +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person)-[r:KNOWS]->(b:Person) + WHERE id(a) < id(b) + RETURN a.name, b.name +$$) AS (aname agtype, bname agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------- + Hash Join + Output: agtype_access_operator(VARIADIC ARRAY[a.properties, '"name"'::agtype]), agtype_access_operator(VARIADIC ARRAY[b.properties, '"name"'::agtype]) + Inner Unique: true + Hash Cond: (r.end_id = b.id) + Join Filter: (a.id < b.id) + -> Hash Join + Output: a.properties, a.id, r.end_id + Inner Unique: true + Hash Cond: (r.start_id = a.id) + -> Seq Scan on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Hash + Output: a.properties, a.id + -> Seq Scan on accessor_opt."Person" a + Output: a.properties, a.id + -> Hash + Output: b.properties, b.id + -> Seq Scan on accessor_opt."Person" b + Output: b.properties, b.id +(19 rows) + +-- Compare edge start_id and end_id +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + WHERE start_id(r) <> end_id(r) + RETURN id(r) +$$) AS (rid agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Merge Join + Output: (r.id)::agtype + Merge Cond: (_age_default_alias_1.id = r.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on accessor_opt._ag_label_vertex _age_default_alias_1_1 + Output: _age_default_alias_1_1.id + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_1_2 + Output: _age_default_alias_1_2.id + -> Materialize + Output: r.id, r.end_id + -> Nested Loop + Output: r.id, r.end_id + -> Index Scan using "KNOWS_end_id_idx" on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + Filter: (r.start_id <> r.end_id) + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex _age_default_alias_0_1 + Output: _age_default_alias_0_1.id + Filter: (_age_default_alias_0_1.id = r.start_id) + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_0_2 + Output: _age_default_alias_0_2.id + Index Cond: (_age_default_alias_0_2.id = r.start_id) +(23 rows) + +-- Property comparison +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WHERE n.age >= 25 + RETURN n.name +$$) AS (name agtype); + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]) + Filter: (agtype_access_operator(VARIADIC ARRAY[n.properties, '"age"'::agtype]) >= '25'::agtype) +(3 rows) + +-- Multiple property comparisons +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WHERE n.age > 20 AND n.name <> 'Unknown' + RETURN n.name +$$) AS (name agtype); + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]) + Filter: ((agtype_access_operator(VARIADIC ARRAY[n.properties, '"age"'::agtype]) > '20'::agtype) AND (agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]) <> '"Unknown"'::agtype)) +(3 rows) + +-- Compare properties of two nodes +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person)-[r:KNOWS]->(b:Person) + WHERE a.age > b.age + RETURN a.name, b.name +$$) AS (aname agtype, bname agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------- + Hash Join + Output: agtype_access_operator(VARIADIC ARRAY[a.properties, '"name"'::agtype]), agtype_access_operator(VARIADIC ARRAY[b.properties, '"name"'::agtype]) + Inner Unique: true + Hash Cond: (r.end_id = b.id) + Join Filter: (agtype_access_operator(VARIADIC ARRAY[a.properties, '"age"'::agtype]) > agtype_access_operator(VARIADIC ARRAY[b.properties, '"age"'::agtype])) + -> Hash Join + Output: a.properties, r.end_id + Inner Unique: true + Hash Cond: (r.start_id = a.id) + -> Seq Scan on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Hash + Output: a.properties, a.id + -> Seq Scan on accessor_opt."Person" a + Output: a.properties, a.id + -> Hash + Output: b.properties, b.id + -> Seq Scan on accessor_opt."Person" b + Output: b.properties, b.id +(19 rows) + +-- Compare id with property +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WHERE id(n) > n.age + RETURN n.name +$$) AS (name agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]) + Filter: (n.id > (agtype_access_operator(VARIADIC ARRAY[n.properties, '"age"'::agtype]))::graphid) +(3 rows) + +-- Edge property comparison +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + WHERE r.since > 2015 + RETURN r.since +$$) AS (since agtype); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------ + Merge Join + Output: agtype_access_operator(VARIADIC ARRAY[r.properties, '"since"'::agtype]) + Merge Cond: (_age_default_alias_1.id = r.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on accessor_opt._ag_label_vertex _age_default_alias_1_1 + Output: _age_default_alias_1_1.id + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_1_2 + Output: _age_default_alias_1_2.id + -> Materialize + Output: r.properties, r.end_id + -> Nested Loop + Output: r.properties, r.end_id + -> Index Scan using "KNOWS_end_id_idx" on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + Filter: (agtype_access_operator(VARIADIC ARRAY[r.properties, '"since"'::agtype]) > '2015'::agtype) + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex _age_default_alias_0_1 + Output: _age_default_alias_0_1.id + Filter: (_age_default_alias_0_1.id = r.start_id) + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_0_2 + Output: _age_default_alias_0_2.id + Index Cond: (_age_default_alias_0_2.id = r.start_id) +(23 rows) + +-- IS NOT NULL on accessor +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WHERE properties(n) IS NOT NULL + RETURN n.name +$$) AS (name agtype); + QUERY PLAN +---------------------------------------------------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]) +(2 rows) + +-- label() comparison +SELECT * FROM cypher('accessor_opt', $$ + MATCH (n) + WHERE label(n) = 'Person' + RETURN n.name +$$) AS (name agtype); + name +--------- + "Alice" + "Bob" +(2 rows) + +-- type() comparison on edge +SELECT * FROM cypher('accessor_opt', $$ + MATCH ()-[r]->() + WHERE type(r) = 'KNOWS' + RETURN id(r) +$$) AS (rid agtype); + rid +------------------ + 1125899906842625 +(1 row) + +-- +-- Accessors in ORDER BY +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN n.name + ORDER BY n.name +$$) AS (plan agtype); + QUERY PLAN +---------------------------------------------------------------------------------------- + Sort + Output: (agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype])) + Sort Key: (agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype])) + -> Seq Scan on accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]) +(5 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN n.name + ORDER BY id(n) +$$) AS (plan agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------- + Subquery Scan on _ + Output: _.name + -> Index Scan using "Person_pkey" on accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]), n.id +(4 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN n.name, n.age + ORDER BY n.age, n.name +$$) AS (name agtype, age agtype); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------- + Sort + Output: (agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype])), (agtype_access_operator(VARIADIC ARRAY[n.properties, '"age"'::agtype])) + Sort Key: (agtype_access_operator(VARIADIC ARRAY[n.properties, '"age"'::agtype])), (agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype])) + -> Seq Scan on accessor_opt."Person" n + Output: agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]), agtype_access_operator(VARIADIC ARRAY[n.properties, '"age"'::agtype]) +(5 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN r.since + ORDER BY start_id(r) +$$) AS (since agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Subquery Scan on _ + Output: _.since + -> Merge Join + Output: agtype_access_operator(VARIADIC ARRAY[r.properties, '"since"'::agtype]), r.start_id + Merge Cond: (_age_default_alias_0.id = r.start_id) + -> Merge Append + Sort Key: _age_default_alias_0.id + -> Index Only Scan using _ag_label_vertex_pkey on accessor_opt._ag_label_vertex _age_default_alias_0_1 + Output: _age_default_alias_0_1.id + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_0_2 + Output: _age_default_alias_0_2.id + -> Materialize + Output: r.properties, r.start_id + -> Nested Loop + Output: r.properties, r.start_id + -> Index Scan using "KNOWS_start_id_idx" on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex _age_default_alias_1_1 + Output: _age_default_alias_1_1.id + Filter: (_age_default_alias_1_1.id = r.end_id) + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" _age_default_alias_1_2 + Output: _age_default_alias_1_2.id + Index Cond: (_age_default_alias_1_2.id = r.end_id) +(24 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person)-[r:KNOWS]->(b:Person) + RETURN a.name, b.name + ORDER BY id(a), id(r), id(b) +$$) AS (aname agtype, bname agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Subquery Scan on _ + Output: _.name, _.name_1 + -> Sort + Output: (agtype_access_operator(VARIADIC ARRAY[a.properties, '"name"'::agtype])), (agtype_access_operator(VARIADIC ARRAY[b.properties, '"name"'::agtype])), a.id, r.id, b.id + Sort Key: a.id, r.id, b.id + -> Hash Join + Output: agtype_access_operator(VARIADIC ARRAY[a.properties, '"name"'::agtype]), agtype_access_operator(VARIADIC ARRAY[b.properties, '"name"'::agtype]), a.id, r.id, b.id + Inner Unique: true + Hash Cond: (r.end_id = b.id) + -> Hash Join + Output: a.properties, a.id, r.id, r.end_id + Inner Unique: true + Hash Cond: (r.start_id = a.id) + -> Seq Scan on accessor_opt."KNOWS" r + Output: r.id, r.start_id, r.end_id, r.properties + -> Hash + Output: a.properties, a.id + -> Seq Scan on accessor_opt."Person" a + Output: a.properties, a.id + -> Hash + Output: b.properties, b.id + -> Seq Scan on accessor_opt."Person" b + Output: b.properties, b.id +(23 rows) + +-- +-- Accessors in expressions +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN [id(n), n.name] +$$) AS (plan agtype); + QUERY PLAN +----------------------------------------------------------------------------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: agtype_build_list(n.id, agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype])) +(2 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN {id: id(n), name: n.name} +$$) AS (plan agtype); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: agtype_build_map_nonull('id'::text, n.id, 'name'::text, agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype])) +(2 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN n {.name, .age} +$$) AS (plan agtype); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Seq Scan on accessor_opt."Person" n + Output: agtype_build_map_nonull('name'::text, agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype]), 'age'::text, agtype_access_operator(VARIADIC ARRAY[n.properties, '"age"'::agtype])) +(2 rows) + +-- +-- Accessor function in multiple match +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person) + WHERE id(a) > 0 + MATCH (a)-[r]->(b) + WHERE id(b) > 0 + RETURN properties(a), start_id(r), end_id(r) +$$) AS (props_a agtype, sid agtype, eid agtype); + QUERY PLAN +------------------------------------------------------------------ + Hash Join + Output: a.properties, (r.start_id)::agtype, (r.end_id)::agtype + Hash Cond: (r.end_id = b.id) + -> Hash Join + Output: a.properties, r.start_id, r.end_id + Inner Unique: true + Hash Cond: (r.start_id = a.id) + -> Append + -> Seq Scan on accessor_opt._ag_label_edge r_1 + Output: r_1.start_id, r_1.end_id + -> Seq Scan on accessor_opt."KNOWS" r_2 + Output: r_2.start_id, r_2.end_id + -> Hash + Output: a.properties, a.id + -> Bitmap Heap Scan on accessor_opt."Person" a + Output: a.properties, a.id + Recheck Cond: (a.id > '0'::graphid) + -> Bitmap Index Scan on "Person_pkey" + Index Cond: (a.id > '0'::graphid) + -> Hash + Output: b.id + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex b_1 + Output: b_1.id + Filter: (b_1.id > '0'::graphid) + -> Bitmap Heap Scan on accessor_opt."Person" b_2 + Output: b_2.id + Recheck Cond: (b_2.id > '0'::graphid) + -> Bitmap Index Scan on "Person_pkey" + Index Cond: (b_2.id > '0'::graphid) +(30 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person) + MATCH (b:Person) + WHERE b.name <> a.name + RETURN a.name, b.name +$$) AS (a_name agtype, b_name agtype); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: agtype_access_operator(VARIADIC ARRAY[a.properties, '"name"'::agtype]), agtype_access_operator(VARIADIC ARRAY[b.properties, '"name"'::agtype]) + Join Filter: (agtype_access_operator(VARIADIC ARRAY[b.properties, '"name"'::agtype]) <> agtype_access_operator(VARIADIC ARRAY[a.properties, '"name"'::agtype])) + -> Seq Scan on accessor_opt."Person" a + Output: a.id, a.properties + -> Materialize + Output: b.properties + -> Seq Scan on accessor_opt."Person" b + Output: b.properties +(9 rows) + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person) + MATCH (a)-[r1]->(b) + MATCH (b)-[r2]->(c) + RETURN id(a), id(b), id(c), a.name +$$) AS (a_id agtype, b_id agtype, c_id agtype, a_name agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- + Merge Join + Output: (a.id)::agtype, (b.id)::agtype, (c.id)::agtype, agtype_access_operator(VARIADIC ARRAY[a.properties, '"name"'::agtype]) + Merge Cond: (r2.start_id = r1.end_id) + -> Nested Loop + Output: r2.start_id, c.id + -> Merge Append + Sort Key: r2.start_id + -> Index Scan using _ag_label_edge_start_id_idx on accessor_opt._ag_label_edge r2_1 + Output: r2_1.start_id, r2_1.end_id + -> Index Scan using "KNOWS_start_id_idx" on accessor_opt."KNOWS" r2_2 + Output: r2_2.start_id, r2_2.end_id + -> Append + -> Seq Scan on accessor_opt._ag_label_vertex c_1 + Output: c_1.id + Filter: (c_1.id = r2.end_id) + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" c_2 + Output: c_2.id + Index Cond: (c_2.id = r2.end_id) + -> Materialize + Output: a.id, a.properties, r1.end_id, b.id + -> Merge Join + Output: a.id, a.properties, r1.end_id, b.id + Merge Cond: (b.id = r1.end_id) + -> Merge Append + Sort Key: b.id + -> Index Only Scan using _ag_label_vertex_pkey on accessor_opt._ag_label_vertex b_1 + Output: b_1.id + -> Index Only Scan using "Person_pkey" on accessor_opt."Person" b_2 + Output: b_2.id + -> Sort + Output: a.id, a.properties, r1.end_id + Sort Key: r1.end_id + -> Hash Join + Output: a.id, a.properties, r1.end_id + Inner Unique: true + Hash Cond: (r1.start_id = a.id) + -> Append + -> Seq Scan on accessor_opt._ag_label_edge r1_1 + Output: r1_1.start_id, r1_1.end_id + -> Seq Scan on accessor_opt."KNOWS" r1_2 + Output: r1_2.start_id, r1_2.end_id + -> Hash + Output: a.id, a.properties + -> Seq Scan on accessor_opt."Person" a + Output: a.id, a.properties +(45 rows) + +-- +-- Composite types, vertex and edge +-- +SELECT * FROM create_graph('composite_types'); +NOTICE: graph "composite_types" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('composite_types', $$ + CREATE (a:Person {name: 'Alice', age: 30})-[r:KNOWS {since: 2020}]->(b:Person {name: 'Bob', age: 25}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- Return vertex as vertex type +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n +$$) AS (n vertex); + n +--------------------------------------------------------------------- + (844424930131969,"""Person""","{""age"": 30, ""name"": ""Alice""}") + (844424930131970,"""Person""","{""age"": 25, ""name"": ""Bob""}") +(2 rows) + +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n + ORDER BY n.name +$$) AS (n vertex); + n +--------------------------------------------------------------------- + (844424930131969,"""Person""","{""age"": 30, ""name"": ""Alice""}") + (844424930131970,"""Person""","{""age"": 25, ""name"": ""Bob""}") +(2 rows) + +-- Return edge as edge type +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r:KNOWS]->() + RETURN r +$$) AS (r edge); + r +------------------------------------------------------------------------------------ + (1125899906842625,"""KNOWS""",844424930131970,844424930131969,"{""since"": 2020}") +(1 row) + +-- Return multiple entities +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person)-[r:KNOWS]->(b:Person) + RETURN a, b +$$) AS (a vertex, b vertex); + a | b +---------------------------------------------------------------------+------------------------------------------------------------------- + (844424930131969,"""Person""","{""age"": 30, ""name"": ""Alice""}") | (844424930131970,"""Person""","{""age"": 25, ""name"": ""Bob""}") +(1 row) + +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person)-[r:KNOWS]->(b:Person) + RETURN a, r, b +$$) AS (a vertex, r edge, b vertex); + a | r | b +---------------------------------------------------------------------+------------------------------------------------------------------------------------+------------------------------------------------------------------- + (844424930131969,"""Person""","{""age"": 30, ""name"": ""Alice""}") | (1125899906842625,"""KNOWS""",844424930131970,844424930131969,"{""since"": 2020}") | (844424930131970,"""Person""","{""age"": 25, ""name"": ""Bob""}") +(1 row) + +-- Mixed return: entity and agtype property +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n, n.name +$$) AS (n vertex, name agtype); + n | name +---------------------------------------------------------------------+--------- + (844424930131969,"""Person""","{""age"": 30, ""name"": ""Alice""}") | "Alice" + (844424930131970,"""Person""","{""age"": 25, ""name"": ""Bob""}") | "Bob" +(2 rows) + +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n, n.name + ORDER BY n.name +$$) AS (n vertex, name agtype); + n | name +---------------------------------------------------------------------+--------- + (844424930131969,"""Person""","{""age"": 30, ""name"": ""Alice""}") | "Alice" + (844424930131970,"""Person""","{""age"": 25, ""name"": ""Bob""}") | "Bob" +(2 rows) + +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r:KNOWS]->() + RETURN r, r.since +$$) AS (r edge, since agtype); + r | since +------------------------------------------------------------------------------------+------- + (1125899906842625,"""KNOWS""",844424930131970,844424930131969,"{""since"": 2020}") | 2020 +(1 row) + +-- IN operator with vertex and edge types +EXPLAIN (COSTS OFF) +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person), (b:Person) + WHERE a <> b AND a IN [a, b] + RETURN a.name +$$) AS (name agtype); + QUERY PLAN +---------------------------------------------------------------------- + Nested Loop + Join Filter: ((a.id <> b.id) AND ((a.id = a.id) OR (a.id = b.id))) + -> Seq Scan on "Person" a + -> Materialize + -> Seq Scan on "Person" b +(5 rows) + +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person), (b:Person) + WHERE a <> b AND a IN [a, b] + RETURN a.name +$$) AS (name agtype); + name +--------- + "Alice" + "Bob" +(2 rows) + +EXPLAIN (COSTS OFF) +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r1:KNOWS]->(), ()-[r2:KNOWS]->() + WHERE r1 IN [r1, r2] + RETURN id(r1) +$$) AS (id agtype); + QUERY PLAN +------------------------------------------------------------------------------------------------------------ + Merge Join + Merge Cond: (_age_default_alias_1.id = r1.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on _ag_label_vertex _age_default_alias_1_1 + -> Index Only Scan using "Person_pkey" on "Person" _age_default_alias_1_2 + -> Materialize + -> Nested Loop + -> Nested Loop + -> Nested Loop + -> Nested Loop + Join Filter: ((r1.id = r1.id) OR (r1.id = r2.id)) + -> Index Scan using "KNOWS_end_id_idx" on "KNOWS" r1 + -> Materialize + -> Seq Scan on "KNOWS" r2 + -> Append + -> Seq Scan on _ag_label_vertex _age_default_alias_2_1 + Filter: (id = r2.start_id) + -> Index Only Scan using "Person_pkey" on "Person" _age_default_alias_2_2 + Index Cond: (id = r2.start_id) + -> Append + -> Seq Scan on _ag_label_vertex _age_default_alias_3_1 + Filter: (id = r2.end_id) + -> Index Only Scan using "Person_pkey" on "Person" _age_default_alias_3_2 + Index Cond: (id = r2.end_id) + -> Append + -> Seq Scan on _ag_label_vertex _age_default_alias_0_1 + Filter: (id = r1.start_id) + -> Index Only Scan using "Person_pkey" on "Person" _age_default_alias_0_2 + Index Cond: (id = r1.start_id) +(30 rows) + +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r1:KNOWS]->(), ()-[r2:KNOWS]->() + WHERE r1 IN [r1, r2] + RETURN id(r1) +$$) AS (id agtype); + id +------------------ + 1125899906842625 +(1 row) + +-- Equality operators with vertex and edge types +EXPLAIN (COSTS OFF) +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person), (b:Person) + WHERE a = b + RETURN a.name +$$) AS (name agtype); + QUERY PLAN +------------------------ + Seq Scan on "Person" b +(1 row) + +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person), (b:Person) + WHERE a = b + RETURN a.name +$$) AS (name agtype); + name +--------- + "Alice" + "Bob" +(2 rows) + +EXPLAIN (COSTS OFF) +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r1:KNOWS]->(), ()-[r2:KNOWS]->() + WHERE r1 = r2 + RETURN id(r1) +$$) AS (id agtype); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------- + Merge Join + Merge Cond: (r1.id = r2.id) + -> Sort + Sort Key: r1.id + -> Merge Join + Merge Cond: (_age_default_alias_1.id = r1.end_id) + -> Merge Append + Sort Key: _age_default_alias_1.id + -> Index Only Scan using _ag_label_vertex_pkey on _ag_label_vertex _age_default_alias_1_1 + -> Index Only Scan using "Person_pkey" on "Person" _age_default_alias_1_2 + -> Materialize + -> Nested Loop + -> Index Scan using "KNOWS_end_id_idx" on "KNOWS" r1 + -> Append + -> Seq Scan on _ag_label_vertex _age_default_alias_0_1 + Filter: (id = r1.start_id) + -> Index Only Scan using "Person_pkey" on "Person" _age_default_alias_0_2 + Index Cond: (id = r1.start_id) + -> Sort + Sort Key: r2.id + -> Merge Join + Merge Cond: (_age_default_alias_3.id = r2.end_id) + -> Merge Append + Sort Key: _age_default_alias_3.id + -> Index Only Scan using _ag_label_vertex_pkey on _ag_label_vertex _age_default_alias_3_1 + -> Index Only Scan using "Person_pkey" on "Person" _age_default_alias_3_2 + -> Materialize + -> Nested Loop + -> Index Scan using "KNOWS_end_id_idx" on "KNOWS" r2 + -> Append + -> Seq Scan on _ag_label_vertex _age_default_alias_2_1 + Filter: (id = r2.start_id) + -> Index Only Scan using "Person_pkey" on "Person" _age_default_alias_2_2 + Index Cond: (id = r2.start_id) +(34 rows) + +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r1:KNOWS]->(), ()-[r2:KNOWS]->() + WHERE r1 = r2 + RETURN id(r1) +$$) AS (id agtype); + id +------------------ + 1125899906842625 +(1 row) + +-- Cast vertex and edge to json +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n +$$) AS (n json); + n +---------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "Person", "properties": {"age": 30, "name": "Alice"}} + {"id": 844424930131970, "label": "Person", "properties": {"age": 25, "name": "Bob"}} +(2 rows) + +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r:KNOWS]->() + RETURN r +$$) AS (r json); + r +----------------------------------------------------------------------------------------------------------------------------------- + {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {"since": 2020}} +(1 row) + +-- Cast vertex and edge to jsonb +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n +$$) AS (n jsonb); + n +---------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "Person", "properties": {"age": 30, "name": "Alice"}} + {"id": 844424930131970, "label": "Person", "properties": {"age": 25, "name": "Bob"}} +(2 rows) + +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r:KNOWS]->() + RETURN r +$$) AS (r jsonb); + r +----------------------------------------------------------------------------------------------------------------------------------- + {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {"since": 2020}} +(1 row) + -- -- Cleanup -- +SELECT * FROM drop_graph('composite_types', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table composite_types._ag_label_vertex +drop cascades to table composite_types._ag_label_edge +drop cascades to table composite_types."Person" +drop cascades to table composite_types."KNOWS" +NOTICE: graph "composite_types" has been dropped + drop_graph +------------ + +(1 row) + +SELECT * FROM drop_graph('accessor_opt', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table accessor_opt._ag_label_vertex +drop cascades to table accessor_opt._ag_label_edge +drop cascades to table accessor_opt."Person" +drop cascades to table accessor_opt."KNOWS" +NOTICE: graph "accessor_opt" has been dropped + drop_graph +------------ + +(1 row) + SELECT * FROM drop_graph('issue_2289', true); NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table issue_2289._ag_label_vertex diff --git a/regress/expected/pgvector.out b/regress/expected/pgvector.out index bbc558349..4af6106ea 100644 --- a/regress/expected/pgvector.out +++ b/regress/expected/pgvector.out @@ -576,7 +576,7 @@ BEGIN USING hnsw ((( agtype_access_operator( VARIADIC ARRAY[ - _agtype_build_vertex(id, _label_name(%L::oid, id), properties), + properties, '"embedding"'::agtype ] )::text @@ -654,7 +654,7 @@ BEGIN USING hnsw (( agtype_access_operator( VARIADIC ARRAY[ - _agtype_build_vertex(id, _label_name(%L::oid, id), properties), + properties, '"embedding"'::agtype ] )::vector(4)) vector_cosine_ops); diff --git a/regress/sql/agtype.sql b/regress/sql/agtype.sql index 505097761..e13edb16d 100644 --- a/regress/sql/agtype.sql +++ b/regress/sql/agtype.sql @@ -829,40 +829,40 @@ SELECT * FROM drop_graph('agtype_null_duplicate_test', true); -- Vertex -- --Basic Vertex Creation -SELECT _agtype_build_vertex('1'::graphid, $$label_name$$, agtype_build_map()); -SELECT _agtype_build_vertex('1'::graphid, $$label$$, agtype_build_map('id', 2)); +SELECT _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map()); +SELECT _agtype_build_vertex('1'::graphid, '"label"', agtype_build_map('id', 2)); --Null properties -SELECT _agtype_build_vertex('1'::graphid, $$label_name$$, NULL); +SELECT _agtype_build_vertex('1'::graphid, '"label_name"', NULL); --Test access operator -SELECT agtype_access_operator(_agtype_build_vertex('1'::graphid, $$label$$, +SELECT agtype_access_operator(_agtype_build_vertex('1'::graphid, '"label"', agtype_build_map('id', 2)), '"id"'); -SELECT _agtype_build_vertex('1'::graphid, $$label$$, agtype_build_list()); +SELECT _agtype_build_vertex('1'::graphid, '"label"', agtype_build_list()); --Vertex in a map SELECT agtype_build_map( 'vertex', - _agtype_build_vertex('1'::graphid, $$label_name$$, agtype_build_map())); + _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map())); SELECT agtype_access_operator( agtype_build_map( - 'vertex', _agtype_build_vertex('1'::graphid, $$label_name$$, + 'vertex', _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map('key', 'value')), - 'other_vertex', _agtype_build_vertex('1'::graphid, $$label_name$$, + 'other_vertex', _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map('key', 'other_value'))), '"vertex"'); --Vertex in a list SELECT agtype_build_list( - _agtype_build_vertex('1'::graphid, $$label_name$$, agtype_build_map()), - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map())); + _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map())); SELECT agtype_access_operator( agtype_build_list( - _agtype_build_vertex('1'::graphid, $$label_name$$, + _agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map('id', 3)), - _agtype_build_vertex('2'::graphid, $$label_name$$, + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map('id', 4))), '0'); -- @@ -870,18 +870,18 @@ SELECT agtype_access_operator( -- --Basic Edge Creation SELECT _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map()); + '"label_name"', agtype_build_map()); SELECT _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)); + '"label"', agtype_build_map('id', 2)); --Null properties SELECT _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, NULL); + '"label_name"', NULL); --Test access operator SELECT agtype_access_operator(_agtype_build_edge('1'::graphid, '2'::graphid, - '3'::graphid, $$label$$, agtype_build_map('id', 2)),'"id"'); + '3'::graphid, '"label"', agtype_build_map('id', 2)),'"id"'); @@ -889,97 +889,97 @@ SELECT agtype_access_operator(_agtype_build_edge('1'::graphid, '2'::graphid, SELECT agtype_build_map( 'edge', _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map())); + '"label_name"', agtype_build_map())); SELECT agtype_access_operator( agtype_build_map( 'edge', _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map('key', 'value')), + '"label_name"', agtype_build_map('key', 'value')), 'other_edge', _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map('key', 'other_value'))), + '"label_name"', agtype_build_map('key', 'other_value'))), '"edge"'); --Edge in a list SELECT agtype_build_list( _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map()), + '"label_name"', agtype_build_map()), _agtype_build_edge('2'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map())); + '"label_name"', agtype_build_map())); SELECT agtype_access_operator( agtype_build_list( - _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, $$label_name$$, + _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, '"label_name"', agtype_build_map('id', 3)), - _agtype_build_edge('2'::graphid, '2'::graphid, '3'::graphid, $$label_name$$, + _agtype_build_edge('2'::graphid, '2'::graphid, '3'::graphid, '"label_name"', agtype_build_map('id', 4))), '0'); -- Path SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), - _agtype_build_vertex('3'::graphid, $$label_name$$, agtype_build_map()) + '"label"', agtype_build_map('id', 2)), + _agtype_build_vertex('3'::graphid, '"label_name"', agtype_build_map()) ); --All these paths should produce Errors SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)) + '"label"', agtype_build_map('id', 2)) ); SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), - _agtype_build_vertex('3'::graphid, $$label_name$$, agtype_build_map()), + '"label"', agtype_build_map('id', 2)), + _agtype_build_vertex('3'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '4'::graphid, '5'::graphid, - $$label$$, agtype_build_map('id', 2)) + '"label"', agtype_build_map('id', 2)) ); SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), + '"label"', agtype_build_map('id', 2)), NULL ); SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), + '"label"', agtype_build_map('id', 2)), 1 ); SELECT _agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), + '"label"', agtype_build_map('id', 2)), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)) + '"label"', agtype_build_map('id', 2)) ); -- -- id, startid, endid -- -SELECT age_id(_agtype_build_vertex('1'::graphid, $$label_name$$, agtype_build_map())); +SELECT age_id(_agtype_build_vertex('1'::graphid, '"label_name"', agtype_build_map())); SELECT age_id(_agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map('id', 2))); + '"label_name"', agtype_build_map('id', 2))); SELECT age_start_id(_agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map('id', 2))); + '"label_name"', agtype_build_map('id', 2))); SELECT age_end_id(_agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label_name$$, agtype_build_map('id', 2))); + '"label_name"', agtype_build_map('id', 2))); SELECT age_id(_agtype_build_path( - _agtype_build_vertex('2'::graphid, $$label_name$$, agtype_build_map()), + _agtype_build_vertex('2'::graphid, '"label_name"', agtype_build_map()), _agtype_build_edge('1'::graphid, '2'::graphid, '3'::graphid, - $$label$$, agtype_build_map('id', 2)), - _agtype_build_vertex('3'::graphid, $$label$$, agtype_build_map('id', 2)) + '"label"', agtype_build_map('id', 2)), + _agtype_build_vertex('3'::graphid, '"label"', agtype_build_map('id', 2)) )); SELECT age_id(agtype_in('1')); diff --git a/regress/sql/cypher_with.sql b/regress/sql/cypher_with.sql index 93413f2c9..25e22b2a2 100644 --- a/regress/sql/cypher_with.sql +++ b/regress/sql/cypher_with.sql @@ -227,6 +227,149 @@ $$) as (a agtype,b agtype); SELECT drop_graph('graph', true); + +-- +-- Test accessor optimizations in WITH clause +-- +SELECT * FROM create_graph('with_accessor_opt'); + +SELECT * FROM cypher('with_accessor_opt', $$ + CREATE (a:Person {name: 'Alice', age: 30})-[r:KNOWS {since: 2020}]->(b:Person {name: 'Bob', age: 25}) +$$) AS (result agtype); + +-- Single with +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH id(n) AS node_id + RETURN node_id +$$) AS (node_id agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH properties(n) AS props + RETURN props +$$) AS (props agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + MATCH (n:Person) + WITH label(n) AS lbl + RETURN lbl +$$) AS (lbl agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n.name AS name, n.age AS age + RETURN name, age +$$) AS (name agtype, age agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + WITH id(r) AS rid + RETURN rid +$$) AS (rid agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n, id(n) AS nid + WITH n.name AS name, nid + RETURN name, nid +$$) AS (name agtype, nid agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + MATCH (n:Person) + WITH n as m + RETURN m +$$) AS (n vertex); + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n + RETURN id(n), n.name +$$) AS (id agtype, name agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + MATCH ()-[r:KNOWS]->() + WITH r + RETURN id(r), type(r) +$$) AS (id agtype, typ agtype); + +-- +-- Chained WITH tests +-- +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person)-[r:KNOWS]->(b:Person) + WITH a, b, id(a) AS aid + WITH a.name AS aname, b.name AS bname, aid + RETURN aname, bname, aid +$$) AS (aname agtype, bname agtype, aid agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n, n.age AS age + WHERE age > 20 + RETURN n.name +$$) AS (name agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n.name as name, id(n) AS nid + ORDER BY nid + RETURN name +$$) AS (name agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH id(n) AS id, count(*) AS cnt + RETURN id, cnt +$$) AS (id agtype, cnt agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + MATCH (n:Person) + WITH DISTINCT label(n) AS lbl + RETURN lbl +$$) AS (lbl agtype); + +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH n + WITH n, id(n) AS nid + WITH n.name AS name, nid + RETURN name, nid +$$) AS (name agtype, nid agtype); + +-- MATCH -> WITH accessors -> MATCH -> RETURN +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person) + WITH a, id(a) AS aid + MATCH (a)-[r:KNOWS]->(b) + RETURN aid, b.name +$$) AS (aid agtype, bname agtype); + +-- WITH + UNWIND with accessors +SELECT * FROM cypher('with_accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WITH collect(id(n)) AS ids + UNWIND ids AS nid + RETURN nid +$$) AS (nid agtype); + +-- Clean up +SELECT drop_graph('with_accessor_opt', true); + + -- -- End of test -- diff --git a/regress/sql/expr.sql b/regress/sql/expr.sql index 7304d4d65..b951a2367 100644 --- a/regress/sql/expr.sql +++ b/regress/sql/expr.sql @@ -3712,9 +3712,409 @@ $$) AS (out agtype); SELECT * FROM create_graph('issue_2289'); SELECT * FROM cypher('issue_2289', $$ RETURN (1 IN []) AS v $$) AS (v agtype); +-- +-- Accessor functions and properties extraction +-- without _agtype_build.. functions +-- +SELECT * FROM create_graph('accessor_opt'); + +SELECT * FROM cypher('accessor_opt', $$ + CREATE (a:Person {name: 'Alice', age: 30})-[r:KNOWS {since: 2020}]->(b:Person {name: 'Bob', age: 25}) +$$) AS (a agtype); + +-- +-- Vertex accessor tests +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN id(n) +$$) AS (plan agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN properties(n) +$$) AS (plan agtype); + +SELECT * FROM cypher('accessor_opt', $$ + MATCH (n:Person) + RETURN label(n) +$$) AS (plan agtype); + +-- should use n.properties in _agtype_access_operator +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN n.name, n.age +$$) AS (a agtype, b agtype); + +-- +-- Edge accessor tests +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN id(r) +$$) AS (plan agtype); + +SELECT * FROM cypher('accessor_opt', $$ + MATCH ()-[r:KNOWS]->() + RETURN type(r) +$$) AS (plan agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN start_id(r) +$$) AS (plan agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN end_id(r) +$$) AS (plan agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN properties(r) +$$) AS (plan agtype); + +-- should use r.properties in _agtype_access_operator +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN r.since +$$) AS (plan agtype); + +-- +-- Multiple accessors in same query +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN id(n), properties(n), n.name +$$) AS (a agtype, c agtype, d agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN id(r), start_id(r), end_id(r), properties(r) +$$) AS (a agtype, c agtype, d agtype, e agtype); + +-- +-- Accessors in WHERE clause +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WHERE id(n) > 0 + RETURN n.name +$$) AS (plan agtype); + +-- Compare two node ids +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person)-[r:KNOWS]->(b:Person) + WHERE id(a) < id(b) + RETURN a.name, b.name +$$) AS (aname agtype, bname agtype); + +-- Compare edge start_id and end_id +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + WHERE start_id(r) <> end_id(r) + RETURN id(r) +$$) AS (rid agtype); + +-- Property comparison +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WHERE n.age >= 25 + RETURN n.name +$$) AS (name agtype); + +-- Multiple property comparisons +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WHERE n.age > 20 AND n.name <> 'Unknown' + RETURN n.name +$$) AS (name agtype); + +-- Compare properties of two nodes +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person)-[r:KNOWS]->(b:Person) + WHERE a.age > b.age + RETURN a.name, b.name +$$) AS (aname agtype, bname agtype); + +-- Compare id with property +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WHERE id(n) > n.age + RETURN n.name +$$) AS (name agtype); + +-- Edge property comparison +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + WHERE r.since > 2015 + RETURN r.since +$$) AS (since agtype); + +-- IS NOT NULL on accessor +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + WHERE properties(n) IS NOT NULL + RETURN n.name +$$) AS (name agtype); + +-- label() comparison +SELECT * FROM cypher('accessor_opt', $$ + MATCH (n) + WHERE label(n) = 'Person' + RETURN n.name +$$) AS (name agtype); + +-- type() comparison on edge +SELECT * FROM cypher('accessor_opt', $$ + MATCH ()-[r]->() + WHERE type(r) = 'KNOWS' + RETURN id(r) +$$) AS (rid agtype); + +-- +-- Accessors in ORDER BY +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN n.name + ORDER BY n.name +$$) AS (plan agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN n.name + ORDER BY id(n) +$$) AS (plan agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN n.name, n.age + ORDER BY n.age, n.name +$$) AS (name agtype, age agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH ()-[r:KNOWS]->() + RETURN r.since + ORDER BY start_id(r) +$$) AS (since agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person)-[r:KNOWS]->(b:Person) + RETURN a.name, b.name + ORDER BY id(a), id(r), id(b) +$$) AS (aname agtype, bname agtype); + +-- +-- Accessors in expressions +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN [id(n), n.name] +$$) AS (plan agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN {id: id(n), name: n.name} +$$) AS (plan agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (n:Person) + RETURN n {.name, .age} +$$) AS (plan agtype); + +-- +-- Accessor function in multiple match +-- +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person) + WHERE id(a) > 0 + MATCH (a)-[r]->(b) + WHERE id(b) > 0 + RETURN properties(a), start_id(r), end_id(r) +$$) AS (props_a agtype, sid agtype, eid agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person) + MATCH (b:Person) + WHERE b.name <> a.name + RETURN a.name, b.name +$$) AS (a_name agtype, b_name agtype); + +SELECT * FROM cypher('accessor_opt', $$ + EXPLAIN (VERBOSE, COSTS OFF) + MATCH (a:Person) + MATCH (a)-[r1]->(b) + MATCH (b)-[r2]->(c) + RETURN id(a), id(b), id(c), a.name +$$) AS (a_id agtype, b_id agtype, c_id agtype, a_name agtype); + +-- +-- Composite types, vertex and edge +-- +SELECT * FROM create_graph('composite_types'); + +SELECT * FROM cypher('composite_types', $$ + CREATE (a:Person {name: 'Alice', age: 30})-[r:KNOWS {since: 2020}]->(b:Person {name: 'Bob', age: 25}) +$$) AS (result agtype); + +-- Return vertex as vertex type +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n +$$) AS (n vertex); + +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n + ORDER BY n.name +$$) AS (n vertex); + +-- Return edge as edge type +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r:KNOWS]->() + RETURN r +$$) AS (r edge); + +-- Return multiple entities +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person)-[r:KNOWS]->(b:Person) + RETURN a, b +$$) AS (a vertex, b vertex); + +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person)-[r:KNOWS]->(b:Person) + RETURN a, r, b +$$) AS (a vertex, r edge, b vertex); + +-- Mixed return: entity and agtype property +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n, n.name +$$) AS (n vertex, name agtype); + +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n, n.name + ORDER BY n.name +$$) AS (n vertex, name agtype); + +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r:KNOWS]->() + RETURN r, r.since +$$) AS (r edge, since agtype); + +-- IN operator with vertex and edge types +EXPLAIN (COSTS OFF) +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person), (b:Person) + WHERE a <> b AND a IN [a, b] + RETURN a.name +$$) AS (name agtype); + +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person), (b:Person) + WHERE a <> b AND a IN [a, b] + RETURN a.name +$$) AS (name agtype); + +EXPLAIN (COSTS OFF) +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r1:KNOWS]->(), ()-[r2:KNOWS]->() + WHERE r1 IN [r1, r2] + RETURN id(r1) +$$) AS (id agtype); + +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r1:KNOWS]->(), ()-[r2:KNOWS]->() + WHERE r1 IN [r1, r2] + RETURN id(r1) +$$) AS (id agtype); + +-- Equality operators with vertex and edge types +EXPLAIN (COSTS OFF) +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person), (b:Person) + WHERE a = b + RETURN a.name +$$) AS (name agtype); + +SELECT * FROM cypher('composite_types', $$ + MATCH (a:Person), (b:Person) + WHERE a = b + RETURN a.name +$$) AS (name agtype); + +EXPLAIN (COSTS OFF) +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r1:KNOWS]->(), ()-[r2:KNOWS]->() + WHERE r1 = r2 + RETURN id(r1) +$$) AS (id agtype); + +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r1:KNOWS]->(), ()-[r2:KNOWS]->() + WHERE r1 = r2 + RETURN id(r1) +$$) AS (id agtype); + +-- Cast vertex and edge to json +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n +$$) AS (n json); + +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r:KNOWS]->() + RETURN r +$$) AS (r json); + +-- Cast vertex and edge to jsonb +SELECT * FROM cypher('composite_types', $$ + MATCH (n:Person) + RETURN n +$$) AS (n jsonb); + +SELECT * FROM cypher('composite_types', $$ + MATCH ()-[r:KNOWS]->() + RETURN r +$$) AS (r jsonb); + -- -- Cleanup -- + +SELECT * FROM drop_graph('composite_types', true); +SELECT * FROM drop_graph('accessor_opt', true); SELECT * FROM drop_graph('issue_2289', true); SELECT * FROM drop_graph('issue_2263', true); SELECT * FROM drop_graph('issue_1988', true); diff --git a/regress/sql/pgvector.sql b/regress/sql/pgvector.sql index 677e78586..9c6c2c412 100644 --- a/regress/sql/pgvector.sql +++ b/regress/sql/pgvector.sql @@ -219,7 +219,7 @@ BEGIN USING hnsw ((( agtype_access_operator( VARIADIC ARRAY[ - _agtype_build_vertex(id, _label_name(%L::oid, id), properties), + properties, '"embedding"'::agtype ] )::text @@ -274,7 +274,7 @@ BEGIN USING hnsw (( agtype_access_operator( VARIADIC ARRAY[ - _agtype_build_vertex(id, _label_name(%L::oid, id), properties), + properties, '"embedding"'::agtype ] )::vector(4)) vector_cosine_ops); diff --git a/sql/age_main.sql b/sql/age_main.sql index 3e9a71c92..12ee304f4 100644 --- a/sql/age_main.sql +++ b/sql/age_main.sql @@ -368,13 +368,6 @@ CREATE FUNCTION ag_catalog._graphid(label_id int, entry_id bigint) PARALLEL SAFE AS 'MODULE_PATHNAME'; -CREATE FUNCTION ag_catalog._label_name(graph_oid oid, graphid) - RETURNS cstring - LANGUAGE c - IMMUTABLE -PARALLEL SAFE -AS 'MODULE_PATHNAME'; - CREATE FUNCTION ag_catalog._extract_label_id(graphid) RETURNS label_id LANGUAGE c diff --git a/sql/age_scalar.sql b/sql/age_scalar.sql index 5014fbbe8..ec098e8ee 100644 --- a/sql/age_scalar.sql +++ b/sql/age_scalar.sql @@ -93,6 +93,15 @@ RETURNS NULL ON NULL INPUT PARALLEL SAFE AS 'MODULE_PATHNAME'; +-- Helper function for optimized startNode/endNode +CREATE FUNCTION ag_catalog._get_vertex_by_graphid(text, graphid) + RETURNS agtype + LANGUAGE c + STABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + CREATE FUNCTION ag_catalog.age_length(agtype) RETURNS agtype LANGUAGE c diff --git a/sql/agtype_graphid.sql b/sql/agtype_graphid.sql index 0887db8a9..2970e6d54 100644 --- a/sql/agtype_graphid.sql +++ b/sql/agtype_graphid.sql @@ -43,6 +43,33 @@ CREATE CAST (agtype AS graphid) WITH FUNCTION ag_catalog.agtype_to_graphid(agtype) AS IMPLICIT; +-- +-- Composite types for vertex and edge +-- +CREATE TYPE ag_catalog.vertex AS ( + id graphid, + label agtype, + properties agtype +); + +CREATE TYPE ag_catalog.edge AS ( + id graphid, + label agtype, + end_id graphid, + start_id graphid, + properties agtype +); + +-- +-- Label name function +-- +CREATE FUNCTION ag_catalog._label_name(graph_oid oid, graphid) + RETURNS agtype + LANGUAGE c + IMMUTABLE +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + -- -- agtype - path -- @@ -57,7 +84,7 @@ AS 'MODULE_PATHNAME'; -- -- agtype - vertex -- -CREATE FUNCTION ag_catalog._agtype_build_vertex(graphid, cstring, agtype) +CREATE FUNCTION ag_catalog._agtype_build_vertex(graphid, agtype, agtype) RETURNS agtype LANGUAGE c IMMUTABLE @@ -69,7 +96,7 @@ AS 'MODULE_PATHNAME'; -- agtype - edge -- CREATE FUNCTION ag_catalog._agtype_build_edge(graphid, graphid, graphid, - cstring, agtype) + agtype, agtype) RETURNS agtype LANGUAGE c IMMUTABLE @@ -77,6 +104,155 @@ CALLED ON NULL INPUT PARALLEL SAFE AS 'MODULE_PATHNAME'; +-- +-- vertex/edge to agtype cast functions +-- +CREATE FUNCTION ag_catalog.vertex_to_agtype(vertex) + RETURNS agtype + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +CREATE FUNCTION ag_catalog.edge_to_agtype(edge) + RETURNS agtype + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +-- +-- Implicit casts from vertex/edge to agtype +-- +CREATE CAST (vertex AS agtype) + WITH FUNCTION ag_catalog.vertex_to_agtype(vertex) +AS IMPLICIT; + +CREATE CAST (edge AS agtype) + WITH FUNCTION ag_catalog.edge_to_agtype(edge) +AS IMPLICIT; + +CREATE FUNCTION ag_catalog.vertex_to_json(vertex) + RETURNS json + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +CREATE FUNCTION ag_catalog.edge_to_json(edge) + RETURNS json + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +CREATE CAST (vertex AS json) + WITH FUNCTION ag_catalog.vertex_to_json(vertex); + +CREATE CAST (edge AS json) + WITH FUNCTION ag_catalog.edge_to_json(edge); + +CREATE FUNCTION ag_catalog.vertex_to_jsonb(vertex) + RETURNS jsonb + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +CREATE FUNCTION ag_catalog.edge_to_jsonb(edge) + RETURNS jsonb + LANGUAGE c + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS 'MODULE_PATHNAME'; + +CREATE CAST (vertex AS jsonb) + WITH FUNCTION ag_catalog.vertex_to_jsonb(vertex); + +CREATE CAST (edge AS jsonb) + WITH FUNCTION ag_catalog.edge_to_jsonb(edge); + +-- +-- Equality operators for vertex and edge (compare by id) +-- +CREATE FUNCTION ag_catalog.vertex_eq(vertex, vertex) + RETURNS boolean + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS $$ SELECT $1.id = $2.id $$; + +CREATE OPERATOR = ( + FUNCTION = ag_catalog.vertex_eq, + LEFTARG = vertex, + RIGHTARG = vertex, + COMMUTATOR = =, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel +); + +CREATE FUNCTION ag_catalog.vertex_ne(vertex, vertex) + RETURNS boolean + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS $$ SELECT $1.id <> $2.id $$; + +CREATE OPERATOR <> ( + FUNCTION = ag_catalog.vertex_ne, + LEFTARG = vertex, + RIGHTARG = vertex, + COMMUTATOR = <>, + NEGATOR = =, + RESTRICT = neqsel, + JOIN = neqjoinsel +); + +CREATE FUNCTION ag_catalog.edge_eq(edge, edge) + RETURNS boolean + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS $$ SELECT $1.id = $2.id $$; + +CREATE OPERATOR = ( + FUNCTION = ag_catalog.edge_eq, + LEFTARG = edge, + RIGHTARG = edge, + COMMUTATOR = =, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel +); + +CREATE FUNCTION ag_catalog.edge_ne(edge, edge) + RETURNS boolean + LANGUAGE sql + IMMUTABLE +RETURNS NULL ON NULL INPUT +PARALLEL SAFE +AS $$ SELECT $1.id <> $2.id $$; + +CREATE OPERATOR <> ( + FUNCTION = ag_catalog.edge_ne, + LEFTARG = edge, + RIGHTARG = edge, + COMMUTATOR = <>, + NEGATOR = =, + RESTRICT = neqsel, + JOIN = neqjoinsel +); + CREATE FUNCTION ag_catalog._ag_enforce_edge_uniqueness2(graphid, graphid) RETURNS bool LANGUAGE c diff --git a/src/backend/catalog/ag_catalog.c b/src/backend/catalog/ag_catalog.c index 79beeb42e..f5276e092 100644 --- a/src/backend/catalog/ag_catalog.c +++ b/src/backend/catalog/ag_catalog.c @@ -160,6 +160,7 @@ static void drop_age_extension(DropStmt *stmt) /* reset global variables for OIDs */ clear_global_Oids_AGTYPE(); clear_global_Oids_GRAPHID(); + clear_global_Oids_VERTEX_EDGE(); } /* Check to see if the Utility Command is to drop the AGE Extension. */ diff --git a/src/backend/catalog/ag_label.c b/src/backend/catalog/ag_label.c index d407626fd..78579fe8d 100644 --- a/src/backend/catalog/ag_label.c +++ b/src/backend/catalog/ag_label.c @@ -168,6 +168,17 @@ char *get_label_seq_relation_name(const char *label_name) return psprintf("%s_id_seq", label_name); } +char *get_label_name(int32 label_id, Oid graph_oid) +{ + label_cache_data *cache_data; + + cache_data = search_label_graph_oid_cache(graph_oid, label_id); + if (cache_data) + return NameStr(cache_data->name); + else + return NULL; +} + PG_FUNCTION_INFO_V1(_label_name); /* @@ -177,9 +188,9 @@ PG_FUNCTION_INFO_V1(_label_name); Datum _label_name(PG_FUNCTION_ARGS) { char *label_name; - label_cache_data *label_cache; Oid graph; uint32 label_id; + agtype *result; if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) { @@ -188,7 +199,7 @@ Datum _label_name(PG_FUNCTION_ARGS) } graph = PG_GETARG_OID(0); - + /* Check if the graph OID is valid */ if (!graph_namespace_exists(graph)) { @@ -196,11 +207,8 @@ Datum _label_name(PG_FUNCTION_ARGS) errmsg("graph with oid %u does not exist", graph))); } - label_id = (int32)(((uint64)AG_GETARG_GRAPHID(1)) >> ENTRY_ID_BITS); - - label_cache = search_label_graph_oid_cache(graph, label_id); - - label_name = NameStr(label_cache->name); + label_id = get_graphid_label_id(AG_GETARG_GRAPHID(1)); + label_name = get_label_name(label_id, graph); /* If label_name is not found, error out */ if (label_name == NULL) @@ -210,10 +218,17 @@ Datum _label_name(PG_FUNCTION_ARGS) label_id, graph))); } + /* Convert cstring to agtype string */ if (IS_AG_DEFAULT_LABEL(label_name)) - PG_RETURN_CSTRING(""); + { + result = DATUM_GET_AGTYPE_P(string_to_agtype("")); + } + else + { + result = DATUM_GET_AGTYPE_P(string_to_agtype(label_name)); + } - PG_RETURN_CSTRING(label_name); + PG_RETURN_POINTER(result); } PG_FUNCTION_INFO_V1(_label_id); diff --git a/src/backend/executor/cypher_create.c b/src/backend/executor/cypher_create.c index 876b6f250..40f446bf6 100644 --- a/src/backend/executor/cypher_create.c +++ b/src/backend/executor/cypher_create.c @@ -436,7 +436,7 @@ static void create_edge(cypher_create_custom_scan_state *css, Datum result; result = make_edge( - id, start_id, end_id, CStringGetDatum(node->label_name), + id, start_id, end_id, string_to_agtype(node->label_name), scanTupleSlot->tts_values[node->prop_attr_num]); if (CYPHER_TARGET_NODE_IN_PATH(node->flags)) @@ -526,7 +526,7 @@ static Datum create_vertex(cypher_create_custom_scan_state *css, scantuple = ps->ps_ExprContext->ecxt_scantuple; /* make the vertex agtype */ - result = make_vertex(id, CStringGetDatum(node->label_name), + result = make_vertex(id, string_to_agtype(node->label_name), scanTupleSlot->tts_values[node->prop_attr_num]); /* append to the path list */ diff --git a/src/backend/executor/cypher_merge.c b/src/backend/executor/cypher_merge.c index 2b3d1f7dd..6f2e9376e 100644 --- a/src/backend/executor/cypher_merge.c +++ b/src/backend/executor/cypher_merge.c @@ -1325,7 +1325,7 @@ static Datum merge_vertex(cypher_merge_custom_scan_state *css, Datum result; /* make the vertex agtype */ - result = make_vertex(id, CStringGetDatum(node->label_name), prop); + result = make_vertex(id, string_to_agtype(node->label_name), prop); /* append to the path list */ if (CYPHER_TARGET_NODE_IN_PATH(node->flags)) @@ -1664,7 +1664,7 @@ static void merge_edge(cypher_merge_custom_scan_state *css, Datum result; result = make_edge(id, start_id, end_id, - CStringGetDatum(node->label_name), prop); + string_to_agtype(node->label_name), prop); /* add the Datum to the list of entities for creating the path variable */ if (CYPHER_TARGET_NODE_IN_PATH(node->flags)) diff --git a/src/backend/executor/cypher_set.c b/src/backend/executor/cypher_set.c index a6e64ba56..fe3633f8d 100644 --- a/src/backend/executor/cypher_set.c +++ b/src/backend/executor/cypher_set.c @@ -28,6 +28,7 @@ #include "executor/cypher_utils.h" #include "utils/age_global_graph.h" #include "catalog/ag_graph.h" +#include "utils/agtype.h" static void begin_cypher_set(CustomScanState *node, EState *estate, int eflags); @@ -644,7 +645,7 @@ void apply_update_list(CustomScanState *node, if (original_entity_value->type == AGTV_VERTEX) { new_entity = make_vertex(GRAPHID_GET_DATUM(id->val.int_value), - CStringGetDatum(label_name), + string_to_agtype(label_name), AGTYPE_P_GET_DATUM(agtype_value_to_agtype(altered_properties))); slot = populate_vertex_tts(slot, id, altered_properties); @@ -657,7 +658,7 @@ void apply_update_list(CustomScanState *node, new_entity = make_edge(GRAPHID_GET_DATUM(id->val.int_value), GRAPHID_GET_DATUM(startid->val.int_value), GRAPHID_GET_DATUM(endid->val.int_value), - CStringGetDatum(label_name), + string_to_agtype(label_name), AGTYPE_P_GET_DATUM(agtype_value_to_agtype(altered_properties))); slot = populate_edge_tts(slot, id, startid, endid, diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 015c0719e..3f1697182 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -39,6 +39,8 @@ #include "parser/parsetree.h" #include "parser/parse_relation.h" #include "rewrite/rewriteHandler.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" #include "catalog/ag_graph.h" #include "catalog/ag_label.h" @@ -350,6 +352,7 @@ static bool isa_special_VLE_case(cypher_path *path); static ParseNamespaceItem *find_pnsi(cypher_parsestate *cpstate, char *varname); static bool has_list_comp_or_subquery(Node *expr, void *context); +static bool clause_is_dml(cypher_clause *clause); static bool clause_chain_has_dml(cypher_clause *clause); static Node *make_false_where_clause(bool volatile_needed); @@ -527,6 +530,19 @@ Query *transform_cypher_clause(cypher_parsestate *cpstate, ereport(ERROR, (errmsg_internal("unexpected Node for cypher_clause"))); } + /* + * Force a terminal data-modifying clause (CREATE/SET/DELETE/MERGE) to emit + * agtype instead of the vertex/edge composite. Its executor and the + * consuming cypher() SRF operate on agtype. Read clauses may emit composites + * directly, so this is restricted to a trailing DML clause. The appended + * clause FuncExpr is already agtype and is skipped by the coercion. + */ + if (clause->next == NULL && clause_is_dml(clause)) + { + coerce_target_entities_to_agtype((ParseState *) cpstate, + result->targetList); + } + result->querySource = QSRC_ORIGINAL; result->canSetTag = true; @@ -1015,6 +1031,15 @@ transform_cypher_union_tree(cypher_parsestate *cpstate, cypher_clause *clause, int32 rescoltypmod; Oid rescolcoll; + /* Cast vertex/edge to agtype for UNION compatibility */ + lcolnode = coerce_entity_to_agtype(pstate, lcolnode); + ltle->expr = (Expr *) lcolnode; + lcoltype = exprType(lcolnode); + + rcolnode = coerce_entity_to_agtype(pstate, rcolnode); + rtle->expr = (Expr *) rcolnode; + rcoltype = exprType(rcolnode); + /* select common type, same as CASE et al */ rescoltype = select_common_type(pstate, list_make2(lcolnode, rcolnode), @@ -2242,12 +2267,14 @@ cypher_update_information *transform_cypher_set_item_list( */ if (IsA(set_item->expr, ColumnRef)) { - List *qualified_name, *args; + List *fname; - qualified_name = list_make2(makeString("ag_catalog"), - makeString("age_properties")); - args = list_make1(set_item->expr); - set_item->expr = (Node *)makeFuncCall(qualified_name, args, + /* + * make unqualified function name so that potential + * optimization can kick in + */ + fname = list_make1(makeString("properties")); + set_item->expr = (Node *)makeFuncCall(fname, list_make1(set_item->expr), COERCE_SQL_SYNTAX, -1); } } @@ -5170,26 +5197,7 @@ static List *transform_match_entities(cypher_parsestate *cpstate, Query *query, */ else if (prop_var != NULL) { - /* - * Remember that prop_var is already transformed. We need - * to built the transform manually. - */ - FuncCall *fc = NULL; - List *targs = NIL; - List *fname = NIL; - - targs = lappend(targs, prop_var); - fname = list_make2(makeString("ag_catalog"), - makeString("age_properties")); - fc = makeFuncCall(fname, targs, COERCE_SQL_SYNTAX, -1); - - /* - * Hand off to ParseFuncOrColumn to create the function - * expression for properties(prop_var) - */ - prop_expr = ParseFuncOrColumn(pstate, fname, targs, - pstate->p_last_srf, fc, false, - -1); + prop_expr = make_properties_expr(prop_var); } if (is_ag_node(node->props, cypher_map)) @@ -5302,26 +5310,7 @@ static List *transform_match_entities(cypher_parsestate *cpstate, Query *query, */ else if (prop_var != NULL) { - /* - * Remember that prop_var is already transformed. We need - * to built the transform manually. - */ - FuncCall *fc = NULL; - List *targs = NIL; - List *fname = NIL; - - targs = lappend(targs, prop_var); - fname = list_make2(makeString("ag_catalog"), - makeString("age_properties")); - fc = makeFuncCall(fname, targs, COERCE_SQL_SYNTAX, -1); - - /* - * Hand off to ParseFuncOrColumn to create the function - * expression for properties(prop_var) - */ - prop_expr = ParseFuncOrColumn(pstate, fname, targs, - pstate->p_last_srf, fc, - false, -1); + prop_expr = make_properties_expr(prop_var); } if (is_ag_node(rel->props, cypher_map)) @@ -5480,17 +5469,25 @@ transform_match_create_path_variable(cypher_parsestate *cpstate, if (entity->expr != NULL) { + Node *entity_expr = (Node *)entity->expr; + /* * Is it a NULL constant, meaning there was an invalid label? * If so, flag it for later */ - if (IsA(entity->expr, Const) && - ((Const*)(entity->expr))->constisnull) + if (IsA(entity_expr, Const) && + ((Const*)(entity_expr))->constisnull) { null_path_entity = true; } - entity_exprs = lappend(entity_exprs, entity->expr); + /* + * If the entity is a vertex/edge, convert it to agtype since + * _agtype_build_path expects agtype arguments. + */ + entity_expr = coerce_entity_to_agtype(pstate, entity_expr); + + entity_exprs = lappend(entity_exprs, entity_expr); } } @@ -5586,8 +5583,36 @@ static Node *make_qual(cypher_parsestate *cpstate, { List *qualified_name, *args; Node *node; + char *entity_name; + + if (is_vertex_or_edge((Node *) entity->expr)) + { + A_Indirection *indir = makeNode(A_Indirection); + ColumnRef *cr = makeNode(ColumnRef); + + if (entity->type == ENT_VERTEX) + { + entity_name = entity->entity.node->name; + } + else if (entity->type == ENT_EDGE) + { + entity_name = entity->entity.rel->name; + } + else + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("unknown entity type"))); + } + + cr->fields = list_make1(makeString(entity_name)); + cr->location = -1; - if (IsA(entity->expr, Var)) + indir->arg = (Node *)cr; + indir->indirection = list_make1(makeString(col_name)); + + node = (Node *)indir; + } + else if (IsA(entity->expr, Var)) { char *function_name; @@ -5596,14 +5621,12 @@ static Node *make_qual(cypher_parsestate *cpstate, qualified_name = list_make2(makeString("ag_catalog"), makeString(function_name)); - args = list_make1(entity->expr); node = (Node *)makeFuncCall(qualified_name, args, COERCE_EXPLICIT_CALL, -1); } else { - char *entity_name; ColumnRef *cr = makeNode(ColumnRef); if (entity->type == ENT_EDGE) @@ -5628,6 +5651,87 @@ static Node *make_qual(cypher_parsestate *cpstate, return node; } +/* + * Helper function to get field number and type for vertex/edge field access. + * Vertex: (id(1), label(2), properties(3)) + * Edge: (id(1), label(2), end_id(3), start_id(4), properties(5)) + */ +void get_record_field_info(char *field_name, Oid entity_type, + AttrNumber *fieldnum, Oid *fieldtype) +{ + bool is_vertex = (entity_type == VERTEXOID); + bool is_edge = (entity_type == EDGEOID); + + Assert(field_name != NULL); + Assert(fieldnum != NULL); + Assert(fieldtype != NULL); + Assert(is_vertex || is_edge); + + *fieldnum = InvalidAttrNumber; + *fieldtype = InvalidOid; + + if (strcasecmp(field_name, "id") == 0) + { + *fieldnum = 1; + *fieldtype = GRAPHIDOID; + } + else if (is_vertex) + { + if (strcasecmp(field_name, "label") == 0) + { + *fieldnum = 2; + *fieldtype = AGTYPEOID; + } + else if (strcasecmp(field_name, "properties") == 0) + { + *fieldnum = 3; + *fieldtype = AGTYPEOID; + } + } + else if (is_edge) + { + if (strcasecmp(field_name, "label") == 0 || + strcasecmp(field_name, "type") == 0) + { + *fieldnum = 2; + *fieldtype = AGTYPEOID; + } + else if (strcasecmp(field_name, "end_id") == 0 || + strcasecmp(field_name, "endnode") == 0) + { + *fieldnum = 3; + *fieldtype = GRAPHIDOID; + } + else if (strcasecmp(field_name, "start_id") == 0 || + strcasecmp(field_name, "startnode") == 0) + { + *fieldnum = 4; + *fieldtype = GRAPHIDOID; + } + else if (strcasecmp(field_name, "properties") == 0) + { + *fieldnum = 5; + *fieldtype = AGTYPEOID; + } + } +} + +/* + * Helper function to build a FieldSelect node + */ +FieldSelect *make_field_select(Var *var, AttrNumber fieldnum, Oid resulttype) +{ + FieldSelect *fselect = makeNode(FieldSelect); + + fselect->arg = (Expr *)copyObject(var); + fselect->fieldnum = fieldnum; + fselect->resulttype = resulttype; + fselect->resulttypmod = -1; + fselect->resultcollid = InvalidOid; + + return fselect; +} + static Expr *transform_cypher_edge(cypher_parsestate *cpstate, cypher_relationship *rel, List **target_list, @@ -6178,17 +6282,14 @@ static Node *make_edge_expr(cypher_parsestate *cpstate, { ParseState *pstate = (ParseState *)cpstate; Oid label_name_func_oid; - Oid func_oid; Node *id, *start_id, *end_id; Const *graph_oid_const; Node *props; - List *args, *label_name_args; - FuncExpr *func_expr; + List *label_name_args; FuncExpr *label_name_func_expr; + RowExpr *row_expr; - func_oid = get_ag_func_oid("_agtype_build_edge", 5, GRAPHIDOID, GRAPHIDOID, - GRAPHIDOID, CSTRINGOID, AGTYPEOID); - + /* Get raw column references */ id = scanNSItemForColumn(pstate, pnsi, 0, AG_EDGE_COLNAME_ID, -1); start_id = scanNSItemForColumn(pstate, pnsi, 0, AG_EDGE_COLNAME_START_ID, -1); @@ -6204,40 +6305,45 @@ static Node *make_edge_expr(cypher_parsestate *cpstate, label_name_args = list_make2(graph_oid_const, id); - label_name_func_expr = makeFuncExpr(label_name_func_oid, CSTRINGOID, + label_name_func_expr = makeFuncExpr(label_name_func_oid, AGTYPEOID, label_name_args, InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); label_name_func_expr->location = -1; props = scanNSItemForColumn(pstate, pnsi, 0, AG_EDGE_COLNAME_PROPERTIES, -1); - args = list_make4(id, start_id, end_id, label_name_func_expr); - args = lappend(args, props); - - func_expr = makeFuncExpr(func_oid, AGTYPEOID, args, InvalidOid, InvalidOid, - COERCE_EXPLICIT_CALL); - func_expr->location = -1; - - return (Node *)func_expr; + /* + * Create a RowExpr with the edge composite type. + * Edge format: (id, label, end_id, start_id, properties) + * Implicit cast to agtype is used when needed. + */ + row_expr = makeNode(RowExpr); + row_expr->args = list_make5(id, label_name_func_expr, end_id, start_id, props); + row_expr->row_typeid = EDGEOID; + row_expr->row_format = COERCE_EXPLICIT_CALL; + row_expr->colnames = list_make5(makeString("id"), + makeString("label"), + makeString("end_id"), + makeString("start_id"), + makeString("properties")); + row_expr->location = -1; + + return (Node *)row_expr; } static Node *make_vertex_expr(cypher_parsestate *cpstate, ParseNamespaceItem *pnsi) { ParseState *pstate = (ParseState *)cpstate; Oid label_name_func_oid; - Oid func_oid; Node *id; Const *graph_oid_const; Node *props; - List *args, *label_name_args; - FuncExpr *func_expr; + List *label_name_args; FuncExpr *label_name_func_expr; + RowExpr *row_expr; Assert(pnsi != NULL); - func_oid = get_ag_func_oid("_agtype_build_vertex", 3, GRAPHIDOID, - CSTRINGOID, AGTYPEOID); - id = scanNSItemForColumn(pstate, pnsi, 0, AG_VERTEX_COLNAME_ID, -1); label_name_func_oid = get_ag_func_oid("_label_name", 2, OIDOID, @@ -6249,7 +6355,7 @@ static Node *make_vertex_expr(cypher_parsestate *cpstate, label_name_args = list_make2(graph_oid_const, id); - label_name_func_expr = makeFuncExpr(label_name_func_oid, CSTRINGOID, + label_name_func_expr = makeFuncExpr(label_name_func_oid, AGTYPEOID, label_name_args, InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); label_name_func_expr->location = -1; @@ -6257,13 +6363,21 @@ static Node *make_vertex_expr(cypher_parsestate *cpstate, props = scanNSItemForColumn(pstate, pnsi, 0, AG_VERTEX_COLNAME_PROPERTIES, -1); - args = list_make3(id, label_name_func_expr, props); - - func_expr = makeFuncExpr(func_oid, AGTYPEOID, args, InvalidOid, InvalidOid, - COERCE_EXPLICIT_CALL); - func_expr->location = -1; - - return (Node *)func_expr; + /* + * Create a RowExpr with the vertex composite type. + * Vertex format: (id, label, properties) + * Implicit cast to agtype is used when needed. + */ + row_expr = makeNode(RowExpr); + row_expr->args = list_make3(id, label_name_func_expr, props); + row_expr->row_typeid = VERTEXOID; + row_expr->row_format = COERCE_EXPLICIT_CALL; + row_expr->colnames = list_make3(makeString("id"), + makeString("label"), + makeString("properties")); + row_expr->location = -1; + + return (Node *)row_expr; } static Query *transform_cypher_create(cypher_parsestate *cpstate, @@ -6312,7 +6426,6 @@ static Query *transform_cypher_create(cypher_parsestate *cpstate, target_nodes->flags |= CYPHER_CLAUSE_FLAG_TERMINAL; } - func_expr = make_clause_func_expr(CREATE_CLAUSE_FUNCTION_NAME, (Node *)target_nodes); @@ -6726,7 +6839,6 @@ static void add_volatile_wrapper_to_target_entry(List *target_list, int resno) TargetEntry *te = (TargetEntry *)lfirst(lc); if (te->resno == resno) { - /* wrap it */ te->expr = add_volatile_wrapper(te->expr); return; } @@ -7069,6 +7181,18 @@ static void advance_transform_entities_to_next_clause(List *entities) } } +/* + * Return true if this single clause is a data-modifying operation + * (CREATE, SET, DELETE, or MERGE). + */ +static bool clause_is_dml(cypher_clause *clause) +{ + return is_ag_node(clause->self, cypher_create) || + is_ag_node(clause->self, cypher_set) || + is_ag_node(clause->self, cypher_delete) || + is_ag_node(clause->self, cypher_merge); +} + /* * Walk the clause chain and return true if any clause is a * data-modifying operation (CREATE, SET, DELETE, or MERGE). @@ -7077,10 +7201,7 @@ static bool clause_chain_has_dml(cypher_clause *clause) { while (clause != NULL) { - if (is_ag_node(clause->self, cypher_create) || - is_ag_node(clause->self, cypher_set) || - is_ag_node(clause->self, cypher_delete) || - is_ag_node(clause->self, cypher_merge)) + if (clause_is_dml(clause)) { return true; } @@ -7282,6 +7403,33 @@ Query *cypher_parse_sub_analyze(Node *parseTree, return query; } +/* + * Resolve prop_expr for each SET item by looking up its target entry. + * The planner may strip SET expression target entries from the plan, + * so we embed the Expr in the update item for direct evaluation. + */ +static void +resolve_merge_set_exprs(List *set_items, List *targetList, + const char *clause_name) +{ + ListCell *lc; + + foreach(lc, set_items) + { + cypher_update_item *item = lfirst(lc); + TargetEntry *set_tle = get_tle_by_resno(targetList, + item->prop_position); + if (set_tle == NULL) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("%s target entry not found at position %d", + clause_name, item->prop_position))); + } + item->prop_expr = (Node *)set_tle->expr; + } +} + /* * Function for transforming MERGE. * @@ -7322,33 +7470,6 @@ Query *cypher_parse_sub_analyze(Node *parseTree, * similar to OPTIONAL MATCH, however with the added feature of creating the * path if not there, rather than just emitting NULL. */ -/* - * Resolve prop_expr for each SET item by looking up its target entry. - * The planner may strip SET expression target entries from the plan, - * so we embed the Expr in the update item for direct evaluation. - */ -static void -resolve_merge_set_exprs(List *set_items, List *targetList, - const char *clause_name) -{ - ListCell *lc; - - foreach(lc, set_items) - { - cypher_update_item *item = lfirst(lc); - TargetEntry *set_tle = get_tle_by_resno(targetList, - item->prop_position); - if (set_tle == NULL) - { - ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("%s target entry not found at position %d", - clause_name, item->prop_position))); - } - item->prop_expr = (Node *)set_tle->expr; - } -} - static Query *transform_cypher_merge(cypher_parsestate *cpstate, cypher_clause *clause) { @@ -7520,6 +7641,36 @@ transform_merge_make_lateral_join(cypher_parsestate *cpstate, Query *query, */ j->larg = transform_clause_for_join(cpstate, clause->prev, &l_rte, &l_nsitem, l_alias); + + /* + * Wrap vertex/edge columns in the left subquery with volatile wrapper + * before adding to namespace and building property expressions since + * our executors are strictly tied to agtype. + * + * This is necessary because: + * 1. The volatile wrapper will be applied to the final targetList later, + * changing column types from VERTEXOID/EDGEOID to AGTYPEOID + * 2. If we build property expressions before this, the Vars will have + * vartype=VERTEXOID but the execution slot will have AGTYPEOID + * 3. By wrapping now, the namespace lookups will see AGTYPEOID columns + * and build expressions with correct type expectations + */ + foreach(lc, l_rte->subquery->targetList) + { + TargetEntry *te = (TargetEntry *)lfirst(lc); + Oid te_type = exprType((Node *)te->expr); + + if (te_type == VERTEXOID || te_type == EDGEOID) + { + /* resno is 1-based */ + int col_idx = te->resno - 1; + + /* Wrap the expression in the subquery targetList */ + te->expr = add_volatile_wrapper(te->expr); + l_nsitem->p_nscolumns[col_idx].p_vartype = AGTYPEOID; + } + } + pstate->p_namespace = lappend(pstate->p_namespace, l_nsitem); /* @@ -7605,9 +7756,15 @@ transform_merge_make_lateral_join(cypher_parsestate *cpstate, Query *query, nscolumns[col_idx].p_varno = join_rtindex; nscolumns[col_idx].p_varattno = col_idx + 1; - nscolumns[col_idx].p_vartype = v->vartype; - nscolumns[col_idx].p_vartypmod = v->vartypmod; - nscolumns[col_idx].p_varcollid = v->varcollid; + nscolumns[col_idx].p_vartype = + (v->vartype == VERTEXOID || v->vartype == EDGEOID) ? + AGTYPEOID : v->vartype; + nscolumns[col_idx].p_vartypmod = + (nscolumns[col_idx].p_vartype == AGTYPEOID) ? -1 : + v->vartypmod; + nscolumns[col_idx].p_varcollid = + (nscolumns[col_idx].p_vartype == AGTYPEOID) ? InvalidOid : + v->varcollid; nscolumns[col_idx].p_varnosyn = join_rtindex; nscolumns[col_idx].p_varattnosyn = col_idx + 1; col_idx++; diff --git a/src/backend/parser/cypher_expr.c b/src/backend/parser/cypher_expr.c index 1ed777486..e62f2c6e7 100644 --- a/src/backend/parser/cypher_expr.c +++ b/src/backend/parser/cypher_expr.c @@ -116,6 +116,7 @@ static bool function_exists(char *funcname, char *extension); static Node *coerce_expr_flexible(ParseState *pstate, Node *expr, Oid source_oid, Oid target_oid, int32 t_typemod, bool error_out); +static Oid get_entity_record_type(Node *node); /* transform a cypher expression */ Node *transform_cypher_expr(cypher_parsestate *cpstate, Node *expr, @@ -537,6 +538,25 @@ static Node *transform_AEXPR_OP(cypher_parsestate *cpstate, A_Expr *a) Node *last_srf = pstate->p_last_srf; Node *lexpr = transform_cypher_expr_recurse(cpstate, a->lexpr); Node *rexpr = transform_cypher_expr_recurse(cpstate, a->rexpr); + char *opname = strVal(linitial(a->name)); + Oid lexpr_type = exprType(lexpr); + Oid rexpr_type = exprType(rexpr); + bool is_eq_op = (strcmp(opname, "=") == 0 || strcmp(opname, "<>") == 0); + + /* + * For equality/inequality operators between same vertex/edge types, + * use native operators instead of coercing to agtype. + */ + if (is_eq_op && (lexpr_type == rexpr_type) && + (lexpr_type == VERTEXOID || lexpr_type == EDGEOID)) + { + return (Node *)make_op(pstate, a->name, lexpr, rexpr, last_srf, + a->location); + } + + /* Cast vertex/edge to agtype for operator compatibility */ + lexpr = coerce_entity_to_agtype(pstate, lexpr); + rexpr = coerce_entity_to_agtype(pstate, rexpr); return (Node *)make_op(pstate, a->name, lexpr, rexpr, last_srf, a->location); @@ -871,8 +891,7 @@ static Node *transform_cypher_map_projection(cypher_parsestate *cpstate, FuncExpr *fexpr_new_map; bool has_all_prop_selector; Node *transformed_map_var; - Oid foid_age_properties; - FuncExpr *fexpr_orig_map; + Node *fexpr_orig_map; pstate = (ParseState *)cpstate; keyvals = NIL; @@ -886,11 +905,8 @@ static Node *transform_cypher_map_projection(cypher_parsestate *cpstate, */ transformed_map_var = transform_cypher_expr_recurse(cpstate, (Node *)cmp->map_var); - foid_age_properties = get_ag_func_oid("age_properties", 1, AGTYPEOID); - fexpr_orig_map = makeFuncExpr(foid_age_properties, AGTYPEOID, - list_make1(transformed_map_var), InvalidOid, - InvalidOid, COERCE_EXPLICIT_CALL); - fexpr_orig_map->location = cmp->location; + + fexpr_orig_map = make_properties_expr(transformed_map_var); /* * Builds a new map. Each map projection element is transformed into a key @@ -1161,6 +1177,67 @@ static Node *transform_cypher_map(cypher_parsestate *cpstate, cypher_map *cm) return (Node *)fexpr; } +/* + * Check if a node is a vertex or edge composite type. + */ +bool is_vertex_or_edge(Node *node) +{ + Oid typeid; + + if (node == NULL) + return false; + + if (IsA(node, RowExpr)) + typeid = ((RowExpr *)node)->row_typeid; + else if (IsA(node, Var)) + typeid = ((Var *)node)->vartype; + else + return false; + + return (typeid == VERTEXOID || typeid == EDGEOID); +} + +/* + * Extract a field from a vertex/edge Var or RowExpr. + */ +Node *extract_field_from_record(Node *node, char *field_name) +{ + AttrNumber fieldnum; + Oid fieldtype; + Oid type_id = get_entity_record_type(node); + + get_record_field_info(field_name, type_id, + &fieldnum, &fieldtype); + + if (fieldnum == InvalidAttrNumber || fieldtype == InvalidOid) + { + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("fieldname '%s' does not exist for the entity", field_name))); + } + + if (IsA(node, Var)) + { + return (Node *)make_field_select((Var *)node, fieldnum, fieldtype); + } + else if (IsA(node, RowExpr)) + { + RowExpr *re = (RowExpr *)node; + + if (list_length(re->args) < fieldnum) + { + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("field '%s' does not exist in RowExpr", field_name))); + } + + /* RowExpr args are 0-indexed, but fieldnum is 1-indexed */ + return (Node *)list_nth(re->args, fieldnum - 1); + } + + return NULL; +} + /* * Helper function to transform a cypher list into an agtype list. The function * will use agtype_add to concatenate argument lists when the number of list @@ -1198,9 +1275,10 @@ static Node *transform_cypher_list(cypher_parsestate *cpstate, cypher_list *cl) foreach (le, cl->elems) { Node *texpr = NULL; + Node *cexpr = lfirst(le); /* transform the argument */ - texpr = transform_cypher_expr_recurse(cpstate, lfirst(le)); + texpr = transform_cypher_expr_recurse(cpstate, cexpr); /* * If we have more than 100 elements we will need to add in the list @@ -1313,21 +1391,21 @@ static Node *transform_column_ref_for_indirection(cypher_parsestate *cpstate, pnsi = refnameNamespaceItem(pstate, NULL, relname, cr->location, &levels_up); - /* - * If we didn't find anything, try looking for a previous variable - * reference. Otherwise, return NULL (colNameToVar will return NULL - * if nothing is found). - */ - if (!pnsi) + if (pnsi) { - Node *prev_var = colNameToVar(pstate, relname, false, cr->location); - - return prev_var; + /* find the properties column of the NSI and return a var for it */ + node = scanNSItemForColumn(pstate, pnsi, levels_up, "properties", + cr->location); + } + else + { + node = colNameToVar(pstate, relname, false, cr->location); } - /* find the properties column of the NSI and return a var for it */ - node = scanNSItemForColumn(pstate, pnsi, levels_up, "properties", - cr->location); + if (node && is_vertex_or_edge(node)) + { + return extract_field_from_record(node, "properties"); + } /* * If there's no "properties" column, continue transforming the @@ -1994,6 +2072,145 @@ static bool function_exists(char *funcname, char *extension) return found; } +/* + * Check if function name is an accessor function that can be optimized + */ +static bool is_accessor_function(const char *func_name) +{ + return (strcasecmp("id", func_name) == 0 || + strcasecmp("properties", func_name) == 0 || + strcasecmp("type", func_name) == 0 || + strcasecmp("label", func_name) == 0 || + strcasecmp("start_id", func_name) == 0 || + strcasecmp("end_id", func_name) == 0 || + strcasecmp("startnode", func_name) == 0 || + strcasecmp("endnode", func_name) == 0); +} + +static Oid get_entity_record_type(Node *node) +{ + Oid typeid = InvalidOid; + + if (IsA(node, RowExpr)) + { + typeid = ((RowExpr *)node)->row_typeid; + } + else if (IsA(node, Var)) + { + typeid = ((Var *)node)->vartype; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("argument must be a vertex or edge record"))); + } + + if (typeid != VERTEXOID && typeid != EDGEOID) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("argument must be a vertex or edge record"))); + } + + return typeid; +} + +/* + * Optimize accessor functions when called on RECORD entities + * Extracts fields directly using FieldSelect instead of building and parsing agtype + */ +static Node *optimize_accessor_function(cypher_parsestate *cpstate, + const char *func_name, + FuncCall *fn, + List *targs) +{ + ParseState *pstate = &cpstate->pstate; + Node *arg; + AttrNumber fieldnum; + Oid fieldtype; + Node *result; + Oid entity_type; + bool is_node_func = (strcasecmp(func_name, "startNode") == 0 || + strcasecmp(func_name, "endNode") == 0); + + if (is_node_func) + { + if (list_length(targs) < 2) + return NULL; + arg = (Node *)lsecond(targs); + } + else + { + if (list_length(targs) < 1) + return NULL; + arg = (Node *)linitial(targs); + } + + if (!is_vertex_or_edge(arg)) + { + return NULL; + } + + entity_type = get_entity_record_type(arg); + get_record_field_info((char *)func_name, entity_type, &fieldnum, &fieldtype); + + if (fieldnum != InvalidAttrNumber && fieldtype != InvalidOid) + { + Node *field = NULL; + + /* Validate RowExpr has enough fields */ + if (IsA(arg, RowExpr)) + { + RowExpr *re = (RowExpr *)arg; + if (list_length(re->args) < fieldnum) + { + return NULL; + } + + /* RowExpr args are 0-indexed, but fieldnum is 1-indexed */ + field = (Node *)list_nth(re->args, fieldnum - 1); + } + else if (IsA(arg, Var)) + { + field = (Node *)make_field_select((Var *)arg, fieldnum, fieldtype); + } + + /* For startNode/endNode functions on edges, look up the vertex */ + if (is_node_func && entity_type == EDGEOID) + { + Oid func_oid; + Node *graph_name; + + graph_name = (Node *) makeConst(TEXTOID, -1, InvalidOid, -1, + CStringGetTextDatum(cpstate->graph_name), + false, false); + + /* Call _get_vertex_by_graphid(graph_name, vertex_id) */ + func_oid = get_ag_func_oid("_get_vertex_by_graphid", 2, TEXTOID, GRAPHIDOID); + result = (Node *)makeFuncExpr(func_oid, AGTYPEOID, + list_make2(graph_name, field), + InvalidOid, InvalidOid, + COERCE_EXPLICIT_CALL); + } + else + { + result = field; + } + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("%s() argument must be an %s or null", + func_name, + (entity_type == VERTEXOID) ? "edge" : "vertex"), + parser_errposition(pstate, fn->location))); + } + + return result; +} + /* * Code borrowed from PG's transformFuncCall and updated for AGE */ @@ -2005,14 +2222,24 @@ static Node *transform_FuncCall(cypher_parsestate *cpstate, FuncCall *fn) List *fname = NIL; ListCell *arg; Node *retval = NULL; + bool is_accessor = false; + + /* Get function name to check if it has special optimizations */ + if (list_length(fn->funcname) == 1) + { + is_accessor = is_accessor_function(strVal(linitial(fn->funcname))); + } /* Transform the list of arguments ... */ foreach(arg, fn->args) { Node *farg = NULL; + Node *transformed_arg = NULL; farg = (Node *)lfirst(arg); - targs = lappend(targs, transform_cypher_expr_recurse(cpstate, farg)); + transformed_arg = transform_cypher_expr_recurse(cpstate, farg); + + targs = lappend(targs, transformed_arg); } /* within group should not happen */ @@ -2044,10 +2271,10 @@ static Node *transform_FuncCall(cypher_parsestate *cpstate, FuncCall *fn) * is not empty. Then prepend the graph name if necessary. */ if ((list_length(targs) != 0) && - (strcmp("startNode", name) == 0 || - strcmp("endNode", name) == 0 || - strcmp("vle", name) == 0 || - strcmp("vertex_stats", name) == 0)) + (strcasecmp("startNode", name) == 0 || + strcasecmp("endNode", name) == 0 || + strcasecmp("vle", name) == 0 || + strcasecmp("vertex_stats", name) == 0)) { char *graph_name = cpstate->graph_name; Datum d = string_to_agtype(graph_name); @@ -2056,6 +2283,16 @@ static Node *transform_FuncCall(cypher_parsestate *cpstate, FuncCall *fn) targs = lcons(c, targs); } + + /* + * Try to optimize accessor functions on RECORD entities + */ + if (is_accessor) + { + Node *optimized = optimize_accessor_function(cpstate, name, fn, targs); + if (optimized != NULL) + return optimized; + } } /* * If it's not in age, check if it's a potential call to some function @@ -2186,8 +2423,16 @@ static Node *transform_CaseExpr(cypher_parsestate *cpstate, CaseExpr /* generate placeholder for test expression */ if (arg) { - if (exprType(arg) == UNKNOWNOID) + Oid argtype = exprType(arg); + + if (argtype == UNKNOWNOID) + { arg = coerce_to_common_type(pstate, arg, TEXTOID, "CASE"); + } + else if (argtype == VERTEXOID || argtype == EDGEOID) + { + arg = coerce_to_common_type(pstate, arg, AGTYPEOID, "CASE"); + } assign_expr_collations(pstate, arg); @@ -2277,15 +2522,33 @@ static Node *transform_CaseExpr(cypher_parsestate *cpstate, CaseExpr /* * we pass a NULL context to select_common_type because the common types can * only be AGTYPEOID or BOOLOID. If it returns invalidoid, we know there is a - * boolean involved. + * boolean or record involved. */ ptype = select_common_type(pstate, resultexprs, NULL, NULL); - /* InvalidOid shows that there is a boolean in the result expr. */ + /* InvalidOid shows that there is a boolean or record in the result expr. */ if (ptype == InvalidOid) { - /* we manually set the type to boolean here to handle the bool casting. */ - ptype = BOOLOID; + /* + * If we have record type, we need to cast it to agtype here, + * else return boolean + */ + ListCell *lc; + bool has_vertex_edge = false; + + foreach(lc, resultexprs) + { + Node *expr = (Node *) lfirst(lc); + Oid exprtype = exprType(expr); + + if (exprtype == VERTEXOID || exprtype == EDGEOID) + { + has_vertex_edge = true; + break; + } + } + + ptype = has_vertex_edge ? AGTYPEOID : BOOLOID; } Assert(OidIsValid(ptype)); @@ -2426,3 +2689,70 @@ static Node *transform_SubLink(cypher_parsestate *cpstate, SubLink *sublink) return result; } + +/* + * coerce_entity_to_agtype + * + * If the node is a vertex or edge type, coerce it to agtype. + * Returns the coerced node, or the original node if no coercion needed. + */ +Node *coerce_entity_to_agtype(ParseState *pstate, Node *node) +{ + Oid node_type = exprType(node); + + if (node_type == VERTEXOID || node_type == EDGEOID) + { + return coerce_to_target_type(pstate, node, node_type, + AGTYPEOID, -1, COERCION_EXPLICIT, + COERCE_IMPLICIT_CAST, -1); + } + + return node; +} + +/* + * coerce_target_entities_to_agtype + * + * Iterate through a target list and coerce any vertex/edge types to agtype. + * Used for terminal clauses (SET, DELETE, CREATE, MERGE) to ensure the + * executor receives agtype. + */ +void coerce_target_entities_to_agtype(ParseState *pstate, List *target_list) +{ + ListCell *lc; + + foreach(lc, target_list) + { + TargetEntry *te = (TargetEntry *)lfirst(lc); + Oid te_type = exprType((Node *)te->expr); + + if (te_type == VERTEXOID || te_type == EDGEOID) + { + te->expr = (Expr *)coerce_to_target_type( + pstate, (Node *)te->expr, te_type, + AGTYPEOID, -1, COERCION_EXPLICIT, + COERCE_IMPLICIT_CAST, -1); + } + } +} + +Node *make_properties_expr(Node *prop_var) +{ + Node *prop_expr; + + if (is_vertex_or_edge(prop_var)) + { + prop_expr = extract_field_from_record(prop_var, "properties"); + } + else + { + /* Call age_properties for non-RECORD types */ + Oid foid = get_ag_func_oid("age_properties", 1, AGTYPEOID); + prop_expr = (Node *)makeFuncExpr(foid, AGTYPEOID, + list_make1(prop_var), + InvalidOid, InvalidOid, + COERCE_EXPLICIT_CALL); + } + + return prop_expr; +} diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index 6700be3f3..624d6a881 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -37,6 +37,7 @@ #include "access/genam.h" #include "access/heapam.h" +#include "access/htup_details.h" #include "catalog/namespace.h" #include "catalog/pg_am_d.h" #include "catalog/pg_collation_d.h" @@ -59,6 +60,7 @@ #include "utils/agtype_raw.h" #include "catalog/ag_graph.h" #include "catalog/ag_label.h" +#include "utils/ag_func.h" /* State structure for Percentile aggregate functions */ typedef struct PercentileGroupAggState @@ -161,7 +163,6 @@ static bool is_array_path(agtype_value *agtv); /* graph entity retrieval */ static Datum get_vertex(const char *graph, const char *vertex_label, int64 graphid); -static char *get_label_name(const char *graph_name, graphid element_graphid); static float8 get_float_compatible_arg(Datum arg, Oid type, char *funcname, bool *is_null); static Numeric get_numeric_compatible_arg(Datum arg, Oid type, char *funcname, @@ -246,6 +247,43 @@ void clear_global_Oids_AGTYPE(void) g_AGTYPEARRAYOID = InvalidOid; } +/* global storage of OID for vertex and edge composite types */ +static Oid g_VERTEXOID = InvalidOid; +static Oid g_EDGEOID = InvalidOid; + +/* helper function to quickly set, if necessary, and retrieve VERTEXOID */ +Oid get_VERTEXOID(void) +{ + if (g_VERTEXOID == InvalidOid) + { + g_VERTEXOID = GetSysCacheOid2(TYPENAMENSP, Anum_pg_type_oid, + CStringGetDatum("vertex"), + ObjectIdGetDatum(ag_catalog_namespace_id())); + } + + return g_VERTEXOID; +} + +/* helper function to quickly set, if necessary, and retrieve EDGEOID */ +Oid get_EDGEOID(void) +{ + if (g_EDGEOID == InvalidOid) + { + g_EDGEOID = GetSysCacheOid2(TYPENAMENSP, Anum_pg_type_oid, + CStringGetDatum("edge"), + ObjectIdGetDatum(ag_catalog_namespace_id())); + } + + return g_EDGEOID; +} + +/* helper function to clear the VERTEX/EDGE OIDs after a drop extension */ +void clear_global_Oids_VERTEX_EDGE(void) +{ + g_VERTEXOID = InvalidOid; + g_EDGEOID = InvalidOid; +} + /* fast helper function to test for AGTV_NULL in an agtype */ bool is_agtype_null(agtype *agt_arg) { @@ -1441,17 +1479,22 @@ static void agtype_categorize_type(Oid typoid, agt_type_category *tcategory, break; default: - /* Check for arrays and composites */ if (typoid == AGTYPEOID) { *tcategory = AGT_TYPE_AGTYPE; } + else if (typoid == VERTEXOID || typoid == EDGEOID) + { + *tcategory = AGT_TYPE_AGTYPE; + *outfuncoid = (typoid == VERTEXOID) ? + get_ag_func_oid("vertex_to_agtype", 1, VERTEXOID) : + get_ag_func_oid("edge_to_agtype", 1, EDGEOID); + } else if (OidIsValid(get_element_type(typoid)) || typoid == ANYARRAYOID || typoid == RECORDARRAYOID) { *tcategory = AGT_TYPE_ARRAY; } - /* includes RECORDOID */ else if (type_is_rowtype(typoid)) { *tcategory = AGT_TYPE_COMPOSITE; @@ -1682,14 +1725,17 @@ static void datum_to_agtype(Datum val, bool is_null, agtype_in_state *result, case AGT_TYPE_AGTYPE: case AGT_TYPE_JSONB: { - agtype *jsonb = DATUM_GET_AGTYPE_P(val); + agtype *jsonb; agtype_iterator *it; + if (OidIsValid(outfuncoid)) + val = OidFunctionCall1(outfuncoid, val); + /* * val is actually jsonb datum but we can handle it as an agtype * datum because agtype is currently an extension of jsonb. */ - + jsonb = DATUM_GET_AGTYPE_P(val); it = agtype_iterator_init(&jsonb->root); if (AGT_ROOT_IS_SCALAR(jsonb)) @@ -2286,12 +2332,14 @@ Datum make_path(List *path) PG_FUNCTION_INFO_V1(_agtype_build_vertex); /* - * SQL function agtype_build_vertex(graphid, cstring, agtype) + * SQL function agtype_build_vertex(graphid, agtype, agtype) */ Datum _agtype_build_vertex(PG_FUNCTION_ARGS) { graphid id; char *label; + agtype *label_agtype; + agtype_value *label_value; agtype *properties; agtype_build_state *bstate; agtype *rawscalar; @@ -2312,7 +2360,25 @@ Datum _agtype_build_vertex(PG_FUNCTION_ARGS) } id = AG_GETARG_GRAPHID(0); - label = PG_GETARG_CSTRING(1); + label_agtype = AG_GET_ARG_AGTYPE_P(1); + + /* Extract the string from the agtype label */ + if (!AGT_ROOT_IS_SCALAR(label_agtype)) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("_agtype_build_vertex() label must be a scalar string"))); + } + + label_value = get_ith_agtype_value_from_container(&label_agtype->root, 0); + if (label_value->type != AGTV_STRING) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("_agtype_build_vertex() label must be a string"))); + } + + label = pnstrdup(label_value->val.string.val, label_value->val.string.len); if (fcinfo->args[2].isnull) { @@ -2347,7 +2413,8 @@ Datum _agtype_build_vertex(PG_FUNCTION_ARGS) rawscalar = build_agtype(bstate); pfree_agtype_build_state(bstate); - PG_FREE_IF_COPY(label, 1); + pfree(label); + PG_FREE_IF_COPY(label_agtype, 1); PG_FREE_IF_COPY(properties, 2); PG_RETURN_POINTER(rawscalar); @@ -2361,7 +2428,7 @@ Datum make_vertex(Datum id, Datum label, Datum properties) PG_FUNCTION_INFO_V1(_agtype_build_edge); /* - * SQL function agtype_build_edge(graphid, graphid, graphid, cstring, agtype) + * SQL function agtype_build_edge(graphid, graphid, graphid, agtype, agtype) */ Datum _agtype_build_edge(PG_FUNCTION_ARGS) { @@ -2369,6 +2436,8 @@ Datum _agtype_build_edge(PG_FUNCTION_ARGS) agtype *edge, *rawscalar; graphid id, start_id, end_id; char *label; + agtype *label_agtype; + agtype_value *label_value; agtype *properties; /* process graph id */ @@ -2385,10 +2454,28 @@ Datum _agtype_build_edge(PG_FUNCTION_ARGS) if (fcinfo->args[3].isnull) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("_agtype_build_vertex() label cannot be NULL"))); + errmsg("_agtype_build_edge() label cannot be NULL"))); } - label = PG_GETARG_CSTRING(3); + label_agtype = AG_GET_ARG_AGTYPE_P(3); + + /* Extract the string from the agtype label */ + if (!AGT_ROOT_IS_SCALAR(label_agtype)) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("_agtype_build_edge() label must be a scalar string"))); + } + + label_value = get_ith_agtype_value_from_container(&label_agtype->root, 0); + if (label_value->type != AGTV_STRING) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("_agtype_build_edge() label must be a string"))); + } + + label = pnstrdup(label_value->val.string.val, label_value->val.string.len); /* process end_id */ if (fcinfo->args[2].isnull) @@ -2450,7 +2537,8 @@ Datum _agtype_build_edge(PG_FUNCTION_ARGS) rawscalar = build_agtype(bstate); pfree_agtype_build_state(bstate); - PG_FREE_IF_COPY(label, 3); + pfree(label); + PG_FREE_IF_COPY(label_agtype, 3); PG_FREE_IF_COPY(properties, 4); PG_RETURN_POINTER(rawscalar); @@ -2463,6 +2551,308 @@ Datum make_edge(Datum id, Datum startid, Datum endid, Datum label, properties); } +PG_FUNCTION_INFO_V1(vertex_to_agtype); + +/* + * Cast function: vertex -> agtype + * Vertex: (id graphid, label agtype, properties agtype) + */ +Datum vertex_to_agtype(PG_FUNCTION_ARGS) +{ + HeapTupleHeader rec; + TupleDesc tupdesc; + HeapTupleData tuple; + Datum *values; + bool *nulls; + graphid id; + agtype *label; + agtype *properties; + Datum result; + + rec = PG_GETARG_HEAPTUPLEHEADER(0); + + tupdesc = lookup_rowtype_tupdesc(HeapTupleHeaderGetTypeId(rec), + HeapTupleHeaderGetTypMod(rec)); + + tuple.t_len = HeapTupleHeaderGetDatumLength(rec); + tuple.t_data = rec; + + values = (Datum *) palloc(3 * sizeof(Datum)); + nulls = (bool *) palloc(3 * sizeof(bool)); + heap_deform_tuple(&tuple, tupdesc, values, nulls); + + if (nulls[0]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("vertex id cannot be NULL"))); + if (nulls[1]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("vertex label cannot be NULL"))); + + id = DatumGetInt64(values[0]); + label = DATUM_GET_AGTYPE_P(values[1]); + properties = nulls[2] ? NULL : DATUM_GET_AGTYPE_P(values[2]); + + if (properties == NULL) + { + agtype_build_state *bstate = init_agtype_build_state(0, AGT_FOBJECT); + properties = build_agtype(bstate); + pfree_agtype_build_state(bstate); + } + + result = make_vertex(Int64GetDatum(id), PointerGetDatum(label), + PointerGetDatum(properties)); + + ReleaseTupleDesc(tupdesc); + pfree(values); + pfree(nulls); + + return result; +} + +PG_FUNCTION_INFO_V1(edge_to_agtype); + +/* + * Cast function: edge -> agtype + * Edge: (id graphid, label agtype, end_id graphid, start_id graphid, properties agtype) + */ +Datum edge_to_agtype(PG_FUNCTION_ARGS) +{ + HeapTupleHeader rec; + TupleDesc tupdesc; + HeapTupleData tuple; + Datum *values; + bool *nulls; + graphid id, start_id, end_id; + agtype *label; + agtype *properties; + Datum result; + + rec = PG_GETARG_HEAPTUPLEHEADER(0); + + tupdesc = lookup_rowtype_tupdesc(HeapTupleHeaderGetTypeId(rec), + HeapTupleHeaderGetTypMod(rec)); + + tuple.t_len = HeapTupleHeaderGetDatumLength(rec); + tuple.t_data = rec; + + values = (Datum *) palloc(5 * sizeof(Datum)); + nulls = (bool *) palloc(5 * sizeof(bool)); + heap_deform_tuple(&tuple, tupdesc, values, nulls); + + if (nulls[0]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("edge id cannot be NULL"))); + if (nulls[1]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("edge label cannot be NULL"))); + if (nulls[2]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("edge end_id cannot be NULL"))); + if (nulls[3]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("edge start_id cannot be NULL"))); + + id = DatumGetInt64(values[0]); + label = DATUM_GET_AGTYPE_P(values[1]); + end_id = DatumGetInt64(values[2]); + start_id = DatumGetInt64(values[3]); + properties = nulls[4] ? NULL : DATUM_GET_AGTYPE_P(values[4]); + + if (properties == NULL) + { + agtype_build_state *bstate = init_agtype_build_state(0, AGT_FOBJECT); + properties = build_agtype(bstate); + pfree_agtype_build_state(bstate); + } + + result = make_edge(Int64GetDatum(id), Int64GetDatum(start_id), + Int64GetDatum(end_id), PointerGetDatum(label), + PointerGetDatum(properties)); + + ReleaseTupleDesc(tupdesc); + pfree(values); + pfree(nulls); + + return result; +} + +/* + * Helper function to build JSON string from vertex composite type + */ +static char *vertex_to_json_string(HeapTupleHeader rec) +{ + TupleDesc tupdesc; + HeapTupleData tuple; + Datum *values; + bool *nulls; + graphid id; + agtype *label; + agtype *properties; + StringInfoData buf; + char *label_str; + char *props_str; + + tupdesc = lookup_rowtype_tupdesc(HeapTupleHeaderGetTypeId(rec), + HeapTupleHeaderGetTypMod(rec)); + + tuple.t_len = HeapTupleHeaderGetDatumLength(rec); + tuple.t_data = rec; + + values = (Datum *) palloc(3 * sizeof(Datum)); + nulls = (bool *) palloc(3 * sizeof(bool)); + heap_deform_tuple(&tuple, tupdesc, values, nulls); + + if (nulls[0]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("vertex id cannot be NULL"))); + if (nulls[1]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("vertex label cannot be NULL"))); + + id = DatumGetInt64(values[0]); + label = DATUM_GET_AGTYPE_P(values[1]); + properties = nulls[2] ? NULL : DATUM_GET_AGTYPE_P(values[2]); + + label_str = agtype_to_cstring(NULL, &label->root, VARSIZE(label)); + + if (properties != NULL) + { + props_str = agtype_to_cstring_worker(NULL, &properties->root, + VARSIZE(properties), false, false); + } + else + { + props_str = "{}"; + } + + initStringInfo(&buf); + appendStringInfo(&buf, "{\"id\": " INT64_FORMAT ", \"label\": %s, \"properties\": %s}", + id, label_str, props_str); + + ReleaseTupleDesc(tupdesc); + pfree(values); + pfree(nulls); + + return buf.data; +} + +/* + * Helper function to build JSON string from edge composite type + * Edge: (id graphid, label agtype, end_id graphid, start_id graphid, properties agtype) + */ +static char *edge_to_json_string(HeapTupleHeader rec) +{ + TupleDesc tupdesc; + HeapTupleData tuple; + Datum *values; + bool *nulls; + graphid id, start_id, end_id; + agtype *label; + agtype *properties; + StringInfoData buf; + char *label_str; + char *props_str; + + tupdesc = lookup_rowtype_tupdesc(HeapTupleHeaderGetTypeId(rec), + HeapTupleHeaderGetTypMod(rec)); + + tuple.t_len = HeapTupleHeaderGetDatumLength(rec); + tuple.t_data = rec; + + values = (Datum *) palloc(5 * sizeof(Datum)); + nulls = (bool *) palloc(5 * sizeof(bool)); + heap_deform_tuple(&tuple, tupdesc, values, nulls); + + if (nulls[0]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("edge id cannot be NULL"))); + if (nulls[1]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("edge label cannot be NULL"))); + if (nulls[2]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("edge end_id cannot be NULL"))); + if (nulls[3]) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("edge start_id cannot be NULL"))); + + id = DatumGetInt64(values[0]); + label = DATUM_GET_AGTYPE_P(values[1]); + end_id = DatumGetInt64(values[2]); + start_id = DatumGetInt64(values[3]); + properties = nulls[4] ? NULL : DATUM_GET_AGTYPE_P(values[4]); + + label_str = agtype_to_cstring(NULL, &label->root, VARSIZE(label)); + + if (properties != NULL) + { + props_str = agtype_to_cstring_worker(NULL, &properties->root, + VARSIZE(properties), false, false); + } + else + { + props_str = "{}"; + } + + initStringInfo(&buf); + appendStringInfo(&buf, "{\"id\": " INT64_FORMAT ", \"label\": %s, \"end_id\": " INT64_FORMAT ", \"start_id\": " INT64_FORMAT ", \"properties\": %s}", + id, label_str, end_id, start_id, props_str); + + ReleaseTupleDesc(tupdesc); + pfree(values); + pfree(nulls); + + return buf.data; +} + +PG_FUNCTION_INFO_V1(vertex_to_json); + +Datum vertex_to_json(PG_FUNCTION_ARGS) +{ + char *json_str = vertex_to_json_string(PG_GETARG_HEAPTUPLEHEADER(0)); + Datum result = DirectFunctionCall1(json_in, CStringGetDatum(json_str)); + + pfree_if_not_null(json_str); + + PG_RETURN_DATUM(result); +} + +PG_FUNCTION_INFO_V1(vertex_to_jsonb); + +Datum vertex_to_jsonb(PG_FUNCTION_ARGS) +{ + char *json_str = vertex_to_json_string(PG_GETARG_HEAPTUPLEHEADER(0)); + Datum result = DirectFunctionCall1(jsonb_in, CStringGetDatum(json_str)); + + pfree_if_not_null(json_str); + + PG_RETURN_DATUM(result); +} + +PG_FUNCTION_INFO_V1(edge_to_json); + +Datum edge_to_json(PG_FUNCTION_ARGS) +{ + char *json_str = edge_to_json_string(PG_GETARG_HEAPTUPLEHEADER(0)); + Datum result = DirectFunctionCall1(json_in, CStringGetDatum(json_str)); + + pfree_if_not_null(json_str); + + PG_RETURN_DATUM(result); +} + +PG_FUNCTION_INFO_V1(edge_to_jsonb); + +Datum edge_to_jsonb(PG_FUNCTION_ARGS) +{ + char *json_str = edge_to_json_string(PG_GETARG_HEAPTUPLEHEADER(0)); + Datum result = DirectFunctionCall1(jsonb_in, CStringGetDatum(json_str)); + + pfree_if_not_null(json_str); + + PG_RETURN_DATUM(result); +} + static agtype_value *agtype_build_map_as_agtype_value(FunctionCallInfo fcinfo) { int nargs; @@ -5258,6 +5648,7 @@ Datum agtype_typecast_vertex(PG_FUNCTION_ARGS) agtype *arg_agt; agtype_value agtv_key; agtype_value *agtv_graphid, *agtv_label, *agtv_properties; + agtype *label_agtype; Datum result; int count; @@ -5316,9 +5707,10 @@ Datum agtype_typecast_vertex(PG_FUNCTION_ARGS) errmsg("vertex typecast object has invalid or missing properties"))); /* Hand it off to the build vertex routine */ + label_agtype = agtype_value_to_agtype(agtv_label); result = DirectFunctionCall3(_agtype_build_vertex, Int64GetDatum(agtv_graphid->val.int_value), - CStringGetDatum(agtv_label->val.string.val), + PointerGetDatum(label_agtype), PointerGetDatum(agtype_value_to_agtype(agtv_properties))); return result; } @@ -5333,6 +5725,7 @@ Datum agtype_typecast_edge(PG_FUNCTION_ARGS) agtype_value agtv_key; agtype_value *agtv_graphid, *agtv_label, *agtv_properties, *agtv_startid, *agtv_endid; + agtype *label_agtype; Datum result; int count; @@ -5409,11 +5802,12 @@ Datum agtype_typecast_edge(PG_FUNCTION_ARGS) errmsg("edge typecast object has an invalid or missing end_id"))); /* Hand it off to the build edge routine */ + label_agtype = agtype_value_to_agtype(agtv_label); result = DirectFunctionCall5(_agtype_build_edge, Int64GetDatum(agtv_graphid->val.int_value), Int64GetDatum(agtv_startid->val.int_value), Int64GetDatum(agtv_endid->val.int_value), - CStringGetDatum(agtv_label->val.string.val), + PointerGetDatum(label_agtype), PointerGetDatum(agtype_value_to_agtype(agtv_properties))); return result; } @@ -5671,66 +6065,6 @@ Datum column_get_datum(TupleDesc tupdesc, HeapTuple tuple, int column, return result; } -/* - * Function to retrieve a label name, given the graph name and graphid of the - * node or edge. The function returns a pointer to a duplicated string that - * needs to be freed when you are finished using it. - */ -static char *get_label_name(const char *graph_name, graphid element_graphid) -{ - ScanKeyData scan_keys[2]; - Relation ag_label; - SysScanDesc scan_desc; - HeapTuple tuple; - TupleDesc tupdesc; - char *result = NULL; - bool column_is_null = false; - Oid graph_oid = get_graph_oid(graph_name); - int32 label_id = get_graphid_label_id(element_graphid); - - /* scankey for first match in ag_label, column 2, graphoid, BTEQ, OidEQ */ - ScanKeyInit(&scan_keys[0], Anum_ag_label_graph, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(graph_oid)); - /* scankey for second match in ag_label, column 3, label id, BTEQ, Int4EQ */ - ScanKeyInit(&scan_keys[1], Anum_ag_label_id, BTEqualStrategyNumber, - F_INT4EQ, Int32GetDatum(label_id)); - - ag_label = table_open(ag_label_relation_id(), ShareLock); - scan_desc = systable_beginscan(ag_label, ag_label_graph_oid_index_id(), true, - NULL, 2, scan_keys); - - tuple = systable_getnext(scan_desc); - if (!HeapTupleIsValid(tuple)) - { - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_SCHEMA), - errmsg("graphid %lu does not exist", element_graphid))); - } - - /* get the tupdesc - we don't need to release this one */ - tupdesc = RelationGetDescr(ag_label); - - /* bail if the number of columns differs */ - if (tupdesc->natts != Natts_ag_label) - { - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_TABLE), - errmsg("Invalid number of attributes for ag_catalog.ag_label"))); - } - - /* get the label name */ - result = NameStr(*DatumGetName(heap_getattr(tuple, Anum_ag_label_name, - tupdesc, &column_is_null))); - /* duplicate it */ - result = pstrdup(result); - - /* end the scan and close the relation */ - systable_endscan(scan_desc); - table_close(ag_label, ShareLock); - - return result; -} - static Datum get_vertex(const char *graph, const char *vertex_label, int64 graphid) { @@ -5744,14 +6078,18 @@ static Datum get_vertex(const char *graph, const char *vertex_label, TupleTableSlot *slot; Oid index_oid; bool should_free_tuple = false; + agtype *label_agtype; + Oid graph_namespace_oid; + Oid vertex_label_table_oid; + Snapshot snapshot; /* get the specific graph namespace (schema) */ - Oid graph_namespace_oid = get_namespace_oid(graph, false); + graph_namespace_oid = get_namespace_oid(graph, false); /* get the specific vertex label table (schema.vertex_label) */ - Oid vertex_label_table_oid = get_relname_relid(vertex_label, - graph_namespace_oid); + vertex_label_table_oid = get_relname_relid(vertex_label, + graph_namespace_oid); /* get the active snapshot */ - Snapshot snapshot = GetActiveSnapshot(); + snapshot = GetActiveSnapshot(); /* check for SELECT permission on the table */ aclresult = pg_class_aclcheck(vertex_label_table_oid, GetUserId(), @@ -5849,8 +6187,9 @@ static Datum get_vertex(const char *graph, const char *vertex_label, properties = column_get_datum(tupdesc, tuple, 1, "properties", AGTYPEOID, true); /* reconstruct the vertex */ + label_agtype = DATUM_GET_AGTYPE_P(string_to_agtype((char *)vertex_label)); result = DirectFunctionCall3(_agtype_build_vertex, id, - CStringGetDatum(vertex_label), properties); + PointerGetDatum(label_agtype), properties); /* end the scan and close the relation with new cleanup logic */ if (scan_desc) @@ -5877,6 +6216,7 @@ Datum age_startnode(PG_FUNCTION_ARGS) char *graph_name = NULL; char *label_name = NULL; graphid start_id; + int label_id; Datum result; /* we need the graph name */ @@ -5921,7 +6261,9 @@ Datum age_startnode(PG_FUNCTION_ARGS) start_id = agtv_value->val.int_value; /* get the label */ - label_name = get_label_name(graph_name, start_id); + label_id = get_graphid_label_id(start_id); + label_name = get_label_name(label_id, get_graph_oid(graph_name)); + /* it must not be null and must be a string */ Assert(label_name != NULL); @@ -5940,6 +6282,7 @@ Datum age_endnode(PG_FUNCTION_ARGS) char *graph_name = NULL; char *label_name = NULL; graphid end_id; + int32 label_id; Datum result; /* we need the graph name */ @@ -5984,7 +6327,9 @@ Datum age_endnode(PG_FUNCTION_ARGS) end_id = agtv_value->val.int_value; /* get the label */ - label_name = get_label_name(graph_name, end_id); + label_id = get_graphid_label_id(end_id); + label_name = get_label_name(label_id, get_graph_oid(graph_name)); + /* it must not be null and must be a string */ Assert(label_name != NULL); @@ -5993,6 +6338,42 @@ Datum age_endnode(PG_FUNCTION_ARGS) return result; } +PG_FUNCTION_INFO_V1(_get_vertex_by_graphid); + +/* + * Helper function for optimized startNode/endNode + * Fetches a vertex given graph name and vertex graphid + */ +Datum _get_vertex_by_graphid(PG_FUNCTION_ARGS) +{ + char *graph_name = NULL; + graphid vertex_id; + int32 label_id; + char *label_name = NULL; + Datum result; + + /* check for nulls */ + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) + PG_RETURN_NULL(); + + /* get the graph name from text */ + graph_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + /* get the vertex graphid */ + vertex_id = PG_GETARG_INT64(1); + + /* get the label name */ + label_id = get_graphid_label_id(vertex_id); + label_name = get_label_name(label_id, get_graph_oid(graph_name)); + if (label_name == NULL) + PG_RETURN_NULL(); + + /* fetch and return the vertex */ + result = get_vertex(graph_name, label_name, vertex_id); + + return result; +} + PG_FUNCTION_INFO_V1(age_head); Datum age_head(PG_FUNCTION_ARGS) @@ -7843,8 +8224,7 @@ Datum age_reverse(PG_FUNCTION_ARGS) else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("reverse() unsupported argument type %d", - type))); + errmsg("reverse() unsupported argument type"))); } } else @@ -7880,8 +8260,7 @@ Datum age_reverse(PG_FUNCTION_ARGS) else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("reverse() unsupported argument agtype %d", - agtv_value->type))); + errmsg("reverse() unsupported argument agtype"))); } } else if (AGT_ROOT_IS_ARRAY(agt_arg)) @@ -12494,11 +12873,12 @@ Datum agtype_volatile_wrapper(PG_FUNCTION_ARGS) agtv_result.type = AGTV_BOOL; agtv_result.val.boolean = DatumGetBool(arg); } - else if (type == INT2OID || type == INT4OID || type == INT8OID) + else if (type == INT2OID || type == INT4OID || + type == INT8OID || type == GRAPHIDOID) { agtv_result.type = AGTV_INTEGER; - if (type == INT8OID) + if (type == INT8OID || type == GRAPHIDOID) { agtv_result.val.int_value = DatumGetInt64(arg); } @@ -12541,6 +12921,14 @@ Datum agtype_volatile_wrapper(PG_FUNCTION_ARGS) agtv_result.val.string.val = text_to_cstring(DatumGetTextPP(arg)); agtv_result.val.string.len = strlen(agtv_result.val.string.val); } + else if (type == VERTEXOID) + { + PG_RETURN_DATUM(DirectFunctionCall1(vertex_to_agtype, arg)); + } + else if (type == EDGEOID) + { + PG_RETURN_DATUM(DirectFunctionCall1(edge_to_agtype, arg)); + } else { ereport(ERROR, diff --git a/src/include/catalog/ag_label.h b/src/include/catalog/ag_label.h index 0a8480b1a..e6b9a1ca7 100644 --- a/src/include/catalog/ag_label.h +++ b/src/include/catalog/ag_label.h @@ -74,7 +74,7 @@ Oid get_label_relation(const char *label_name, Oid graph_oid); char *get_label_relation_name(const char *label_name, Oid graph_oid); char get_label_kind(const char *label_name, Oid label_graph); char *get_label_seq_relation_name(const char *label_name); - +char *get_label_name(int32 label_id, Oid graph_oid); bool label_id_exists(Oid graph_oid, int32 label_id); RangeVar *get_label_range_var(char *graph_name, Oid graph_oid, diff --git a/src/include/parser/cypher_clause.h b/src/include/parser/cypher_clause.h index 461a664ca..d0fc3ca22 100644 --- a/src/include/parser/cypher_clause.h +++ b/src/include/parser/cypher_clause.h @@ -21,6 +21,7 @@ #define AG_CYPHER_CLAUSE_H #include "parser/cypher_parse_node.h" +#include "parser/cypher_transform_entity.h" typedef struct cypher_clause cypher_clause; @@ -39,4 +40,12 @@ Query *cypher_parse_sub_analyze(Node *parseTree, CommonTableExpr *parentCTE, bool locked_from_parent, bool resolve_unknowns); + +/* + * Helper to get field info for vertex/edge entity field access + */ +void get_record_field_info(char *field_name, Oid entity_type, + AttrNumber *fieldnum, Oid *fieldtype); +FieldSelect *make_field_select(Var *var, AttrNumber fieldnum, Oid resulttype); + #endif diff --git a/src/include/parser/cypher_expr.h b/src/include/parser/cypher_expr.h index cde40cc87..e069abc6d 100644 --- a/src/include/parser/cypher_expr.h +++ b/src/include/parser/cypher_expr.h @@ -29,5 +29,10 @@ Node *transform_cypher_expr(cypher_parsestate *cpstate, Node *expr, ParseExprKind expr_kind); +Node *make_properties_expr(Node *prop_var); +bool is_vertex_or_edge(Node *node); +Node *extract_field_from_record(Node *node, char *field_name); +Node *coerce_entity_to_agtype(ParseState *pstate, Node *node); +void coerce_target_entities_to_agtype(ParseState *pstate, List *target_list); #endif diff --git a/src/include/utils/agtype.h b/src/include/utils/agtype.h index 807d3795e..b10474ce0 100644 --- a/src/include/utils/agtype.h +++ b/src/include/utils/agtype.h @@ -669,4 +669,11 @@ void clear_global_Oids_AGTYPE(void); #define AGTYPEOID get_AGTYPEOID() #define AGTYPEARRAYOID get_AGTYPEARRAYOID() +/* Oid accessors for vertex and edge composite types */ +Oid get_VERTEXOID(void); +Oid get_EDGEOID(void); +void clear_global_Oids_VERTEX_EDGE(void); +#define VERTEXOID get_VERTEXOID() +#define EDGEOID get_EDGEOID() + #endif From 795a881d20d6b7cf74439a8be06289ac3e390dc4 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Mon, 1 Jun 2026 11:52:20 -0700 Subject: [PATCH 070/100] perf(agtype): arena passthrough + binary-direct id extraction (#2424) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Targets the agtype value-tree allocation cost that dominates LDBC IC4 and other sort/comparator-bound queries. Two complementary optimizations (A1 arena, A2 fast-id) plus the sanitizer issues uncovered while verifying the change. A1 — per-tuple agtype arena (Pattern B applied): Add agt_arena_begin / AGT_ARENA_BEGIN_SHARED / agt_arena_reset / agt_arena_end helpers (agtype_util.c, agtype.h). Two usage patterns: disposable per-call arenas for one-shot tree builds, and a long-lived shared arena (file-static, parented under CacheMemoryContext) for hot inner loops that would otherwise pay AllocSetCreate cost per call. Apply to compare_agtype_scalar_containers: when the comparator deserializes VERTEX/EDGE/PATH composites via fill_agtype_value_no_copy, route those allocations into a shared arena and reset (O(1)) at the end of the comparison instead of recursively pfree'ing the tree (O(N)). Master IC4 spent 17.64% of total CPU in pfree_agtype_value_content from this comparator; the arena collapses that walk to one MemoryContextReset. A2 — binary-direct id extraction: When compare_agtype_scalar_containers compares two VERTEX or two EDGE values, only the graphid 'id' determines order (compare_agtype_scalar_values ignores all other fields). Master IC4 spent 73.48% inclusive in ag_deserialize_composite called from this comparator, building the entire agtype_value tree (label, properties, nested values) just to read one int64. Add extract_composite_id_fast: walks the binary VERTEX/EDGE container directly and reads the int64 id at field index 0 (always 'id' due to length-sorted key ordering established by uniqueify_agtype_object — the same invariant PR #2302 leveraged for direct array indexing). Wired into compare_agtype_scalar_containers as a fast path before the existing arena+slow path. Falls through on cross-type comparisons (VERTEX vs EDGE), PATH, and malformed extended-type entries. ASAN/UBSan fixes (10 issues found while verifying A1+A2 under -fsanitize=address -fsanitize=undefined; 1 introduced by A2, 9 pre-existing). The 11th finding (uninitialized parent_cpstate fields in cypher_analyze.c) was independently fixed upstream by PR #2423; this branch is rebased on top of that fix and no longer carries a duplicate change. - HIGH: heap-buffer-overflow in agtype_raw.c:write_container — VARSIZE included the 4-byte varlena header but copy started past it, reading VARHDRSZ bytes past source allocation. Subtract VARHDRSZ. ASan flagged 4 times in installcheck. - 7 unaligned int64/float8 reads/writes (AGT_HEADER is 4 bytes, leaving payload 4-byte- but not 8-byte-aligned). UB under strict-alignment rules. Replaced typed loads/stores with memcpy in agtype_ext.c (4 sites), agtype_util.c (3 sites incl. extract_composite_id_fast), agtype_raw.c (write_graphid). - agtype.c:agtype_hash_cmp: reading r->val.array.raw_scalar at WAGT_END_ARRAY where the iterator does not populate *val. Replaced with explicit raw_scalar bool stack tracked across BEGIN/END pairs. Hash output unchanged. - agtype_util.c:copy_to_buffer: memcpy(NULL, NULL, 0) is UB. Guard with if (len > 0). Performance (SF3 LDBC, paired same-session vs master): IC sum: 201,930 -> 157,129 ms = -22.19% IC4: 50,570 -> 7,615 ms = -84.94% IC5: 21,779 -> 20,646 ms = -5.20% IS, IU: within noise (<= +/-3% on sub-second queries) Tests: 34/34 installcheck on production PG18.3 34/34 installcheck on ASAN+UBSan PG18.3 (zero unique runtime errors, zero heap-buffer-overflows; the original audit found 10 unique runtime issues plus 1 HBO with 4 instances, all addressed here or upstream) Co-authored-by: Claude noreply@anthropic.com modified: src/backend/utils/adt/agtype.c modified: src/backend/utils/adt/agtype_ext.c modified: src/backend/utils/adt/agtype_raw.c modified: src/backend/utils/adt/agtype_util.c modified: src/include/utils/agtype.h --- src/backend/utils/adt/agtype.c | 57 +++++- src/backend/utils/adt/agtype_ext.c | 21 ++- src/backend/utils/adt/agtype_raw.c | 28 ++- src/backend/utils/adt/agtype_util.c | 282 ++++++++++++++++++++++++++-- src/include/utils/agtype.h | 59 ++++++ 5 files changed, 422 insertions(+), 25 deletions(-) diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index 624d6a881..cfff8138f 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -5261,6 +5261,22 @@ Datum agtype_hash_cmp(PG_FUNCTION_ARGS) agtype_iterator_token tok; agtype_value *r; uint64 seed = 0xF0F0F0F0; + /* + * Stack of "is the current open array a raw_scalar pseudo-array?" so + * that the WAGT_END_ARRAY case can match the choice we made at + * WAGT_BEGIN_ARRAY. The iterator does not populate *val on END tokens + * (see comment above agtype_iterator_next), so r->val.array.raw_scalar + * is uninitialized at that point and reading it as a bool is undefined + * behavior (UBSan flagged values like 127 here). + * + * Arrays in agtype are bounded by AGTYPE_CONTAINER_SIZE which fits in + * AGT_CMASK; a fixed-size stack of 64 levels is far more than any real + * agtype graph value will ever produce. Bail out with an error if we + * exceed it rather than silently corrupting the hash. + */ +#define HASH_RAW_SCALAR_STACK_DEPTH 64 + bool raw_scalar_stack[HASH_RAW_SCALAR_STACK_DEPTH]; + int raw_scalar_top = -1; /* this function returns INTEGER which is 32 bits */ if (PG_ARGISNULL(0)) @@ -5277,18 +5293,51 @@ Datum agtype_hash_cmp(PG_FUNCTION_ARGS) { if (IS_A_AGTYPE_SCALAR(r) && AGTYPE_ITERATOR_TOKEN_IS_HASHABLE(tok)) agtype_hash_scalar_value_extended(r, &hash, seed); - else if (tok == WAGT_BEGIN_ARRAY && !r->val.array.raw_scalar) - seed = LEFT_ROTATE(seed, 4); + else if (tok == WAGT_BEGIN_ARRAY) + { + /* push raw_scalar state for the matching END token */ + raw_scalar_top++; + if (raw_scalar_top >= HASH_RAW_SCALAR_STACK_DEPTH) + { + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("agtype_hash_cmp: array nesting depth exceeds %d", + HASH_RAW_SCALAR_STACK_DEPTH))); + } + raw_scalar_stack[raw_scalar_top] = r->val.array.raw_scalar; + if (!r->val.array.raw_scalar) + { + seed = LEFT_ROTATE(seed, 4); + } + } else if (tok == WAGT_BEGIN_OBJECT) seed = LEFT_ROTATE(seed, 6); - else if (tok == WAGT_END_ARRAY && !r->val.array.raw_scalar) - seed = RIGHT_ROTATE(seed, 4); + else if (tok == WAGT_END_ARRAY) + { + bool was_raw_scalar; + + /* pop matching BEGIN's raw_scalar state */ + if (raw_scalar_top < 0) + { + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("agtype_hash_cmp: WAGT_END_ARRAY without matching WAGT_BEGIN_ARRAY"))); + } + was_raw_scalar = raw_scalar_stack[raw_scalar_top]; + raw_scalar_top--; + if (!was_raw_scalar) + { + seed = RIGHT_ROTATE(seed, 4); + } + } else if (tok == WAGT_END_OBJECT) seed = RIGHT_ROTATE(seed, 4); seed = LEFT_ROTATE(seed, 1); } +#undef HASH_RAW_SCALAR_STACK_DEPTH + pfree_if_not_null(r); PG_FREE_IF_COPY(agt, 0); diff --git a/src/backend/utils/adt/agtype_ext.c b/src/backend/utils/adt/agtype_ext.c index 7a0ea991d..e4a38e3d1 100644 --- a/src/backend/utils/adt/agtype_ext.c +++ b/src/backend/utils/adt/agtype_ext.c @@ -57,7 +57,14 @@ bool ag_serialize_extended_type(StringInfo buffer, agtentry *agtentry, /* copy in the int_value data */ numlen = sizeof(int64); offset = reserve_from_buffer(buffer, numlen); - *((int64 *)(buffer->data + offset)) = scalar_val->val.int_value; + /* + * Use memcpy because the AGT_HEADER (4 bytes) leaves the buffer + * write position 4-byte-aligned but not necessarily 8-byte-aligned. + * Direct *((int64 *) ...) = ... is undefined behavior under strict + * alignment rules (UBSan flags it; works in practice on x86_64). + */ + memcpy(buffer->data + offset, &scalar_val->val.int_value, + sizeof(int64)); *agtentry = AGTENTRY_IS_AGTYPE | (padlen + numlen + AGT_HEADER_SIZE); break; @@ -68,7 +75,8 @@ bool ag_serialize_extended_type(StringInfo buffer, agtentry *agtentry, /* copy in the float_value data */ numlen = sizeof(scalar_val->val.float_value); offset = reserve_from_buffer(buffer, numlen); - *((float8 *)(buffer->data + offset)) = scalar_val->val.float_value; + memcpy(buffer->data + offset, &scalar_val->val.float_value, + sizeof(float8)); *agtentry = AGTENTRY_IS_AGTYPE | (padlen + numlen + AGT_HEADER_SIZE); break; @@ -154,12 +162,17 @@ void ag_deserialize_extended_type(char *base_addr, uint32 offset, { case AGT_HEADER_INTEGER: result->type = AGTV_INTEGER; - result->val.int_value = *((int64 *)(base + AGT_HEADER_SIZE)); + /* See comment in ag_serialize_extended_type. The 4-byte AGT_HEADER + * leaves the int64 at a 4-byte-aligned but not 8-byte-aligned + * address, so a direct typed load is undefined behavior. */ + memcpy(&result->val.int_value, base + AGT_HEADER_SIZE, + sizeof(int64)); break; case AGT_HEADER_FLOAT: result->type = AGTV_FLOAT; - result->val.float_value = *((float8 *)(base + AGT_HEADER_SIZE)); + memcpy(&result->val.float_value, base + AGT_HEADER_SIZE, + sizeof(float8)); break; case AGT_HEADER_VERTEX: diff --git a/src/backend/utils/adt/agtype_raw.c b/src/backend/utils/adt/agtype_raw.c index d8bad3d24..3d56d94d5 100644 --- a/src/backend/utils/adt/agtype_raw.c +++ b/src/backend/utils/adt/agtype_raw.c @@ -197,9 +197,20 @@ void write_graphid(agtype_build_state *bstate, graphid graphid) write_const(AGT_HEADER_INTEGER, AGT_HEADER_TYPE); length += AGT_HEADER_SIZE; - /* graphid value */ - write_const(graphid, int64); - length += sizeof(int64); + /* + * graphid value (int64). The 4-byte AGT_HEADER above leaves the buffer + * write position 4-byte-aligned but not 8-byte-aligned, so the typed + * write done by write_const(graphid, int64) is undefined behavior under + * strict alignment rules. Use memcpy. (UBSan flags the typed write as + * "store to misaligned address ... requires 8 byte alignment".) + */ + { + int numlen = sizeof(int64); + int g_offset = BUFFER_RESERVE(numlen); + + memcpy(bstate->buffer->data + g_offset, &graphid, sizeof(int64)); + length += numlen; + } /* agtentry */ write_agt(AGTENTRY_IS_AGTYPE | length); @@ -214,8 +225,15 @@ void write_container(agtype_build_state *bstate, agtype *agtype) /* padding */ length += BUFFER_WRITE_PAD(); - /* varlen data */ - length += write_ptr((char *) &agtype->root, VARSIZE(agtype)); + /* + * Copy the inner agtype_container only, NOT the outer varlena header. + * VARSIZE(agtype) reports the total varlena size (including the 4-byte + * vl_len_ header), but we are starting our copy at &agtype->root, which + * is already past that header. Subtracting VARHDRSZ avoids reading + * VARHDRSZ bytes past the source allocation (caught by ASan as + * heap-buffer-overflow in __interceptor_memcpy from write_pointer). + */ + length += write_ptr((char *) &agtype->root, VARSIZE(agtype) - VARHDRSZ); /* agtentry */ write_agt(AGTENTRY_IS_CONTAINER | length); diff --git a/src/backend/utils/adt/agtype_util.c b/src/backend/utils/adt/agtype_util.c index b39723413..90e540708 100644 --- a/src/backend/utils/adt/agtype_util.c +++ b/src/backend/utils/adt/agtype_util.c @@ -103,6 +103,66 @@ static int get_type_sort_priority(enum agtype_value_type type); static void pfree_iterator_agtype_value_token(agtype_iterator_token token, agtype_value *agtv); +/* + * Per-call agtype build arena. + * + * Hot Datum functions in agtype.c build an agtype_value tree, serialize it + * via agtype_value_to_agtype(), then recursively free it with + * pfree_agtype_value_content(). The recursive free is O(N) in the number of + * tree nodes; the arena helpers below let callers replace it with an O(K) + * block reset (K = number of allocated AllocSet blocks; typically 1 for + * small trees) by allocating the entire tree inside a short-lived + * MemoryContext. + * + * The arena is sized for typical agtype_value trees; it grows on demand. + * Cost of creating and deleting the context is amortized against the + * eliminated per-node pfree walk and per-node AllocSetFree calls. + * + * agt_arena_end() is a no-op on NULL, allowing simple cleanup paths. + * + * The AGT_ARENA_BEGIN_SHARED() macro (declared in agtype.h) returns a + * context parented under CacheMemoryContext (process lifetime) so the + * caller may stash it in a file-static variable for reuse across many + * invocations. The first caller pays AllocSetContextCreate cost; + * subsequent callers reuse the context and pay only MemoryContextReset + * (cheap; ~100 ns when nothing was allocated since last reset). Use this + * for hot inner loops (sort comparators, per-tuple deep-copy workspaces). + */ +MemoryContext agt_arena_begin(void) +{ + /* + * ALLOCSET_SMALL_SIZES (initial=1KB, max=8KB). Most agtype value trees + * fit in a single 1KB block. + */ + return AllocSetContextCreate(CurrentMemoryContext, + "agtype build arena", + ALLOCSET_SMALL_SIZES); +} + +/* + * Pattern B (shared, long-lived arena) is exposed as the macro + * AGT_ARENA_BEGIN_SHARED in agtype.h, because PG's AllocSetContextCreate + * enforces a compile-time-constant name via StaticAssertStmt. The macro + * inlines the AllocSet creation at the call site so the name literal is + * visible to the StaticAssert. + */ + +void agt_arena_reset(MemoryContext arena) +{ + if (arena != NULL) + { + MemoryContextReset(arena); + } +} + +void agt_arena_end(MemoryContext arena) +{ + if (arena != NULL) + { + MemoryContextDelete(arena); + } +} + /* * Turn an in-memory agtype_value into an agtype for on-disk storage. * @@ -834,12 +894,18 @@ static void fill_agtype_value_no_copy(agtype_container *container, int index, { case AGT_HEADER_INTEGER: result->type = AGTV_INTEGER; - result->val.int_value = *((int64 *)(base + AGT_HEADER_SIZE)); + /* See ag_serialize_extended_type: the 4-byte AGT_HEADER leaves + * the int64 at a 4-byte-aligned but not 8-byte-aligned address. + * Use memcpy to avoid undefined behavior on strict alignment + * platforms (caught by UBSan). */ + memcpy(&result->val.int_value, base + AGT_HEADER_SIZE, + sizeof(int64)); break; case AGT_HEADER_FLOAT: result->type = AGTV_FLOAT; - result->val.float_value = *((float8 *)(base + AGT_HEADER_SIZE)); + memcpy(&result->val.float_value, base + AGT_HEADER_SIZE, + sizeof(float8)); break; default: @@ -880,8 +946,122 @@ static void fill_agtype_value_no_copy(agtype_container *container, int index, * of the full iterator machinery. It extracts the scalar values using no-copy * fill and compares them directly. * + * For composite types (VERTEX, EDGE, PATH) the no-copy fill still + * deserializes into newly-allocated memory. To avoid the per-call + * pfree_agtype_value_content recursive walk on every sort comparator + * invocation, those allocations are routed through a long-lived shared + * arena that is reset (O(1)) at the end of each call. The arena is + * created lazily on the first call that needs it (i.e. the first + * VERTEX/EDGE/PATH comparison) and reused across all subsequent calls in + * the process. + * + * Additional fast path (A2): when both containers are VERTEX or EDGE of + * the same type, the comparator only needs the graphid `id` field + * (compare_agtype_scalar_values does this; all other fields are ignored). + * Skip the full ag_deserialize_composite agtype_value tree build entirely + * and walk the binary representation directly to read just the int64 id. + * This is the dominant cost of sort-bound queries (notably IC4) since the + * tree build is O(properties + label) per row when only one int64 is + * needed for ordering. + * * Returns: negative if a < b, 0 if a == b, positive if a > b */ +static MemoryContext compare_scalar_arena = NULL; + +/* + * Fast int64-id extraction from a binary VERTEX or EDGE container. + * + * Layout of an extended VERTEX/EDGE entry (set up by ag_serialize_extended_type + * with convert_extended_object): a 4-byte AGT_HEADER followed by a standard + * agtype_container holding an object whose first field (sorted by key length + * ascending) is "id". The id is itself stored as an extended AGTV_INTEGER: + * a 4-byte AGT_HEADER_INTEGER followed by an int64. + * + * Pair layout in an object container: children[0..N-1] are the keys, then + * children[N..2N-1] are the values; data follows. The "id" key is at index 0 + * because "id" (length 2) is the shortest key in both VERTEX (id, label, + * properties) and EDGE (id, label, end_id, start_id, properties), and the + * agtype object representation sorts pairs by key length ascending. + * + * On success, writes the extracted graphid through *out and returns true. + * Returns false (leaving *out untouched) if the id field is not a well-formed + * extended AGTV_INTEGER, signaling the caller to fall back to the slow path. + * A bool result is used instead of a sentinel return value because a valid + * graphid can legitimately take any int64 value (e.g. label_id 0x8000 with + * entry_id 0 equals PG_INT64_MIN), so no in-band sentinel is safe. + * + * The caller is responsible for ensuring the input container actually holds a + * VERTEX or EDGE (i.e. this is only called after AGTE_IS_AGTYPE indicates an + * extended type and the AGT_HEADER matches AGT_HEADER_VERTEX or + * AGT_HEADER_EDGE). + */ +static bool extract_composite_id_fast(const agtype_container *outer_scalar, + graphid *out) +{ + const char *base_addr; + const agtype_container *obj; + int n_pairs; + const agtentry *children; + uint32 id_value_offset; + char *id_value_base; + char *id_extended_base; + AGT_HEADER_TYPE id_header; + int id_value_index; + + /* outer_scalar is a single-element pseudo-array; element 0 is our extended type */ + base_addr = (const char *)&outer_scalar->children[1]; + + /* + * Element 0 is an extended type. Its data starts at base_addr + 0 and + * begins with AGT_HEADER_VERTEX or AGT_HEADER_EDGE; the inner + * agtype_container follows immediately. + */ + obj = (const agtype_container *)((const char *)base_addr + AGT_HEADER_SIZE); + n_pairs = AGTYPE_CONTAINER_SIZE(obj); + children = obj->children; + + /* + * Pair value 0 ("id") lives at child index n_pairs (values come after + * keys). Compute its offset within the object's data area. + */ + id_value_index = n_pairs; + id_value_offset = get_agtype_offset(obj, id_value_index); + + /* Data area for this object starts after children[2*n_pairs] */ + id_value_base = (char *)&children[2 * n_pairs]; + + /* + * The id value is an extended AGTV_INTEGER: AGT_HEADER (aligned) followed + * by the int64. INTALIGN matches what ag_deserialize_extended_type does. + */ + id_extended_base = id_value_base + INTALIGN(id_value_offset); + /* + * The header is uint32 (4-byte) and lives at a 4-byte-aligned address, + * so a typed load is fine. The int64 that follows starts at offset + * AGT_HEADER_SIZE = 4 from id_extended_base, so it is 4-byte- but not + * necessarily 8-byte-aligned; use memcpy to avoid alignment UB. + */ + memcpy(&id_header, id_extended_base, sizeof(AGT_HEADER_TYPE)); + + /* + * Defensive: if the value isn't actually an extended integer, fall back + * to the slow deserialize path by signaling. AGT_HEADER_INTEGER is the + * only valid header for the id field of a well-formed VERTEX/EDGE. + */ + if (id_header != AGT_HEADER_INTEGER) + { + return false; + } + + { + graphid id; + + memcpy(&id, id_extended_base + AGT_HEADER_SIZE, sizeof(int64)); + *out = id; + return true; + } +} + static int compare_agtype_scalar_containers(agtype_container *a, agtype_container *b) { @@ -892,6 +1072,7 @@ static int compare_agtype_scalar_containers(agtype_container *a, int result; bool need_free_a = false; bool need_free_b = false; + MemoryContext saved_ctx = NULL; Assert(AGTYPE_CONTAINER_IS_SCALAR(a)); Assert(AGTYPE_CONTAINER_IS_SCALAR(b)); @@ -900,13 +1081,77 @@ static int compare_agtype_scalar_containers(agtype_container *a, base_addr_a = (char *)&a->children[1]; base_addr_b = (char *)&b->children[1]; + /* + * A2 fast path: when both sides are extended-type scalars holding the + * same composite kind (VERTEX or EDGE), we only need the int64 id field + * to determine ordering (see compare_agtype_scalar_values' AGTV_VERTEX + * and AGTV_EDGE cases). Skip the full agtype_value tree build entirely. + */ + if (AGTE_IS_AGTYPE(a->children[0]) && AGTE_IS_AGTYPE(b->children[0])) + { + AGT_HEADER_TYPE ha; + AGT_HEADER_TYPE hb; + + /* + * The header is uint32 at an INTALIGN'd (4-byte) address, so a typed + * load would be safe, but use memcpy for consistency with the rest of + * this comparator (extract_composite_id_fast reads the same header via + * memcpy) and to stay robust if AGT_HEADER_TYPE is ever widened. + */ + memcpy(&ha, + base_addr_a + INTALIGN(get_agtype_offset(a, 0)), + sizeof(AGT_HEADER_TYPE)); + memcpy(&hb, + base_addr_b + INTALIGN(get_agtype_offset(b, 0)), + sizeof(AGT_HEADER_TYPE)); + + if (ha == hb && + (ha == AGT_HEADER_VERTEX || ha == AGT_HEADER_EDGE)) + { + graphid ida; + graphid idb; + + /* + * extract_composite_id_fast returns false on a malformed id + * field; fall through to the slow path in that rare case rather + * than producing a wrong comparison. + */ + if (extract_composite_id_fast(a, &ida) && + extract_composite_id_fast(b, &idb)) + { + if (ida == idb) + { + return 0; + } + return (ida > idb) ? 1 : -1; + } + } + } + + /* + * Peek at the entry types without filling, so we can route any composite + * allocations into the shared arena instead of CurrentMemoryContext. + * This avoids the recursive pfree walk after every comparison. + */ + if (AGTE_IS_AGTYPE(a->children[0]) || AGTE_IS_AGTYPE(b->children[0])) + { + if (compare_scalar_arena == NULL) + { + compare_scalar_arena = + AGT_ARENA_BEGIN_SHARED("agtype scalar comparator"); + } + saved_ctx = MemoryContextSwitchTo(compare_scalar_arena); + } + /* Use no-copy fill to avoid allocations for simple types */ fill_agtype_value_no_copy(a, 0, base_addr_a, 0, &va); fill_agtype_value_no_copy(b, 0, base_addr_b, 0, &vb); /* - * Check if we need to free the values after comparison. - * Only VERTEX, EDGE, and PATH types allocate memory in no-copy mode. + * Track which sides allocated composite content (VERTEX/EDGE/PATH). + * For the arena-backed allocations we still need this so we know to + * reset the arena at the end; for the (rare) caller-context-backed + * fallback path we still need to recursively free. */ if (va.type == AGTV_VERTEX || va.type == AGTV_EDGE || va.type == AGTV_PATH) { @@ -936,14 +1181,17 @@ static int compare_agtype_scalar_containers(agtype_container *a, get_type_sort_priority(vb.type)) ? -1 : 1; } - /* Free any allocated memory from composite types */ - if (need_free_a) - { - pfree_agtype_value_content(&va); - } - if (need_free_b) + /* + * Cleanup. If we allocated into the shared arena, reset it (O(1)); + * otherwise the no-copy fill made no allocations to free. + */ + if (saved_ctx != NULL) { - pfree_agtype_value_content(&vb); + MemoryContextSwitchTo(saved_ctx); + if (need_free_a || need_free_b) + { + agt_arena_reset(compare_scalar_arena); + } } return result; @@ -2127,7 +2375,17 @@ int reserve_from_buffer(StringInfo buffer, int len) static void copy_to_buffer(StringInfo buffer, int offset, const char *data, int len) { - memcpy(buffer->data + offset, data, len); + /* + * Guard against memcpy(dst, NULL, 0). Some callers (notably empty PATH + * scalars with no element bytes) reach here with data == NULL and len + * == 0; the C standard says memcpy with a NULL pointer is undefined even + * when len == 0. UBSan flags this as "null pointer passed as argument 2, + * which is declared to never be null". + */ + if (len > 0) + { + memcpy(buffer->data + offset, data, len); + } } /* diff --git a/src/include/utils/agtype.h b/src/include/utils/agtype.h index b10474ce0..4bddbe4f6 100644 --- a/src/include/utils/agtype.h +++ b/src/include/utils/agtype.h @@ -32,6 +32,7 @@ #define AG_AGTYPE_H #include "utils/array.h" +#include "utils/memutils.h" #include "utils/numeric.h" #include "utils/graphid.h" @@ -581,6 +582,64 @@ agtype_iterator_token agtype_iterator_next(agtype_iterator **it, agtype *agtype_value_to_agtype(agtype_value *val); bool agtype_deep_contains(agtype_iterator **val, agtype_iterator **m_contained, bool skip_nested); + +/* + * Per-call agtype build arena. + * + * Many top-level Datum functions in agtype.c follow the pattern: + * 1. Allocate / build an agtype_value tree + * 2. Serialize it via agtype_value_to_agtype() + * 3. Recursively pfree the tree + * + * Step 3 (pfree_agtype_value_content) is an O(N) walk that frees each tree + * node individually. By building the tree inside a dedicated AllocSet + * MemoryContext and resetting the context after serialization, we trade + * the O(N) walk for an O(K) block reset (K << N) and eliminate per-node + * AllocSetFree overhead. The AllocSetAlloc cost is also reduced because + * the arena uses a small initial block size and grows on demand. + * + * Two usage patterns: + * + * Pattern A — disposable per-call arena (one-shot): + * MemoryContext arena = agt_arena_begin(); + * MemoryContext caller = MemoryContextSwitchTo(arena); + * ... build tree ... + * MemoryContextSwitchTo(caller); + * result = agtype_value_to_agtype(val); // copies into caller + * agt_arena_end(arena); // resets / discards + * + * Pattern B — long-lived shared arena (sort comparator, hot inner loop): + * static MemoryContext shared_arena = NULL; + * if (shared_arena == NULL) + * { + * shared_arena = AGT_ARENA_BEGIN_SHARED("comparator"); + * } + * MemoryContext caller = MemoryContextSwitchTo(shared_arena); + * ... allocate / use ... + * MemoryContextSwitchTo(caller); + * agt_arena_reset(shared_arena); // O(1) cheap reset + * + * Pattern A creates a fresh context per call (microseconds). Pattern B + * amortizes the create cost across many calls (single AllocSetCreate at + * the very first call), and pays only a MemoryContextReset (~100 ns) per + * subsequent call. Use Pattern B when the function is invoked many times + * per query (e.g. a sort comparator called O(N log N) times). + */ +MemoryContext agt_arena_begin(void); +/* + * Pattern-B shared-arena creator. Implemented as a macro because PG's + * AllocSetContextCreate enforces memory-context names be compile-time + * constants via a StaticAssert. Use with a string literal: + * static MemoryContext my_arena = NULL; + * if (my_arena == NULL) + * { + * my_arena = AGT_ARENA_BEGIN_SHARED("my comparator workspace"); + * } + */ +#define AGT_ARENA_BEGIN_SHARED(_name) \ + AllocSetContextCreate(CacheMemoryContext, (_name), ALLOCSET_SMALL_SIZES) +void agt_arena_reset(MemoryContext arena); +void agt_arena_end(MemoryContext arena); void agtype_hash_scalar_value(const agtype_value *scalar_val, uint32 *hash); void agtype_hash_scalar_value_extended(const agtype_value *scalar_val, uint64 *hash, uint64 seed); From 5a254d6869d8b2c271f025ea158c0fee2cfacfa3 Mon Sep 17 00:00:00 2001 From: Sai Asish Y Date: Mon, 1 Jun 2026 17:05:17 -0700 Subject: [PATCH 071/100] python driver: preserve trailing quotes in agtype string values (#2425) * python driver: preserve trailing quotes in agtype string values str.strip('"') in visitStringValue() and visitPair() removes every '"' on either side of the token, not just the outer delimiters, so a value ending in an escaped quote (e.g. '"foo \"bar\""') loses its trailing backslash-escaped '"' character. The Agtype grammar guarantees STRING tokens always carry exactly one delimiter on each side, so slice with [1:-1] to strip them precisely. Fixes #2418 Signed-off-by: SAY-5 * fix(python-driver): centralize string delimiter trim and clean test docstring --------- Signed-off-by: SAY-5 --- drivers/python/age/builder.py | 11 +++++++++-- drivers/python/test_agtypes.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/drivers/python/age/builder.py b/drivers/python/age/builder.py index 08a40c252..b4f746e7a 100644 --- a/drivers/python/age/builder.py +++ b/drivers/python/age/builder.py @@ -105,9 +105,16 @@ def visitAgValue(self, ctx:AgtypeParser.AgValueContext): return valueCtx.accept(self) + @staticmethod + def _stripStringDelimiters(stringToken): + # The STRING token always has surrounding '"' delimiters per the + # Agtype grammar; slice rather than strip('"') so escaped quotes + # at the boundaries are preserved. + return stringToken.getText()[1:-1] + # Visit a parse tree produced by AgtypeParser#StringValue. def visitStringValue(self, ctx:AgtypeParser.StringValueContext): - return ctx.STRING().getText().strip('"') + return self._stripStringDelimiters(ctx.STRING()) # Visit a parse tree produced by AgtypeParser#IntegerValue. @@ -182,7 +189,7 @@ def visitPair(self, ctx:AgtypeParser.PairContext): raise AGTypeError(ctx.getText(), "Missing key in object pair") if agValNode is None: raise AGTypeError(ctx.getText(), "Missing value in object pair") - return (strNode.getText().strip('"') , agValNode) + return (self._stripStringDelimiters(strNode), agValNode) # Visit a parse tree produced by AgtypeParser#array. diff --git a/drivers/python/test_agtypes.py b/drivers/python/test_agtypes.py index 97f7972d1..7de0f5f52 100644 --- a/drivers/python/test_agtypes.py +++ b/drivers/python/test_agtypes.py @@ -245,6 +245,22 @@ def test_array_of_mixed_types(self): self.assertEqual(result[4], [1, 2, 3]) self.assertEqual(result[5], {"key": "val"}) + def test_string_value_preserves_inner_quotes(self): + """Issue #2418: preserve escaped boundary quotes when stripping. + + visitStringValue must strip only the outer delimiters; otherwise + values like '"foo \\"bar\\""' lose their trailing escaped quote. + """ + self.assertEqual(self.parse('"foo \\"bar\\""'), 'foo \\"bar\\"') + self.assertEqual(self.parse('"\\"leading"'), '\\"leading') + self.assertEqual(self.parse('"trailing\\""'), 'trailing\\"') + self.assertEqual(self.parse('""'), '') + # Same fix applies to visitPair() for object keys. + self.assertEqual( + self.parse('{"key\\"q": 1}'), + {'key\\"q': 1}, + ) + def test_malformed_vertex_raises_agtypeerror_or_recovers(self): """Issue #2367: Malformed agtype must raise AGTypeError or recover gracefully.""" from age.exceptions import AGTypeError From 73d0705e4ca7b3866b30436248b292b0170fb67b Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Wed, 3 Jun 2026 01:08:54 -0700 Subject: [PATCH 072/100] =?UTF-8?q?perf:=20VLE=20hash-adjacency=20overhaul?= =?UTF-8?q?=20=E2=80=94=20agehash=20+=20flat-array=20VertexEdgeArray=20(#2?= =?UTF-8?q?421)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the per-graph adjacency map with a Robin Hood open-addressing hashtable (agehash) and an embedded flat-array edge list, removing the hottest dynahash path on IC1 and shrinking the largest hashtable AGE keeps. Stages land as one commit: S1 MurmurHash3 fmix64 for graphid hashtables (replaces tag_hash) S2 Precompute graphid hash; share across paired DFS lookups S3 Replace ListGraphId adjacency with embedded flat-array VertexEdgeArray (single palloc, contiguous iteration) S4 Batched MLP lookup pipeline in add_valid_vertex_edges S5/C1 agehash library: INLINE Robin Hood hashtable with _with_hash API, freeze, iter, and a regress-only selftest S5/C2 Wire global graph edge_hashtable through agehash; drop edge_id from edge_entry (key lives in slot header); AGEHASH_MAX_LOAD=0.85; MemoryContextAllocHuge for SF10+ Performance (SF3 LDBC SNB, 5 runs/3 warmup, vs clean master baseline_v2): IC1 8,625 → 7,117 ms −17.49 % (the headline; hashtable-bound) IU1 40 → 35 ms −11.86 % (heaviest update; lookup-bound) IC sum 198,958 → 197,367 ms −0.80 % (suite-level noise) IS sum 1,009 → 1,028 ms +1.86 % (IS3 jitter; sub-ms) IU sum 77 → 72 ms −6.64 % IC2/3/4/5/6/7/8/9/10/11/12: parity (within ±3.3 %, mostly ±1.5 %) The VLE-DFS-heavy queries (IC3/5/6/9/11) sit at parity: with hash_search_with_hash_value at ≤1 % inclusive on their baseline flames, no hashtable swap can recover meaningful wall-time on them. Memory: removing edge_id from edge_entry saves ~416 MB on SF3 and ~1.4 GB on SF10 for the global graph's edge_hashtable. Slot capacity uses MemoryContextAllocHuge so SF10+ edge tables can be built. Adds: src/backend/utils/cache/agehash.c, src/include/utils/agehash.h regress/sql/agehash.sql + expected/agehash.out (boundary selftest) _agehash_self_test() in both fresh-install and upgrade SQL Tested on PostgreSQL 18.3 (REL_18_STABLE): all 35 regression tests pass (installcheck), warning-free build. modified: Makefile modified: age--1.7.0--y.y.y.sql new file: regress/expected/agehash.out new file: regress/sql/agehash.sql modified: sql/age_main.sql modified: src/backend/utils/adt/age_global_graph.c modified: src/backend/utils/adt/age_vle.c new file: src/backend/utils/cache/agehash.c modified: src/include/utils/age_global_graph.h new file: src/include/utils/agehash.h --- Makefile | 2 + age--1.7.0--y.y.y.sql | 9 + regress/expected/agehash.out | 14 + regress/sql/agehash.sql | 9 + sql/age_main.sql | 10 + src/backend/utils/adt/age_global_graph.c | 283 ++++++--- src/backend/utils/adt/age_vle.c | 273 +++++--- src/backend/utils/cache/agehash.c | 756 +++++++++++++++++++++++ src/include/utils/age_global_graph.h | 49 +- src/include/utils/agehash.h | 216 +++++++ 10 files changed, 1448 insertions(+), 173 deletions(-) create mode 100644 regress/expected/agehash.out create mode 100644 regress/sql/agehash.sql create mode 100644 src/backend/utils/cache/agehash.c create mode 100644 src/include/utils/agehash.h diff --git a/Makefile b/Makefile index 7e5fb3d2d..b79dbd3bb 100644 --- a/Makefile +++ b/Makefile @@ -127,6 +127,7 @@ OBJS = src/backend/age.o \ src/backend/utils/ag_func.o \ src/backend/utils/graph_generation.o \ src/backend/utils/cache/ag_cache.o \ + src/backend/utils/cache/agehash.o \ src/backend/utils/load/ag_load_labels.o \ src/backend/utils/load/ag_load_edges.o \ src/backend/utils/load/age_load.o \ @@ -153,6 +154,7 @@ REGRESS = scan \ graphid \ agtype \ agtype_hash_cmp \ + agehash \ catalog \ cypher \ expr \ diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index 4e718d426..74b84d604 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -753,3 +753,12 @@ RETURNS NULL ON NULL INPUT PARALLEL SAFE AS 'MODULE_PATHNAME'; +-- Internal selftest for the agehash open-addressing hashtable. Returns "OK" +-- on success or "FAIL: ..." with a diagnostic message. Intended for the +-- agehash regression test only. +CREATE FUNCTION ag_catalog._agehash_self_test() + RETURNS text + LANGUAGE c + VOLATILE + PARALLEL UNSAFE +AS 'MODULE_PATHNAME'; diff --git a/regress/expected/agehash.out b/regress/expected/agehash.out new file mode 100644 index 000000000..a6e647978 --- /dev/null +++ b/regress/expected/agehash.out @@ -0,0 +1,14 @@ +/* + * agehash internal selftest. + * + * Exercises the Robin Hood open-addressing hashtable used by AGE's hot-path + * caches at boundary sizes (1, 7, 8, 9, 63, 64, 65, 1024, ...) and across + * grow thresholds. A success returns the literal "OK"; any failure returns + * "FAIL: " describing the offending case. + */ +SELECT ag_catalog._agehash_self_test(); + _agehash_self_test +-------------------- + OK +(1 row) + diff --git a/regress/sql/agehash.sql b/regress/sql/agehash.sql new file mode 100644 index 000000000..4e6f827e8 --- /dev/null +++ b/regress/sql/agehash.sql @@ -0,0 +1,9 @@ +/* + * agehash internal selftest. + * + * Exercises the Robin Hood open-addressing hashtable used by AGE's hot-path + * caches at boundary sizes (1, 7, 8, 9, 63, 64, 65, 1024, ...) and across + * grow thresholds. A success returns the literal "OK"; any failure returns + * "FAIL: " describing the offending case. + */ +SELECT ag_catalog._agehash_self_test(); diff --git a/sql/age_main.sql b/sql/age_main.sql index 12ee304f4..72f420002 100644 --- a/sql/age_main.sql +++ b/sql/age_main.sql @@ -86,6 +86,16 @@ CREATE FUNCTION ag_catalog._label_id(graph_name name, label_name name) PARALLEL SAFE AS 'MODULE_PATHNAME'; +-- Internal selftest for the agehash open-addressing hashtable. Returns "OK" +-- on success or "FAIL: ..." with a diagnostic message. Intended for the +-- agehash regression test only. +CREATE FUNCTION ag_catalog._agehash_self_test() + RETURNS text + LANGUAGE c + VOLATILE + PARALLEL UNSAFE +AS 'MODULE_PATHNAME'; + -- -- utility functions -- diff --git a/src/backend/utils/adt/age_global_graph.c b/src/backend/utils/adt/age_global_graph.c index a9b9b7111..1c4a4601b 100644 --- a/src/backend/utils/adt/age_global_graph.c +++ b/src/backend/utils/adt/age_global_graph.c @@ -40,6 +40,7 @@ #endif #include "utils/age_global_graph.h" +#include "utils/agehash.h" #include "catalog/ag_graph.h" #include "catalog/ag_label.h" #include "utils/ag_cache.h" @@ -48,7 +49,6 @@ /* defines */ #define VERTEX_HTAB_NAME "Vertex to edge lists " /* added a space at end for */ -#define EDGE_HTAB_NAME "Edge to vertex mapping " /* the graph name to follow */ #define VERTEX_HTAB_INITIAL_SIZE 10000 #define EDGE_HTAB_INITIAL_SIZE 10000 @@ -104,17 +104,26 @@ static GraphVersionState *shmem_version_state = NULL; typedef struct vertex_entry { graphid vertex_id; /* vertex id, it is also the hash key */ - ListGraphId *edges_in; /* List of entering edges graphids (int64) */ - ListGraphId *edges_out; /* List of exiting edges graphids (int64) */ - ListGraphId *edges_self; /* List of selfloop edges graphids (int64) */ + VertexEdgeArray edges_in; /* incoming edge graphids (flat array) */ + VertexEdgeArray edges_out; /* outgoing edge graphids (flat array) */ + VertexEdgeArray edges_self; /* self-loop edge graphids (flat array) */ Oid vertex_label_table_oid; /* the label table oid */ ItemPointerData tid; /* physical tuple location for lazy fetch */ } vertex_entry; -/* edge entry for the edge_hashtable */ +/* + * edge entry for the edge_table. + * + * The edge_id is the hash key and is stored in the agehash slot header + * (immediately before the payload). It is intentionally NOT a field on this + * payload struct: duplicating it would add 8 bytes per edge to the slot, + * which on SF10 (~175M edges) is over a gigabyte of overhead. Use + * get_edge_entry_id(ee) when you need the id of an entry returned by + * get_edge_entry / get_edge_entry_with_hash; that helper recovers the key + * from the slot via agehash_key_from_payload. + */ typedef struct edge_entry { - graphid edge_id; /* edge id, it is also the hash key */ Oid edge_label_table_oid; /* the label table oid */ ItemPointerData tid; /* physical tuple location for lazy fetch */ graphid start_vertex_id; /* start vertex */ @@ -131,7 +140,8 @@ typedef struct GRAPH_global_context char *graph_name; /* graph name */ Oid graph_oid; /* graph oid for searching */ HTAB *vertex_hashtable; /* hashtable to hold vertex edge lists */ - HTAB *edge_hashtable; /* hashtable to hold edge to vertex map */ + AgeHashTable *edge_table; /* edge to vertex map (Robin Hood) */ + MemoryContext edge_table_mcxt; /* private context owning edge_table */ uint64 graph_version; /* version counter for cache invalidation */ TransactionId xmin; /* snapshot fallback: transaction xmin */ TransactionId xmax; /* snapshot fallback: transaction xmax */ @@ -155,6 +165,50 @@ typedef struct GRAPH_global_context_container /* global variable to hold the per process GRAPH global contexts */ static GRAPH_global_context_container global_graph_contexts_container = {0}; +/* + * VertexEdgeArray helpers — flat-array adjacency container used by + * vertex_entry's edges_in / edges_out / edges_self. + * + * Growth policy: start at 4 slots on first append, then double on each + * overflow. This keeps the average cost of n appends amortised O(n) and + * keeps the memory waste bounded by 2x. + */ +#define VEA_INITIAL_CAPACITY 4 + +static inline void vea_append(VertexEdgeArray *vea, graphid edge_id) +{ + if (vea->size == vea->capacity) + { + int32 new_capacity = (vea->capacity == 0) + ? VEA_INITIAL_CAPACITY + : vea->capacity * 2; + + if (vea->array == NULL) + { + vea->array = (graphid *) palloc(new_capacity * sizeof(graphid)); + } + else + { + vea->array = (graphid *) repalloc(vea->array, + new_capacity * sizeof(graphid)); + } + + vea->capacity = new_capacity; + } + vea->array[vea->size++] = edge_id; +} + +static inline void vea_free(VertexEdgeArray *vea) +{ + if (vea->array != NULL) + { + pfree(vea->array); + vea->array = NULL; + } + vea->size = 0; + vea->capacity = 0; +} + /* declarations */ /* GRAPH global context functions */ static bool free_specific_GRAPH_global_context(GRAPH_global_context *ggctx); @@ -222,6 +276,57 @@ bool is_ggctx_invalid(GRAPH_global_context *ggctx) ggctx->curcid != snap->curcid); } } +/* + * Fast hash function for graphid (int64) keys. + * + * Replaces dynahash's tag_hash (Jenkins lookup3 → ~17 mixing ops) with the + * MurmurHash3 fmix64 finalizer (5 ops: 3 xorshifts + 2 multiplies). + * + * Quality: fmix64 is the avalanche stage of MurmurHash3 and passes all SMHasher + * tests for 64-bit integer inputs. The output is truncated to uint32 to match + * dynahash's HashValueFunc signature; bits 0..31 of fmix64 are well-mixed. + * + * Performance rationale: graphid lookups dominate hash_search_with_hash_value + * time (≈41% IC1 on SF3). Reducing the per-call mixing cost cuts both insert + * and lookup overhead in age_global_graph and age_vle hashtables. + */ +uint32 graphid_hash(const void *key, Size keysize) +{ + uint64 k; + + /* keysize is always sizeof(int64) for every graphid hashtable; assert in debug. */ + Assert(keysize == sizeof(int64)); + (void) keysize; + + /* graphid keys are stored as int64; load aligned (callers pass &graphid). */ + memcpy(&k, key, sizeof(uint64)); + + /* MurmurHash3 fmix64 (Austin Appleby, public domain). */ + k ^= k >> 33; + k *= UINT64CONST(0xff51afd7ed558ccd); + k ^= k >> 33; + k *= UINT64CONST(0xc4ceb9fe1a85ec53); + k ^= k >> 33; + + return (uint32) k; +} + +/* + * agehash key-equality callback for graphid (int64) keys. + * + * graphid_hash collisions are rare but real (32-bit hash space, billions of + * possible keys), so the equality check has to compare the full 8 bytes. + * memcmp on a fixed 8-byte length compiles to a single load + cmp on x86, + * which is just as fast as an int64 cast and avoids any alignment risk on + * other architectures. + */ +bool graphid_keyeq(const void *a, const void *b, Size keysize) +{ + Assert(keysize == sizeof(int64)); + (void) keysize; + return memcmp(a, b, sizeof(int64)) == 0; +} + /* * Helper function to create the global vertex and edge hashtables. One * hashtable will hold the vertex, its edges (both incoming and exiting) as a @@ -231,49 +336,50 @@ bool is_ggctx_invalid(GRAPH_global_context *ggctx) static void create_GRAPH_global_hashtables(GRAPH_global_context *ggctx) { HASHCTL vertex_ctl; - HASHCTL edge_ctl; char *graph_name = NULL; char *vhn = NULL; - char *ehn = NULL; int glen; int vlen; - int elen; /* get the graph name and length */ graph_name = ggctx->graph_name; glen = strlen(graph_name); /* get the vertex htab name length */ vlen = strlen(VERTEX_HTAB_NAME); - /* get the edge htab name length */ - elen = strlen(EDGE_HTAB_NAME); - /* allocate the space and build the names */ + /* allocate the space and build the name */ vhn = palloc0(vlen + glen + 1); - ehn = palloc0(elen + glen + 1); - /* copy in the names */ strcpy(vhn, VERTEX_HTAB_NAME); - strcpy(ehn, EDGE_HTAB_NAME); - /* add in the graph name */ vhn = strncat(vhn, graph_name, glen); - ehn = strncat(ehn, graph_name, glen); /* initialize the vertex hashtable */ MemSet(&vertex_ctl, 0, sizeof(vertex_ctl)); vertex_ctl.keysize = sizeof(int64); vertex_ctl.entrysize = sizeof(vertex_entry); - vertex_ctl.hash = tag_hash; + vertex_ctl.hash = graphid_hash; ggctx->vertex_hashtable = hash_create(vhn, VERTEX_HTAB_INITIAL_SIZE, &vertex_ctl, HASH_ELEM | HASH_FUNCTION); pfree_if_not_null(vhn); - /* initialize the edge hashtable */ - MemSet(&edge_ctl, 0, sizeof(edge_ctl)); - edge_ctl.keysize = sizeof(int64); - edge_ctl.entrysize = sizeof(edge_entry); - edge_ctl.hash = tag_hash; - ggctx->edge_hashtable = hash_create(ehn, EDGE_HTAB_INITIAL_SIZE, &edge_ctl, - HASH_ELEM | HASH_FUNCTION); - pfree_if_not_null(ehn); + /* + * Initialize the edge_table (agehash, INLINE mode). + * + * Owns its own MemoryContext as a child of CurrentMemoryContext (which, + * at the call site, is TopMemoryContext for the lifetime of the cached + * GRAPH_global_context). Cleanup is a single MemoryContextDelete in + * free_specific_GRAPH_global_context, so an elog during build cannot + * leak slots. + */ + ggctx->edge_table_mcxt = + AllocSetContextCreate(CurrentMemoryContext, + "AGE edge_table", + ALLOCSET_DEFAULT_SIZES); + ggctx->edge_table = agehash_create_inline(ggctx->edge_table_mcxt, + sizeof(graphid), + sizeof(edge_entry), + EDGE_HTAB_INITIAL_SIZE, + graphid_hash, + graphid_keyeq); } /* helper function to get a List of all label names for the specified graph */ @@ -420,10 +526,10 @@ static bool insert_edge_entry(GRAPH_global_context *ggctx, graphid edge_id, bool found = false; /* search for the edge */ - ee = (edge_entry *)hash_search(ggctx->edge_hashtable, (void *)&edge_id, - HASH_ENTER, &found); + ee = (edge_entry *) agehash_insert(ggctx->edge_table, + (void *) &edge_id, &found); - /* if the hash enter returned is NULL, error out */ + /* agehash never returns NULL on insert; a NULL would indicate a bug. */ if (ee == NULL) { elog(ERROR, "insert_edge_entry: hash table returned NULL for ee"); @@ -445,20 +551,17 @@ static bool insert_edge_entry(GRAPH_global_context *ggctx, graphid edge_id, ereport(WARNING, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("previous edge: [id: %ld, start: %ld, end: %ld, label oid: %d]", - ee->edge_id, ee->start_vertex_id, ee->end_vertex_id, + edge_id, ee->start_vertex_id, ee->end_vertex_id, ee->edge_label_table_oid))); return false; } - /* not sure if we really need to zero out the entry, as we set everything */ - MemSet(ee, 0, sizeof(edge_entry)); - /* - * Set the edge id - this is important as this is the hash key value used - * for hash function collisions. + * agehash_insert zero-fills the payload on a fresh insert, so we can fill + * in only the fields we care about. The hash key (edge_id) lives in the + * slot header; recoverable via get_edge_entry_id() if needed. */ - ee->edge_id = edge_id; ee->tid = tid; ee->start_vertex_id = start_vertex_id; ee->end_vertex_id = end_vertex_id; @@ -520,10 +623,10 @@ static bool insert_vertex_entry(GRAPH_global_context *ggctx, graphid vertex_id, ve->vertex_label_table_oid = vertex_label_table_oid; /* set the TID for lazy property fetch */ ve->tid = tid; - /* set the NIL edge list */ - ve->edges_in = NULL; - ve->edges_out = NULL; - ve->edges_self = NULL; + /* + * MemSet above already zeroed the embedded VertexEdgeArray fields + * (array=NULL, size=0, capacity=0); no explicit NIL assignment needed. + */ /* we also need to store the vertex id for clean up of vertex lists */ ggctx->vertices = append_graphid(ggctx->vertices, vertex_id); @@ -561,7 +664,7 @@ static bool insert_vertex_edge(GRAPH_global_context *ggctx, */ if (start_found && is_selfloop) { - value->edges_self = append_graphid(value->edges_self, edge_id); + vea_append(&value->edges_self, edge_id); return true; } /* @@ -570,7 +673,7 @@ static bool insert_vertex_edge(GRAPH_global_context *ggctx, */ else if (start_found) { - value->edges_out = append_graphid(value->edges_out, edge_id); + vea_append(&value->edges_out, edge_id); } /* search for the end vertex of the edge */ @@ -584,7 +687,7 @@ static bool insert_vertex_edge(GRAPH_global_context *ggctx, */ if (start_found && end_found) { - value->edges_in = append_graphid(value->edges_in, edge_id); + vea_append(&value->edges_in, edge_id); return true; } /* @@ -836,7 +939,7 @@ static void load_edge_hashtable(GRAPH_global_context *ggctx) static void freeze_GRAPH_global_hashtables(GRAPH_global_context *ggctx) { hash_freeze(ggctx->vertex_hashtable); - hash_freeze(ggctx->edge_hashtable); + agehash_freeze(ggctx->edge_table); } /* @@ -885,14 +988,10 @@ static bool free_specific_GRAPH_global_context(GRAPH_global_context *ggctx) return false; } - /* free the edge list associated with this vertex */ - free_ListGraphId(value->edges_in); - free_ListGraphId(value->edges_out); - free_ListGraphId(value->edges_self); - - value->edges_in = NULL; - value->edges_out = NULL; - value->edges_self = NULL; + /* free the edge arrays associated with this vertex */ + vea_free(&value->edges_in); + vea_free(&value->edges_out); + vea_free(&value->edges_self); /* move to the next vertex */ curr_vertex = next_vertex; @@ -904,10 +1003,18 @@ static bool free_specific_GRAPH_global_context(GRAPH_global_context *ggctx) /* free the hashtables */ hash_destroy(ggctx->vertex_hashtable); - hash_destroy(ggctx->edge_hashtable); + /* + * The edge_table and all of its slots live entirely inside + * edge_table_mcxt, so a single MemoryContextDelete reclaims them. + */ + if (ggctx->edge_table_mcxt != NULL) + { + MemoryContextDelete(ggctx->edge_table_mcxt); + } ggctx->vertex_hashtable = NULL; - ggctx->edge_hashtable = NULL; + ggctx->edge_table = NULL; + ggctx->edge_table_mcxt = NULL; /* free the context */ pfree_if_not_null(ggctx); @@ -1201,17 +1308,33 @@ vertex_entry *get_vertex_entry(GRAPH_global_context *ggctx, graphid vertex_id) return ve; } -/* helper function to retrieve an edge_entry from the graph's edge hash table */ +/* helper function to retrieve an edge_entry from the graph's edge table */ edge_entry *get_edge_entry(GRAPH_global_context *ggctx, graphid edge_id) { - edge_entry *ee = NULL; - bool found = false; + edge_entry *ee; - /* retrieve the current edge entry */ - ee = (edge_entry *)hash_search(ggctx->edge_hashtable, (void *)&edge_id, - HASH_FIND, &found); + ee = (edge_entry *) agehash_lookup(ggctx->edge_table, (void *) &edge_id); /* it should be found, otherwise we have problems */ - Assert(found); + Assert(ee != NULL); + + return ee; +} + +/* + * Variant of get_edge_entry that uses a precomputed hash value to skip the + * agehash internal hash callback. The caller is responsible for ensuring + * hashvalue == graphid_hash(&edge_id, sizeof(int64)). Used by the VLE DFS + * hot loop where the same edge_id is also looked up in edge_state_hashtable. + */ +edge_entry *get_edge_entry_with_hash(GRAPH_global_context *ggctx, + graphid edge_id, uint32 hashvalue) +{ + edge_entry *ee; + + ee = (edge_entry *) agehash_lookup_with_hash(ggctx->edge_table, + (void *) &edge_id, + hashvalue); + Assert(ee != NULL); return ee; } @@ -1264,19 +1387,19 @@ graphid get_vertex_entry_id(vertex_entry *ve) return ve->vertex_id; } -ListGraphId *get_vertex_entry_edges_in(vertex_entry *ve) +VertexEdgeArray *get_vertex_entry_edges_in_array(vertex_entry *ve) { - return ve->edges_in; + return &ve->edges_in; } -ListGraphId *get_vertex_entry_edges_out(vertex_entry *ve) +VertexEdgeArray *get_vertex_entry_edges_out_array(vertex_entry *ve) { - return ve->edges_out; + return &ve->edges_out; } -ListGraphId *get_vertex_entry_edges_self(vertex_entry *ve) +VertexEdgeArray *get_vertex_entry_edges_self_array(vertex_entry *ve) { - return ve->edges_self; + return &ve->edges_self; } @@ -1343,7 +1466,15 @@ Datum get_vertex_entry_properties(vertex_entry *ve) /* edge_entry accessor functions */ graphid get_edge_entry_id(edge_entry *ee) { - return ee->edge_id; + /* + * The edge_id is stored as the agehash slot key, immediately preceding + * the payload pointer we hand back as `edge_entry *`. Recover it via + * the public agehash_key_from_payload helper to avoid a redundant + * 8-byte field on every entry (saves ~400MB on SF3, ~1.4GB on SF10). + */ + graphid k; + memcpy(&k, agehash_key_from_payload(ee, sizeof(graphid)), sizeof(graphid)); + return k; } Oid get_edge_entry_label_table_oid(edge_entry *ee) @@ -1450,7 +1581,7 @@ Datum age_vertex_stats(PG_FUNCTION_ARGS) { GRAPH_global_context *ggctx = NULL; vertex_entry *ve = NULL; - ListGraphId *edges = NULL; + VertexEdgeArray *edges = NULL; agtype_value *agtv_vertex = NULL; agtype_value *agtv_temp = NULL; agtype_value agtv_integer; @@ -1530,24 +1661,24 @@ Datum age_vertex_stats(PG_FUNCTION_ARGS) agtv_temp->val.int_value = 0; /* get and store the self_loops */ - edges = get_vertex_entry_edges_self(ve); - self_loops = (edges != NULL) ? get_list_size(edges) : 0; + edges = get_vertex_entry_edges_self_array(ve); + self_loops = edges->size; agtv_temp->val.int_value = self_loops; result.res = push_agtype_value(&result.parse_state, WAGT_KEY, string_to_agtype_value("self_loops")); result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp); /* get and store the in_degree */ - edges = get_vertex_entry_edges_in(ve); - degree = (edges != NULL) ? get_list_size(edges) : 0; + edges = get_vertex_entry_edges_in_array(ve); + degree = edges->size; agtv_temp->val.int_value = degree + self_loops; result.res = push_agtype_value(&result.parse_state, WAGT_KEY, string_to_agtype_value("in_degree")); result.res = push_agtype_value(&result.parse_state, WAGT_VALUE, agtv_temp); /* get and store the out_degree */ - edges = get_vertex_entry_edges_out(ve); - degree = (edges != NULL) ? get_list_size(edges) : 0; + edges = get_vertex_entry_edges_out_array(ve); + degree = edges->size; agtv_temp->val.int_value = degree + self_loops; result.res = push_agtype_value(&result.parse_state, WAGT_KEY, string_to_agtype_value("out_degree")); diff --git a/src/backend/utils/adt/age_vle.c b/src/backend/utils/adt/age_vle.c index c312a62ee..3bc276cfc 100644 --- a/src/backend/utils/adt/age_vle.c +++ b/src/backend/utils/adt/age_vle.c @@ -195,8 +195,9 @@ static VLE_local_context *build_local_vle_context(FunctionCallInfo fcinfo, static void create_VLE_local_state_hashtable(VLE_local_context *vlelctx); static void free_VLE_local_context(VLE_local_context *vlelctx); /* VLE graph traversal functions */ -static edge_state_entry *get_edge_state(VLE_local_context *vlelctx, - graphid edge_id); +static edge_state_entry *get_edge_state_with_hash(VLE_local_context *vlelctx, + graphid edge_id, + uint32 hashvalue); /* graphid data structures */ static void load_initial_dfs_stacks(VLE_local_context *vlelctx); static bool dfs_find_a_path_between(VLE_local_context *vlelctx); @@ -390,7 +391,7 @@ static void create_VLE_local_state_hashtable(VLE_local_context *vlelctx) MemSet(&edge_state_ctl, 0, sizeof(edge_state_ctl)); edge_state_ctl.keysize = sizeof(int64); edge_state_ctl.entrysize = sizeof(edge_state_entry); - edge_state_ctl.hash = tag_hash; + edge_state_ctl.hash = graphid_hash; vlelctx->edge_state_hashtable = hash_create(eshn, EDGE_STATE_HTAB_INITIAL_SIZE, &edge_state_ctl, @@ -932,23 +933,24 @@ static VLE_local_context *build_local_vle_context(FunctionCallInfo fcinfo, } /* - * Helper function to get the specified edge's state. If it does not find it, it - * creates and initializes it. + * Helper function to get the specified edge's state, using a precomputed hash + * value. Callers can compute graphid_hash() once and reuse it for lookups in + * both the dynahash edge_state_hashtable here and the agehash-backed + * edge_table on the global-graph lookup path elsewhere. */ -static edge_state_entry *get_edge_state(VLE_local_context *vlelctx, - graphid edge_id) +static edge_state_entry *get_edge_state_with_hash(VLE_local_context *vlelctx, + graphid edge_id, + uint32 hashvalue) { edge_state_entry *ese = NULL; bool found = false; - /* retrieve the edge_state_entry from the edge state hashtable */ - ese = (edge_state_entry *)hash_search(vlelctx->edge_state_hashtable, - (void *)&edge_id, HASH_ENTER, &found); - - /* if it isn't found, it needs to be created and initialized */ + ese = (edge_state_entry *)hash_search_with_hash_value( + vlelctx->edge_state_hashtable, + (void *)&edge_id, hashvalue, + HASH_ENTER, &found); if (!found) { - /* the edge id is also the hash key for resolving collisions */ ese->edge_id = edge_id; ese->used_in_path = false; ese->has_been_matched = false; @@ -1047,11 +1049,18 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) edge_state_entry *ese = NULL; edge_entry *ee = NULL; bool found = false; + uint32 edge_hashvalue; /* get an edge, but leave it on the stack for now */ edge_id = gid_stack_peek(edge_stack); + /* + * Compute the hash for edge_id once and reuse it for both the + * edge_state_hashtable lookup and (later) the edge_hashtable lookup. + * Both tables key on graphid using graphid_hash(). + */ + edge_hashvalue = graphid_hash(&edge_id, sizeof(int64)); /* get the edge's state */ - ese = get_edge_state(vlelctx, edge_id); + ese = get_edge_state_with_hash(vlelctx, edge_id, edge_hashvalue); /* * If the edge is already in use, it means that the edge is in the path. * So, we need to see if it is the last path entry (we are backing up - @@ -1099,7 +1108,7 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) gid_stack_push(path_stack, edge_id); /* now get the edge entry so we can get the next vertex to move to */ - ee = get_edge_entry(vlelctx->ggctx, edge_id); + ee = get_edge_entry_with_hash(vlelctx->ggctx, edge_id, edge_hashvalue); next_vertex_id = get_next_vertex(vlelctx, ee); /* @@ -1175,11 +1184,18 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) edge_state_entry *ese = NULL; edge_entry *ee = NULL; bool found = false; + uint32 edge_hashvalue; /* get an edge, but leave it on the stack for now */ edge_id = gid_stack_peek(edge_stack); + /* + * Compute the hash for edge_id once and reuse it for both the + * edge_state_hashtable lookup and (later) the edge_hashtable lookup. + * Both tables key on graphid using graphid_hash(). + */ + edge_hashvalue = graphid_hash(&edge_id, sizeof(int64)); /* get the edge's state */ - ese = get_edge_state(vlelctx, edge_id); + ese = get_edge_state_with_hash(vlelctx, edge_id, edge_hashvalue); /* * If the edge is already in use, it means that the edge is in the path. * So, we need to see if it is the last path entry (we are backing up - @@ -1227,7 +1243,7 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) gid_stack_push(path_stack, edge_id); /* now get the edge entry so we can get the next vertex to move to */ - ee = get_edge_entry(vlelctx->ggctx, edge_id); + ee = get_edge_entry_with_hash(vlelctx->ggctx, edge_id, edge_hashvalue); next_vertex_id = get_next_vertex(vlelctx, ee); /* @@ -1291,16 +1307,51 @@ static bool is_edge_in_path(VLE_local_context *vlelctx, graphid edge_id) * * Note: The vertex must exist. */ +/* + * Batched candidate buffer size for the adjacency lookup pipeline below. + * 8 was chosen because it comfortably fits within the OoO window and the + * per-core L1 MSHR count of modern Xeons (12+), so the K back-to-back + * hashtable misses overlap in a single MLP wave. + */ +#define VLE_LOOKUP_BATCH 8 + static void add_valid_vertex_edges(VLE_local_context *vlelctx, graphid vertex_id) { GraphIdStack *vertex_stack = NULL; GraphIdStack *edge_stack = NULL; - ListGraphId *edges = NULL; vertex_entry *ve = NULL; - GraphIdNode *edge_in = NULL; - GraphIdNode *edge_out = NULL; - GraphIdNode *edge_self = NULL; + /* + * Three flat-array adjacency lists, walked in parallel via integer + * indices. An empty (or direction-disabled) list has size == 0 so its + * branch never fires. This replaces the previous GraphIdNode pointer + * walk with a contiguous-memory traversal — significantly better for + * cache and branch-predictor behaviour on the DFS hot path. + */ + graphid *arr_out = NULL; + int32 sz_out = 0; + int32 idx_out = 0; + graphid *arr_in = NULL; + int32 sz_in = 0; + int32 idx_in = 0; + graphid *arr_self = NULL; + int32 sz_self = 0; + int32 idx_self = 0; + VertexEdgeArray *vea = NULL; + + /* + * Per-batch scratch arrays for the MLP lookup pipeline. Each iteration + * gathers up to VLE_LOOKUP_BATCH not-already-in-path candidate edges, + * then issues their edge_table (agehash) and edge_state_hashtable + * (dynahash) lookups in two tight back-to-back loops. The CPU's + * out-of-order engine overlaps the K independent cache misses inside + * each loop, hiding memory latency that the original one-edge-at-a-time + * loop serialized. + */ + graphid batch_eids[VLE_LOOKUP_BATCH]; + uint32 batch_hashes[VLE_LOOKUP_BATCH]; + edge_entry *batch_ee[VLE_LOOKUP_BATCH]; + edge_state_entry *batch_ese[VLE_LOOKUP_BATCH]; /* get the vertex entry */ ve = get_vertex_entry(vlelctx->ggctx, vertex_id); @@ -1314,82 +1365,128 @@ static void add_valid_vertex_edges(VLE_local_context *vlelctx, vertex_stack = vlelctx->dfs_vertex_stack; edge_stack = vlelctx->dfs_edge_stack; - /* set to the first edge for each edge list for the specified direction */ + /* set up walked arrays for the requested direction(s) */ if (vlelctx->edge_direction == CYPHER_REL_DIR_RIGHT || vlelctx->edge_direction == CYPHER_REL_DIR_NONE) { - edges = get_vertex_entry_edges_out(ve); - edge_out = (edges != NULL) ? get_list_head(edges) : NULL; + vea = get_vertex_entry_edges_out_array(ve); + arr_out = vea->array; + sz_out = vea->size; } if (vlelctx->edge_direction == CYPHER_REL_DIR_LEFT || vlelctx->edge_direction == CYPHER_REL_DIR_NONE) { - edges = get_vertex_entry_edges_in(ve); - edge_in = (edges != NULL) ? get_list_head(edges) : NULL; + vea = get_vertex_entry_edges_in_array(ve); + arr_in = vea->array; + sz_in = vea->size; } - /* set to the first selfloop edge */ - edges = get_vertex_entry_edges_self(ve); - edge_self = (edges != NULL) ? get_list_head(edges) : NULL; + /* selfloops are always traversed */ + vea = get_vertex_entry_edges_self_array(ve); + arr_self = vea->array; + sz_self = vea->size; - /* add in valid vertex edges */ - while (edge_out != NULL || edge_in != NULL || edge_self != NULL) + /* + * Outer loop: drain the three flat arrays via a 5-phase pipeline. + * 1. Gather: pull up to VLE_LOOKUP_BATCH next edge_ids that survive + * the cheap is_edge_in_path() early-skip. + * 2. Hash: compute graphid_hash for the batch (pure compute). + * 3. Lookup: K back-to-back edge_table (agehash) lookups via + * get_edge_entry_with_hash() — MLP window 1 (the CPU overlaps + * the K slot misses). + * 4. State: K back-to-back edge_state_hashtable (dynahash) HASH_ENTER + * calls — MLP window 2 (different table, different bucket misses). + * 5. Apply: per-edge match/state-update/stack-push, now operating + * on cache-warm ee/ese pointers. + * Phase 5 preserves the exact processing order of the original loop + * (out direction first, then in, then self), so DFS stack ordering and + * therefore path enumeration are identical to the previous version. + */ + while (idx_out < sz_out || idx_in < sz_in || idx_self < sz_self) { - edge_entry *ee = NULL; - edge_state_entry *ese = NULL; - graphid edge_id; + int batch_n = 0; + int i; - /* get the edge_id from the next available edge*/ - if (edge_out != NULL) - { - edge_id = get_graphid(edge_out); - } - else if (edge_in != NULL) - { - edge_id = get_graphid(edge_in); - } - else + /* Phase 1: gather */ + while (batch_n < VLE_LOOKUP_BATCH && + (idx_out < sz_out || idx_in < sz_in || idx_self < sz_self)) { - edge_id = get_graphid(edge_self); - } + graphid edge_id; - /* - * This is a fast existence check, relative to the hash search, for when - * the path stack is small. If the edge is in the path, we skip it. - */ - if (gid_stack_size(vlelctx->dfs_path_stack) < 10 && - is_edge_in_path(vlelctx, edge_id)) - { - /* set to the next available edge */ - if (edge_out != NULL) + if (idx_out < sz_out) { - edge_out = next_GraphIdNode(edge_out); + edge_id = arr_out[idx_out++]; } - else if (edge_in != NULL) + else if (idx_in < sz_in) { - edge_in = next_GraphIdNode(edge_in); + edge_id = arr_in[idx_in++]; } else { - edge_self = next_GraphIdNode(edge_self); + edge_id = arr_self[idx_self++]; } - continue; + + /* + * Fast early-skip when the path stack is small: avoids two + * hashtable lookups for edges already on the path. + */ + if (gid_stack_size(vlelctx->dfs_path_stack) < 10 && + is_edge_in_path(vlelctx, edge_id)) + { + continue; + } + + batch_eids[batch_n++] = edge_id; } - /* get the edge entry */ - ee = get_edge_entry(vlelctx->ggctx, edge_id); - /* it better exist */ - if (ee == NULL) + if (batch_n == 0) { - elog(ERROR, "add_valid_vertex_edges: no edge found"); + break; } - /* get its state */ - ese = get_edge_state(vlelctx, edge_id); - /* - * Don't add any edges that we have already seen because they will - * cause a loop to form. - */ - if (!ese->used_in_path) + + /* Phase 2: compute hashes (pure compute, no misses) */ + for (i = 0; i < batch_n; i++) { + batch_hashes[i] = graphid_hash(&batch_eids[i], sizeof(int64)); + } + + /* Phase 3: K back-to-back edge_table (agehash) lookups (MLP wave 1) */ + for (i = 0; i < batch_n; i++) + { + batch_ee[i] = get_edge_entry_with_hash(vlelctx->ggctx, + batch_eids[i], + batch_hashes[i]); + } + + /* Phase 4: K back-to-back edge_state_hashtable lookups (MLP wave 2) */ + for (i = 0; i < batch_n; i++) + { + batch_ese[i] = get_edge_state_with_hash(vlelctx, + batch_eids[i], + batch_hashes[i]); + } + + /* Phase 5: process the batch sequentially */ + for (i = 0; i < batch_n; i++) + { + edge_entry *ee = batch_ee[i]; + edge_state_entry *ese = batch_ese[i]; + graphid edge_id = batch_eids[i]; + + /* it better exist */ + if (ee == NULL) + { + elog(ERROR, "add_valid_vertex_edges: no edge found"); + } + + /* + * Don't add any edges that we have already seen because they + * will cause a loop to form. + */ + if (ese->used_in_path) + { + continue; + } + /* validate the edge if it hasn't been already */ if (!ese->has_been_matched && is_an_edge_match(vlelctx, ee)) { @@ -1401,37 +1498,25 @@ static void add_valid_vertex_edges(VLE_local_context *vlelctx, ese->has_been_matched = true; ese->matched = false; } + /* if it is a match, add it */ if (ese->has_been_matched && ese->matched) { /* - * We need to maintain our source vertex for each edge added - * if the edge_direction is CYPHER_REL_DIR_NONE. This is due - * to the edges having a fixed direction and the dfs + * We need to maintain our source vertex for each edge + * added if the edge_direction is CYPHER_REL_DIR_NONE. This + * is due to the edges having a fixed direction and the dfs * algorithm working strictly through edges. With an * un-directional VLE edge, you don't know the vertex that * you just came from. So, we need to store it. */ - if (vlelctx->edge_direction == CYPHER_REL_DIR_NONE) - { - gid_stack_push(vertex_stack, get_vertex_entry_id(ve)); - } - gid_stack_push(edge_stack, edge_id); + if (vlelctx->edge_direction == CYPHER_REL_DIR_NONE) + { + gid_stack_push(vertex_stack, get_vertex_entry_id(ve)); + } + gid_stack_push(edge_stack, edge_id); } } - /* get the next working edge */ - if (edge_out != NULL) - { - edge_out = next_GraphIdNode(edge_out); - } - else if (edge_in != NULL) - { - edge_in = next_GraphIdNode(edge_in); - } - else - { - edge_self = next_GraphIdNode(edge_self); - } } } @@ -2531,7 +2616,7 @@ Datum _ag_enforce_edge_uniqueness(PG_FUNCTION_ARGS) MemSet(&exists_ctl, 0, sizeof(exists_ctl)); exists_ctl.keysize = sizeof(int64); exists_ctl.entrysize = sizeof(int64); - exists_ctl.hash = tag_hash; + exists_ctl.hash = graphid_hash; /* create exists_hash table */ exists_hash = hash_create(EXISTS_HTAB_NAME, EXISTS_HTAB_NAME_INITIAL_SIZE, diff --git a/src/backend/utils/cache/agehash.c b/src/backend/utils/cache/agehash.c new file mode 100644 index 000000000..2f3f256e0 --- /dev/null +++ b/src/backend/utils/cache/agehash.c @@ -0,0 +1,756 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * agehash.c - Robin Hood open-addressing hashtable for AGE. + * + * See agehash.h for the public contract. This file implements the INLINE + * mode only. + * + * Internal slot layout (INLINE): + * + * bytes 0..1 uint16 probe_dist (AGEHASH_EMPTY = 0xFFFF marks empty) + * bytes 2..3 uint16 reserved (future tombstone / flag bits) + * bytes 4..7 uint32 pad (forces key to 8-byte alignment) + * bytes 8..K+7 key + * bytes K+8.. payload + * + * slot_size = MAXALIGN(8 + key_size + payload_size). + */ + +#include "postgres.h" + +#include "fmgr.h" +#include "utils/agehash.h" +#include "utils/builtins.h" +#include "utils/memutils.h" + +/* ------------------------------------------------------------------------- */ + +struct AgeHashTable +{ + /* Slot array: capacity * slot_size bytes, palloc'd in mcxt. */ + char *slots; + uint32 capacity; /* always a power of two */ + uint32 capacity_mask; /* capacity - 1 */ + uint32 size; /* live entries */ + uint32 max_size; /* size at which we grow */ + uint32 slot_size; /* total bytes per slot */ + uint32 key_size; + uint32 payload_size; + uint32 payload_offset; /* AGEHASH_SLOT_KEY_OFFSET + key_size */ + AgeHashMode mode; + bool frozen; + agehash_hash_fn hash_fn; + agehash_keyeq_fn keyeq_fn; + MemoryContext mcxt; +}; + +/* ------------------------------------------------------------------------- */ +/* Slot accessors. */ + +static inline char * +slot_at(AgeHashTable *t, uint32 idx) +{ + return t->slots + (Size) idx * t->slot_size; +} + +static inline uint16 +slot_probe_dist(const char *slot) +{ + uint16 d; + memcpy(&d, slot, sizeof(uint16)); + return d; +} + +static inline void +slot_set_probe_dist(char *slot, uint16 d) +{ + memcpy(slot, &d, sizeof(uint16)); +} + +static inline char * +slot_key_ptr(AgeHashTable *t, char *slot) +{ + (void) t; + return slot + AGEHASH_SLOT_KEY_OFFSET; +} + +static inline char * +slot_payload_ptr(AgeHashTable *t, char *slot) +{ + return slot + t->payload_offset; +} + +/* ------------------------------------------------------------------------- */ +/* Construction. */ + +static uint32 +next_pow2(uint32 v) +{ + uint32 p = 1; + while (p < v) + p <<= 1; + return p; +} + +AgeHashTable * +agehash_create_inline(MemoryContext mcxt, + Size key_size, + Size payload_size, + uint32 capacity_hint, + agehash_hash_fn hash_fn, + agehash_keyeq_fn keyeq_fn) +{ + AgeHashTable *t; + MemoryContext oldctx; + uint32 min_cap; + uint32 cap; + + Assert(mcxt != NULL); + Assert(key_size > 0 && key_size <= 64); + Assert(payload_size > 0 && payload_size <= 4096); + Assert(hash_fn != NULL); + Assert(keyeq_fn != NULL); + + /* + * Runtime enforcement of the on-stack carrier limits used by the inline + * Robin Hood path (carry_key[64], carry_payload[4096]). Asserts above + * give early diagnostics in debug builds; this elog covers production + * builds where Asserts compile out and the same caller would otherwise + * trigger a stack-buffer overflow during insert. + */ + if (key_size == 0 || key_size > 64) + { + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("agehash inline key size %zu out of range (must be 1..64)", + (size_t) key_size))); + } + if (payload_size == 0 || payload_size > 4096) + { + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("agehash inline payload size %zu out of range (must be 1..4096)", + (size_t) payload_size))); + } + + oldctx = MemoryContextSwitchTo(mcxt); + + t = palloc0(sizeof(AgeHashTable)); + t->mcxt = mcxt; + t->mode = AGEHASH_INLINE; + t->frozen = false; + t->hash_fn = hash_fn; + t->keyeq_fn = keyeq_fn; + t->key_size = (uint32) key_size; + t->payload_size = (uint32) payload_size; + /* + * MAXALIGN payload_offset so that the typed payload pointer returned + * by slot_payload_ptr() is suitably aligned for any C type the caller + * might cast to. Without this, a key_size that is not a multiple of + * MAXIMUM_ALIGNOF (e.g. a 12-byte key) would leave the payload at a + * misaligned address — undefined behavior under strict alignment rules + * even though it works in practice on x86_64. + */ + t->payload_offset = MAXALIGN(AGEHASH_SLOT_KEY_OFFSET + (uint32) key_size); + t->slot_size = MAXALIGN(t->payload_offset + (uint32) payload_size); + /* + * agehash_key_from_payload() recovers the key as (payload - key_size), + * which is only valid when the payload abuts the key with no MAXALIGN + * padding between them. That holds iff key_size is a multiple of + * MAXIMUM_ALIGNOF. Enforce the invariant here so a future non-aligned + * key trips in DEBUG builds rather than silently handing the macro a + * wrong pointer. + */ + if (key_size % MAXIMUM_ALIGNOF != 0) + { + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("agehash inline key size %zu must be a multiple of %d for key recovery", + (size_t) key_size, MAXIMUM_ALIGNOF))); + } + Assert(t->payload_offset == AGEHASH_SLOT_KEY_OFFSET + (uint32) key_size); + + /* + * Capacity floor of 64 keeps tiny tables out of degenerate-load territory + * and avoids a flurry of grows on the first few inserts. + */ + if (capacity_hint == 0) + min_cap = 64; + else + { + /* size capacity_hint at MAX_LOAD so we don't immediately grow */ + min_cap = (uint32) ((double) capacity_hint / AGEHASH_MAX_LOAD) + 1; + if (min_cap < 64) + min_cap = 64; + } + cap = next_pow2(min_cap); + Assert((cap & (cap - 1)) == 0); + + t->capacity = cap; + t->capacity_mask = cap - 1; + t->size = 0; + t->max_size = (uint32) ((double) cap * AGEHASH_MAX_LOAD); + /* + * The slot array can comfortably exceed 1 GiB on production graphs + * (the SF3 ldbc_snb edge_table is multiple GiB at the 0.85 load + * factor). Use the HUGE allocator to bypass the standard MaxAllocSize + * check. + */ + t->slots = (char *) MemoryContextAllocHuge(mcxt, + (Size) cap * t->slot_size); + + /* Mark every slot empty. */ + { + uint32 i; + for (i = 0; i < cap; i++) + slot_set_probe_dist(slot_at(t, i), AGEHASH_EMPTY); + } + + MemoryContextSwitchTo(oldctx); + return t; +} + +/* ------------------------------------------------------------------------- */ +/* Insert. Robin Hood with rich-poor swap. */ + +static void agehash_grow(AgeHashTable *t); + +static void * +agehash_insert_internal(AgeHashTable *t, const void *key, uint32 hashvalue, + bool *found) +{ + uint32 i; + uint16 d; + /* + * Carrier for the entry currently being placed. Starts as the caller's + * key with a fresh, zero-filled payload; gets overwritten on each + * Robin Hood swap. + */ + char carry_key[64]; + char carry_payload[4096]; + void *result_payload = NULL; + bool placed_caller = false; + + /* + * agehash_freeze() is documented to make subsequent inserts/grows fail + * loudly. The Assert below catches violations in debug builds; the + * runtime check guarantees the contract in production builds where + * Asserts compile out, preventing silent slot reordering that would + * invalidate any payload pointers callers consider stable post-freeze. + */ + if (t->frozen) + { + elog(ERROR, "agehash: insert into frozen table"); + } + Assert(!t->frozen); + Assert(t->key_size <= sizeof(carry_key)); + Assert(t->payload_size <= sizeof(carry_payload)); + + /* Grow before insert if at threshold. */ + if (t->size >= t->max_size) + agehash_grow(t); + + /* Initialize carry buffers with the caller's key and an empty payload. */ + memcpy(carry_key, key, t->key_size); + memset(carry_payload, 0, t->payload_size); + + i = hashvalue & t->capacity_mask; + d = 0; + + for (;;) + { + char *slot = slot_at(t, i); + uint16 sd = slot_probe_dist(slot); + + if (sd == AGEHASH_EMPTY) + { + /* Place the carrier here and we're done. */ + slot_set_probe_dist(slot, d); + memcpy(slot_key_ptr(t, slot), carry_key, t->key_size); + memcpy(slot_payload_ptr(t, slot), carry_payload, t->payload_size); + t->size++; + if (!placed_caller) + { + /* The caller's slot landed here. */ + if (found != NULL) + *found = false; + return slot_payload_ptr(t, slot); + } + /* + * The caller was placed earlier via a swap; result_payload + * already points at their final slot. + */ + Assert(result_payload != NULL); + return result_payload; + } + + if (sd == d && + !placed_caller && + t->keyeq_fn(slot_key_ptr(t, slot), carry_key, t->key_size)) + { + /* + * Existing entry with the caller's key. Note: this match check + * is only relevant before we've performed a swap; once we've + * placed the caller into a slot, the key in `carry` is some + * displaced entry that, by RH invariant on insert from a fresh + * key, cannot already exist in the table. + */ + if (found != NULL) + *found = true; + return slot_payload_ptr(t, slot); + } + + if (sd < d) + { + /* + * Rich-poor swap: this slot's owner is closer to its ideal + * bucket than we are. Take its place and continue with the + * displaced entry. If we have not yet placed the caller, this + * is where they end up; remember the pointer so we can return + * it once the displaced chain finishes. + */ + char tmp_key[64]; + char tmp_payload[4096]; + uint16 tmp_d = sd; + + memcpy(tmp_key, slot_key_ptr(t, slot), t->key_size); + memcpy(tmp_payload, slot_payload_ptr(t, slot), t->payload_size); + + slot_set_probe_dist(slot, d); + memcpy(slot_key_ptr(t, slot), carry_key, t->key_size); + memcpy(slot_payload_ptr(t, slot), carry_payload, t->payload_size); + + if (!placed_caller) + { + placed_caller = true; + result_payload = slot_payload_ptr(t, slot); + /* Notify caller: this insert is a fresh entry. */ + if (found != NULL) + { + *found = false; + found = NULL; /* don't write again */ + } + } + + /* Continue with the displaced entry as the new carrier. */ + memcpy(carry_key, tmp_key, t->key_size); + memcpy(carry_payload, tmp_payload, t->payload_size); + d = tmp_d; + } + + i = (i + 1) & t->capacity_mask; + d++; + + /* + * Probe distance overflow guard. With AGEHASH_MAX_LOAD = 0.85 and a + * non-degenerate hash function, max probe is expected to remain far + * below this limit in practice. The 0xFE00 ceiling reserves + * headroom while leaving probe_dist well clear of the AGEHASH_EMPTY + * sentinel. + */ + Assert(d < 0xFE00); + if (unlikely(d >= 0xFE00)) + elog(ERROR, "agehash: probe distance overflow (likely a bad hash function)"); + } +} + +void * +agehash_insert(AgeHashTable *t, const void *key, bool *found) +{ + uint32 h = t->hash_fn(key, t->key_size); + return agehash_insert_internal(t, key, h, found); +} + +void * +agehash_insert_with_hash(AgeHashTable *t, const void *key, + uint32 hashvalue, bool *found) +{ + return agehash_insert_internal(t, key, hashvalue, found); +} + +/* ------------------------------------------------------------------------- */ +/* Grow: double the capacity and rehash. */ + +static void +agehash_grow(AgeHashTable *t) +{ + char *old_slots; + uint32 old_cap; + uint32 old_slot_size; + uint32 new_cap; + MemoryContext oldctx; + uint32 i; + + old_slots = t->slots; + old_cap = t->capacity; + old_slot_size = t->slot_size; + + if (t->frozen) + { + elog(ERROR, "agehash: grow on frozen table"); + } + Assert(!t->frozen); + if (old_cap > (UINT32_MAX >> 1)) + { + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("agehash capacity overflow: cannot grow beyond %u slots", + old_cap))); + } + new_cap = old_cap << 1; + Assert(new_cap > old_cap); /* overflow guard */ + + oldctx = MemoryContextSwitchTo(t->mcxt); + + t->capacity = new_cap; + t->capacity_mask = new_cap - 1; + t->max_size = (uint32) ((double) new_cap * AGEHASH_MAX_LOAD); + /* HUGE allocator: see agehash_create_inline for the rationale. */ + t->slots = (char *) MemoryContextAllocHuge(t->mcxt, + (Size) new_cap * t->slot_size); + for (i = 0; i < new_cap; i++) + slot_set_probe_dist(slot_at(t, i), AGEHASH_EMPTY); + + /* Reset size; we re-insert below (which will increment it). */ + t->size = 0; + for (i = 0; i < old_cap; i++) + { + char *src = old_slots + (Size) i * old_slot_size; + if (slot_probe_dist(src) != AGEHASH_EMPTY) + { + void *src_key = src + AGEHASH_SLOT_KEY_OFFSET; + void *src_payload = src + t->payload_offset; + uint32 h = t->hash_fn(src_key, t->key_size); + void *dst_payload = agehash_insert_internal(t, src_key, h, NULL); + memcpy(dst_payload, src_payload, t->payload_size); + } + } + + pfree(old_slots); + MemoryContextSwitchTo(oldctx); +} + +/* ------------------------------------------------------------------------- */ +/* Lookup. */ + +void * +agehash_lookup_with_hash(AgeHashTable *t, const void *key, uint32 hashvalue) +{ + uint32 i = hashvalue & t->capacity_mask; + uint16 d = 0; + + for (;;) + { + char *slot = slot_at(t, i); + uint16 sd = slot_probe_dist(slot); + + if (sd == AGEHASH_EMPTY) + return NULL; + /* + * Robin Hood invariant: probe_dist values along a probe sequence + * are non-increasing as we move from an entry's home slot. If the + * slot we land on has a smaller probe_dist than ours, the key + * we're looking for can't be anywhere later in the sequence. + */ + if (sd < d) + return NULL; + if (t->keyeq_fn(slot_key_ptr(t, slot), key, t->key_size)) + return slot_payload_ptr(t, slot); + + i = (i + 1) & t->capacity_mask; + d++; + Assert(d < 0xFE00); + } +} + +void * +agehash_lookup(AgeHashTable *t, const void *key) +{ + uint32 h = t->hash_fn(key, t->key_size); + return agehash_lookup_with_hash(t, key, h); +} + +/* ------------------------------------------------------------------------- */ +/* Misc accessors. */ + +void +agehash_freeze(AgeHashTable *t) +{ + t->frozen = true; +} + +bool +agehash_is_frozen(const AgeHashTable *t) +{ + return t->frozen; +} + +uint32 +agehash_size(const AgeHashTable *t) +{ + return t->size; +} + +uint32 +agehash_capacity(const AgeHashTable *t) +{ + return t->capacity; +} + +void +agehash_iter_init(AgeHashTable *t, AgeHashIter *it) +{ + it->t = t; + it->idx = 0; + it->key = NULL; + it->payload = NULL; +} + +bool +agehash_iter_next(AgeHashIter *it) +{ + AgeHashTable *t = it->t; + while (it->idx < t->capacity) + { + char *slot = slot_at(t, it->idx); + uint32 idx = it->idx++; + (void) idx; + if (slot_probe_dist(slot) != AGEHASH_EMPTY) + { + it->key = slot_key_ptr(t, slot); + it->payload = slot_payload_ptr(t, slot); + return true; + } + } + it->key = NULL; + it->payload = NULL; + return false; +} + +/* ------------------------------------------------------------------------- */ +/* Self-test. Exercises insert / lookup / grow / iterate at small + medium + * sizes and verifies invariants. Returns a string in CurrentMemoryContext. */ + +/* MurmurHash3 fmix64, identical to graphid_hash. */ +static uint32 +selftest_hash(const void *key, Size keysize) +{ + uint64 k; + Assert(keysize == sizeof(uint64)); + memcpy(&k, key, sizeof(uint64)); + k ^= k >> 33; + k *= UINT64CONST(0xff51afd7ed558ccd); + k ^= k >> 33; + k *= UINT64CONST(0xc4ceb9fe1a85ec53); + k ^= k >> 33; + return (uint32) k; +} + +static bool +selftest_keyeq(const void *a, const void *b, Size keysize) +{ + return memcmp(a, b, keysize) == 0; +} + +typedef struct selftest_payload +{ + uint64 mirror_key; + uint64 marker; +} selftest_payload; + +static const char * +selftest_run_one(MemoryContext parent, uint32 n, uint32 hint) +{ + MemoryContext mcxt; + AgeHashTable *t; + selftest_payload *p; + bool found; + uint32 i; + uint32 seen; + AgeHashIter it; + + mcxt = AllocSetContextCreate(parent, "agehash selftest", ALLOCSET_DEFAULT_SIZES); + t = agehash_create_inline(mcxt, sizeof(uint64), sizeof(selftest_payload), + hint, selftest_hash, selftest_keyeq); + + /* Insert n keys. */ + for (i = 0; i < n; i++) + { + uint64 k = ((uint64) 0xa5a5 << 48) | (i + 1); + p = (selftest_payload *) agehash_insert(t, &k, &found); + if (found) + { + MemoryContextDelete(mcxt); + return psprintf("FAIL: duplicate insert at i=%u", i); + } + p->mirror_key = k; + p->marker = (uint64) 0xdeadbeef00000000ULL | i; + } + if (agehash_size(t) != n) + { + MemoryContextDelete(mcxt); + return psprintf("FAIL: size %u != %u after inserts", + agehash_size(t), n); + } + + /* Lookup all n keys. */ + for (i = 0; i < n; i++) + { + uint64 k = ((uint64) 0xa5a5 << 48) | (i + 1); + p = (selftest_payload *) agehash_lookup(t, &k); + if (p == NULL) + { + MemoryContextDelete(mcxt); + return psprintf("FAIL: lookup miss at i=%u", i); + } + if (p->mirror_key != k || + p->marker != ((uint64) 0xdeadbeef00000000ULL | i)) + { + MemoryContextDelete(mcxt); + return psprintf("FAIL: payload corruption at i=%u", i); + } + } + + /* Lookup n keys that should not exist. */ + for (i = 0; i < n; i++) + { + uint64 k = ((uint64) 0xb6b6 << 48) | (i + 1); + p = (selftest_payload *) agehash_lookup(t, &k); + if (p != NULL) + { + MemoryContextDelete(mcxt); + return psprintf("FAIL: phantom lookup hit at i=%u", i); + } + } + + /* Re-insert (HASH_ENTER semantics) — should report found = true. */ + for (i = 0; i < n; i++) + { + uint64 k = ((uint64) 0xa5a5 << 48) | (i + 1); + p = (selftest_payload *) agehash_insert(t, &k, &found); + if (!found) + { + MemoryContextDelete(mcxt); + return psprintf("FAIL: re-insert reported !found at i=%u", i); + } + if (p->mirror_key != k) + { + MemoryContextDelete(mcxt); + return psprintf("FAIL: re-insert payload mismatch at i=%u", i); + } + } + if (agehash_size(t) != n) + { + MemoryContextDelete(mcxt); + return psprintf("FAIL: size %u != %u after re-inserts", + agehash_size(t), n); + } + + /* Iterate and count. */ + seen = 0; + agehash_iter_init(t, &it); + while (agehash_iter_next(&it)) + { + selftest_payload *pp = it.payload; + uint64 k; + memcpy(&k, it.key, sizeof(uint64)); + if (pp->mirror_key != k) + { + MemoryContextDelete(mcxt); + return psprintf("FAIL: iter payload mismatch at seen=%u", seen); + } + seen++; + } + if (seen != n) + { + MemoryContextDelete(mcxt); + return psprintf("FAIL: iter saw %u of %u", seen, n); + } + + /* Freeze and confirm lookups still work. */ + agehash_freeze(t); + if (!agehash_is_frozen(t)) + { + MemoryContextDelete(mcxt); + return "FAIL: agehash_is_frozen returned false after freeze"; + } + { + uint64 k = ((uint64) 0xa5a5 << 48) | 1; + p = (selftest_payload *) agehash_lookup(t, &k); + if (p == NULL) + { + MemoryContextDelete(mcxt); + return "FAIL: lookup failed after freeze"; + } + } + + MemoryContextDelete(mcxt); + return NULL; /* OK */ +} + +const char * +agehash_self_test(void) +{ + static const struct { uint32 n; uint32 hint; } cases[] = { + { 1, 0 }, + { 7, 0 }, + { 8, 0 }, + { 9, 0 }, + { 63, 0 }, + { 64, 0 }, + { 65, 0 }, + { 1023, 0 }, /* forces grow from 64 floor */ + { 1024, 0 }, + { 1025, 0 }, + { 10000, 0 }, /* forces multiple grows */ + { 10000, 8192 }, /* with capacity hint, no grow expected */ + { 50000, 0 }, /* larger; multiple grows */ + { 1000000, 0 }, /* exercises grow at multi-MB allocations */ + /* + * NOTE: this set is bounded so 'make installcheck' completes + * quickly. The library has been manually verified up to 256M + * entries (multi-GiB slot arrays via MemoryContextAllocHuge). + */ + }; + const size_t ncases = sizeof(cases) / sizeof(cases[0]); + size_t i; + + for (i = 0; i < ncases; i++) + { + const char *r = selftest_run_one(CurrentMemoryContext, + cases[i].n, cases[i].hint); + if (r != NULL) + return psprintf("%s [n=%u hint=%u]", r, cases[i].n, cases[i].hint); + } + return "OK"; +} + +/* ------------------------------------------------------------------------- */ +/* SQL-callable wrapper: SELECT ag_catalog._agehash_self_test(); */ + +PG_FUNCTION_INFO_V1(_agehash_self_test); + +Datum +_agehash_self_test(PG_FUNCTION_ARGS) +{ + const char *r = agehash_self_test(); + PG_RETURN_TEXT_P(cstring_to_text(r)); +} diff --git a/src/include/utils/age_global_graph.h b/src/include/utils/age_global_graph.h index 92044fc7e..d68530a91 100644 --- a/src/include/utils/age_global_graph.h +++ b/src/include/utils/age_global_graph.h @@ -22,6 +22,25 @@ #include "utils/age_graphid_ds.h" +/* + * Flat dynamic-array adjacency container for vertex edges. Replaces a + * linked-list (ListGraphId) of GraphIdNodes for vertex_entry::edges_*. + * + * Storage: a single palloc'd graphid array, doubled on growth. The struct + * itself is embedded by value in vertex_entry so that the (array, size, + * capacity) triple lives in the same cache line as the surrounding entry + * fields, saving one indirection on the DFS hot path. + * + * Empty arrays carry array == NULL, size == 0, capacity == 0 and incur no + * allocation until the first append. + */ +typedef struct VertexEdgeArray +{ + graphid *array; /* contiguous edge graphid array; NULL when empty */ + int32 size; /* number of edges currently stored */ + int32 capacity; /* allocated capacity (in graphid slots) */ +} VertexEdgeArray; + /* * We declare the graph nodes and edges here, and in this way, so that it may be * used elsewhere. However, we keep the contents private by defining it in @@ -46,13 +65,27 @@ ListGraphId *get_graph_vertices(GRAPH_global_context *ggctx); vertex_entry *get_vertex_entry(GRAPH_global_context *ggctx, graphid vertex_id); edge_entry *get_edge_entry(GRAPH_global_context *ggctx, graphid edge_id); + +/* + * Variant of get_edge_entry that accepts a precomputed hash value, allowing + * the same hash to be reused across multiple lookups of the same graphid + * (e.g. edge_state_hashtable + edge_hashtable in the VLE DFS hot loop). + */ +edge_entry *get_edge_entry_with_hash(GRAPH_global_context *ggctx, + graphid edge_id, uint32 hashvalue); /* vertex entry accessor functions*/ graphid get_vertex_entry_id(vertex_entry *ve); -ListGraphId *get_vertex_entry_edges_in(vertex_entry *ve); -ListGraphId *get_vertex_entry_edges_out(vertex_entry *ve); -ListGraphId *get_vertex_entry_edges_self(vertex_entry *ve); Oid get_vertex_entry_label_table_oid(vertex_entry *ve); Datum get_vertex_entry_properties(vertex_entry *ve); + +/* + * Flat-array adjacency accessors. Returned pointer is into the entry's + * embedded VertexEdgeArray and is therefore non-NULL for a valid entry, + * but the underlying VertexEdgeArray::array may be NULL when size == 0. + */ +VertexEdgeArray *get_vertex_entry_edges_out_array(vertex_entry *ve); +VertexEdgeArray *get_vertex_entry_edges_in_array(vertex_entry *ve); +VertexEdgeArray *get_vertex_entry_edges_self_array(vertex_entry *ve); /* edge entry accessor functions */ graphid get_edge_entry_id(edge_entry *ee); Oid get_edge_entry_label_table_oid(edge_entry *ee); @@ -65,6 +98,16 @@ uint64 get_graph_version(Oid graph_oid); void increment_graph_version(Oid graph_oid); Oid get_graph_oid_for_table(Oid table_oid); +/* + * Fast hash function for graphid (int64) keys used in dynahash tables. + * Replaces tag_hash with the MurmurHash3 fmix64 finalizer for better + * distribution and lower instruction count on modern x86_64. + */ +uint32 graphid_hash(const void *key, Size keysize); + +/* Equality predicate for graphid (int64) keys; agehash_keyeq_fn signature. */ +bool graphid_keyeq(const void *a, const void *b, Size keysize); + /* Shared memory initialization for PG < 17 (shmem_request_hook path) */ #if PG_VERSION_NUM < 170000 void age_graph_version_shmem_request(void); diff --git a/src/include/utils/agehash.h b/src/include/utils/agehash.h new file mode 100644 index 000000000..9f0ba2793 --- /dev/null +++ b/src/include/utils/agehash.h @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * agehash.h - Robin Hood open-addressing hashtable for AGE's hot-path caches. + * + * This is an internal utility used by the global graph cache (vertex and edge + * tables) to replace dynahash on the lookup-dominated read-only-after-build + * path. Each lookup traverses a single contiguous slot array with no chain + * pointer-chasing, which on AGE workloads roughly halves lookup latency + * relative to dynahash (see Stage 5 microbench). + * + * Mode: only AGEHASH_INLINE is supported by this initial revision. INLINE + * stores the payload directly in the slot, suitable for tables that are + * never mutated after the build phase. A future revision will add an + * AGEHASH_INDIRECT mode for tables that support insert and pointer-stable + * payloads during queries. + * + * Memory: every allocation lives in a caller-supplied MemoryContext. Free is + * a single MemoryContextDelete by the caller; agehash itself never frees + * piecewise. This makes leak-on-elog impossible: PG unwinds the context. + * + * Capacity: always a power of two; grown by doubling when size exceeds + * AGEHASH_MAX_LOAD * capacity. After agehash_freeze() is called, all + * insert paths are forbidden (asserted in DEBUG builds), which guarantees + * that lookups can never observe a partially-rehashed structure. + */ + +#ifndef AG_AGEHASH_H +#define AG_AGEHASH_H + +#include "postgres.h" +#include "utils/memutils.h" +#include "utils/palloc.h" + +/* Sentinel probe distance marking an empty slot. */ +#define AGEHASH_EMPTY 0xFFFFu + +/* + * Load factor above which we grow. + * + * 0.85 balances three goals on AGE's hot tables: + * - Memory: the edge_table on SF3 is ~52M entries; at 0.85 the slot array + * is ~67M slots which is roughly the same total bytes as the dynahash + * bucket array + per-entry HASHELEMENT chain headers it replaces. + * - Probe distance: Robin Hood at 0.85 still keeps average probes near 1 + * and max probes well below the 0xFE00 overflow guard. + * - Grow cadence: a higher threshold means fewer doublings during the + * edge cache build (each doubling rehashes the entire table). + * + * If you change this, re-run the rh_microbench harness on the VM and the + * SF3 paired benchmark; both are sensitive to the load factor. + */ +#define AGEHASH_MAX_LOAD 0.85 + +/* + * Caller-supplied hash callback. keysize is constant for a given table; we + * still pass it so callers can reuse one function across multiple tables + * with different key types if desired. + */ +typedef uint32 (*agehash_hash_fn)(const void *key, Size keysize); + +/* Caller-supplied key-equality callback. Returns true iff a == b. */ +typedef bool (*agehash_keyeq_fn)(const void *a, const void *b, Size keysize); + +/* + * Layout mode. Only INLINE is implemented in this revision; INDIRECT is + * declared so the public enum values stay stable when it lands. + */ +typedef enum AgeHashMode +{ + AGEHASH_INLINE = 0, + AGEHASH_INDIRECT = 1 +} AgeHashMode; + +/* Opaque table handle. */ +typedef struct AgeHashTable AgeHashTable; + +/* + * Slot layout (INLINE mode), packed: + * + * offset 0 : uint16 probe_dist (AGEHASH_EMPTY == empty) + * offset 2 : uint16 _reserved (future flags / tombstones) + * offset 4 : uint32 _pad (force key to 8-byte alignment) + * offset 8 : key (key_size bytes) + * offset 8+key_size : payload (payload_size bytes) + * + * The header is 8 bytes; total slot bytes = 8 + key_size + payload_size, + * rounded up to MAXIMUM_ALIGNOF. + */ + +#define AGEHASH_SLOT_HDR_BYTES 8 +#define AGEHASH_SLOT_KEY_OFFSET AGEHASH_SLOT_HDR_BYTES + +/* + * Recover a key pointer from a payload pointer. INLINE-mode tables store + * the key immediately before the payload, so this is pure pointer + * arithmetic and does not need the table handle. The caller must know the + * key size at this site; this is the case for every AGE caller (each table + * has a single fixed key type). + * + * Invariant: this is correct only when the payload directly abuts the key, + * i.e. when payload_offset == AGEHASH_SLOT_KEY_OFFSET + key_size with no + * MAXALIGN padding. That holds iff key_size is a multiple of + * MAXIMUM_ALIGNOF (all current AGE callers use an 8-byte graphid key). + * agehash_create_inline() asserts this invariant in DEBUG builds. + */ +#define agehash_key_from_payload(payload, key_size) \ + ((const void *) ((const char *) (payload) - (Size) (key_size))) + +/* + * Construction. capacity_hint is a number of entries; the actual capacity + * will be the next power of two >= capacity_hint / AGEHASH_MAX_LOAD, with a + * floor of 64 slots. Pass 0 to let the table start at the floor. + */ +extern AgeHashTable *agehash_create_inline(MemoryContext mcxt, + Size key_size, + Size payload_size, + uint32 capacity_hint, + agehash_hash_fn hash_fn, + agehash_keyeq_fn keyeq_fn); + +/* + * Reserve / find. If the key is not present, allocates a fresh slot + * (rebalancing via Robin Hood swaps), zero-fills the payload, sets + * *found = false, and returns a pointer to the payload region. The caller + * fills it in. If the key is present, sets *found = true and returns the + * existing payload pointer. + * + * The returned payload pointer is *not* stable across subsequent + * agehash_insert calls in INLINE mode (a later insert may swap this slot). + * Callers requiring stable pointers must use INDIRECT mode (future). + * + * Asserts that the table has not been frozen (DEBUG builds). + */ +extern void *agehash_insert(AgeHashTable *t, const void *key, bool *found); + +/* Variant that accepts a precomputed hash value, skipping the hash callback. */ +extern void *agehash_insert_with_hash(AgeHashTable *t, const void *key, + uint32 hashvalue, bool *found); + +/* + * Lookup. Returns a pointer to the payload region, or NULL if absent. + * The pointer is stable as long as no further insert touches the table. + */ +extern void *agehash_lookup(AgeHashTable *t, const void *key); + +/* Variant accepting a precomputed hash value. */ +extern void *agehash_lookup_with_hash(AgeHashTable *t, const void *key, + uint32 hashvalue); + +/* + * Freeze the table: subsequent insert/grow attempts are an Assert failure + * in DEBUG and an elog(ERROR) in production. This is the contract that + * lets read-only-after-build callers hand out long-lived payload pointers. + */ +extern void agehash_freeze(AgeHashTable *t); + +/* True after agehash_freeze(); useful for caller-side asserts. */ +extern bool agehash_is_frozen(const AgeHashTable *t); + +/* Live entry count. */ +extern uint32 agehash_size(const AgeHashTable *t); + +/* Allocated slot count (capacity). */ +extern uint32 agehash_capacity(const AgeHashTable *t); + +/* + * Iteration. Usage: + * + * AgeHashIter it; + * for (agehash_iter_init(t, &it); agehash_iter_next(&it); ) + * { + * graphid k = *(graphid *) it.key; + * my_payload *p = it.payload; + * ... + * } + * + * Iteration order is unspecified. Modifying the table during iteration is + * undefined behaviour. + */ +typedef struct AgeHashIter +{ + AgeHashTable *t; + uint32 idx; + void *key; + void *payload; +} AgeHashIter; + +extern void agehash_iter_init(AgeHashTable *t, AgeHashIter *it); +extern bool agehash_iter_next(AgeHashIter *it); + +/* + * Internal self-test. Returns a NUL-terminated diagnostic string allocated + * in CurrentMemoryContext: "OK" on success, "FAIL: " on failure. + * Used by the agehash regression test. + */ +extern const char *agehash_self_test(void); + +#endif /* AG_AGEHASH_H */ From 639c8d5ac1cc930095caccb58b90dd65d9035e12 Mon Sep 17 00:00:00 2001 From: Prashant Chinnam <5108573+crprashant@users.noreply.github.com> Date: Mon, 8 Jun 2026 16:55:50 -0700 Subject: [PATCH 073/100] Restore contsel/contjoinsel for containment & key-existence operators (#2356) (#2417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The containment (`@>`, `<@`, `@>>`, `<<@`) and key-existence (`?`, `?|`, `?&`) operators on `agtype` were bound to `matchingsel`/`matchingjoinsel` on the PG14+ source tree. `matchingsel` is built for pattern operators (LIKE/regex) and during planning invokes the operator's underlying function (`agtype_contains`) once per `pg_statistic` MCV. With realistic statistics targets that produces a planner-time regression that dominates simple OLTP-style point queries. Rebind those operators to the lighter `contsel`/`contjoinsel` estimators, which return fixed selectivity constants without invoking the operator function during planning. This is a deliberate planning-speed vs. estimate-accuracy trade-off. Note it DIVERGES from PostgreSQL core, which keeps jsonb's `@>`, `<@`, `?`, `?|`, `?&` on `matchingsel`/`matchingjoinsel` (verified on REL_16/17/18_STABLE in `pg_operator.dat`); it is an AGE-specific choice favoring workloads where these operators appear in selective point lookups. A future improvement could add a custom `agtype` selectivity function that is both cheap and statistics-aware. Changes: * `sql/agtype_operators.sql`, `sql/agtype_exists.sql`: 10 operators flipped from `matchingsel`/`matchingjoinsel` to `contsel`/`contjoinsel`. * `age--1.7.0--y.y.y.sql`: appended `ALTER OPERATOR ... SET (RESTRICT, JOIN)` for all 10 operators so existing installs flip on `ALTER EXTENSION age UPDATE`. * `regress/sql/containment_selectivity.sql` (+ `expected/.out`): pin the bindings via `pg_operator`, plus a scoped "no leaked matchingsel" guard and functional smoke for all 10 operators. Adds an upgrade-path assertion that simulates a stale (pre-fix) install, replays the shipped `ALTER OPERATOR` block, and confirms every overload flips to `contsel`/`contjoinsel` (run in a rolled-back transaction). * `regress/expected/cypher_match.out`, `regress/expected/cypher_vle.out`: refresh expected to reflect new (and better) plan shapes that the lower-selectivity helper produces — `test_enable_containment` now picks Nested Loop + Index Only Scans over a Seq Scan/Hash Join, and two `MATCH p=...` and `show_list_use_vle` queries flip row order (queries had no `ORDER BY`; result set is unchanged, only ordering). * `Makefile`: register `containment_selectivity` in `REGRESS`. Validation: * Build: clean, `-Werror`. * Regression: 36/37 tests pass under `EXTRA_TESTS="pgvector fuzzystrmatch pg_trgm"`. Only `age_upgrade` fails — pre-existing on master at 774e781b (verified by `git stash && installcheck`). * Reporter's exact methodology (LDBC-SNB-style snb_graph + pgbench on `bench_message_content`) reproduces the regression and the fix: | Metric | matchingsel | contsel | Delta | |----------------------------|-------------|---------|-------| | EXPLAIN planning time (ms) | 1.42 | 0.97 | -32% | | EXPLAIN execution time (ms)| 0.34 | 0.31 | ~equal| | pgbench TPS (8c x 30s) | 5247 | 7378 | +40.6%| Run with `default_statistics_target = 1000` to populate MCV lists, matching the reporter's analyzed-graph conditions. * Upgrade path: validated end-to-end during the benchmark — operator bindings were flipped from `matchingsel` -> `contsel` via the same `ALTER OPERATOR` statements the upgrade SQL ships, while operators remained functional throughout. Driver workflows (python/go/node/jdbc) intentionally not run: this PR only adjusts pg_operator selectivity metadata. There is no C, type, or protocol change that drivers could observe. Closes #2356. --- Makefile | 3 +- age--1.7.0--y.y.y.sql | 38 +++ regress/expected/containment_selectivity.out | 245 +++++++++++++++++++ regress/expected/cypher_match.out | 30 ++- regress/expected/cypher_vle.out | 8 +- regress/sql/containment_selectivity.sql | 156 ++++++++++++ regress/sql/cypher_match.sql | 6 +- regress/sql/cypher_vle.sql | 2 +- sql/agtype_exists.sql | 24 +- sql/agtype_operators.sql | 16 +- 10 files changed, 483 insertions(+), 45 deletions(-) create mode 100644 regress/expected/containment_selectivity.out create mode 100755 regress/sql/containment_selectivity.sql diff --git a/Makefile b/Makefile index b79dbd3bb..f2a0b9a62 100644 --- a/Makefile +++ b/Makefile @@ -183,7 +183,8 @@ REGRESS = scan \ direct_field_access \ security \ reserved_keyword_alias \ - agtype_jsonb_cast + agtype_jsonb_cast \ + containment_selectivity ifneq ($(EXTRA_TESTS),) REGRESS += $(EXTRA_TESTS) diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index 74b84d604..a4cac0c5c 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -762,3 +762,41 @@ CREATE FUNCTION ag_catalog._agehash_self_test() VOLATILE PARALLEL UNSAFE AS 'MODULE_PATHNAME'; + +-- +-- Issue #2356: rebind containment and key-existence operators to the +-- lightweight contsel / contjoinsel selectivity estimators. +-- +-- @>, <@, @>>, <<@, ?, ?|, ?& on agtype were bound to matchingsel / +-- matchingjoinsel. During planning matchingsel invokes the operator's +-- underlying function (agtype_contains) once per pg_statistic MCV entry and +-- histogram bin; with a realistic default_statistics_target that planning +-- cost dominates simple point queries (the regression reported in #2356). +-- +-- contsel / contjoinsel return fixed selectivity constants without calling the +-- operator function, so planning is constant-time. This is a deliberate +-- planning-speed vs. estimate-accuracy trade-off. Note it DIVERGES from +-- PostgreSQL core, which keeps jsonb's @>, <@, ?, ?|, ?& on matchingsel / +-- matchingjoinsel; it is an AGE-specific choice favoring workloads where these +-- operators appear in selective point lookups. +-- +ALTER OPERATOR ag_catalog.@>(agtype, agtype) + SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.<@(agtype, agtype) + SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.@>>(agtype, agtype) + SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.<<@(agtype, agtype) + SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?(agtype, text) + SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?(agtype, agtype) + SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?|(agtype, text[]) + SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?|(agtype, agtype) + SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?&(agtype, text[]) + SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?&(agtype, agtype) + SET (RESTRICT = contsel, JOIN = contjoinsel); diff --git a/regress/expected/containment_selectivity.out b/regress/expected/containment_selectivity.out new file mode 100644 index 000000000..a0626d554 --- /dev/null +++ b/regress/expected/containment_selectivity.out @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Regression coverage for issue #2356: + * The containment (@>, <@, @>>, <<@) and key-existence (?, ?|, ?&) + * operators on agtype must be bound to the lightweight selectivity + * helpers contsel / contjoinsel during planning. Earlier PG14+ + * branches used matchingsel / matchingjoinsel, which caused planning + * to invoke agtype_contains() against pg_statistic MCVs and produced + * a 30%+ planning-time regression on point queries (severe TPS drop + * reported on the PG18 branch). + * + * This test pins the bindings by querying pg_operator directly. If + * someone re-introduces matchingsel here, the test diff is loud and + * precise. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +-- Selectivity helpers for the four containment operators. +SELECT o.oprname, + pg_catalog.format_type(o.oprleft, NULL) AS lhs, + pg_catalog.format_type(o.oprright, NULL) AS rhs, + o.oprrest::text AS restrict_fn, + o.oprjoin::text AS join_fn +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname = 'ag_catalog' + AND o.oprname IN ('@>', '<@', '@>>', '<<@') +ORDER BY o.oprname, lhs, rhs; + oprname | lhs | rhs | restrict_fn | join_fn +---------+--------+--------+-------------+------------- + <<@ | agtype | agtype | contsel | contjoinsel + <@ | agtype | agtype | contsel | contjoinsel + @> | agtype | agtype | contsel | contjoinsel + @>> | agtype | agtype | contsel | contjoinsel +(4 rows) + +-- Selectivity helpers for all key-existence operator overloads +-- (right-hand side may be text, text[], or agtype). +SELECT o.oprname, + pg_catalog.format_type(o.oprleft, NULL) AS lhs, + pg_catalog.format_type(o.oprright, NULL) AS rhs, + o.oprrest::text AS restrict_fn, + o.oprjoin::text AS join_fn +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname = 'ag_catalog' + AND o.oprname IN ('?', '?|', '?&') +ORDER BY o.oprname, lhs, rhs; + oprname | lhs | rhs | restrict_fn | join_fn +---------+--------+--------+-------------+------------- + ? | agtype | agtype | contsel | contjoinsel + ? | agtype | text | contsel | contjoinsel + ?& | agtype | agtype | contsel | contjoinsel + ?& | agtype | text[] | contsel | contjoinsel + ?| | agtype | agtype | contsel | contjoinsel + ?| | agtype | text[] | contsel | contjoinsel +(6 rows) + +-- Scoped guard for issue #2356: assert that none of the specific containment +-- and key-existence operators on agtype are bound to matchingsel / +-- matchingjoinsel. We deliberately limit the check to these operator names +-- (rather than every operator in ag_catalog) so unrelated operators that +-- legitimately use matchingsel for their own semantics are not affected by +-- this regression test. +SELECT COUNT(*) AS leaked_matchingsel_bindings +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname = 'ag_catalog' + AND o.oprname IN ('@>', '<@', '@>>', '<<@', '?', '?|', '?&') + AND (o.oprrest::text = 'matchingsel' + OR o.oprjoin::text = 'matchingjoinsel'); + leaked_matchingsel_bindings +----------------------------- + 0 +(1 row) + +-- Smoke test: each operator still works functionally. Selectivity binding +-- only affects the planner; this guards against an inadvertent operator +-- removal as part of any future cleanup. +SELECT '{"a":1,"b":2}'::agtype @> '{"a":1}'::agtype AS contains_yes; + contains_yes +-------------- + t +(1 row) + +SELECT '{"a":1}'::agtype <@ '{"a":1,"b":2}'::agtype AS contained_yes; + contained_yes +--------------- + t +(1 row) + +SELECT '{"a":{"b":1}}'::agtype @>> '{"a":{"b":1}}'::agtype AS top_contains_yes; + top_contains_yes +------------------ + t +(1 row) + +SELECT '{"a":{"b":1}}'::agtype <<@ '{"a":{"b":1}}'::agtype AS top_contained_yes; + top_contained_yes +------------------- + t +(1 row) + +SELECT '{"a":1}'::agtype ? 'a'::text AS exists_text_yes; + exists_text_yes +----------------- + t +(1 row) + +SELECT '{"a":1}'::agtype ? '"a"'::agtype AS exists_agtype_yes; + exists_agtype_yes +------------------- + t +(1 row) + +SELECT '{"a":1,"b":2}'::agtype ?| ARRAY['a','c'] AS exists_any_text_yes; + exists_any_text_yes +--------------------- + t +(1 row) + +SELECT '{"a":1,"b":2}'::agtype ?| '["a","c"]'::agtype AS exists_any_agtype_yes; + exists_any_agtype_yes +----------------------- + t +(1 row) + +SELECT '{"a":1,"b":2}'::agtype ?& ARRAY['a','b'] AS exists_all_text_yes; + exists_all_text_yes +--------------------- + t +(1 row) + +SELECT '{"a":1,"b":2}'::agtype ?& '["a","b"]'::agtype AS exists_all_agtype_yes; + exists_all_agtype_yes +----------------------- + t +(1 row) + +-- Upgrade-path assertion for issue #2356. +-- +-- The checks above cover a FRESH install: contsel / contjoinsel come straight +-- from agtype_operators.sql and agtype_exists.sql. Existing installs instead +-- pick up the fix from the ALTER OPERATOR ... SET (RESTRICT, JOIN) block that +-- age--1.7.0--y.y.y.sql ships and "ALTER EXTENSION age UPDATE" replays. Nothing +-- above exercises that block, so a silent regression in it would go unnoticed. +-- +-- We replay the shipped ALTER OPERATOR statements directly rather than running +-- ALTER EXTENSION age UPDATE: the dev upgrade script targets the placeholder +-- version "y.y.y" and is not a stable version-chain target inside the +-- regression harness. The whole section runs in a transaction that is rolled +-- back, so it observes the flip without permanently mutating the operator +-- catalog (PostgreSQL DDL is transactional). +BEGIN; +-- Simulate a stale (pre-fix) install: force all ten overloads back onto +-- matchingsel / matchingjoinsel. +ALTER OPERATOR ag_catalog.@>(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.<@(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.@>>(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.<<@(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?(agtype, text) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?|(agtype, text[]) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?|(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?&(agtype, text[]) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?&(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +-- Stale state: every overload now reports matchingsel / matchingjoinsel. +SELECT o.oprname, + pg_catalog.format_type(o.oprleft, NULL) AS lhs, + pg_catalog.format_type(o.oprright, NULL) AS rhs, + o.oprrest::text AS restrict_fn, + o.oprjoin::text AS join_fn +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname = 'ag_catalog' + AND o.oprname IN ('@>', '<@', '@>>', '<<@', '?', '?|', '?&') +ORDER BY o.oprname, lhs, rhs; + oprname | lhs | rhs | restrict_fn | join_fn +---------+--------+--------+-------------+----------------- + <<@ | agtype | agtype | matchingsel | matchingjoinsel + <@ | agtype | agtype | matchingsel | matchingjoinsel + ? | agtype | agtype | matchingsel | matchingjoinsel + ? | agtype | text | matchingsel | matchingjoinsel + ?& | agtype | agtype | matchingsel | matchingjoinsel + ?& | agtype | text[] | matchingsel | matchingjoinsel + ?| | agtype | agtype | matchingsel | matchingjoinsel + ?| | agtype | text[] | matchingsel | matchingjoinsel + @> | agtype | agtype | matchingsel | matchingjoinsel + @>> | agtype | agtype | matchingsel | matchingjoinsel +(10 rows) + +-- Replay the exact ALTER OPERATOR block shipped in age--1.7.0--y.y.y.sql. +ALTER OPERATOR ag_catalog.@>(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.<@(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.@>>(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.<<@(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?(agtype, text) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?|(agtype, text[]) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?|(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?&(agtype, text[]) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?&(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +-- After the upgrade replay every overload is back on contsel / contjoinsel. +SELECT o.oprname, + pg_catalog.format_type(o.oprleft, NULL) AS lhs, + pg_catalog.format_type(o.oprright, NULL) AS rhs, + o.oprrest::text AS restrict_fn, + o.oprjoin::text AS join_fn +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname = 'ag_catalog' + AND o.oprname IN ('@>', '<@', '@>>', '<<@', '?', '?|', '?&') +ORDER BY o.oprname, lhs, rhs; + oprname | lhs | rhs | restrict_fn | join_fn +---------+--------+--------+-------------+------------- + <<@ | agtype | agtype | contsel | contjoinsel + <@ | agtype | agtype | contsel | contjoinsel + ? | agtype | agtype | contsel | contjoinsel + ? | agtype | text | contsel | contjoinsel + ?& | agtype | agtype | contsel | contjoinsel + ?& | agtype | text[] | contsel | contjoinsel + ?| | agtype | agtype | contsel | contjoinsel + ?| | agtype | text[] | contsel | contjoinsel + @> | agtype | agtype | contsel | contjoinsel + @>> | agtype | agtype | contsel | contjoinsel +(10 rows) + +ROLLBACK; diff --git a/regress/expected/cypher_match.out b/regress/expected/cypher_match.out index 33c728f2a..ab51486b3 100644 --- a/regress/expected/cypher_match.out +++ b/regress/expected/cypher_match.out @@ -2404,21 +2404,21 @@ SELECT * FROM cypher('cypher_match', $$ MATCH (a {name:a.name}) MATCH (a {age:a. {"id": 281474976710659, "label": "", "properties": {"age": 3, "name": "orphan"}}::vertex (3 rows) -SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relationship}]->(b) RETURN p $$) as (a agtype); +SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relationship}]->(b) RETURN p ORDER BY id(u) $$) as (a agtype); a ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 281474976710661, "label": "", "properties": {"age": 4, "name": "T"}}::vertex, {"id": 4785074604081153, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710661, "properties": {"years": 3, "relationship": "friends"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path [{"id": 281474976710659, "label": "", "properties": {"age": 3, "name": "orphan"}}::vertex, {"id": 4785074604081154, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710659, "properties": {"years": 4, "relationship": "enemies"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path (2 rows) -SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relationship, years: u.years}]->(b) RETURN p $$) as (a agtype); +SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relationship, years: u.years}]->(b) RETURN p ORDER BY id(u) $$) as (a agtype); a ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 281474976710661, "label": "", "properties": {"age": 4, "name": "T"}}::vertex, {"id": 4785074604081153, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710661, "properties": {"years": 3, "relationship": "friends"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path [{"id": 281474976710659, "label": "", "properties": {"age": 3, "name": "orphan"}}::vertex, {"id": 4785074604081154, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710659, "properties": {"years": 4, "relationship": "enemies"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path (2 rows) -SELECT * FROM cypher('cypher_match', $$ MATCH p=(a {name:a.name})-[u {relationship: u.relationship}]->(b {age:b.age}) RETURN p $$) as (a agtype); +SELECT * FROM cypher('cypher_match', $$ MATCH p=(a {name:a.name})-[u {relationship: u.relationship}]->(b {age:b.age}) RETURN p ORDER BY id(u) $$) as (a agtype); a ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 281474976710661, "label": "", "properties": {"age": 4, "name": "T"}}::vertex, {"id": 4785074604081153, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710661, "properties": {"years": 3, "relationship": "friends"}}::edge, {"id": 281474976710666, "label": "", "properties": {"age": 6}}::vertex]::path @@ -3398,19 +3398,17 @@ SELECT count(*) FROM cypher('test_enable_containment', $$ MATCH p=(x:Customer)-[ (1 row) SELECT * FROM cypher('test_enable_containment', $$ EXPLAIN (costs off) MATCH (x:Customer)-[:bought ={store: 'Amazon', addr:{city: 'Vancouver', street: 30}}]->(y:Product) RETURN 0 $$) as (a agtype); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------- - Hash Join - Hash Cond: (y.id = _age_default_alias_0.end_id) - -> Seq Scan on "Product" y - -> Hash - -> Hash Join - Hash Cond: (x.id = _age_default_alias_0.start_id) - -> Seq Scan on "Customer" x - -> Hash - -> Seq Scan on bought _age_default_alias_0 - Filter: (properties @>> '{"addr": {"city": "Vancouver", "street": 30}, "store": "Amazon"}'::agtype) -(10 rows) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------- + Nested Loop + -> Nested Loop + -> Seq Scan on bought _age_default_alias_0 + Filter: (properties @>> '{"addr": {"city": "Vancouver", "street": 30}, "store": "Amazon"}'::agtype) + -> Index Only Scan using "Customer_pkey" on "Customer" x + Index Cond: (id = _age_default_alias_0.start_id) + -> Index Only Scan using "Product_pkey" on "Product" y + Index Cond: (id = _age_default_alias_0.end_id) +(8 rows) SELECT * FROM cypher('test_enable_containment', $$ EXPLAIN (costs off) MATCH (x:Customer ={school: { name: 'XYZ College',program: { major: 'Psyc', degree: 'BSc'} },phone: [ 123456789, 987654321, 456987123 ]}) RETURN 0 $$) as (a agtype); QUERY PLAN diff --git a/regress/expected/cypher_vle.out b/regress/expected/cypher_vle.out index 27e3bb822..a09cd4aa9 100644 --- a/regress/expected/cypher_vle.out +++ b/regress/expected/cypher_vle.out @@ -691,7 +691,7 @@ BEGIN RETURN QUERY SELECT * FROM cypher('mygraph', $CYPHER$ MATCH (h:head {name: $list_name})-[e:next*]->(v:node) - RETURN v + RETURN v ORDER BY id(v) $CYPHER$, ag_param) AS (node agtype); END $$; -- create a list @@ -726,8 +726,8 @@ SELECT prepend_node('list01', 'b'); SELECT * FROM show_list_use_vle('list01'); node ----------------------------------------------------------------------------------- - {"id": 1407374883553282, "label": "node", "properties": {"content": "b"}}::vertex {"id": 1407374883553281, "label": "node", "properties": {"content": "a"}}::vertex + {"id": 1407374883553282, "label": "node", "properties": {"content": "b"}}::vertex (2 rows) -- prepend a node 'c' @@ -741,9 +741,9 @@ SELECT prepend_node('list01', 'c'); SELECT * FROM show_list_use_vle('list01'); node ----------------------------------------------------------------------------------- - {"id": 1407374883553283, "label": "node", "properties": {"content": "c"}}::vertex - {"id": 1407374883553282, "label": "node", "properties": {"content": "b"}}::vertex {"id": 1407374883553281, "label": "node", "properties": {"content": "a"}}::vertex + {"id": 1407374883553282, "label": "node", "properties": {"content": "b"}}::vertex + {"id": 1407374883553283, "label": "node", "properties": {"content": "c"}}::vertex (3 rows) DROP FUNCTION show_list_use_vle; diff --git a/regress/sql/containment_selectivity.sql b/regress/sql/containment_selectivity.sql new file mode 100755 index 000000000..c35ad3fc8 --- /dev/null +++ b/regress/sql/containment_selectivity.sql @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Regression coverage for issue #2356: + * The containment (@>, <@, @>>, <<@) and key-existence (?, ?|, ?&) + * operators on agtype must be bound to the lightweight selectivity + * helpers contsel / contjoinsel during planning. Earlier PG14+ + * branches used matchingsel / matchingjoinsel, which caused planning + * to invoke agtype_contains() against pg_statistic MCVs and produced + * a 30%+ planning-time regression on point queries (severe TPS drop + * reported on the PG18 branch). + * + * This test pins the bindings by querying pg_operator directly. If + * someone re-introduces matchingsel here, the test diff is loud and + * precise. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +-- Selectivity helpers for the four containment operators. +SELECT o.oprname, + pg_catalog.format_type(o.oprleft, NULL) AS lhs, + pg_catalog.format_type(o.oprright, NULL) AS rhs, + o.oprrest::text AS restrict_fn, + o.oprjoin::text AS join_fn +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname = 'ag_catalog' + AND o.oprname IN ('@>', '<@', '@>>', '<<@') +ORDER BY o.oprname, lhs, rhs; + +-- Selectivity helpers for all key-existence operator overloads +-- (right-hand side may be text, text[], or agtype). +SELECT o.oprname, + pg_catalog.format_type(o.oprleft, NULL) AS lhs, + pg_catalog.format_type(o.oprright, NULL) AS rhs, + o.oprrest::text AS restrict_fn, + o.oprjoin::text AS join_fn +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname = 'ag_catalog' + AND o.oprname IN ('?', '?|', '?&') +ORDER BY o.oprname, lhs, rhs; + +-- Scoped guard for issue #2356: assert that none of the specific containment +-- and key-existence operators on agtype are bound to matchingsel / +-- matchingjoinsel. We deliberately limit the check to these operator names +-- (rather than every operator in ag_catalog) so unrelated operators that +-- legitimately use matchingsel for their own semantics are not affected by +-- this regression test. +SELECT COUNT(*) AS leaked_matchingsel_bindings +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname = 'ag_catalog' + AND o.oprname IN ('@>', '<@', '@>>', '<<@', '?', '?|', '?&') + AND (o.oprrest::text = 'matchingsel' + OR o.oprjoin::text = 'matchingjoinsel'); + +-- Smoke test: each operator still works functionally. Selectivity binding +-- only affects the planner; this guards against an inadvertent operator +-- removal as part of any future cleanup. +SELECT '{"a":1,"b":2}'::agtype @> '{"a":1}'::agtype AS contains_yes; +SELECT '{"a":1}'::agtype <@ '{"a":1,"b":2}'::agtype AS contained_yes; +SELECT '{"a":{"b":1}}'::agtype @>> '{"a":{"b":1}}'::agtype AS top_contains_yes; +SELECT '{"a":{"b":1}}'::agtype <<@ '{"a":{"b":1}}'::agtype AS top_contained_yes; +SELECT '{"a":1}'::agtype ? 'a'::text AS exists_text_yes; +SELECT '{"a":1}'::agtype ? '"a"'::agtype AS exists_agtype_yes; +SELECT '{"a":1,"b":2}'::agtype ?| ARRAY['a','c'] AS exists_any_text_yes; +SELECT '{"a":1,"b":2}'::agtype ?| '["a","c"]'::agtype AS exists_any_agtype_yes; +SELECT '{"a":1,"b":2}'::agtype ?& ARRAY['a','b'] AS exists_all_text_yes; +SELECT '{"a":1,"b":2}'::agtype ?& '["a","b"]'::agtype AS exists_all_agtype_yes; + +-- Upgrade-path assertion for issue #2356. +-- +-- The checks above cover a FRESH install: contsel / contjoinsel come straight +-- from agtype_operators.sql and agtype_exists.sql. Existing installs instead +-- pick up the fix from the ALTER OPERATOR ... SET (RESTRICT, JOIN) block that +-- age--1.7.0--y.y.y.sql ships and "ALTER EXTENSION age UPDATE" replays. Nothing +-- above exercises that block, so a silent regression in it would go unnoticed. +-- +-- We replay the shipped ALTER OPERATOR statements directly rather than running +-- ALTER EXTENSION age UPDATE: the dev upgrade script targets the placeholder +-- version "y.y.y" and is not a stable version-chain target inside the +-- regression harness. The whole section runs in a transaction that is rolled +-- back, so it observes the flip without permanently mutating the operator +-- catalog (PostgreSQL DDL is transactional). +BEGIN; + +-- Simulate a stale (pre-fix) install: force all ten overloads back onto +-- matchingsel / matchingjoinsel. +ALTER OPERATOR ag_catalog.@>(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.<@(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.@>>(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.<<@(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?(agtype, text) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?|(agtype, text[]) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?|(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?&(agtype, text[]) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); +ALTER OPERATOR ag_catalog.?&(agtype, agtype) SET (RESTRICT = matchingsel, JOIN = matchingjoinsel); + +-- Stale state: every overload now reports matchingsel / matchingjoinsel. +SELECT o.oprname, + pg_catalog.format_type(o.oprleft, NULL) AS lhs, + pg_catalog.format_type(o.oprright, NULL) AS rhs, + o.oprrest::text AS restrict_fn, + o.oprjoin::text AS join_fn +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname = 'ag_catalog' + AND o.oprname IN ('@>', '<@', '@>>', '<<@', '?', '?|', '?&') +ORDER BY o.oprname, lhs, rhs; + +-- Replay the exact ALTER OPERATOR block shipped in age--1.7.0--y.y.y.sql. +ALTER OPERATOR ag_catalog.@>(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.<@(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.@>>(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.<<@(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?(agtype, text) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?|(agtype, text[]) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?|(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?&(agtype, text[]) SET (RESTRICT = contsel, JOIN = contjoinsel); +ALTER OPERATOR ag_catalog.?&(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); + +-- After the upgrade replay every overload is back on contsel / contjoinsel. +SELECT o.oprname, + pg_catalog.format_type(o.oprleft, NULL) AS lhs, + pg_catalog.format_type(o.oprright, NULL) AS rhs, + o.oprrest::text AS restrict_fn, + o.oprjoin::text AS join_fn +FROM pg_catalog.pg_operator o +JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace +WHERE n.nspname = 'ag_catalog' + AND o.oprname IN ('@>', '<@', '@>>', '<<@', '?', '?|', '?&') +ORDER BY o.oprname, lhs, rhs; + +ROLLBACK; diff --git a/regress/sql/cypher_match.sql b/regress/sql/cypher_match.sql index e56aafac8..8da012ff8 100644 --- a/regress/sql/cypher_match.sql +++ b/regress/sql/cypher_match.sql @@ -1068,9 +1068,9 @@ SELECT * FROM cypher('cypher_match', $$ MATCH (a {name:a.name}) RETURN a $$) as SELECT * FROM cypher('cypher_match', $$ MATCH (a {name:a.name, age:a.age}) RETURN a $$) as (a agtype); SELECT * FROM cypher('cypher_match', $$ MATCH (a {name:a.name}) MATCH (a {age:a.age}) RETURN a $$) as (a agtype); -SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relationship}]->(b) RETURN p $$) as (a agtype); -SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relationship, years: u.years}]->(b) RETURN p $$) as (a agtype); -SELECT * FROM cypher('cypher_match', $$ MATCH p=(a {name:a.name})-[u {relationship: u.relationship}]->(b {age:b.age}) RETURN p $$) as (a agtype); +SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relationship}]->(b) RETURN p ORDER BY id(u) $$) as (a agtype); +SELECT * FROM cypher('cypher_match', $$ MATCH p=(a)-[u {relationship: u.relationship, years: u.years}]->(b) RETURN p ORDER BY id(u) $$) as (a agtype); +SELECT * FROM cypher('cypher_match', $$ MATCH p=(a {name:a.name})-[u {relationship: u.relationship}]->(b {age:b.age}) RETURN p ORDER BY id(u) $$) as (a agtype); SELECT * FROM cypher('cypher_match', $$ CREATE () WITH * MATCH (x{n0:x.n1}) RETURN 0 $$) as (a agtype); diff --git a/regress/sql/cypher_vle.sql b/regress/sql/cypher_vle.sql index c960aa7a4..b121234e1 100644 --- a/regress/sql/cypher_vle.sql +++ b/regress/sql/cypher_vle.sql @@ -264,7 +264,7 @@ BEGIN RETURN QUERY SELECT * FROM cypher('mygraph', $CYPHER$ MATCH (h:head {name: $list_name})-[e:next*]->(v:node) - RETURN v + RETURN v ORDER BY id(v) $CYPHER$, ag_param) AS (node agtype); END $$; diff --git a/sql/agtype_exists.sql b/sql/agtype_exists.sql index 441af1755..f4ec4660b 100644 --- a/sql/agtype_exists.sql +++ b/sql/agtype_exists.sql @@ -32,8 +32,8 @@ CREATE OPERATOR ? ( LEFTARG = agtype, RIGHTARG = text, FUNCTION = ag_catalog.agtype_exists, - RESTRICT = matchingsel, - JOIN = matchingjoinsel + RESTRICT = contsel, + JOIN = contjoinsel ); CREATE FUNCTION ag_catalog.agtype_exists_agtype(agtype, agtype) @@ -48,8 +48,8 @@ CREATE OPERATOR ? ( LEFTARG = agtype, RIGHTARG = agtype, FUNCTION = ag_catalog.agtype_exists_agtype, - RESTRICT = matchingsel, - JOIN = matchingjoinsel + RESTRICT = contsel, + JOIN = contjoinsel ); CREATE FUNCTION ag_catalog.agtype_exists_any(agtype, text[]) @@ -64,8 +64,8 @@ CREATE OPERATOR ?| ( LEFTARG = agtype, RIGHTARG = text[], FUNCTION = ag_catalog.agtype_exists_any, - RESTRICT = matchingsel, - JOIN = matchingjoinsel + RESTRICT = contsel, + JOIN = contjoinsel ); CREATE FUNCTION ag_catalog.agtype_exists_any_agtype(agtype, agtype) @@ -80,8 +80,8 @@ CREATE OPERATOR ?| ( LEFTARG = agtype, RIGHTARG = agtype, FUNCTION = ag_catalog.agtype_exists_any_agtype, - RESTRICT = matchingsel, - JOIN = matchingjoinsel + RESTRICT = contsel, + JOIN = contjoinsel ); CREATE FUNCTION ag_catalog.agtype_exists_all(agtype, text[]) @@ -96,8 +96,8 @@ CREATE OPERATOR ?& ( LEFTARG = agtype, RIGHTARG = text[], FUNCTION = ag_catalog.agtype_exists_all, - RESTRICT = matchingsel, - JOIN = matchingjoinsel + RESTRICT = contsel, + JOIN = contjoinsel ); CREATE FUNCTION ag_catalog.agtype_exists_all_agtype(agtype, agtype) @@ -112,6 +112,6 @@ CREATE OPERATOR ?& ( LEFTARG = agtype, RIGHTARG = agtype, FUNCTION = ag_catalog.agtype_exists_all_agtype, - RESTRICT = matchingsel, - JOIN = matchingjoinsel + RESTRICT = contsel, + JOIN = contjoinsel ); diff --git a/sql/agtype_operators.sql b/sql/agtype_operators.sql index 36fedfe80..3fbc52f33 100644 --- a/sql/agtype_operators.sql +++ b/sql/agtype_operators.sql @@ -33,8 +33,8 @@ CREATE OPERATOR @> ( RIGHTARG = agtype, FUNCTION = ag_catalog.agtype_contains, COMMUTATOR = '<@', - RESTRICT = matchingsel, - JOIN = matchingjoinsel + RESTRICT = contsel, + JOIN = contjoinsel ); CREATE FUNCTION ag_catalog.agtype_contained_by(agtype, agtype) @@ -50,8 +50,8 @@ CREATE OPERATOR <@ ( RIGHTARG = agtype, FUNCTION = ag_catalog.agtype_contained_by, COMMUTATOR = '@>', - RESTRICT = matchingsel, - JOIN = matchingjoinsel + RESTRICT = contsel, + JOIN = contjoinsel ); CREATE FUNCTION ag_catalog.agtype_contains_top_level(agtype, agtype) @@ -67,8 +67,8 @@ CREATE OPERATOR @>> ( RIGHTARG = agtype, FUNCTION = ag_catalog.agtype_contains_top_level, COMMUTATOR = '<<@', - RESTRICT = matchingsel, - JOIN = matchingjoinsel + RESTRICT = contsel, + JOIN = contjoinsel ); CREATE FUNCTION ag_catalog.agtype_contained_by_top_level(agtype, agtype) @@ -84,6 +84,6 @@ CREATE OPERATOR <<@ ( RIGHTARG = agtype, FUNCTION = ag_catalog.agtype_contained_by_top_level, COMMUTATOR = '@>>', - RESTRICT = matchingsel, - JOIN = matchingjoinsel + RESTRICT = contsel, + JOIN = contjoinsel ); \ No newline at end of file From 23cbe57a1eedee9167221a0359b09e538ff6fc87 Mon Sep 17 00:00:00 2001 From: crdv7 <49877075+crdv7@users.noreply.github.com> Date: Wed, 10 Jun 2026 01:44:15 +0800 Subject: [PATCH 074/100] fix: remove pthread_mutex that causes self-deadlock on VLE queries (#2433) The pthread_mutex in manage_GRAPH_global_contexts() causes permanent self-deadlock when ereport(ERROR) triggers siglongjmp while the mutex is held, skipping pthread_mutex_unlock(). Any subsequent VLE query on the same backend connection hangs forever in pthread_mutex_lock() with __owner == own PID. The mutex was introduced in PR #1881 (fix for issue #1878) but is unnecessary: it protects a process-local static variable in PostgreSQL's single-threaded backend model where no concurrent access exists. The actual fix for #1878 was the Assert-to-runtime-check conversion and strndup defensive copy, which remain untouched. --- src/backend/utils/adt/age_global_graph.c | 63 +++++------------------- 1 file changed, 13 insertions(+), 50 deletions(-) diff --git a/src/backend/utils/adt/age_global_graph.c b/src/backend/utils/adt/age_global_graph.c index 1c4a4601b..397e0511c 100644 --- a/src/backend/utils/adt/age_global_graph.c +++ b/src/backend/utils/adt/age_global_graph.c @@ -45,7 +45,6 @@ #include "catalog/ag_label.h" #include "utils/ag_cache.h" -#include /* defines */ #define VERTEX_HTAB_NAME "Vertex to edge lists " /* added a space at end for */ @@ -152,18 +151,8 @@ typedef struct GRAPH_global_context struct GRAPH_global_context *next; /* next graph */ } GRAPH_global_context; -/* container for GRAPH_global_context and its mutex */ -typedef struct GRAPH_global_context_container -{ - /* head of the list */ - GRAPH_global_context *contexts; - - /* mutex to protect the list */ - pthread_mutex_t mutex_lock; -} GRAPH_global_context_container; - /* global variable to hold the per process GRAPH global contexts */ -static GRAPH_global_context_container global_graph_contexts_container = {0}; +static GRAPH_global_context *global_graph_contexts = NULL; /* * VertexEdgeArray helpers — flat-array adjacency container used by @@ -1054,12 +1043,10 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char *graph_name, * 5) One or more other contexts do exist but, one or more are invalid. */ - /* lock the global contexts list */ - pthread_mutex_lock(&global_graph_contexts_container.mutex_lock); /* free the invalidated GRAPH global contexts first */ prev_ggctx = NULL; - curr_ggctx = global_graph_contexts_container.contexts; + curr_ggctx = global_graph_contexts; while (curr_ggctx != NULL) { GRAPH_global_context *next_ggctx = curr_ggctx->next; @@ -1076,7 +1063,7 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char *graph_name, */ if (prev_ggctx == NULL) { - global_graph_contexts_container.contexts = next_ggctx; + global_graph_contexts = next_ggctx; } else { @@ -1089,8 +1076,6 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char *graph_name, /* if it wasn't successfull, there was a missing vertex entry */ if (!success) { - /* unlock the mutex so we don't get a deadlock */ - pthread_mutex_unlock(&global_graph_contexts_container.mutex_lock); ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("missing vertex or edge entry during free"))); @@ -1106,7 +1091,7 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char *graph_name, } /* find our graph's context. if it exists, we are done */ - curr_ggctx = global_graph_contexts_container.contexts; + curr_ggctx = global_graph_contexts; while (curr_ggctx != NULL) { if (curr_ggctx->graph_oid == graph_oid) @@ -1114,8 +1099,6 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char *graph_name, /* switch our context back */ MemoryContextSwitchTo(oldctx); - /* we are done unlock the global contexts list */ - pthread_mutex_unlock(&global_graph_contexts_container.mutex_lock); return curr_ggctx; } @@ -1125,9 +1108,9 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char *graph_name, /* otherwise, we need to create one and possibly attach it */ new_ggctx = palloc0(sizeof(GRAPH_global_context)); - if (global_graph_contexts_container.contexts != NULL) + if (global_graph_contexts != NULL) { - new_ggctx->next = global_graph_contexts_container.contexts; + new_ggctx->next = global_graph_contexts; } else { @@ -1135,7 +1118,7 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char *graph_name, } /* set the global context variable */ - global_graph_contexts_container.contexts = new_ggctx; + global_graph_contexts = new_ggctx; /* set the graph name and oid */ new_ggctx->graph_name = pstrdup(graph_name); @@ -1157,8 +1140,6 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char *graph_name, load_GRAPH_global_hashtables(new_ggctx); freeze_GRAPH_global_hashtables(new_ggctx); - /* unlock the global contexts list */ - pthread_mutex_unlock(&global_graph_contexts_container.mutex_lock); /* switch back to the previous memory context */ MemoryContextSwitchTo(oldctx); @@ -1170,18 +1151,16 @@ GRAPH_global_context *manage_GRAPH_global_contexts(char *graph_name, * Helper function to delete all of the global graph contexts used by the * process. When done the global global_graph_contexts will be NULL. * - * NOTE: Function uses a MUTEX global_graph_contexts_mutex + * */ static bool delete_GRAPH_global_contexts(void) { GRAPH_global_context *curr_ggctx = NULL; bool retval = false; - /* lock contexts list */ - pthread_mutex_lock(&global_graph_contexts_container.mutex_lock); /* get the first context, if any */ - curr_ggctx = global_graph_contexts_container.contexts; + curr_ggctx = global_graph_contexts; /* free all GRAPH global contexts */ while (curr_ggctx != NULL) @@ -1195,8 +1174,6 @@ static bool delete_GRAPH_global_contexts(void) /* if it wasn't successfull, there was a missing vertex entry */ if (!success) { - /* unlock the mutex so we don't get a deadlock */ - pthread_mutex_unlock(&global_graph_contexts_container.mutex_lock); ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("missing vertex or edge entry during free"))); @@ -1209,10 +1186,8 @@ static bool delete_GRAPH_global_contexts(void) } /* reset the head of the contexts to NULL */ - global_graph_contexts_container.contexts = NULL; + global_graph_contexts = NULL; - /* unlock the global contexts list */ - pthread_mutex_unlock(&global_graph_contexts_container.mutex_lock); return retval; } @@ -1235,11 +1210,9 @@ static bool delete_specific_GRAPH_global_contexts(char *graph_name) /* get the graph oid */ graph_oid = get_graph_oid(graph_name); - /* lock the global contexts list */ - pthread_mutex_lock(&global_graph_contexts_container.mutex_lock); /* get the first context, if any */ - curr_ggctx = global_graph_contexts_container.contexts; + curr_ggctx = global_graph_contexts; /* find the specified GRAPH global context */ while (curr_ggctx != NULL) @@ -1256,7 +1229,7 @@ static bool delete_specific_GRAPH_global_contexts(char *graph_name) */ if (prev_ggctx == NULL) { - global_graph_contexts_container.contexts = next_ggctx; + global_graph_contexts = next_ggctx; } else { @@ -1266,8 +1239,6 @@ static bool delete_specific_GRAPH_global_contexts(char *graph_name) /* free the current graph context */ success = free_specific_GRAPH_global_context(curr_ggctx); - /* unlock the global contexts list */ - pthread_mutex_unlock(&global_graph_contexts_container.mutex_lock); /* if it wasn't successfull, there was a missing vertex entry */ if (!success) @@ -1285,8 +1256,6 @@ static bool delete_specific_GRAPH_global_contexts(char *graph_name) curr_ggctx = next_ggctx; } - /* unlock the global contexts list */ - pthread_mutex_unlock(&global_graph_contexts_container.mutex_lock); /* we didn't find it, return false */ return false; @@ -1347,19 +1316,15 @@ GRAPH_global_context *find_GRAPH_global_context(Oid graph_oid) { GRAPH_global_context *ggctx = NULL; - /* lock the global contexts lists */ - pthread_mutex_lock(&global_graph_contexts_container.mutex_lock); /* get the root */ - ggctx = global_graph_contexts_container.contexts; + ggctx = global_graph_contexts; while(ggctx != NULL) { /* if we found it return it */ if (ggctx->graph_oid == graph_oid) { - /* unlock the global contexts lists */ - pthread_mutex_unlock(&global_graph_contexts_container.mutex_lock); return ggctx; } @@ -1368,8 +1333,6 @@ GRAPH_global_context *find_GRAPH_global_context(Oid graph_oid) ggctx = ggctx->next; } - /* unlock the global contexts lists */ - pthread_mutex_unlock(&global_graph_contexts_container.mutex_lock); /* we did not find it so return NULL */ return NULL; From 12e2a31c6716073fa046efe6fd3b1f930efabca6 Mon Sep 17 00:00:00 2001 From: Prashant Chinnam <5108573+crprashant@users.noreply.github.com> Date: Thu, 11 Jun 2026 06:58:30 -0700 Subject: [PATCH 075/100] Fix VLE [*0..N] zero-hop self-binding when edge label is missing (#2382) (#2419) A variable-length relationship pattern with a zero lower bound, e.g. `(p)-[:LABEL*0..N]-(f)`, must produce the zero-hop self-binding row (`f` = `p`) regardless of whether any edge of `LABEL` exists in the graph. This matches Neo4j/openCypher semantics. Previously, when the edge label did not exist in the label cache, AGE short-circuited the entire MATCH to zero rows (or NULL-extended rows for OPTIONAL MATCH). The fix has three parts: 1. parser/cypher_clause.c: A new helper `is_zero_lower_bound_vle()` inspects the FuncCall produced by `build_VLE_relation()` and reports whether the relationship is a zero-bound VLE. It is intentionally defensive about the FuncCall shape so that any future parser changes fall back to the existing short-circuit safely. `match_check_valid_label()` and `path_check_valid_label()` now treat a missing edge label as fatal only when the relationship requires at least one edge of that label. Patterns mixing a zero-bound segment with another impossible segment (e.g. `(a)-[:NOEXIST*0..1]-(b)-[:STILL_MISSING]-(c)`) still correctly resolve to zero rows because the second segment independently fails the label check. 2. utils/adt/age_vle.c: `is_an_edge_match()` now returns false early when the user requested a specific label that does not exist (`edge_label_name != NULL && edge_label_name_oid == InvalidOid`). This prevents a zero-bound traversal of `[:NOEXIST*0..N]` from incorrectly walking arbitrary other-label edges via the existing "no constraints -> match all" fast path. The zero-hop case itself is unaffected because it is generated by `build_VLE_zero_container()` without ever consulting `is_an_edge_match()`. 3. regress/sql/cypher_vle.sql: Adds seven regression cases that lock in the new behaviour, including the rubber-duck scenarios where another label exists in the graph (must NOT be matched by the missing-label VLE), where another segment is unsatisfiable (must still produce zero rows), and where the label exists (sanity check, unchanged behaviour). --- regress/expected/cypher_vle.out | 115 +++++++++++++++++++++++++++++ regress/sql/cypher_vle.sql | 69 +++++++++++++++++ src/backend/parser/cypher_clause.c | 73 +++++++++++++++++- src/backend/utils/adt/age_vle.c | 14 ++++ 4 files changed, 269 insertions(+), 2 deletions(-) diff --git a/regress/expected/cypher_vle.out b/regress/expected/cypher_vle.out index a09cd4aa9..0f564015b 100644 --- a/regress/expected/cypher_vle.out +++ b/regress/expected/cypher_vle.out @@ -1219,6 +1219,121 @@ NOTICE: graph "cypher_vle" has been dropped (1 row) +-- +-- Issue #2382: variable-length relationships with a zero lower bound must +-- still produce the zero-hop self-binding even when the edge label does not +-- exist in the graph (Neo4j/openCypher semantics). +-- +SELECT create_graph('issue_2382'); +NOTICE: graph "issue_2382" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('issue_2382', $$ + CREATE (:Person {name: 'Alice'})-[:KNOWS]->(:Person {name: 'Bob'}) +$$) AS (v agtype); + v +--- +(0 rows) + +-- Plain MATCH on a non-existent edge label with [*0..N] must return the +-- zero-hop self-binding row (Alice -> Alice). It must NOT match arbitrary +-- edges of other labels (e.g. KNOWS). +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + MATCH (p)-[:NOEXIST*0..1]-(f:Person) + RETURN p.name AS person, f.name AS friend +$$) AS (person agtype, friend agtype); + person | friend +---------+--------- + "Alice" | "Alice" +(1 row) + +-- OPTIONAL MATCH form (the exact shape from the issue report). +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + OPTIONAL MATCH (p)-[:NOEXIST*0..1]-(f:Person) + RETURN p.name AS person, f.name AS friend +$$) AS (person agtype, friend agtype); + person | friend +---------+--------- + "Alice" | "Alice" +(1 row) + +-- [*0..0] still emits exactly the zero-hop self-binding. +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + MATCH (p)-[:NOEXIST*0..0]-(f:Person) + RETURN p.name AS person, f.name AS friend +$$) AS (person agtype, friend agtype); + person | friend +---------+--------- + "Alice" | "Alice" +(1 row) + +-- Fixed-length (lower bound > 0) on a missing label must still return zero +-- rows: there is no edge of that label, so the pattern is unsatisfiable. +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + MATCH (p)-[:NOEXIST*1..1]-(f:Person) + RETURN p.name AS person, f.name AS friend +$$) AS (person agtype, friend agtype); + person | friend +--------+-------- +(0 rows) + +-- OPTIONAL MATCH on the unsatisfiable fixed-length pattern still preserves +-- the outer row with NULL bindings. +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + OPTIONAL MATCH (p)-[:NOEXIST*1..1]-(f:Person) + RETURN p.name AS person, f.name AS friend +$$) AS (person agtype, friend agtype); + person | friend +---------+-------- + "Alice" | +(1 row) + +-- Mixed pattern: a zero-bound VLE on a missing label combined with another +-- fixed-length missing label segment must still yield zero rows. The other +-- segment is impossible regardless of the zero-hop case. +SELECT * FROM cypher('issue_2382', $$ + MATCH (a:Person {name: 'Alice'}) + MATCH (a)-[:NOEXIST*0..1]-(b:Person)-[:STILL_MISSING]-(c:Person) + RETURN a.name, b.name, c.name +$$) AS (a agtype, b agtype, c agtype); + a | b | c +---+---+--- +(0 rows) + +-- Sanity: zero-bound VLE on an EXISTING label still works the way it did +-- before (Alice via zero-hop, Bob via 1-hop KNOWS). +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + MATCH (p)-[:KNOWS*0..1]-(f:Person) + RETURN p.name AS person, f.name AS friend + ORDER BY f.name +$$) AS (person agtype, friend agtype); + person | friend +---------+--------- + "Alice" | "Alice" + "Alice" | "Bob" +(2 rows) + +SELECT drop_graph('issue_2382', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table issue_2382._ag_label_vertex +drop cascades to table issue_2382._ag_label_edge +drop cascades to table issue_2382."Person" +drop cascades to table issue_2382."KNOWS" +NOTICE: graph "issue_2382" has been dropped + drop_graph +------------ + +(1 row) + -- -- End -- diff --git a/regress/sql/cypher_vle.sql b/regress/sql/cypher_vle.sql index b121234e1..4592a7fbd 100644 --- a/regress/sql/cypher_vle.sql +++ b/regress/sql/cypher_vle.sql @@ -417,6 +417,75 @@ SELECT drop_graph('issue_2092', true); DROP TABLE start_and_end_points; SELECT drop_graph('cypher_vle', true); +-- +-- Issue #2382: variable-length relationships with a zero lower bound must +-- still produce the zero-hop self-binding even when the edge label does not +-- exist in the graph (Neo4j/openCypher semantics). +-- +SELECT create_graph('issue_2382'); + +SELECT * FROM cypher('issue_2382', $$ + CREATE (:Person {name: 'Alice'})-[:KNOWS]->(:Person {name: 'Bob'}) +$$) AS (v agtype); + +-- Plain MATCH on a non-existent edge label with [*0..N] must return the +-- zero-hop self-binding row (Alice -> Alice). It must NOT match arbitrary +-- edges of other labels (e.g. KNOWS). +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + MATCH (p)-[:NOEXIST*0..1]-(f:Person) + RETURN p.name AS person, f.name AS friend +$$) AS (person agtype, friend agtype); + +-- OPTIONAL MATCH form (the exact shape from the issue report). +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + OPTIONAL MATCH (p)-[:NOEXIST*0..1]-(f:Person) + RETURN p.name AS person, f.name AS friend +$$) AS (person agtype, friend agtype); + +-- [*0..0] still emits exactly the zero-hop self-binding. +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + MATCH (p)-[:NOEXIST*0..0]-(f:Person) + RETURN p.name AS person, f.name AS friend +$$) AS (person agtype, friend agtype); + +-- Fixed-length (lower bound > 0) on a missing label must still return zero +-- rows: there is no edge of that label, so the pattern is unsatisfiable. +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + MATCH (p)-[:NOEXIST*1..1]-(f:Person) + RETURN p.name AS person, f.name AS friend +$$) AS (person agtype, friend agtype); + +-- OPTIONAL MATCH on the unsatisfiable fixed-length pattern still preserves +-- the outer row with NULL bindings. +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + OPTIONAL MATCH (p)-[:NOEXIST*1..1]-(f:Person) + RETURN p.name AS person, f.name AS friend +$$) AS (person agtype, friend agtype); + +-- Mixed pattern: a zero-bound VLE on a missing label combined with another +-- fixed-length missing label segment must still yield zero rows. The other +-- segment is impossible regardless of the zero-hop case. +SELECT * FROM cypher('issue_2382', $$ + MATCH (a:Person {name: 'Alice'}) + MATCH (a)-[:NOEXIST*0..1]-(b:Person)-[:STILL_MISSING]-(c:Person) + RETURN a.name, b.name, c.name +$$) AS (a agtype, b agtype, c agtype); + +-- Sanity: zero-bound VLE on an EXISTING label still works the way it did +-- before (Alice via zero-hop, Bob via 1-hop KNOWS). +SELECT * FROM cypher('issue_2382', $$ + MATCH (p:Person {name: 'Alice'}) + MATCH (p)-[:KNOWS*0..1]-(f:Person) + RETURN p.name AS person, f.name AS friend + ORDER BY f.name +$$) AS (person agtype, friend agtype); + +SELECT drop_graph('issue_2382', true); -- -- End diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 3f1697182..dcd31b9df 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -132,6 +132,60 @@ static Expr *transform_cypher_edge(cypher_parsestate *cpstate, static Expr *transform_cypher_node(cypher_parsestate *cpstate, cypher_node *node, List **target_list, bool output_node, bool valid_label); +/* + * Issue #2382: For variable-length relationships with a lower bound of 0 + * (e.g., [:LABEL*0..N]), the zero-hop self-binding case must succeed even + * when LABEL is missing from the cache, because Neo4j/openCypher semantics + * say a zero-hop pattern matches the same node regardless of any edges. + * + * By the time match_check_valid_label() runs, build_VLE_relation() (in + * cypher_gram.y) has rewritten cypher_relationship.varlen from A_Indices + * into a FuncCall named "vle" whose argument list is: + * (start_id, end_id, edge_match_proto, lidx, uidx, dir, unique_id) + * so the lower-bound is the 4th argument (1-based). + * + * This helper is intentionally defensive: every assumption about the shape + * of the FuncCall is guarded so any parser refactor that changes it will + * fall back to "not zero-bound", which is the safe behaviour (the existing + * false-where short-circuit will still kick in for impossible patterns). + */ +static bool is_zero_lower_bound_vle(Node *varlen) +{ + FuncCall *fc; + String *fname; + Node *lidx_node; + A_Const *lidx; + + if (varlen == NULL || !IsA(varlen, FuncCall)) + return false; + + fc = (FuncCall *) varlen; + + if (list_length(fc->funcname) != 1) + return false; + fname = (String *) linitial(fc->funcname); + if (fname == NULL || !IsA(fname, String)) + return false; + if (strcmp(strVal(fname), "vle") != 0) + return false; + + /* args = {start, end, edge_match, lidx, uidx, dir, uniq} */ + if (list_length(fc->args) < 5) + return false; + + lidx_node = (Node *) list_nth(fc->args, 3); + if (lidx_node == NULL || !IsA(lidx_node, A_Const)) + return false; + + lidx = (A_Const *) lidx_node; + if (lidx->isnull) + return false; + if (lidx->val.ival.type != T_Integer) + return false; + + return lidx->val.ival.ival == 0; +} + static bool match_check_valid_label(cypher_match *match, cypher_parsestate *cpstate); static Node *make_vertex_expr(cypher_parsestate *cpstate, @@ -2927,7 +2981,14 @@ static bool match_check_valid_label(cypher_match *match, if (lcd == NULL || lcd->kind != LABEL_KIND_EDGE) { - return false; + /* + * Issue #2382: a missing edge label is fatal only if + * the pattern actually requires an edge of that label. + * For VLE with lower bound 0, the zero-hop self-bind + * case must still produce rows. + */ + if (!is_zero_lower_bound_vle(rel->varlen)) + return false; } } } @@ -5047,7 +5108,15 @@ static bool path_check_valid_label(cypher_path *path, if (lcd == NULL || lcd->kind != LABEL_KIND_EDGE) { - return false; + /* + * Issue #2382: Don't invalidate the whole path just + * because a VLE edge with lower bound 0 references a + * missing label. The zero-hop self-binding semantics + * still allow the surrounding nodes to bind, so the + * other vertex labels in this path must be honoured. + */ + if (!is_zero_lower_bound_vle(rel->varlen)) + return false; } } } diff --git a/src/backend/utils/adt/age_vle.c b/src/backend/utils/adt/age_vle.c index 3bc276cfc..9e433b9e2 100644 --- a/src/backend/utils/adt/age_vle.c +++ b/src/backend/utils/adt/age_vle.c @@ -417,6 +417,20 @@ static bool is_an_edge_match(VLE_local_context *vlelctx, edge_entry *ee) /* get the number of conditions from the prototype edge */ num_edge_property_constraints = AGT_ROOT_COUNT(vlelctx->edge_property_constraint); + /* + * Issue #2382: If the user asked for a specific edge label but that label + * does not exist in the graph (edge_label_name_oid == InvalidOid while + * edge_label_name is non-NULL), no real edge can match. Returning false + * here ensures that for VLE patterns like [:NOEXIST*0..N] we do not + * traverse arbitrary other-label edges. Zero-hop self-binding is handled + * separately via build_VLE_zero_container() so this does not break it. + */ + if (vlelctx->edge_label_name != NULL && + vlelctx->edge_label_name_oid == InvalidOid) + { + return false; + } + /* * We only care about verifying that we have all of the property conditions. * We don't care about extra unmatched properties. If there aren't any edge From 14732bfaf4a63a1ac9775d7dc7a2c0eed3f47ab2 Mon Sep 17 00:00:00 2001 From: serdarmumcu Date: Fri, 12 Jun 2026 00:05:26 +0300 Subject: [PATCH 076/100] Fix locale-dependent string comparison in direct_field_access test (#2439) Replace 'test' > 'TEST' with deterministic 'abd' > 'abc'. The original comparison returns different results depending on system collation settings (true under C locale, false under en_US.UTF-8). --- regress/expected/direct_field_access.out | 2 +- regress/sql/direct_field_access.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/regress/expected/direct_field_access.out b/regress/expected/direct_field_access.out index 0a059cdd9..25082d9fa 100644 --- a/regress/expected/direct_field_access.out +++ b/regress/expected/direct_field_access.out @@ -78,7 +78,7 @@ $$) AS (lt agtype, gt agtype, eq agtype, ne agtype); (1 row) SELECT * FROM cypher('direct_access', $$ - RETURN 'hello world' < 'hello worlds', 'test' > 'TEST' + RETURN 'hello world' < 'hello worlds', 'abd' > 'abc' $$) AS (lt agtype, gt agtype); lt | gt ------+------ diff --git a/regress/sql/direct_field_access.sql b/regress/sql/direct_field_access.sql index c8060be4a..3ca2259f4 100644 --- a/regress/sql/direct_field_access.sql +++ b/regress/sql/direct_field_access.sql @@ -61,7 +61,7 @@ SELECT * FROM cypher('direct_access', $$ $$) AS (lt agtype, gt agtype, eq agtype, ne agtype); SELECT * FROM cypher('direct_access', $$ - RETURN 'hello world' < 'hello worlds', 'test' > 'TEST' + RETURN 'hello world' < 'hello worlds', 'abd' > 'abc' $$) AS (lt agtype, gt agtype); -- Boolean comparisons From a17bf457fe5856e1e7f538400c67d8669d828e54 Mon Sep 17 00:00:00 2001 From: serdarmumcu Date: Sat, 20 Jun 2026 19:53:25 +0300 Subject: [PATCH 077/100] Make age extension usable from shared_preload_libraries (#2438) When AGE is loaded via shared_preload_libraries, its hooks (post_parse_analyze, set_rel_pathlist, object_access) are active before CREATE EXTENSION age is run. This causes errors when non-Cypher queries trigger those hooks and ag_catalog does not yet exist. Changes: - Add is_age_extension_exist() with a relcache callback cache so that checking pg_extension is not repeated on every hook invocation. - Guard post_parse_analyze, set_rel_pathlist, and object_access hooks with is_age_extension_exist() so they become no-ops when the extension is not installed. - Refactor ag_ProcessUtility_hook to detect CREATE/DROP EXTENSION age and broadcast a relcache invalidation via CacheInvalidateRelcacheByRelid(ExtensionRelationId) so other backends update their cached extension state. - Wrap DROP EXTENSION processing in PG_TRY/PG_CATCH to restore object_access_hook if the drop fails (e.g. dependent objects). - Skip _PG_init during pg_upgrade (IsBinaryUpgrade) to avoid hook registration when the binary-upgrade machinery is running. - Add regression tests that verify hooks do not error when ag_catalog schema is absent. --- regress/expected/drop.out | 12 +- regress/sql/drop.sql | 9 ++ src/backend/age.c | 9 +- src/backend/catalog/ag_catalog.c | 216 ++++++++++++++++++++------- src/backend/optimizer/cypher_paths.c | 8 + src/backend/parser/cypher_analyze.c | 6 + src/include/catalog/ag_catalog.h | 2 + 7 files changed, 202 insertions(+), 60 deletions(-) diff --git a/regress/expected/drop.out b/regress/expected/drop.out index 3cfa2cf28..43e0bf41a 100644 --- a/regress/expected/drop.out +++ b/regress/expected/drop.out @@ -40,6 +40,14 @@ SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = 'ag_catalog'; ----------- (0 rows) +-- When ag_catalog is missing extension hooks shouldn't fail with the +-- ERROR schema "ag_catalog" does not exist. +-- It might happen when 'age' is loaded but extension isn't created yet. +SET client_min_messages TO WARNING; +DROP SCHEMA IF EXISTS ag_catalog CASCADE; +RESET client_min_messages; +CREATE SCHEMA _regress_drop; +DROP SCHEMA _regress_drop; -- should'n produce the ERROR -- Recreate the extension and validate we can recreate a graph CREATE EXTENSION age; SELECT create_graph('drop'); @@ -115,7 +123,7 @@ NOTICE: label "issue_1305"."r" has been dropped (1 row) SELECT drop_label('issue_1305', 'r'); -ERROR: rel_name not found for label "r" +ERROR: label "r" does not exist SELECT drop_label('issue_1305', 'n', false); NOTICE: label "issue_1305"."n" has been dropped drop_label @@ -124,7 +132,7 @@ NOTICE: label "issue_1305"."n" has been dropped (1 row) SELECT drop_label('issue_1305', 'n'); -ERROR: rel_name not found for label "n" +ERROR: label "n" does not exist SELECT * FROM drop_graph('issue_1305', true); NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table issue_1305._ag_label_vertex diff --git a/regress/sql/drop.sql b/regress/sql/drop.sql index 564492bbc..a71d5a74c 100644 --- a/regress/sql/drop.sql +++ b/regress/sql/drop.sql @@ -28,6 +28,15 @@ SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname = 'drop'; SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = 'ag_catalog'; +-- When ag_catalog is missing extension hooks shouldn't fail with the +-- ERROR schema "ag_catalog" does not exist. +-- It might happen when 'age' is loaded but extension isn't created yet. +SET client_min_messages TO WARNING; +DROP SCHEMA IF EXISTS ag_catalog CASCADE; +RESET client_min_messages; +CREATE SCHEMA _regress_drop; +DROP SCHEMA _regress_drop; -- should'n produce the ERROR + -- Recreate the extension and validate we can recreate a graph CREATE EXTENSION age; diff --git a/src/backend/age.c b/src/backend/age.c index 18085302c..24ae8456b 100644 --- a/src/backend/age.c +++ b/src/backend/age.c @@ -17,6 +17,9 @@ * under the License. */ +#include "postgres.h" +#include "miscadmin.h" + #include "catalog/ag_catalog.h" #include "nodes/ag_nodes.h" #include "optimizer/cypher_paths.h" @@ -25,7 +28,6 @@ #include "utils/age_global_graph.h" #if PG_VERSION_NUM < 170000 -#include "miscadmin.h" /* saved hook pointers for PG < 17 shmem path */ static shmem_request_hook_type prev_shmem_request_hook = NULL; @@ -56,6 +58,11 @@ void _PG_init(void); void _PG_init(void) { + if (IsBinaryUpgrade) + { + return; + } + register_ag_nodes(); set_rel_pathlist_init(); object_access_hook_init(); diff --git a/src/backend/catalog/ag_catalog.c b/src/backend/catalog/ag_catalog.c index f5276e092..107de370d 100644 --- a/src/backend/catalog/ag_catalog.c +++ b/src/backend/catalog/ag_catalog.c @@ -19,14 +19,18 @@ #include "postgres.h" +#include "access/xact.h" #include "catalog/dependency.h" #include "catalog/namespace.h" #include "catalog/objectaccess.h" #include "catalog/pg_class_d.h" +#include "catalog/pg_extension_d.h" #include "catalog/pg_namespace_d.h" #include "commands/defrem.h" +#include "commands/extension.h" #include "nodes/parsenodes.h" #include "tcop/utility.h" +#include "utils/inval.h" #include "utils/lsyscache.h" #include "catalog/ag_graph.h" @@ -34,6 +38,8 @@ #include "utils/ag_cache.h" #include "utils/age_global_graph.h" +static bool extension_cache_is_valid = false; +static bool age_extension_exists = false; static object_access_hook_type prev_object_access_hook; static ProcessUtility_hook_type prev_process_utility_hook; static bool prev_object_hook_is_set; @@ -45,8 +51,46 @@ void ag_ProcessUtility_hook(PlannedStmt *pstmt, const char *queryString, bool re QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); -static bool is_age_drop(PlannedStmt *pstmt); -static void drop_age_extension(DropStmt *stmt); +static bool is_age_drop(DropStmt *drop_stmt); + +static void +invalidate_extension_cache_callback(Datum argument, Oid relationId) +{ + if (!OidIsValid(relationId) || relationId == ExtensionRelationId) + { + extension_cache_is_valid = false; + } +} + +/* + * We don't want most of hooks to do anything if the "age" extension isn't + * created. However, scanning pg_extension is a costly operation, therefore we + * implement a caching mechanism and reset it with the help of the relcache + * callback mechanism. + * + * Please also see ag_ProcessUtility_hook() function for more details. + */ +bool +is_age_extension_exists(void) +{ + static bool callback_registered = false; + + if (extension_cache_is_valid) + return age_extension_exists; + + if (!callback_registered) + { + CacheRegisterRelcacheCallback(invalidate_extension_cache_callback, + (Datum) 0); + callback_registered = true; + } + + age_extension_exists = OidIsValid(get_extension_oid("age", true)); + + extension_cache_is_valid = true; + + return age_extension_exists; +} void object_access_hook_init(void) { @@ -86,50 +130,97 @@ void process_utility_hook_fini(void) * information in the indexes and tables being dropped. To prevent an error * from being thrown, we need to disable the object_access_hook before dropping * the extension. + * + * Besides that, we want to notify other backends about the fact that "age" + * extension was probably created/dropped so that they can enable/disable + * hooks. */ void ag_ProcessUtility_hook(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc) { - if (is_age_drop(pstmt)) - { - drop_age_extension((DropStmt *)pstmt->utilityStmt); - } - else + bool creating_age = false; + bool dropping_age = false; + + if (!IsAbortedTransactionBlockState()) { - /* - * Check for TRUNCATE on graph label tables. If any truncated - * table is a graph label table, increment the version counter - * for that graph to invalidate VLE caches. We do this before - * the truncate executes so the cache is invalidated regardless. - */ - if (IsA(pstmt->utilityStmt, TruncateStmt)) + Node *parsetree = pstmt->utilityStmt; + + switch (nodeTag(parsetree)) { - TruncateStmt *tstmt = (TruncateStmt *) pstmt->utilityStmt; - ListCell *lc; + case T_CreateExtensionStmt: + { + CreateExtensionStmt *stmt = + (CreateExtensionStmt *) parsetree; + creating_age = strcmp(stmt->extname, "age") == 0; + } + break; + case T_DropStmt: + { + DropStmt *stmt = (DropStmt *) parsetree; - foreach(lc, tstmt->relations) - { - RangeVar *rv = (RangeVar *) lfirst(lc); - Oid rel_oid = RangeVarGetRelid(rv, AccessShareLock, true); + if (stmt->removeType != OBJECT_EXTENSION) + break; - if (OidIsValid(rel_oid)) - { - Oid graph_oid = get_graph_oid_for_table(rel_oid); + if (!is_age_drop(stmt)) + break; - if (OidIsValid(graph_oid)) + dropping_age = true; + } + break; + case T_TruncateStmt: + { + /* + * Check for TRUNCATE on graph label tables. If any + * truncated table is a graph label table, increment the + * version counter for that graph to invalidate VLE caches. + * We do this before the truncate executes so the cache is + * invalidated regardless. + */ + TruncateStmt *tstmt = (TruncateStmt *) parsetree; + ListCell *lc; + + foreach(lc, tstmt->relations) { - increment_graph_version(graph_oid); + RangeVar *rv = (RangeVar *) lfirst(lc); + Oid rel_oid = RangeVarGetRelid(rv, AccessShareLock, + true); + + if (OidIsValid(rel_oid)) + { + Oid graph_oid = + get_graph_oid_for_table(rel_oid); + + if (OidIsValid(graph_oid)) + { + increment_graph_version(graph_oid); + } + } } } - } + break; + default: + break; } + } + + if (dropping_age) + { + /* Remove all graphs */ + drop_graphs(get_graphnames()); + /* Remove the object access hook */ + object_access_hook_fini(); + } + + PG_TRY(); + { if (prev_process_utility_hook) { (*prev_process_utility_hook) (pstmt, queryString, readOnlyTree, - context, params, queryEnv, dest, qc); + context, params, queryEnv, dest, + qc); } else { @@ -141,38 +232,47 @@ void ag_ProcessUtility_hook(PlannedStmt *pstmt, const char *queryString, params, queryEnv, dest, qc); } } -} - -static void drop_age_extension(DropStmt *stmt) -{ - /* Remove all graphs */ - drop_graphs(get_graphnames()); + PG_CATCH(); + { + if (dropping_age) + { + /* + * We have to restore the disabled object_access_hook if + * DROP EXTENSION age failed. + */ + object_access_hook_init(); + } + PG_RE_THROW(); + } + PG_END_TRY(); - /* Remove the object access hook */ - object_access_hook_fini(); + if (dropping_age) + { + /* reset global variables for OIDs */ + clear_global_Oids_AGTYPE(); + clear_global_Oids_GRAPHID(); + clear_global_Oids_VERTEX_EDGE(); - /* - * Run Postgres' logic to perform the remaining work to drop the - * extension. - */ - RemoveObjects(stmt); + /* Restore the object access hook */ + object_access_hook_init(); + } - /* reset global variables for OIDs */ - clear_global_Oids_AGTYPE(); - clear_global_Oids_GRAPHID(); - clear_global_Oids_VERTEX_EDGE(); + if (creating_age || dropping_age) + { + /* Notify all backends that pg_extension was modified. */ + CacheInvalidateRelcacheByRelid(ExtensionRelationId); + } } /* Check to see if the Utility Command is to drop the AGE Extension. */ -static bool is_age_drop(PlannedStmt *pstmt) +static bool is_age_drop(DropStmt *drop_stmt) { ListCell *lc; - DropStmt *drop_stmt; - if (!IsA(pstmt->utilityStmt, DropStmt)) + if (!is_age_extension_exists()) + { return false; - - drop_stmt = (DropStmt *)pstmt->utilityStmt; + } foreach(lc, drop_stmt->objects) { @@ -183,8 +283,10 @@ static bool is_age_drop(PlannedStmt *pstmt) String *val = (String *)obj; char *str = val->sval; - if (!pg_strcasecmp(str, "age")) + if (strcmp(str, "age") == 0) + { return true; + } } } @@ -205,16 +307,16 @@ static void object_access(ObjectAccessType access, Oid class_id, Oid object_id, if (prev_object_access_hook) prev_object_access_hook(access, class_id, object_id, sub_id, arg); - /* We are interested in DROP SCHEMA and DROP TABLE commands. */ - if (access != OAT_DROP) + if (!is_age_extension_exists()) + { return; + } - /* - * Age might be installed into shared_preload_libraries before extension is - * created. In this case we must bail out from this hook. - */ - if (!OidIsValid(get_namespace_oid("ag_catalog", true))) + /* We are interested in DROP SCHEMA and DROP TABLE commands. */ + if (access != OAT_DROP) + { return; + } drop_arg = arg; diff --git a/src/backend/optimizer/cypher_paths.c b/src/backend/optimizer/cypher_paths.c index 6c4fd7e07..6d2e400c0 100644 --- a/src/backend/optimizer/cypher_paths.c +++ b/src/backend/optimizer/cypher_paths.c @@ -19,6 +19,7 @@ #include "postgres.h" +#include "catalog/ag_catalog.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" @@ -64,7 +65,14 @@ static void set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { if (prev_set_rel_pathlist_hook) + { prev_set_rel_pathlist_hook(root, rel, rti, rte); + } + + if (!is_age_extension_exists()) + { + return; + } switch (get_cypher_clause_kind(rte)) { diff --git a/src/backend/parser/cypher_analyze.c b/src/backend/parser/cypher_analyze.c index b2c9256ce..5b72f332e 100644 --- a/src/backend/parser/cypher_analyze.c +++ b/src/backend/parser/cypher_analyze.c @@ -19,6 +19,7 @@ #include "postgres.h" +#include "catalog/ag_catalog.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "parser/analyze.h" @@ -86,6 +87,11 @@ static void post_parse_analyze(ParseState *pstate, Query *query, JumbleState *js prev_post_parse_analyze_hook(pstate, query, jstate); } + if (!is_age_extension_exists()) + { + return; + } + /* * extra_node is set in the parsing stage to keep track of EXPLAIN. * So it needs to be set to NULL prior to any cypher parsing. diff --git a/src/include/catalog/ag_catalog.h b/src/include/catalog/ag_catalog.h index 56aa84700..a9ced279d 100644 --- a/src/include/catalog/ag_catalog.h +++ b/src/include/catalog/ag_catalog.h @@ -24,6 +24,8 @@ #include "utils/agtype.h" +bool is_age_extension_exists(void); + void object_access_hook_init(void); void object_access_hook_fini(void); From 310dc3f680a034d81baaf7176ca3359b67b44918 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Sat, 20 Jun 2026 09:55:09 -0700 Subject: [PATCH 078/100] feature: add create_subgraph() (#2441) Add the feature create_subgraph() for materialized induced-subgraph extraction. Add ag_catalog.create_subgraph(new_graph, from_graph, node_filter, relationship_filter) which materializes a new, persistent, fully Cypher-queryable AGE graph as the induced subgraph of an existing graph. Selection follows the graph-theory induced-subgraph definition as operationalized by Neo4j GDS gds.graph.filter(): * a vertex is kept iff node_filter holds ('*' keeps all); * an edge is kept iff relationship_filter holds AND both of its endpoints were kept (no dangling edges). Filters are arbitrary Cypher predicates bound to `n` (nodes) and `r` (relationships) and are evaluated by AGE's own Cypher engine against the source graph, so the full predicate language is available; label selection uses label(n)/label(r) since the match pattern is fixed. Implementation notes: * Result is a real, ACID, registered graph (create_graph + create_v/ elabel), not a virtual view; it composes with cypher() and itself. * Entity graphids are reassigned from the destination labels' own sequences (graphid encodes a per-graph label id), and edge endpoints are remapped through an old->new vertex map, enforcing the induced rule via inner joins. * Source label tables are read with FROM ONLY to avoid double-copying children under PostgreSQL table inheritance. * Properties of any agtype are preserved; self-loops and parallel edges (multigraph structure) are retained. * SECURITY INVOKER: reads respect the caller's table privileges and RLS; the new graph is owned by the caller. * Validates NULL/identical graph names, missing source, pre-existing destination, and a reserved dollar-quote token in predicates. Wire-up: * sql/age_subgraph.sql (new) registered in sql/sql_files after age_pg_upgrade; identical body added to age--1.7.0--y.y.y.sql so the upgrade-path catalog comparison matches. * regress/sql/subgraph.sql + expected output (new), added to REGRESS. Covers full copy, vertex-induced, node+rel, label-only edge drop, bipartite, empty result, composability, self-loops/parallel edges, property fidelity, and error cases over a ~4500-vertex / 2000-edge source graph. All 38 regression tests pass against PostgreSQL 18. Co-authored-by: GitHub Copilot (Claude Opus 4.8) <[email protected]> modified: Makefile modified: age--1.7.0--y.y.y.sql new file: regress/expected/subgraph.out new file: regress/sql/subgraph.sql new file: sql/age_subgraph.sql modified: sql/sql_files --- Makefile | 3 +- age--1.7.0--y.y.y.sql | 257 +++++++++++++++++++++++++ regress/expected/subgraph.out | 341 ++++++++++++++++++++++++++++++++++ regress/sql/subgraph.sql | 189 +++++++++++++++++++ sql/age_subgraph.sql | 294 +++++++++++++++++++++++++++++ sql/sql_files | 1 + 6 files changed, 1084 insertions(+), 1 deletion(-) create mode 100644 regress/expected/subgraph.out create mode 100644 regress/sql/subgraph.sql create mode 100644 sql/age_subgraph.sql diff --git a/Makefile b/Makefile index f2a0b9a62..3ea9236a6 100644 --- a/Makefile +++ b/Makefile @@ -184,7 +184,8 @@ REGRESS = scan \ security \ reserved_keyword_alias \ agtype_jsonb_cast \ - containment_selectivity + containment_selectivity \ + subgraph ifneq ($(EXTRA_TESTS),) REGRESS += $(EXTRA_TESTS) diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index a4cac0c5c..282eaa0f9 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -800,3 +800,260 @@ ALTER OPERATOR ag_catalog.?&(agtype, text[]) SET (RESTRICT = contsel, JOIN = contjoinsel); ALTER OPERATOR ag_catalog.?&(agtype, agtype) SET (RESTRICT = contsel, JOIN = contjoinsel); + +-- +-- create_subgraph(): materialized subgraph extraction (see sql/age_subgraph.sql). +-- Induced-subgraph semantics matching Neo4j GDS gds.graph.filter(): a vertex is +-- kept iff node_filter holds ('*' = all); an edge is kept iff relationship_filter +-- holds AND both endpoints are kept. Produces a persistent, Cypher-queryable graph. +-- +CREATE FUNCTION ag_catalog.create_subgraph(new_graph name, + from_graph name, + node_filter text DEFAULT '*', + relationship_filter text DEFAULT '*') + RETURNS TABLE(node_count bigint, relationship_count bigint) + LANGUAGE plpgsql + VOLATILE + SET search_path = ag_catalog, pg_catalog + AS $function$ +DECLARE + from_oid oid; + new_oid oid; + v_node_count bigint := 0; + v_rel_count bigint := 0; + rec RECORD; + cypher_q text; + where_clause text; + dst_label_id int; + dst_seq_fqn text; + dst_relation text; + inserted bigint; + has_rows boolean; +BEGIN + -- Argument validation. + IF new_graph IS NULL THEN + RAISE EXCEPTION 'new graph name must not be NULL'; + END IF; + IF from_graph IS NULL THEN + RAISE EXCEPTION 'source graph name must not be NULL'; + END IF; + IF new_graph = from_graph THEN + RAISE EXCEPTION 'cannot extract a subgraph of "%" into itself', from_graph; + END IF; + + -- NULL predicate is treated as the '*' wildcard (keep all). + IF node_filter IS NULL THEN + node_filter := '*'; + END IF; + IF relationship_filter IS NULL THEN + relationship_filter := '*'; + END IF; + + -- The predicates are embedded into a dollar-quoted cypher() query using the + -- $age_subgraph$ tag; reject predicates that contain the tag to keep the + -- quoting unambiguous. + IF position('$age_subgraph$' IN node_filter) > 0 + OR position('$age_subgraph$' IN relationship_filter) > 0 THEN + RAISE EXCEPTION 'filter predicate must not contain the reserved token $age_subgraph$'; + END IF; + + -- Validate source graph exists. + SELECT graphid INTO from_oid + FROM ag_catalog.ag_graph WHERE name = from_graph; + IF from_oid IS NULL THEN + RAISE EXCEPTION 'graph "%" does not exist', from_graph; + END IF; + + -- Validate destination graph does not exist (create_graph also enforces + -- naming rules and uniqueness, but we give a clear early error). + IF EXISTS (SELECT 1 FROM ag_catalog.ag_graph WHERE name = new_graph) THEN + RAISE EXCEPTION 'graph "%" already exists', new_graph; + END IF; + + -- Create the destination graph (default labels are created automatically). + PERFORM ag_catalog.create_graph(new_graph); + + SELECT graphid INTO new_oid + FROM ag_catalog.ag_graph WHERE name = new_graph; + + -- Working sets / mapping (uniquely named to avoid colliding with user temps). + DROP TABLE IF EXISTS _ag_sg_kept_v; + DROP TABLE IF EXISTS _ag_sg_kept_e; + DROP TABLE IF EXISTS _ag_sg_vmap; + DROP TABLE IF EXISTS _ag_sg_vstage; + DROP TABLE IF EXISTS _ag_sg_estage; + + -- + -- Kept vertices: evaluate node_filter with AGE's Cypher engine. The node + -- variable `n` is bound exactly as in the spec; '*' selects all vertices. + -- + IF node_filter IS NULL OR btrim(node_filter) = '*' THEN + where_clause := ''; + ELSE + where_clause := ' WHERE ' || node_filter; + END IF; + cypher_q := 'MATCH (n)' || where_clause || ' RETURN id(n)'; + + EXECUTE format( + 'CREATE TEMP TABLE _ag_sg_kept_v ON COMMIT DROP AS ' + 'SELECT DISTINCT ag_catalog.agtype_to_graphid(vid) AS gid ' + 'FROM ag_catalog.cypher(%L, $age_subgraph$%s$age_subgraph$) AS (vid agtype)', + from_graph, cypher_q); + CREATE INDEX ON _ag_sg_kept_v (gid); + + -- + -- Kept edges: evaluate relationship_filter with AGE's Cypher engine. The + -- relationship variable `r` is bound exactly as in the spec. + -- + IF relationship_filter IS NULL OR btrim(relationship_filter) = '*' THEN + where_clause := ''; + ELSE + where_clause := ' WHERE ' || relationship_filter; + END IF; + cypher_q := 'MATCH ()-[r]->()' || where_clause || ' RETURN id(r)'; + + EXECUTE format( + 'CREATE TEMP TABLE _ag_sg_kept_e ON COMMIT DROP AS ' + 'SELECT DISTINCT ag_catalog.agtype_to_graphid(eid) AS gid ' + 'FROM ag_catalog.cypher(%L, $age_subgraph$%s$age_subgraph$) AS (eid agtype)', + from_graph, cypher_q); + CREATE INDEX ON _ag_sg_kept_e (gid); + + -- old -> new vertex id mapping (graphid is unique within a graph). + CREATE TEMP TABLE _ag_sg_vmap (old_id graphid PRIMARY KEY, + new_id graphid NOT NULL) ON COMMIT DROP; + + -- + -- PASS 1: copy kept vertices, label by label, assigning new graphids and + -- recording the old->new mapping for edge remapping. + -- + FOR rec IN + SELECT name, id, relation, seq_name + FROM ag_catalog.ag_label + WHERE graph = from_oid AND kind = 'v' + ORDER BY id + LOOP + -- Skip labels with no surviving vertices. Read ONLY this label's own + -- rows: AGE label tables use table inheritance (custom labels inherit + -- from _ag_label_vertex), so a plain scan of a parent would also return + -- its children and copy them twice. + EXECUTE format( + 'SELECT EXISTS (SELECT 1 FROM ONLY %s t ' + 'WHERE EXISTS (SELECT 1 FROM _ag_sg_kept_v k WHERE k.gid = t.id))', + rec.relation::regclass::text) + INTO has_rows; + IF NOT has_rows THEN + CONTINUE; + END IF; + + -- Ensure the label exists in the destination graph. + IF rec.name <> '_ag_label_vertex' THEN + PERFORM 1 FROM ag_catalog.ag_label + WHERE graph = new_oid AND name = rec.name; + IF NOT FOUND THEN + EXECUTE format('SELECT ag_catalog.create_vlabel(%L, %L)', + new_graph, rec.name); + END IF; + END IF; + + SELECT id, seq_name, relation::regclass::text + INTO dst_label_id, dst_seq_fqn, dst_relation + FROM ag_catalog.ag_label + WHERE graph = new_oid AND name = rec.name; + dst_seq_fqn := format('%I.%I', new_graph, dst_seq_fqn); + + -- Stage surviving vertices with freshly generated ids in a real temp + -- table (single evaluation), then copy to the label table and record + -- the old->new mapping. A materialized stage avoids any ambiguity from + -- referencing a nextval-bearing CTE more than once. + DROP TABLE IF EXISTS _ag_sg_vstage; + EXECUTE format( + 'CREATE TEMP TABLE _ag_sg_vstage ON COMMIT DROP AS ' + 'SELECT t.id AS old_id, ' + ' ag_catalog._graphid(%s, nextval(%L)) AS new_id, ' + ' t.properties AS props ' + 'FROM ONLY %s t ' + 'WHERE EXISTS (SELECT 1 FROM _ag_sg_kept_v k WHERE k.gid = t.id)', + dst_label_id, dst_seq_fqn, rec.relation::regclass::text); + + EXECUTE format('INSERT INTO %s (id, properties) ' + 'SELECT new_id, props FROM _ag_sg_vstage', dst_relation); + + INSERT INTO _ag_sg_vmap (old_id, new_id) + SELECT old_id, new_id FROM _ag_sg_vstage; + + DROP TABLE _ag_sg_vstage; + END LOOP; + + SELECT count(*) INTO v_node_count FROM _ag_sg_vmap; + + -- + -- PASS 2: copy kept edges, remapping endpoints. The joins to _ag_sg_vmap + -- enforce the induced rule (an edge survives only if BOTH endpoints were + -- kept); membership in _ag_sg_kept_e applies relationship_filter. + -- + FOR rec IN + SELECT name, id, relation, seq_name + FROM ag_catalog.ag_label + WHERE graph = from_oid AND kind = 'e' + ORDER BY id + LOOP + -- Skip labels with no surviving edges. Read ONLY this label's own rows + -- (see the vertex pass for why inheritance requires ONLY). + EXECUTE format( + 'SELECT EXISTS (' + ' SELECT 1 FROM ONLY %s x ' + ' JOIN _ag_sg_vmap vs ON vs.old_id = x.start_id ' + ' JOIN _ag_sg_vmap ve ON ve.old_id = x.end_id ' + ' WHERE EXISTS (SELECT 1 FROM _ag_sg_kept_e k WHERE k.gid = x.id))', + rec.relation::regclass::text) + INTO has_rows; + IF NOT has_rows THEN + CONTINUE; + END IF; + + IF rec.name <> '_ag_label_edge' THEN + PERFORM 1 FROM ag_catalog.ag_label + WHERE graph = new_oid AND name = rec.name; + IF NOT FOUND THEN + EXECUTE format('SELECT ag_catalog.create_elabel(%L, %L)', + new_graph, rec.name); + END IF; + END IF; + + SELECT id, seq_name, relation::regclass::text + INTO dst_label_id, dst_seq_fqn, dst_relation + FROM ag_catalog.ag_label + WHERE graph = new_oid AND name = rec.name; + dst_seq_fqn := format('%I.%I', new_graph, dst_seq_fqn); + + -- Stage surviving edges, remapping endpoints through _ag_sg_vmap. The + -- joins enforce the induced rule (both endpoints kept); membership in + -- _ag_sg_kept_e applies relationship_filter. + DROP TABLE IF EXISTS _ag_sg_estage; + EXECUTE format( + 'CREATE TEMP TABLE _ag_sg_estage ON COMMIT DROP AS ' + 'SELECT ag_catalog._graphid(%s, nextval(%L)) AS new_id, ' + ' vs.new_id AS new_start, ve.new_id AS new_end, ' + ' x.properties AS props ' + 'FROM ONLY %s x ' + 'JOIN _ag_sg_vmap vs ON vs.old_id = x.start_id ' + 'JOIN _ag_sg_vmap ve ON ve.old_id = x.end_id ' + 'WHERE EXISTS (SELECT 1 FROM _ag_sg_kept_e k WHERE k.gid = x.id)', + dst_label_id, dst_seq_fqn, rec.relation::regclass::text); + + EXECUTE format('INSERT INTO %s (id, start_id, end_id, properties) ' + 'SELECT new_id, new_start, new_end, props ' + 'FROM _ag_sg_estage', dst_relation); + GET DIAGNOSTICS inserted = ROW_COUNT; + v_rel_count := v_rel_count + inserted; + + DROP TABLE _ag_sg_estage; + END LOOP; + + RETURN QUERY SELECT v_node_count, v_rel_count; +END; +$function$; + +COMMENT ON FUNCTION ag_catalog.create_subgraph(name, name, text, text) IS +'Materializes a new persistent graph as the induced subgraph of from_graph selected by a Cypher node predicate (on n) and relationship predicate (on r); ''*'' keeps all. An edge is kept only if its predicate holds and both endpoints are kept. Returns (node_count, relationship_count).'; diff --git a/regress/expected/subgraph.out b/regress/expected/subgraph.out new file mode 100644 index 000000000..a27569a5d --- /dev/null +++ b/regress/expected/subgraph.out @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +-- Suppress the create_graph / create_vlabel NOTICE chatter so the assertions +-- below are the deterministic output. (The feature is exercised regardless.) +SET client_min_messages = warning; +-- +-- Build a "somewhat large" source graph with NO MATCH (fast bulk CREATE): +-- * 2000 isolated components, each (:Person{pid,age})-[:KNOWS{w}]->(:Friend{pid}) +-- => 2000 Person + 2000 Friend vertices, 2000 KNOWS edges +-- * 500 isolated :Company vertices (no edges) +-- Totals: 4500 vertices, 2000 edges, label set {Person,Friend,Company,KNOWS}. +-- +SELECT create_graph('sg_src'); + create_graph +-------------- + +(1 row) + +SELECT count(*) FROM cypher('sg_src', $$ + UNWIND range(1, 2000) AS i + CREATE (:Person {pid: i, age: i % 100})-[:KNOWS {w: i}]->(:Friend {pid: i}) +$$) AS (a agtype); + count +------- + 0 +(1 row) + +SELECT count(*) FROM cypher('sg_src', $$ + UNWIND range(1, 500) AS i CREATE (:Company {cid: i}) +$$) AS (a agtype); + count +------- + 0 +(1 row) + +-- Source baseline (printed for reference; deterministic). +SELECT + (SELECT count(*) FROM cypher('sg_src', $$ MATCH (n) RETURN n $$) AS (n agtype)) AS src_vertices, + (SELECT count(*) FROM cypher('sg_src', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) AS src_edges; + src_vertices | src_edges +--------------+----------- + 4500 | 2000 +(1 row) + +-- +-- 1. Full copy ('*','*'): counts equal the source, and the new graph round-trips. +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_all', 'sg_src', '*', '*'); + node_count | relationship_count +------------+-------------------- + 4500 | 2000 +(1 row) + +SELECT + (SELECT count(*) FROM cypher('sg_all', $$ MATCH (n) RETURN n $$) AS (n agtype)) + = (SELECT count(*) FROM cypher('sg_src', $$ MATCH (n) RETURN n $$) AS (n agtype)) AS nodes_match, + (SELECT count(*) FROM cypher('sg_all', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) + = (SELECT count(*) FROM cypher('sg_src', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) AS edges_match; + nodes_match | edges_match +-------------+------------- + t | t +(1 row) + +-- +-- 2. Vertex-induced (node filter only): keep pid <= 1000. An edge survives iff +-- BOTH endpoints survive (induced rule), with no relationship filter. +-- node_count is asserted against the function return; correctness is verified +-- by recomputing the induced set from the source (robust booleans). +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_v', 'sg_src', 'n.pid <= 1000', '*'); + node_count | relationship_count +------------+-------------------- + 2000 | 1000 +(1 row) + +SELECT + (SELECT count(*) FROM cypher('sg_v', $$ MATCH (n) RETURN n $$) AS (n agtype)) + = (SELECT count(*) FROM cypher('sg_src', + $$ MATCH (n) WHERE n.pid <= 1000 RETURN n $$) AS (n agtype)) AS nodes_ok, + (SELECT count(*) FROM cypher('sg_v', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) + = (SELECT count(*) FROM cypher('sg_src', + $$ MATCH (a)-[r]->(b) WHERE a.pid <= 1000 AND b.pid <= 1000 RETURN r $$) + AS (r agtype)) AS edges_ok; + nodes_ok | edges_ok +----------+---------- + t | t +(1 row) + +-- +-- 3. Node + relationship predicate: keep pid <= 1000 vertices and w <= 300 edges. +-- Edge survives iff w<=300 AND both endpoints pid<=1000. +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_nr', 'sg_src', 'n.pid <= 1000', 'r.w <= 300'); + node_count | relationship_count +------------+-------------------- + 2000 | 300 +(1 row) + +SELECT + (SELECT count(*) FROM cypher('sg_nr', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) + = (SELECT count(*) FROM cypher('sg_src', + $$ MATCH (a)-[r]->(b) WHERE r.w <= 300 AND a.pid <= 1000 AND b.pid <= 1000 + RETURN r $$) AS (r agtype)) AS edges_ok; + edges_ok +---------- + t +(1 row) + +-- +-- 4. Label filter excludes one endpoint type: keep only :Person. Every KNOWS +-- edge points Person->Friend, so all edges must be dropped (induced rule). +-- (AGE evaluates label predicates with label(n); GDS uses n:Person -- same +-- containment semantics, different predicate syntax.) +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_person', 'sg_src', $f$label(n) = 'Person'$f$, '*'); + node_count | relationship_count +------------+-------------------- + 2000 | 0 +(1 row) + +-- +-- 5. Bipartite (type filter): keep Person+Friend and KNOWS edges => all 2000. +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_bip', 'sg_src', + $f$label(n) = 'Person' OR label(n) = 'Friend'$f$, + $f$label(r) = 'KNOWS'$f$); + node_count | relationship_count +------------+-------------------- + 4000 | 2000 +(1 row) + +-- +-- 6. Empty result: a predicate matching nothing yields an empty subgraph +-- (not an error), with the default labels only. +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_empty', 'sg_src', 'n.pid < 0', '*'); + node_count | relationship_count +------------+-------------------- + 0 | 0 +(1 row) + +SELECT count(*) AS empty_vertices +FROM cypher('sg_empty', $$ MATCH (n) RETURN n $$) AS (n agtype); + empty_vertices +---------------- + 0 +(1 row) + +-- +-- 7. Composability: extract a subgraph from an already-extracted subgraph. +-- From sg_v (pid<=1000) keep pid<=500; verify against recomputation on sg_v. +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_v2', 'sg_v', 'n.pid <= 500', '*'); + node_count | relationship_count +------------+-------------------- + 1000 | 500 +(1 row) + +SELECT + (SELECT count(*) FROM cypher('sg_v2', $$ MATCH (n) RETURN n $$) AS (n agtype)) + = (SELECT count(*) FROM cypher('sg_v', + $$ MATCH (n) WHERE n.pid <= 500 RETURN n $$) AS (n agtype)) AS nodes_ok, + (SELECT count(*) FROM cypher('sg_v2', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) + = (SELECT count(*) FROM cypher('sg_v', + $$ MATCH (a)-[r]->(b) WHERE a.pid <= 500 AND b.pid <= 500 RETURN r $$) + AS (r agtype)) AS edges_ok; + nodes_ok | edges_ok +----------+---------- + t | t +(1 row) + +-- +-- 8. Self-loops and parallel edges (multigraph structure) are preserved. +-- +SELECT create_graph('sg_multi'); + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('sg_multi', $$ + CREATE (a:N {k: 1}) CREATE (a)-[:E {t: 1}]->(a) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT * FROM cypher('sg_multi', $$ + CREATE (a:N {k: 2}), (b:N {k: 3}), + (a)-[:E {t: 2}]->(b), (a)-[:E {t: 3}]->(b) +$$) AS (a agtype); + a +--- +(0 rows) + +SELECT node_count, relationship_count +FROM create_subgraph('sg_multi_sub', 'sg_multi', '*', '*'); + node_count | relationship_count +------------+-------------------- + 3 | 3 +(1 row) + +-- self-loop preserved (exactly one edge from a node to itself) +SELECT count(*) AS self_loops +FROM cypher('sg_multi_sub', $$ MATCH (a)-[r]->(a) RETURN r $$) AS (r agtype); + self_loops +------------ + 1 +(1 row) + +-- parallel edges preserved (two edges between k=2 and k=3) +SELECT count(*) AS parallel_edges +FROM cypher('sg_multi_sub', $$ MATCH (a {k: 2})-[r]->(b {k: 3}) RETURN r $$) AS (r agtype); + parallel_edges +---------------- + 2 +(1 row) + +-- +-- 9. Property fidelity: a copied vertex keeps its properties verbatim. +-- +SELECT count(*) AS person_500_age_ok +FROM cypher('sg_v', $$ MATCH (n:Person {pid: 500}) WHERE n.age = 0 RETURN n $$) AS (n agtype); + person_500_age_ok +------------------- + 1 +(1 row) + +-- +-- 10. Error handling / edge cases. +-- +-- NULL graph name +SELECT create_subgraph(NULL, 'sg_src', '*', '*'); +ERROR: new graph name must not be NULL +CONTEXT: PL/pgSQL function create_subgraph(name,name,text,text) line 18 at RAISE +-- source does not exist +SELECT create_subgraph('sg_x', 'no_such_graph', '*', '*'); +ERROR: graph "no_such_graph" does not exist +CONTEXT: PL/pgSQL function create_subgraph(name,name,text,text) line 47 at RAISE +-- extracting into the source itself +SELECT create_subgraph('sg_src', 'sg_src', '*', '*'); +ERROR: cannot extract a subgraph of "sg_src" into itself +CONTEXT: PL/pgSQL function create_subgraph(name,name,text,text) line 24 at RAISE +-- destination already exists +SELECT create_subgraph('sg_all', 'sg_src', '*', '*'); +ERROR: graph "sg_all" already exists +CONTEXT: PL/pgSQL function create_subgraph(name,name,text,text) line 53 at RAISE +-- invalid Cypher predicate is reported (propagated from the engine) +SELECT create_subgraph('sg_bad', 'sg_src', 'n.pid <<>> 1', '*'); +ERROR: operator does not exist: agtype <<>> agtype +LINE 1: ...her('sg_src', $age_subgraph$MATCH (n) WHERE n.pid <<>> 1 RET... + ^ +HINT: No operator matches the given name and argument types. You might need to add explicit type casts. +QUERY: CREATE TEMP TABLE _ag_sg_kept_v ON COMMIT DROP AS SELECT DISTINCT ag_catalog.agtype_to_graphid(vid) AS gid FROM ag_catalog.cypher('sg_src', $age_subgraph$MATCH (n) WHERE n.pid <<>> 1 RETURN id(n)$age_subgraph$) AS (vid agtype) +CONTEXT: PL/pgSQL function create_subgraph(name,name,text,text) line 80 at EXECUTE +-- cleanup +SELECT drop_graph('sg_v2', true); + drop_graph +------------ + +(1 row) + +SELECT drop_graph('sg_multi_sub', true); + drop_graph +------------ + +(1 row) + +SELECT drop_graph('sg_multi', true); + drop_graph +------------ + +(1 row) + +SELECT drop_graph('sg_empty', true); + drop_graph +------------ + +(1 row) + +SELECT drop_graph('sg_bip', true); + drop_graph +------------ + +(1 row) + +SELECT drop_graph('sg_person', true); + drop_graph +------------ + +(1 row) + +SELECT drop_graph('sg_nr', true); + drop_graph +------------ + +(1 row) + +SELECT drop_graph('sg_v', true); + drop_graph +------------ + +(1 row) + +SELECT drop_graph('sg_all', true); + drop_graph +------------ + +(1 row) + +SELECT drop_graph('sg_src', true); + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/subgraph.sql b/regress/sql/subgraph.sql new file mode 100644 index 000000000..0d01dfe60 --- /dev/null +++ b/regress/sql/subgraph.sql @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +-- Suppress the create_graph / create_vlabel NOTICE chatter so the assertions +-- below are the deterministic output. (The feature is exercised regardless.) +SET client_min_messages = warning; + +-- +-- Build a "somewhat large" source graph with NO MATCH (fast bulk CREATE): +-- * 2000 isolated components, each (:Person{pid,age})-[:KNOWS{w}]->(:Friend{pid}) +-- => 2000 Person + 2000 Friend vertices, 2000 KNOWS edges +-- * 500 isolated :Company vertices (no edges) +-- Totals: 4500 vertices, 2000 edges, label set {Person,Friend,Company,KNOWS}. +-- +SELECT create_graph('sg_src'); + +SELECT count(*) FROM cypher('sg_src', $$ + UNWIND range(1, 2000) AS i + CREATE (:Person {pid: i, age: i % 100})-[:KNOWS {w: i}]->(:Friend {pid: i}) +$$) AS (a agtype); + +SELECT count(*) FROM cypher('sg_src', $$ + UNWIND range(1, 500) AS i CREATE (:Company {cid: i}) +$$) AS (a agtype); + +-- Source baseline (printed for reference; deterministic). +SELECT + (SELECT count(*) FROM cypher('sg_src', $$ MATCH (n) RETURN n $$) AS (n agtype)) AS src_vertices, + (SELECT count(*) FROM cypher('sg_src', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) AS src_edges; + +-- +-- 1. Full copy ('*','*'): counts equal the source, and the new graph round-trips. +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_all', 'sg_src', '*', '*'); + +SELECT + (SELECT count(*) FROM cypher('sg_all', $$ MATCH (n) RETURN n $$) AS (n agtype)) + = (SELECT count(*) FROM cypher('sg_src', $$ MATCH (n) RETURN n $$) AS (n agtype)) AS nodes_match, + (SELECT count(*) FROM cypher('sg_all', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) + = (SELECT count(*) FROM cypher('sg_src', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) AS edges_match; + +-- +-- 2. Vertex-induced (node filter only): keep pid <= 1000. An edge survives iff +-- BOTH endpoints survive (induced rule), with no relationship filter. +-- node_count is asserted against the function return; correctness is verified +-- by recomputing the induced set from the source (robust booleans). +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_v', 'sg_src', 'n.pid <= 1000', '*'); + +SELECT + (SELECT count(*) FROM cypher('sg_v', $$ MATCH (n) RETURN n $$) AS (n agtype)) + = (SELECT count(*) FROM cypher('sg_src', + $$ MATCH (n) WHERE n.pid <= 1000 RETURN n $$) AS (n agtype)) AS nodes_ok, + (SELECT count(*) FROM cypher('sg_v', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) + = (SELECT count(*) FROM cypher('sg_src', + $$ MATCH (a)-[r]->(b) WHERE a.pid <= 1000 AND b.pid <= 1000 RETURN r $$) + AS (r agtype)) AS edges_ok; + +-- +-- 3. Node + relationship predicate: keep pid <= 1000 vertices and w <= 300 edges. +-- Edge survives iff w<=300 AND both endpoints pid<=1000. +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_nr', 'sg_src', 'n.pid <= 1000', 'r.w <= 300'); + +SELECT + (SELECT count(*) FROM cypher('sg_nr', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) + = (SELECT count(*) FROM cypher('sg_src', + $$ MATCH (a)-[r]->(b) WHERE r.w <= 300 AND a.pid <= 1000 AND b.pid <= 1000 + RETURN r $$) AS (r agtype)) AS edges_ok; + +-- +-- 4. Label filter excludes one endpoint type: keep only :Person. Every KNOWS +-- edge points Person->Friend, so all edges must be dropped (induced rule). +-- (AGE evaluates label predicates with label(n); GDS uses n:Person -- same +-- containment semantics, different predicate syntax.) +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_person', 'sg_src', $f$label(n) = 'Person'$f$, '*'); + +-- +-- 5. Bipartite (type filter): keep Person+Friend and KNOWS edges => all 2000. +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_bip', 'sg_src', + $f$label(n) = 'Person' OR label(n) = 'Friend'$f$, + $f$label(r) = 'KNOWS'$f$); + +-- +-- 6. Empty result: a predicate matching nothing yields an empty subgraph +-- (not an error), with the default labels only. +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_empty', 'sg_src', 'n.pid < 0', '*'); + +SELECT count(*) AS empty_vertices +FROM cypher('sg_empty', $$ MATCH (n) RETURN n $$) AS (n agtype); + +-- +-- 7. Composability: extract a subgraph from an already-extracted subgraph. +-- From sg_v (pid<=1000) keep pid<=500; verify against recomputation on sg_v. +-- +SELECT node_count, relationship_count +FROM create_subgraph('sg_v2', 'sg_v', 'n.pid <= 500', '*'); + +SELECT + (SELECT count(*) FROM cypher('sg_v2', $$ MATCH (n) RETURN n $$) AS (n agtype)) + = (SELECT count(*) FROM cypher('sg_v', + $$ MATCH (n) WHERE n.pid <= 500 RETURN n $$) AS (n agtype)) AS nodes_ok, + (SELECT count(*) FROM cypher('sg_v2', $$ MATCH ()-[r]->() RETURN r $$) AS (r agtype)) + = (SELECT count(*) FROM cypher('sg_v', + $$ MATCH (a)-[r]->(b) WHERE a.pid <= 500 AND b.pid <= 500 RETURN r $$) + AS (r agtype)) AS edges_ok; + +-- +-- 8. Self-loops and parallel edges (multigraph structure) are preserved. +-- +SELECT create_graph('sg_multi'); +SELECT * FROM cypher('sg_multi', $$ + CREATE (a:N {k: 1}) CREATE (a)-[:E {t: 1}]->(a) +$$) AS (a agtype); +SELECT * FROM cypher('sg_multi', $$ + CREATE (a:N {k: 2}), (b:N {k: 3}), + (a)-[:E {t: 2}]->(b), (a)-[:E {t: 3}]->(b) +$$) AS (a agtype); + +SELECT node_count, relationship_count +FROM create_subgraph('sg_multi_sub', 'sg_multi', '*', '*'); + +-- self-loop preserved (exactly one edge from a node to itself) +SELECT count(*) AS self_loops +FROM cypher('sg_multi_sub', $$ MATCH (a)-[r]->(a) RETURN r $$) AS (r agtype); + +-- parallel edges preserved (two edges between k=2 and k=3) +SELECT count(*) AS parallel_edges +FROM cypher('sg_multi_sub', $$ MATCH (a {k: 2})-[r]->(b {k: 3}) RETURN r $$) AS (r agtype); + +-- +-- 9. Property fidelity: a copied vertex keeps its properties verbatim. +-- +SELECT count(*) AS person_500_age_ok +FROM cypher('sg_v', $$ MATCH (n:Person {pid: 500}) WHERE n.age = 0 RETURN n $$) AS (n agtype); + +-- +-- 10. Error handling / edge cases. +-- +-- NULL graph name +SELECT create_subgraph(NULL, 'sg_src', '*', '*'); +-- source does not exist +SELECT create_subgraph('sg_x', 'no_such_graph', '*', '*'); +-- extracting into the source itself +SELECT create_subgraph('sg_src', 'sg_src', '*', '*'); +-- destination already exists +SELECT create_subgraph('sg_all', 'sg_src', '*', '*'); +-- invalid Cypher predicate is reported (propagated from the engine) +SELECT create_subgraph('sg_bad', 'sg_src', 'n.pid <<>> 1', '*'); + +-- cleanup +SELECT drop_graph('sg_v2', true); +SELECT drop_graph('sg_multi_sub', true); +SELECT drop_graph('sg_multi', true); +SELECT drop_graph('sg_empty', true); +SELECT drop_graph('sg_bip', true); +SELECT drop_graph('sg_person', true); +SELECT drop_graph('sg_nr', true); +SELECT drop_graph('sg_v', true); +SELECT drop_graph('sg_all', true); +SELECT drop_graph('sg_src', true); diff --git a/sql/age_subgraph.sql b/sql/age_subgraph.sql new file mode 100644 index 000000000..960790ded --- /dev/null +++ b/sql/age_subgraph.sql @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +-- +-- create_subgraph(): materialized subgraph extraction. +-- +-- Builds a new, persistent AGE graph that is the subgraph of an existing graph +-- selected by a node predicate and a relationship predicate. The semantics +-- follow the graph-theory "induced subgraph" definition as operationalized by +-- Neo4j GDS gds.graph.filter(): +-- +-- * a vertex is kept iff node_filter evaluates true ('*' keeps all); +-- * an edge is kept iff relationship_filter evaluates true AND BOTH of its +-- endpoints were kept (the induced rule -- no dangling edges). +-- +-- Unlike the Neo4j in-memory projection, the result is a real, ACID, +-- fully-Cypher-queryable AGE graph; properties of any agtype are preserved, and +-- self-loops / parallel edges (multigraph structure) are kept. +-- +-- node_filter / relationship_filter are Cypher predicates bound to a single +-- entity -- the node variable is `n`, the relationship variable is `r` -- or +-- the literal '*' to keep all. They are evaluated by AGE's own Cypher engine +-- against the source graph, so the full Cypher predicate language is available. +-- +-- Internal entity ids (graphids) are reassigned in the new graph (a graphid +-- encodes the source graph's label id, which differs in the destination), and +-- edge endpoints are remapped accordingly. Properties are copied verbatim. +-- +CREATE FUNCTION ag_catalog.create_subgraph(new_graph name, + from_graph name, + node_filter text DEFAULT '*', + relationship_filter text DEFAULT '*') + RETURNS TABLE(node_count bigint, relationship_count bigint) + LANGUAGE plpgsql + VOLATILE + SET search_path = ag_catalog, pg_catalog + AS $function$ +DECLARE + from_oid oid; + new_oid oid; + v_node_count bigint := 0; + v_rel_count bigint := 0; + rec RECORD; + cypher_q text; + where_clause text; + dst_label_id int; + dst_seq_fqn text; + dst_relation text; + inserted bigint; + has_rows boolean; +BEGIN + -- Argument validation. + IF new_graph IS NULL THEN + RAISE EXCEPTION 'new graph name must not be NULL'; + END IF; + IF from_graph IS NULL THEN + RAISE EXCEPTION 'source graph name must not be NULL'; + END IF; + IF new_graph = from_graph THEN + RAISE EXCEPTION 'cannot extract a subgraph of "%" into itself', from_graph; + END IF; + + -- NULL predicate is treated as the '*' wildcard (keep all). + IF node_filter IS NULL THEN + node_filter := '*'; + END IF; + IF relationship_filter IS NULL THEN + relationship_filter := '*'; + END IF; + + -- The predicates are embedded into a dollar-quoted cypher() query using the + -- $age_subgraph$ tag; reject predicates that contain the tag to keep the + -- quoting unambiguous. + IF position('$age_subgraph$' IN node_filter) > 0 + OR position('$age_subgraph$' IN relationship_filter) > 0 THEN + RAISE EXCEPTION 'filter predicate must not contain the reserved token $age_subgraph$'; + END IF; + + -- Validate source graph exists. + SELECT graphid INTO from_oid + FROM ag_catalog.ag_graph WHERE name = from_graph; + IF from_oid IS NULL THEN + RAISE EXCEPTION 'graph "%" does not exist', from_graph; + END IF; + + -- Validate destination graph does not exist (create_graph also enforces + -- naming rules and uniqueness, but we give a clear early error). + IF EXISTS (SELECT 1 FROM ag_catalog.ag_graph WHERE name = new_graph) THEN + RAISE EXCEPTION 'graph "%" already exists', new_graph; + END IF; + + -- Create the destination graph (default labels are created automatically). + PERFORM ag_catalog.create_graph(new_graph); + + SELECT graphid INTO new_oid + FROM ag_catalog.ag_graph WHERE name = new_graph; + + -- Working sets / mapping (uniquely named to avoid colliding with user temps). + DROP TABLE IF EXISTS _ag_sg_kept_v; + DROP TABLE IF EXISTS _ag_sg_kept_e; + DROP TABLE IF EXISTS _ag_sg_vmap; + DROP TABLE IF EXISTS _ag_sg_vstage; + DROP TABLE IF EXISTS _ag_sg_estage; + + -- + -- Kept vertices: evaluate node_filter with AGE's Cypher engine. The node + -- variable `n` is bound exactly as in the spec; '*' selects all vertices. + -- + IF node_filter IS NULL OR btrim(node_filter) = '*' THEN + where_clause := ''; + ELSE + where_clause := ' WHERE ' || node_filter; + END IF; + cypher_q := 'MATCH (n)' || where_clause || ' RETURN id(n)'; + + EXECUTE format( + 'CREATE TEMP TABLE _ag_sg_kept_v ON COMMIT DROP AS ' + 'SELECT DISTINCT ag_catalog.agtype_to_graphid(vid) AS gid ' + 'FROM ag_catalog.cypher(%L, $age_subgraph$%s$age_subgraph$) AS (vid agtype)', + from_graph, cypher_q); + CREATE INDEX ON _ag_sg_kept_v (gid); + + -- + -- Kept edges: evaluate relationship_filter with AGE's Cypher engine. The + -- relationship variable `r` is bound exactly as in the spec. + -- + IF relationship_filter IS NULL OR btrim(relationship_filter) = '*' THEN + where_clause := ''; + ELSE + where_clause := ' WHERE ' || relationship_filter; + END IF; + cypher_q := 'MATCH ()-[r]->()' || where_clause || ' RETURN id(r)'; + + EXECUTE format( + 'CREATE TEMP TABLE _ag_sg_kept_e ON COMMIT DROP AS ' + 'SELECT DISTINCT ag_catalog.agtype_to_graphid(eid) AS gid ' + 'FROM ag_catalog.cypher(%L, $age_subgraph$%s$age_subgraph$) AS (eid agtype)', + from_graph, cypher_q); + CREATE INDEX ON _ag_sg_kept_e (gid); + + -- old -> new vertex id mapping (graphid is unique within a graph). + CREATE TEMP TABLE _ag_sg_vmap (old_id graphid PRIMARY KEY, + new_id graphid NOT NULL) ON COMMIT DROP; + + -- + -- PASS 1: copy kept vertices, label by label, assigning new graphids and + -- recording the old->new mapping for edge remapping. + -- + FOR rec IN + SELECT name, id, relation, seq_name + FROM ag_catalog.ag_label + WHERE graph = from_oid AND kind = 'v' + ORDER BY id + LOOP + -- Skip labels with no surviving vertices. Read ONLY this label's own + -- rows: AGE label tables use table inheritance (custom labels inherit + -- from _ag_label_vertex), so a plain scan of a parent would also return + -- its children and copy them twice. + EXECUTE format( + 'SELECT EXISTS (SELECT 1 FROM ONLY %s t ' + 'WHERE EXISTS (SELECT 1 FROM _ag_sg_kept_v k WHERE k.gid = t.id))', + rec.relation::regclass::text) + INTO has_rows; + IF NOT has_rows THEN + CONTINUE; + END IF; + + -- Ensure the label exists in the destination graph. + IF rec.name <> '_ag_label_vertex' THEN + PERFORM 1 FROM ag_catalog.ag_label + WHERE graph = new_oid AND name = rec.name; + IF NOT FOUND THEN + EXECUTE format('SELECT ag_catalog.create_vlabel(%L, %L)', + new_graph, rec.name); + END IF; + END IF; + + SELECT id, seq_name, relation::regclass::text + INTO dst_label_id, dst_seq_fqn, dst_relation + FROM ag_catalog.ag_label + WHERE graph = new_oid AND name = rec.name; + dst_seq_fqn := format('%I.%I', new_graph, dst_seq_fqn); + + -- Stage surviving vertices with freshly generated ids in a real temp + -- table (single evaluation), then copy to the label table and record + -- the old->new mapping. A materialized stage avoids any ambiguity from + -- referencing a nextval-bearing CTE more than once. + DROP TABLE IF EXISTS _ag_sg_vstage; + EXECUTE format( + 'CREATE TEMP TABLE _ag_sg_vstage ON COMMIT DROP AS ' + 'SELECT t.id AS old_id, ' + ' ag_catalog._graphid(%s, nextval(%L)) AS new_id, ' + ' t.properties AS props ' + 'FROM ONLY %s t ' + 'WHERE EXISTS (SELECT 1 FROM _ag_sg_kept_v k WHERE k.gid = t.id)', + dst_label_id, dst_seq_fqn, rec.relation::regclass::text); + + EXECUTE format('INSERT INTO %s (id, properties) ' + 'SELECT new_id, props FROM _ag_sg_vstage', dst_relation); + + INSERT INTO _ag_sg_vmap (old_id, new_id) + SELECT old_id, new_id FROM _ag_sg_vstage; + + DROP TABLE _ag_sg_vstage; + END LOOP; + + SELECT count(*) INTO v_node_count FROM _ag_sg_vmap; + + -- + -- PASS 2: copy kept edges, remapping endpoints. The joins to _ag_sg_vmap + -- enforce the induced rule (an edge survives only if BOTH endpoints were + -- kept); membership in _ag_sg_kept_e applies relationship_filter. + -- + FOR rec IN + SELECT name, id, relation, seq_name + FROM ag_catalog.ag_label + WHERE graph = from_oid AND kind = 'e' + ORDER BY id + LOOP + -- Skip labels with no surviving edges. Read ONLY this label's own rows + -- (see the vertex pass for why inheritance requires ONLY). + EXECUTE format( + 'SELECT EXISTS (' + ' SELECT 1 FROM ONLY %s x ' + ' JOIN _ag_sg_vmap vs ON vs.old_id = x.start_id ' + ' JOIN _ag_sg_vmap ve ON ve.old_id = x.end_id ' + ' WHERE EXISTS (SELECT 1 FROM _ag_sg_kept_e k WHERE k.gid = x.id))', + rec.relation::regclass::text) + INTO has_rows; + IF NOT has_rows THEN + CONTINUE; + END IF; + + IF rec.name <> '_ag_label_edge' THEN + PERFORM 1 FROM ag_catalog.ag_label + WHERE graph = new_oid AND name = rec.name; + IF NOT FOUND THEN + EXECUTE format('SELECT ag_catalog.create_elabel(%L, %L)', + new_graph, rec.name); + END IF; + END IF; + + SELECT id, seq_name, relation::regclass::text + INTO dst_label_id, dst_seq_fqn, dst_relation + FROM ag_catalog.ag_label + WHERE graph = new_oid AND name = rec.name; + dst_seq_fqn := format('%I.%I', new_graph, dst_seq_fqn); + + -- Stage surviving edges, remapping endpoints through _ag_sg_vmap. The + -- joins enforce the induced rule (both endpoints kept); membership in + -- _ag_sg_kept_e applies relationship_filter. + DROP TABLE IF EXISTS _ag_sg_estage; + EXECUTE format( + 'CREATE TEMP TABLE _ag_sg_estage ON COMMIT DROP AS ' + 'SELECT ag_catalog._graphid(%s, nextval(%L)) AS new_id, ' + ' vs.new_id AS new_start, ve.new_id AS new_end, ' + ' x.properties AS props ' + 'FROM ONLY %s x ' + 'JOIN _ag_sg_vmap vs ON vs.old_id = x.start_id ' + 'JOIN _ag_sg_vmap ve ON ve.old_id = x.end_id ' + 'WHERE EXISTS (SELECT 1 FROM _ag_sg_kept_e k WHERE k.gid = x.id)', + dst_label_id, dst_seq_fqn, rec.relation::regclass::text); + + EXECUTE format('INSERT INTO %s (id, start_id, end_id, properties) ' + 'SELECT new_id, new_start, new_end, props ' + 'FROM _ag_sg_estage', dst_relation); + GET DIAGNOSTICS inserted = ROW_COUNT; + v_rel_count := v_rel_count + inserted; + + DROP TABLE _ag_sg_estage; + END LOOP; + + RETURN QUERY SELECT v_node_count, v_rel_count; +END; +$function$; + +COMMENT ON FUNCTION ag_catalog.create_subgraph(name, name, text, text) IS +'Materializes a new persistent graph as the induced subgraph of from_graph selected by a Cypher node predicate (on n) and relationship predicate (on r); ''*'' keeps all. An edge is kept only if its predicate holds and both endpoints are kept. Returns (node_count, relationship_count).'; diff --git a/sql/sql_files b/sql/sql_files index 32f9a7099..996ad4b46 100644 --- a/sql/sql_files +++ b/sql/sql_files @@ -15,3 +15,4 @@ age_trig age_aggregate agtype_typecast age_pg_upgrade +age_subgraph From 932d26f2543278ce9ce060b42d6765713aec27f2 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Sat, 20 Jun 2026 09:56:48 -0700 Subject: [PATCH 079/100] cypher_vle: add ORDER BY to non-deterministic RETURN queries (#2434) Several VLE regression queries RETURN multiple rows without an ORDER BY, so their row order depends on traversal/scan order and can vary between runs and platforms. Add ORDER BY ASC to those queries (on the path, edge-list, or graphid as appropriate) so the expected output is stable. The queries are pinned by path (p), edge list (e), or graphid (id(u)/id(v)/id(e[n])) depending on what each RETURN projects. Full audit of cypher_vle: all 38 multi-row result blocks were checked. After this change, every multi-row RETURN is deterministically ordered except the two SELECT * FROM show_list_use_vle('list01') calls, which are already deterministic because the function body orders its results with RETURN v ORDER BY id(v) (added in #2417); their result blocks are unchanged by this commit. This is a test-only change (regress/sql/cypher_vle.sql and regress/expected/cypher_vle.out); no extension C code or SQL is modified. Row counts are unchanged (pure reordering). All 37 regression tests pass (installcheck) on PostgreSQL 18.3. Co-authored-by: GitHub Copilot modified: regress/expected/cypher_vle.out modified: regress/sql/cypher_vle.sql --- regress/expected/cypher_vle.out | 156 ++++++++++++++++---------------- regress/sql/cypher_vle.sql | 78 ++++++++-------- 2 files changed, 117 insertions(+), 117 deletions(-) diff --git a/regress/expected/cypher_vle.out b/regress/expected/cypher_vle.out index 0f564015b..de85176b6 100644 --- a/regress/expected/cypher_vle.out +++ b/regress/expected/cypher_vle.out @@ -281,7 +281,7 @@ SELECT * FROM cypher('cypher_vle', $$MATCH ()-[*]->(v) RETURN count(*) $$) AS (e (1 row) -- Should find 2 -SELECT * FROM cypher('cypher_vle', $$MATCH (u:begin)<-[e*]-(v:end) RETURN e $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH (u:begin)<-[e*]-(v:end) RETURN e ORDER BY e ASC $$) AS (e agtype); e ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge] @@ -289,7 +289,7 @@ SELECT * FROM cypher('cypher_vle', $$MATCH (u:begin)<-[e*]-(v:end) RETURN e $$) (2 rows) -- Should find 5 -SELECT * FROM cypher('cypher_vle', $$MATCH p=(:begin)<-[*1..1]-()-[]-() RETURN p ORDER BY p $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(:begin)<-[*1..1]-()-[]-() RETURN p ORDER BY p ASC $$) AS (e agtype); e --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path @@ -307,54 +307,54 @@ SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[*]->(v) RETURN count(*) $$) AS (1 row) -- Should find 2 -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[*3..3]->(v:end) RETURN p $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[*3..3]->(v:end) RETURN p ORDER BY p ASC $$) AS (e agtype); e ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path (2 rows) -- Should find 12 -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[*3..3]-(v:end) RETURN p $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[*3..3]-(v:end) RETURN p ORDER BY p ASC $$) AS (e agtype); e ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path (12 rows) -- Each should find 2 -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[*]-(v:end) RETURN p $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[*]-(v:end) RETURN p ORDER BY p ASC $$) AS (e agtype); e ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path (2 rows) -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[e*]-(v:end) RETURN p $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[e*]-(v:end) RETURN p ORDER BY p ASC $$) AS (e agtype); e ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path (2 rows) -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[e*]-(v:end) RETURN e $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[e*]-(v:end) RETURN e ORDER BY e ASC $$) AS (e agtype); e ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge] [{"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge] (2 rows) -SELECT * FROM cypher('cypher_vle', $$MATCH p=(:begin)<-[*]-()<-[]-(:end) RETURN p $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(:begin)<-[*]-()<-[]-(:end) RETURN p ORDER BY p ASC $$) AS (e agtype); e ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path @@ -460,31 +460,31 @@ $$) AS (e1 agtype, e2 agtype); (1 row) -- Should return 1 path -SELECT * FROM cypher('cypher_vle', $$ MATCH p=()<-[e1*]-(:end)-[e2*]->(:begin) RETURN p $$) AS (result agtype); +SELECT * FROM cypher('cypher_vle', $$ MATCH p=()<-[e1*]-(:end)-[e2*]->(:begin) RETURN p ORDER BY p ASC $$) AS (result agtype); result ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path (1 row) -- Each should return 3 -SELECT * FROM cypher('cypher_vle', $$MATCH (u:begin)-[e*0..1]->(v) RETURN id(u), e, id(v) $$) AS (u agtype, e agtype, v agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH (u:begin)-[e*0..1]->(v) RETURN id(u), e, id(v) ORDER BY id(u) ASC, e ASC, id(v) ASC $$) AS (u agtype, e agtype, v agtype); u | e | v -----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------ 844424930131969 | [] | 844424930131969 - 844424930131969 | [{"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge] | 1407374883553281 844424930131969 | [{"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge] | 1407374883553281 + 844424930131969 | [{"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge] | 1407374883553281 (3 rows) -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[e*0..1]->(v) RETURN p $$) AS (p agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[e*0..1]->(v) RETURN p ORDER BY p ASC $$) AS (p agtype); p -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path - [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path + [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path (3 rows) -- Each should return 5 -SELECT * FROM cypher('cypher_vle', $$MATCH (u)-[e*0..0]->(v) RETURN id(u), e, id(v) $$) AS (u agtype, e agtype, v agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH (u)-[e*0..0]->(v) RETURN id(u), e, id(v) ORDER BY id(u) ASC, e ASC, id(v) ASC $$) AS (u agtype, e agtype, v agtype); u | e | v ------------------+----+------------------ 844424930131969 | [] | 844424930131969 @@ -494,7 +494,7 @@ SELECT * FROM cypher('cypher_vle', $$MATCH (u)-[e*0..0]->(v) RETURN id(u), e, id 1688849860263937 | [] | 1688849860263937 (5 rows) -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u)-[e*0..0]->(v) RETURN id(u), p, id(v) $$) AS (u agtype, p agtype, v agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u)-[e*0..0]->(v) RETURN id(u), p, id(v) ORDER BY id(u) ASC, p ASC, id(v) ASC $$) AS (u agtype, p agtype, v agtype); u | p | v ------------------+-------------------------------------------------------------------------------+------------------ 844424930131969 | [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path | 844424930131969 @@ -505,13 +505,13 @@ SELECT * FROM cypher('cypher_vle', $$MATCH p=(u)-[e*0..0]->(v) RETURN id(u), p, (5 rows) -- Each should return 13 and will be the same -SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[*0..0]->()-[]->() RETURN p $$) AS (p agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[*0..0]->()-[]->() RETURN p ORDER BY p ASC $$) AS (p agtype); p ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1970324836974593, "label": "self_loop", "end_id": 1407374883553281, "start_id": 1407374883553281, "properties": {"name": "self loop", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1970324836974593, "label": "self_loop", "end_id": 1407374883553281, "start_id": 1407374883553281, "properties": {"name": "self loop", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path @@ -519,26 +519,26 @@ SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[*0..0]->()-[]->() RETURN p $$) [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path (13 rows) -SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[]->()-[*0..0]->() RETURN p $$) AS (p agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[]->()-[*0..0]->() RETURN p ORDER BY p ASC $$) AS (p agtype); p ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 1125899906842628, "label": "edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "main edge", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1970324836974593, "label": "self_loop", "end_id": 1407374883553281, "start_id": 1407374883553281, "properties": {"name": "self loop", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 844424930131969, "label": "begin", "properties": {}}::vertex, {"id": 2251799813685249, "label": "alternate_edge", "end_id": 1407374883553281, "start_id": 844424930131969, "properties": {"name": "alternate edge", "number": 1, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842627, "label": "edge", "end_id": 1407374883553282, "start_id": 1407374883553281, "properties": {"name": "main edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1407374883553281, "label": "middle", "properties": {}}::vertex, {"id": 1970324836974593, "label": "self_loop", "end_id": 1407374883553281, "start_id": 1407374883553281, "properties": {"name": "self loop", "number": 1, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553281, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842626, "label": "edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "main edge", "number": 3, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685250, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1407374883553282, "properties": {"name": "alternate edge", "number": 2, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path - [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395906, "label": "bypass_edge", "end_id": 844424930131969, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 2, "packages": [1, 3, 5, 7], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 844424930131969, "label": "begin", "properties": {}}::vertex]::path [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 1125899906842625, "label": "edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "main edge", "number": 4, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685251, "label": "alternate_edge", "end_id": 1688849860263937, "start_id": 1407374883553283, "properties": {"name": "alternate edge", "number": 3, "packages": [2, 4, 6], "dangerous": {"type": "poisons", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path - [{"id": 1407374883553282, "label": "middle", "properties": {}}::vertex, {"id": 2533274790395905, "label": "bypass_edge", "end_id": 1688849860263937, "start_id": 1407374883553282, "properties": {"name": "bypass edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1407374883553283, "label": "middle", "properties": {}}::vertex, {"id": 2251799813685253, "label": "alternate_edge", "end_id": 1407374883553282, "start_id": 1407374883553283, "properties": {"name": "backup edge", "number": 2, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553282, "label": "middle", "properties": {}}::vertex]::path + [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 1970324836974594, "label": "self_loop", "end_id": 1688849860263937, "start_id": 1688849860263937, "properties": {"name": "self loop", "number": 2, "dangerous": {"type": "all", "level": "all"}}}::edge, {"id": 1688849860263937, "label": "end", "properties": {}}::vertex]::path + [{"id": 1688849860263937, "label": "end", "properties": {}}::vertex, {"id": 2251799813685252, "label": "alternate_edge", "end_id": 1407374883553283, "start_id": 1688849860263937, "properties": {"name": "backup edge", "number": 1, "packages": [1, 3, 5, 7]}}::edge, {"id": 1407374883553283, "label": "middle", "properties": {}}::vertex]::path (13 rows) -- @@ -563,7 +563,7 @@ $$) AS (g1 agtype); /* should return 1 path with 1 edge */ SELECT * FROM cypher('mygraph', $$ MATCH p = ()-[:Edge*]->() - RETURN p + RETURN p ORDER BY p ASC $$) AS (g2 agtype); g2 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- @@ -583,24 +583,24 @@ $$) AS (g3 agtype); /* should find 2 paths with 1 edge */ SELECT * FROM cypher('mygraph', $$ MATCH p = ()-[:Edge]->() - RETURN p + RETURN p ORDER BY p ASC $$) AS (g4 agtype); g4 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex, {"id": 1125899906842626, "label": "Edge", "end_id": 844424930131970, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131970, "label": "Node", "properties": {"name": "c"}}::vertex]::path [{"id": 844424930131969, "label": "Node", "properties": {"name": "a"}}::vertex, {"id": 1125899906842627, "label": "Edge", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex]::path + [{"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex, {"id": 1125899906842626, "label": "Edge", "end_id": 844424930131970, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131970, "label": "Node", "properties": {"name": "c"}}::vertex]::path (2 rows) /* should return 3 paths, 2 with 1 edge, 1 with 2 edges */ SELECT * FROM cypher('mygraph', $$ MATCH p = ()-[:Edge*]->() - RETURN p + RETURN p ORDER BY p ASC $$) AS (g5 agtype); g5 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 844424930131969, "label": "Node", "properties": {"name": "a"}}::vertex, {"id": 1125899906842627, "label": "Edge", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex, {"id": 1125899906842626, "label": "Edge", "end_id": 844424930131970, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131970, "label": "Node", "properties": {"name": "c"}}::vertex]::path - [{"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex, {"id": 1125899906842626, "label": "Edge", "end_id": 844424930131970, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131970, "label": "Node", "properties": {"name": "c"}}::vertex]::path [{"id": 844424930131969, "label": "Node", "properties": {"name": "a"}}::vertex, {"id": 1125899906842627, "label": "Edge", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex]::path + [{"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex, {"id": 1125899906842626, "label": "Edge", "end_id": 844424930131970, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131970, "label": "Node", "properties": {"name": "c"}}::vertex]::path + [{"id": 844424930131969, "label": "Node", "properties": {"name": "a"}}::vertex, {"id": 1125899906842627, "label": "Edge", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131971, "label": "Node", "properties": {"name": "b"}}::vertex, {"id": 1125899906842626, "label": "Edge", "end_id": 844424930131970, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131970, "label": "Node", "properties": {"name": "c"}}::vertex]::path (3 rows) SELECT drop_graph('mygraph', true); @@ -842,22 +842,22 @@ SELECT * FROM cypher('access',$$ CREATE ()-[:knows {id:2, arry:[0,1,2,3,{name: " --------- (0 rows) -SELECT * FROM cypher('access', $$ MATCH (u)-[e*]->(v) RETURN e $$)as (edges agtype); +SELECT * FROM cypher('access', $$ MATCH (u)-[e*]->(v) RETURN e ORDER BY e ASC $$)as (edges agtype); edges ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 844424930131969, "label": "knows", "end_id": 281474976710658, "start_id": 281474976710657, "properties": {}}::edge] + [{"id": 844424930131970, "label": "knows", "end_id": 281474976710661, "start_id": 281474976710660, "properties": {}}::edge] [{"id": 844424930131971, "label": "knows", "end_id": 281474976710660, "start_id": 281474976710659, "properties": {}}::edge] [{"id": 844424930131971, "label": "knows", "end_id": 281474976710660, "start_id": 281474976710659, "properties": {}}::edge, {"id": 844424930131970, "label": "knows", "end_id": 281474976710661, "start_id": 281474976710660, "properties": {}}::edge] - [{"id": 844424930131970, "label": "knows", "end_id": 281474976710661, "start_id": 281474976710660, "properties": {}}::edge] + [{"id": 844424930131972, "label": "knows", "end_id": 281474976710664, "start_id": 281474976710663, "properties": {"id": 1}}::edge] [{"id": 844424930131973, "label": "knows", "end_id": 281474976710663, "start_id": 281474976710662, "properties": {"id": 0}}::edge] [{"id": 844424930131973, "label": "knows", "end_id": 281474976710663, "start_id": 281474976710662, "properties": {"id": 0}}::edge, {"id": 844424930131972, "label": "knows", "end_id": 281474976710664, "start_id": 281474976710663, "properties": {"id": 1}}::edge] - [{"id": 844424930131972, "label": "knows", "end_id": 281474976710664, "start_id": 281474976710663, "properties": {"id": 1}}::edge] + [{"id": 844424930131974, "label": "knows", "end_id": 281474976710667, "start_id": 281474976710666, "properties": {"id": 3, "arry": [1, 3, {"name": "john", "stats": {"age": 1000}}]}}::edge] [{"id": 844424930131975, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710665, "properties": {"id": 2, "arry": [0, 1, 2, 3, {"name": "joe"}]}}::edge] [{"id": 844424930131975, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710665, "properties": {"id": 2, "arry": [0, 1, 2, 3, {"name": "joe"}]}}::edge, {"id": 844424930131974, "label": "knows", "end_id": 281474976710667, "start_id": 281474976710666, "properties": {"id": 3, "arry": [1, 3, {"name": "john", "stats": {"age": 1000}}]}}::edge] - [{"id": 844424930131974, "label": "knows", "end_id": 281474976710667, "start_id": 281474976710666, "properties": {"id": 3, "arry": [1, 3, {"name": "john", "stats": {"age": 1000}}]}}::edge] (10 rows) -SELECT * FROM cypher('access', $$ MATCH (u)-[e*2..2]->(v) RETURN e $$)as (edges agtype); +SELECT * FROM cypher('access', $$ MATCH (u)-[e*2..2]->(v) RETURN e ORDER BY e ASC $$)as (edges agtype); edges ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [{"id": 844424930131971, "label": "knows", "end_id": 281474976710660, "start_id": 281474976710659, "properties": {}}::edge, {"id": 844424930131970, "label": "knows", "end_id": 281474976710661, "start_id": 281474976710660, "properties": {}}::edge] @@ -865,7 +865,7 @@ SELECT * FROM cypher('access', $$ MATCH (u)-[e*2..2]->(v) RETURN e $$)as (edges [{"id": 844424930131975, "label": "knows", "end_id": 281474976710666, "start_id": 281474976710665, "properties": {"id": 2, "arry": [0, 1, 2, 3, {"name": "joe"}]}}::edge, {"id": 844424930131974, "label": "knows", "end_id": 281474976710667, "start_id": 281474976710666, "properties": {"id": 3, "arry": [1, 3, {"name": "john", "stats": {"age": 1000}}]}}::edge] (3 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[0]) $$) as (prop_first_edge agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[0]) ORDER BY id(e[0]) ASC $$) as (prop_first_edge agtype); prop_first_edge -------------------------------------------------- {} @@ -873,7 +873,7 @@ SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[0]) $ {"id": 2, "arry": [0, 1, 2, 3, {"name": "joe"}]} (3 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[0].id $$) as (results agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[0].id ORDER BY id(e[0]) ASC $$) as (results agtype); results --------- @@ -881,7 +881,7 @@ SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[0].id $$) as (re 2 (3 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[0].arry[2] $$) as (results agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[0].arry[2] ORDER BY id(e[0]) ASC $$) as (results agtype); results --------- @@ -889,7 +889,7 @@ SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[0].arry[2] $$) a 2 (3 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[1]) $$) as (prop_second_edge agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[1]) ORDER BY id(e[1]) ASC $$) as (prop_second_edge agtype); prop_second_edge --------------------------------------------------------------------- {} @@ -897,7 +897,7 @@ SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[1]) $ {"id": 3, "arry": [1, 3, {"name": "john", "stats": {"age": 1000}}]} (3 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].id $$) as (results agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].id ORDER BY id(e[1]) ASC $$) as (results agtype); results --------- @@ -905,7 +905,7 @@ SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].id $$) as (re 3 (3 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].arry[2] $$) as (results agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].arry[2] ORDER BY id(e[1]) ASC $$) as (results agtype); results ------------------------------------------ @@ -913,7 +913,7 @@ SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].arry[2] $$) a {"name": "john", "stats": {"age": 1000}} (3 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].arry[2].stats $$) as (results agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].arry[2].stats ORDER BY id(e[1]) ASC $$) as (results agtype); results --------------- @@ -921,7 +921,7 @@ SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].arry[2].stats {"age": 1000} (3 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[2]) $$) as (prop_third_edge agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[2]) ORDER BY id(e[2]) ASC $$) as (prop_third_edge agtype); prop_third_edge ----------------- @@ -929,37 +929,37 @@ SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[2]) $ (3 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN properties(e[0]), properties(e[1]) $$) as (prop_1st agtype, prop_2nd agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN properties(e[0]), properties(e[1]) ORDER BY id(e[0]) ASC, id(e[1]) ASC $$) as (prop_1st agtype, prop_2nd agtype); prop_1st | prop_2nd ---------------------------------------------------------------------+--------------------------------------------------------------------- {} | {} | {} | {} {} | - {"id": 0} | - {"id": 0} | {"id": 1} {"id": 1} | - {"id": 2, "arry": [0, 1, 2, 3, {"name": "joe"}]} | - {"id": 2, "arry": [0, 1, 2, 3, {"name": "joe"}]} | {"id": 3, "arry": [1, 3, {"name": "john", "stats": {"age": 1000}}]} + {"id": 0} | {"id": 1} + {"id": 0} | {"id": 3, "arry": [1, 3, {"name": "john", "stats": {"age": 1000}}]} | + {"id": 2, "arry": [0, 1, 2, 3, {"name": "joe"}]} | {"id": 3, "arry": [1, 3, {"name": "john", "stats": {"age": 1000}}]} + {"id": 2, "arry": [0, 1, 2, 3, {"name": "joe"}]} | (10 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].id, e[1].id $$) as (results_1st agtype, results_2nd agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].id, e[1].id ORDER BY id(e[0]) ASC, id(e[1]) ASC $$) as (results_1st agtype, results_2nd agtype); results_1st | results_2nd -------------+------------- | | | | - 0 | - 0 | 1 1 | - 2 | - 2 | 3 + 0 | 1 + 0 | 3 | + 2 | 3 + 2 | (10 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].arry, e[1].arry $$) as (results_1st agtype, results_2nd agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].arry, e[1].arry ORDER BY id(e[0]) ASC, id(e[1]) ASC $$) as (results_1st agtype, results_2nd agtype); results_1st | results_2nd --------------------------------------------------+-------------------------------------------------- | @@ -969,12 +969,12 @@ SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].arry, e[1].arry $ | | | - [0, 1, 2, 3, {"name": "joe"}] | - [0, 1, 2, 3, {"name": "joe"}] | [1, 3, {"name": "john", "stats": {"age": 1000}}] [1, 3, {"name": "john", "stats": {"age": 1000}}] | + [0, 1, 2, 3, {"name": "joe"}] | [1, 3, {"name": "john", "stats": {"age": 1000}}] + [0, 1, 2, 3, {"name": "joe"}] | (10 rows) -SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].arry[2], e[1].arry[2] $$) as (results_1st agtype, results_2nd agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].arry[2], e[1].arry[2] ORDER BY id(e[0]) ASC, id(e[1]) ASC $$) as (results_1st agtype, results_2nd agtype); results_1st | results_2nd ------------------------------------------+------------------------------------------ | @@ -984,9 +984,9 @@ SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].arry[2], e[1].arr | | | - 2 | - 2 | {"name": "john", "stats": {"age": 1000}} {"name": "john", "stats": {"age": 1000}} | + 2 | {"name": "john", "stats": {"age": 1000}} + 2 | (10 rows) SELECT drop_graph('access', true); @@ -1013,7 +1013,7 @@ SELECT * FROM cypher('issue_1043', $$ CREATE (n)-[:KNOWS {n:'hello'}]->({n:'hell --- (0 rows) -SELECT * FROM cypher('issue_1043', $$ MATCH (x)<-[y *]-(),({n:y[0].n}) RETURN x $$) as (a agtype); +SELECT * FROM cypher('issue_1043', $$ MATCH (x)<-[y *]-(),({n:y[0].n}) RETURN x ORDER BY id(x) ASC $$) as (a agtype); a ---------------------------------------------------------------------------- {"id": 281474976710658, "label": "", "properties": {"n": "hello"}}::vertex @@ -1024,13 +1024,13 @@ SELECT * FROM cypher('issue_1043', $$ CREATE (n)-[:KNOWS {n:'hello'}]->({n:'hell --- (0 rows) -SELECT * FROM cypher('issue_1043', $$ MATCH (x)<-[y *]-(),({n:y[0].n}) RETURN x $$) as (a agtype); +SELECT * FROM cypher('issue_1043', $$ MATCH (x)<-[y *]-(),({n:y[0].n}) RETURN x ORDER BY id(x) ASC $$) as (a agtype); a ---------------------------------------------------------------------------- {"id": 281474976710658, "label": "", "properties": {"n": "hello"}}::vertex - {"id": 281474976710660, "label": "", "properties": {"n": "hello"}}::vertex {"id": 281474976710658, "label": "", "properties": {"n": "hello"}}::vertex {"id": 281474976710660, "label": "", "properties": {"n": "hello"}}::vertex + {"id": 281474976710660, "label": "", "properties": {"n": "hello"}}::vertex (4 rows) SELECT drop_graph('issue_1043', true); @@ -1053,7 +1053,7 @@ NOTICE: graph "issue_1910" has been created (1 row) SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*1]-({name: 'Willem Defoe'})) - RETURN n.full_name $$) AS (full_name agtype); + RETURN n.full_name ORDER BY id(n) ASC $$) AS (full_name agtype); full_name ----------- (0 rows) @@ -1075,7 +1075,7 @@ SELECT * FROM cypher('issue_1910', $$ MATCH (u {name: 'John Doe'}) (0 rows) SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*]-({name: 'Willem Defoe'})) - RETURN n.name $$) AS (name agtype); + RETURN n.name ORDER BY id(n) ASC $$) AS (name agtype); name ---------------- "Jane Doe" @@ -1084,7 +1084,7 @@ SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*]-({name: 'Wi (3 rows) SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*1]-({name: 'Willem Defoe'})) - RETURN n.name $$) AS (name agtype); + RETURN n.name ORDER BY id(n) ASC $$) AS (name agtype); name ---------------- "John Doe" @@ -1092,7 +1092,7 @@ SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*1]-({name: 'W (2 rows) SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*2..2]-({name: 'Willem Defoe'})) - RETURN n.name $$) AS (name agtype); + RETURN n.name ORDER BY id(n) ASC $$) AS (name agtype); name ------------ "Jane Doe" diff --git a/regress/sql/cypher_vle.sql b/regress/sql/cypher_vle.sql index 4592a7fbd..5f3f54ed2 100644 --- a/regress/sql/cypher_vle.sql +++ b/regress/sql/cypher_vle.sql @@ -105,20 +105,20 @@ SELECT * FROM cypher('cypher_vle', $$MATCH ()-[*]->() RETURN count(*) $$) AS (e SELECT * FROM cypher('cypher_vle', $$MATCH (u)-[*]->() RETURN count(*) $$) AS (e agtype); SELECT * FROM cypher('cypher_vle', $$MATCH ()-[*]->(v) RETURN count(*) $$) AS (e agtype); -- Should find 2 -SELECT * FROM cypher('cypher_vle', $$MATCH (u:begin)<-[e*]-(v:end) RETURN e $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH (u:begin)<-[e*]-(v:end) RETURN e ORDER BY e ASC $$) AS (e agtype); -- Should find 5 -SELECT * FROM cypher('cypher_vle', $$MATCH p=(:begin)<-[*1..1]-()-[]-() RETURN p ORDER BY p $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(:begin)<-[*1..1]-()-[]-() RETURN p ORDER BY p ASC $$) AS (e agtype); -- Should find 2922 SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[*]->(v) RETURN count(*) $$) AS (e agtype); -- Should find 2 -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[*3..3]->(v:end) RETURN p $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[*3..3]->(v:end) RETURN p ORDER BY p ASC $$) AS (e agtype); -- Should find 12 -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[*3..3]-(v:end) RETURN p $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[*3..3]-(v:end) RETURN p ORDER BY p ASC $$) AS (e agtype); -- Each should find 2 -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[*]-(v:end) RETURN p $$) AS (e agtype); -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[e*]-(v:end) RETURN p $$) AS (e agtype); -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[e*]-(v:end) RETURN e $$) AS (e agtype); -SELECT * FROM cypher('cypher_vle', $$MATCH p=(:begin)<-[*]-()<-[]-(:end) RETURN p $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[*]-(v:end) RETURN p ORDER BY p ASC $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[e*]-(v:end) RETURN p ORDER BY p ASC $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)<-[e*]-(v:end) RETURN e ORDER BY e ASC $$) AS (e agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(:begin)<-[*]-()<-[]-(:end) RETURN p ORDER BY p ASC $$) AS (e agtype); -- Each should return 31 SELECT count(*) FROM cypher('cypher_vle', $$ MATCH ()-[e1]->(v)-[e2]->() RETURN e1,e2 $$) AS (e1 agtype, e2 agtype); SELECT count(*) FROM cypher('cypher_vle', $$ @@ -163,16 +163,16 @@ FROM cypher('cypher_vle', $$ RETURN a, e $$) AS (e1 agtype, e2 agtype); -- Should return 1 path -SELECT * FROM cypher('cypher_vle', $$ MATCH p=()<-[e1*]-(:end)-[e2*]->(:begin) RETURN p $$) AS (result agtype); +SELECT * FROM cypher('cypher_vle', $$ MATCH p=()<-[e1*]-(:end)-[e2*]->(:begin) RETURN p ORDER BY p ASC $$) AS (result agtype); -- Each should return 3 -SELECT * FROM cypher('cypher_vle', $$MATCH (u:begin)-[e*0..1]->(v) RETURN id(u), e, id(v) $$) AS (u agtype, e agtype, v agtype); -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[e*0..1]->(v) RETURN p $$) AS (p agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH (u:begin)-[e*0..1]->(v) RETURN id(u), e, id(v) ORDER BY id(u) ASC, e ASC, id(v) ASC $$) AS (u agtype, e agtype, v agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u:begin)-[e*0..1]->(v) RETURN p ORDER BY p ASC $$) AS (p agtype); -- Each should return 5 -SELECT * FROM cypher('cypher_vle', $$MATCH (u)-[e*0..0]->(v) RETURN id(u), e, id(v) $$) AS (u agtype, e agtype, v agtype); -SELECT * FROM cypher('cypher_vle', $$MATCH p=(u)-[e*0..0]->(v) RETURN id(u), p, id(v) $$) AS (u agtype, p agtype, v agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH (u)-[e*0..0]->(v) RETURN id(u), e, id(v) ORDER BY id(u) ASC, e ASC, id(v) ASC $$) AS (u agtype, e agtype, v agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=(u)-[e*0..0]->(v) RETURN id(u), p, id(v) ORDER BY id(u) ASC, p ASC, id(v) ASC $$) AS (u agtype, p agtype, v agtype); -- Each should return 13 and will be the same -SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[*0..0]->()-[]->() RETURN p $$) AS (p agtype); -SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[]->()-[*0..0]->() RETURN p $$) AS (p agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[*0..0]->()-[]->() RETURN p ORDER BY p ASC $$) AS (p agtype); +SELECT * FROM cypher('cypher_vle', $$MATCH p=()-[]->()-[*0..0]->() RETURN p ORDER BY p ASC $$) AS (p agtype); -- -- Test VLE inside of a BEGIN/COMMIT block @@ -189,7 +189,7 @@ $$) AS (g1 agtype); /* should return 1 path with 1 edge */ SELECT * FROM cypher('mygraph', $$ MATCH p = ()-[:Edge*]->() - RETURN p + RETURN p ORDER BY p ASC $$) AS (g2 agtype); /* should delete the original path and replace it with a path with 2 edges */ @@ -202,13 +202,13 @@ $$) AS (g3 agtype); /* should find 2 paths with 1 edge */ SELECT * FROM cypher('mygraph', $$ MATCH p = ()-[:Edge]->() - RETURN p + RETURN p ORDER BY p ASC $$) AS (g4 agtype); /* should return 3 paths, 2 with 1 edge, 1 with 2 edges */ SELECT * FROM cypher('mygraph', $$ MATCH p = ()-[:Edge*]->() - RETURN p + RETURN p ORDER BY p ASC $$) AS (g5 agtype); SELECT drop_graph('mygraph', true); @@ -312,48 +312,48 @@ SELECT * FROM cypher('access',$$ CREATE ()-[:knows]->() $$) as (results agtype); SELECT * FROM cypher('access',$$ CREATE ()-[:knows]->()-[:knows]->()$$) as (results agtype); SELECT * FROM cypher('access',$$ CREATE ()-[:knows {id:0}]->()-[:knows {id: 1}]->() $$) as (results agtype); SELECT * FROM cypher('access',$$ CREATE ()-[:knows {id:2, arry:[0,1,2,3,{name: "joe"}]}]->()-[:knows {id: 3, arry:[1,3,{name:"john", stats: {age: 1000}}]}]->() $$) as (results agtype); -SELECT * FROM cypher('access', $$ MATCH (u)-[e*]->(v) RETURN e $$)as (edges agtype); -SELECT * FROM cypher('access', $$ MATCH (u)-[e*2..2]->(v) RETURN e $$)as (edges agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[0]) $$) as (prop_first_edge agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[0].id $$) as (results agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[0].arry[2] $$) as (results agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[1]) $$) as (prop_second_edge agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].id $$) as (results agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].arry[2] $$) as (results agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].arry[2].stats $$) as (results agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[2]) $$) as (prop_third_edge agtype); - -SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN properties(e[0]), properties(e[1]) $$) as (prop_1st agtype, prop_2nd agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].id, e[1].id $$) as (results_1st agtype, results_2nd agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].arry, e[1].arry $$) as (results_1st agtype, results_2nd agtype); -SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].arry[2], e[1].arry[2] $$) as (results_1st agtype, results_2nd agtype); +SELECT * FROM cypher('access', $$ MATCH (u)-[e*]->(v) RETURN e ORDER BY e ASC $$)as (edges agtype); +SELECT * FROM cypher('access', $$ MATCH (u)-[e*2..2]->(v) RETURN e ORDER BY e ASC $$)as (edges agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[0]) ORDER BY id(e[0]) ASC $$) as (prop_first_edge agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[0].id ORDER BY id(e[0]) ASC $$) as (results agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[0].arry[2] ORDER BY id(e[0]) ASC $$) as (results agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[1]) ORDER BY id(e[1]) ASC $$) as (prop_second_edge agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].id ORDER BY id(e[1]) ASC $$) as (results agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].arry[2] ORDER BY id(e[1]) ASC $$) as (results agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN e[1].arry[2].stats ORDER BY id(e[1]) ASC $$) as (results agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*2..2]->() RETURN properties(e[2]) ORDER BY id(e[2]) ASC $$) as (prop_third_edge agtype); + +SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN properties(e[0]), properties(e[1]) ORDER BY id(e[0]) ASC, id(e[1]) ASC $$) as (prop_1st agtype, prop_2nd agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].id, e[1].id ORDER BY id(e[0]) ASC, id(e[1]) ASC $$) as (results_1st agtype, results_2nd agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].arry, e[1].arry ORDER BY id(e[0]) ASC, id(e[1]) ASC $$) as (results_1st agtype, results_2nd agtype); +SELECT * FROM cypher('access',$$ MATCH ()-[e*]->() RETURN e[0].arry[2], e[1].arry[2] ORDER BY id(e[0]) ASC, id(e[1]) ASC $$) as (results_1st agtype, results_2nd agtype); SELECT drop_graph('access', true); -- issue 1043 SELECT create_graph('issue_1043'); SELECT * FROM cypher('issue_1043', $$ CREATE (n)-[:KNOWS {n:'hello'}]->({n:'hello'}) $$) as (a agtype); -SELECT * FROM cypher('issue_1043', $$ MATCH (x)<-[y *]-(),({n:y[0].n}) RETURN x $$) as (a agtype); +SELECT * FROM cypher('issue_1043', $$ MATCH (x)<-[y *]-(),({n:y[0].n}) RETURN x ORDER BY id(x) ASC $$) as (a agtype); SELECT * FROM cypher('issue_1043', $$ CREATE (n)-[:KNOWS {n:'hello'}]->({n:'hello'}) $$) as (a agtype); -SELECT * FROM cypher('issue_1043', $$ MATCH (x)<-[y *]-(),({n:y[0].n}) RETURN x $$) as (a agtype); +SELECT * FROM cypher('issue_1043', $$ MATCH (x)<-[y *]-(),({n:y[0].n}) RETURN x ORDER BY id(x) ASC $$) as (a agtype); SELECT drop_graph('issue_1043', true); -- issue 1910 SELECT create_graph('issue_1910'); SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*1]-({name: 'Willem Defoe'})) - RETURN n.full_name $$) AS (full_name agtype); + RETURN n.full_name ORDER BY id(n) ASC $$) AS (full_name agtype); SELECT * FROM cypher('issue_1910', $$ CREATE ({name: 'Jane Doe'})-[:KNOWS]->({name: 'John Doe'}) $$) AS (result agtype); SELECT * FROM cypher('issue_1910', $$ CREATE ({name: 'Donald Defoe'})-[:KNOWS]->({name: 'Willem Defoe'}) $$) AS (result agtype); SELECT * FROM cypher('issue_1910', $$ MATCH (u {name: 'John Doe'}) MERGE (u)-[:KNOWS]->({name: 'Willem Defoe'}) $$) AS (result agtype); SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*]-({name: 'Willem Defoe'})) - RETURN n.name $$) AS (name agtype); + RETURN n.name ORDER BY id(n) ASC $$) AS (name agtype); SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*1]-({name: 'Willem Defoe'})) - RETURN n.name $$) AS (name agtype); + RETURN n.name ORDER BY id(n) ASC $$) AS (name agtype); SELECT * FROM cypher('issue_1910', $$ MATCH (n) WHERE EXISTS((n)-[*2..2]-({name: 'Willem Defoe'})) - RETURN n.name $$) AS (name agtype); + RETURN n.name ORDER BY id(n) ASC $$) AS (name agtype); SELECT drop_graph('issue_1910', true); From 34483efaf1eb0724bf92adda18f06973a1f4a3e1 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Sat, 20 Jun 2026 09:57:33 -0700 Subject: [PATCH 080/100] age_global_graph: stabilize regression tests (#2431) age_global_graph: stabilize regression tests under concurrent xid load Wrap both vertex_stats() context-building phases in a single BEGIN ISOLATION LEVEL REPEATABLE READ; ... COMMIT; transaction so the three calls share one snapshot. This prevents the snapshot-fallback path in is_ggctx_invalid() from purging an already-built graph context when concurrent xid activity (autovacuum, parallel installcheck, replication, shared CI) advances the snapshot between calls, which would otherwise make the targeted delete_global_graphs(name) checks return false instead of the expected true. Read Committed is insufficient because it acquires a fresh snapshot per statement; REPEATABLE READ pins one snapshot for the whole transaction. Also add explicit ORDER BY id to the three direct-SQL label-table SELECTs (_ag_label_vertex x2, _ag_label_edge) that return multiple rows, so their output no longer depends on heap scan order. This is a test-only change (regress/sql/age_global_graph.sql and regress/expected/age_global_graph.out); no extension C code or SQL is modified. All 37 regression tests pass (installcheck) on PostgreSQL 18.3. Co-authored-by: GitHub Copilot modified: regress/expected/age_global_graph.out modified: regress/sql/age_global_graph.sql --- regress/expected/age_global_graph.out | 19 ++++++++++++++++--- regress/sql/age_global_graph.sql | 19 ++++++++++++++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/regress/expected/age_global_graph.out b/regress/expected/age_global_graph.out index cbfeb6f3c..4833511a7 100644 --- a/regress/expected/age_global_graph.out +++ b/regress/expected/age_global_graph.out @@ -44,6 +44,14 @@ SELECT * FROM cypher('ag_graph_3', $$ CREATE (v:vertex3) RETURN v $$) AS (v agt (1 row) -- load contexts using the vertex_stats command +-- Build all three graph contexts under one snapshot. The vertex_stats() +-- calls are wrapped in a single REPEATABLE READ transaction so they share +-- one snapshot; this keeps the snapshot-fallback path in is_ggctx_invalid() +-- from purging an already-built context when concurrent xid activity +-- (autovacuum, parallel installcheck, replication) advances the snapshot +-- between calls. Read Committed is insufficient: it takes a fresh snapshot +-- per statement. +BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); result ----------------------------------------------------------------------------------------------- @@ -62,6 +70,7 @@ SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY {"id": 844424930131969, "label": "vertex1", "in_degree": 0, "out_degree": 0, "self_loops": 0} (1 row) +COMMIT; --- loading undefined contexts --- should throw exception - graph "ag_graph_4" does not exist SELECT * FROM cypher('ag_graph_4', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); @@ -130,6 +139,9 @@ LINE 1: SELECT * FROM cypher('ag_graph_4', $$ RETURN delete_global_g... -- delete_GRAPH_global_contexts -- -- load contexts again +-- Same REPEATABLE READ wrap as the first build phase above, for the same +-- snapshot-stability reason. +BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); result ----------------------------------------------------------------------------------------------- @@ -148,6 +160,7 @@ SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY {"id": 844424930131969, "label": "vertex1", "in_degree": 0, "out_degree": 0, "self_loops": 0} (1 row) +COMMIT; -- delete all graph contexts -- should return true SELECT * FROM cypher('ag_graph_1', $$ RETURN delete_global_graphs(NULL) $$) AS (result agtype); @@ -306,7 +319,7 @@ SELECT * FROM cypher('ag_graph_1', $$ RETURN graph_stats('ag_graph_1') $$) AS (r (1 row) -- remove some vertices -SELECT * FROM ag_graph_1._ag_label_vertex; +SELECT * FROM ag_graph_1._ag_label_vertex ORDER BY id; id | properties -----------------+-------------------------------------- 281474976710657 | {} @@ -325,7 +338,7 @@ SELECT * FROM ag_graph_1._ag_label_vertex; DELETE FROM ag_graph_1._ag_label_vertex WHERE id::text = '281474976710661'; DELETE FROM ag_graph_1._ag_label_vertex WHERE id::text = '281474976710662'; DELETE FROM ag_graph_1._ag_label_vertex WHERE id::text = '281474976710664'; -SELECT * FROM ag_graph_1._ag_label_vertex; +SELECT * FROM ag_graph_1._ag_label_vertex ORDER BY id; id | properties -----------------+-------------------------------------- 281474976710657 | {} @@ -338,7 +351,7 @@ SELECT * FROM ag_graph_1._ag_label_vertex; 844424930131969 | {} (8 rows) -SELECT * FROM ag_graph_1._ag_label_edge; +SELECT * FROM ag_graph_1._ag_label_edge ORDER BY id; id | start_id | end_id | properties ------------------+-----------------+-----------------+------------ 1125899906842625 | 281474976710659 | 281474976710660 | {} diff --git a/regress/sql/age_global_graph.sql b/regress/sql/age_global_graph.sql index 6ee25e1f3..9f4a1ce2d 100644 --- a/regress/sql/age_global_graph.sql +++ b/regress/sql/age_global_graph.sql @@ -16,9 +16,18 @@ SELECT * FROM create_graph('ag_graph_3'); SELECT * FROM cypher('ag_graph_3', $$ CREATE (v:vertex3) RETURN v $$) AS (v agtype); -- load contexts using the vertex_stats command +-- Build all three graph contexts under one snapshot. The vertex_stats() +-- calls are wrapped in a single REPEATABLE READ transaction so they share +-- one snapshot; this keeps the snapshot-fallback path in is_ggctx_invalid() +-- from purging an already-built context when concurrent xid activity +-- (autovacuum, parallel installcheck, replication) advances the snapshot +-- between calls. Read Committed is insufficient: it takes a fresh snapshot +-- per statement. +BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); SELECT * FROM cypher('ag_graph_2', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); +COMMIT; --- loading undefined contexts --- should throw exception - graph "ag_graph_4" does not exist @@ -55,9 +64,13 @@ SELECT * FROM cypher('ag_graph_4', $$ RETURN delete_global_graphs('ag_graph_4') -- -- load contexts again +-- Same REPEATABLE READ wrap as the first build phase above, for the same +-- snapshot-stability reason. +BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT * FROM cypher('ag_graph_3', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); SELECT * FROM cypher('ag_graph_2', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); SELECT * FROM cypher('ag_graph_1', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) AS (result agtype); +COMMIT; -- delete all graph contexts -- should return true @@ -115,12 +128,12 @@ SELECT * FROM cypher('ag_graph_1', $$ MATCH (u)-[e]->(v) RETURN u, e, v ORDER BY -- what is there now? SELECT * FROM cypher('ag_graph_1', $$ RETURN graph_stats('ag_graph_1') $$) AS (result agtype); -- remove some vertices -SELECT * FROM ag_graph_1._ag_label_vertex; +SELECT * FROM ag_graph_1._ag_label_vertex ORDER BY id; DELETE FROM ag_graph_1._ag_label_vertex WHERE id::text = '281474976710661'; DELETE FROM ag_graph_1._ag_label_vertex WHERE id::text = '281474976710662'; DELETE FROM ag_graph_1._ag_label_vertex WHERE id::text = '281474976710664'; -SELECT * FROM ag_graph_1._ag_label_vertex; -SELECT * FROM ag_graph_1._ag_label_edge; +SELECT * FROM ag_graph_1._ag_label_vertex ORDER BY id; +SELECT * FROM ag_graph_1._ag_label_edge ORDER BY id; -- The graph_stats query below will produce warnings for the dangling edges -- created by the DELETE commands above. The warnings appear in nondeterministic -- order because they come from iterating edge label tables (knows, stalks), From 581d236eb300b94c75ac26bc2e319c6152987140 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Sat, 20 Jun 2026 09:58:14 -0700 Subject: [PATCH 081/100] Makefile: add installcheck-existing target and improve readability (#2437) Add an "installcheck-existing" target that runs the regression suite against an already-running PostgreSQL server, complementing the default "installcheck" (which builds a private temp instance). It points pg_regress at the server via the standard libpq variables (PGHOST/PGPORT/PGUSER; PGDATABASE defaults to contrib_regression) and lets pg_regress create the database and load the extension through --load-extension=age. It deliberately avoids --use-existing -- that option skips database creation and disables --load-extension -- so no manual CREATE EXTENSION step is required. The upgrade test (age_upgrade) is excluded because it stages synthetic extension files into the local sharedir that a running server would not see. Readability and maintainability improvements (no behavior change): - Derive age_sql from AGE_CURR_VER (read from age.control) so the version number is defined in exactly one place. - Add section banners and a top-of-file layout/target index; group the scattered upgrade-test pieces and move the ag_scanner flex rule in with the other parser-generation rules. - Wrap the long REGRESS_OPTS and EXTRA_CLEAN assignments across lines. - Fix the DATA filter-out pattern to use a double dash (age--%--y.y.y.sql) matching the actual template filename; the prior single-dash pattern only matched via greedy '%' expansion. - Anchor the age.control version regex (/^default_version/). - Replace the hardcoded "31 tests" comment with generic wording and add a "help" target listing the common targets. Verified on PostgreSQL 18: make installcheck (temp instance) and make installcheck-existing both pass; clean rebuild and make clean unaffected. Co-authored-by: GitHub Copilot modified: Makefile --- Makefile | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 125 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 3ea9236a6..a4a93aaa1 100644 --- a/Makefile +++ b/Makefile @@ -15,11 +15,38 @@ # specific language governing permissions and limitations # under the License. +# =========================================================================== +# Apache AGE extension build +# +# File layout (top to bottom): +# * Module +# * Upgrade regression-test support (1/2: variables) +# * Extension SQL & data files +# * Regression test suite (REGRESS / REGRESS_OPTS) +# * PGXS include +# * Build rules +# * Upgrade regression-test support (2/2: rules + installcheck lifecycle) +# * installcheck-existing (run against a running server) +# * help +# +# Common targets: +# all Build the extension (default) +# install Install into the PostgreSQL tree +# installcheck Run regression tests in a private temp instance +# installcheck-existing Run regression tests against a running server +# clean Remove build artifacts +# help Show the target list +# =========================================================================== + +# ===== Module ===== MODULE_big = age -age_sql = age--1.7.0.sql - -# --- Extension upgrade regression test support --- +# ===== Upgrade regression-test support (1/2: variables) ===== +# +# This feature spans two sections (the PGXS include forces the split): +# * 1/2 (here, pre-include): variables -- must be defined before DATA, +# REGRESS, and EXTRA_CLEAN reference them. +# * 2/2 (below the PGXS include): build rules + installcheck lifecycle. # # Validates the upgrade template (age----y.y.y.sql) by simulating an # extension version upgrade entirely within "make installcheck". The test: @@ -53,7 +80,7 @@ age_sql = age--1.7.0.sql # (e.g., age--1.7.0--1.8.0.sql is committed): the synthetic test is # redundant because the real script ships with the extension. # Current version from age.control (e.g., "1.7.0") -AGE_CURR_VER := $(shell awk -F"'" '/default_version/ {print $$2}' age.control 2>/dev/null) +AGE_CURR_VER := $(shell awk -F"'" '/^default_version/ {print $$2}' age.control 2>/dev/null) # Git commit that last changed age.control — the "initial release" commit AGE_VER_COMMIT := $(shell git log -1 --format=%H -- age.control 2>/dev/null) # Synthetic initial version: current version with _initial suffix @@ -80,6 +107,7 @@ AGE_REAL_UPGRADE := $(shell git ls-files 'age--$(AGE_CURR_VER)--*.sql' 2>/dev/nu # supersedes the synthetic one and has its own validation path. AGE_HAS_UPGRADE_TEST = $(and $(AGE_VER_COMMIT),$(AGE_UPGRADE_TEMPLATE),$(if $(AGE_REAL_UPGRADE),,yes)) +# ===== Object files ===== OBJS = src/backend/age.o \ src/backend/catalog/ag_catalog.o \ src/backend/catalog/ag_graph.o \ @@ -134,6 +162,7 @@ OBJS = src/backend/age.o \ src/backend/utils/name_validation.o \ src/backend/utils/ag_guc.o +# ===== Extension SQL & data files ===== EXTENSION = age # to allow cleaning of previous (old) age--.sql files @@ -143,12 +172,18 @@ SQLS := $(shell cat sql/sql_files) SQLS := $(addprefix sql/,$(SQLS)) SQLS := $(addsuffix .sql,$(SQLS)) +# Name of the generated install SQL (age--.sql). +# Derived from AGE_CURR_VER (read from age.control above) so the version +# number lives in exactly one place. +age_sql = age--$(AGE_CURR_VER).sql + DATA_built = $(age_sql) # Git-tracked upgrade scripts shipped with the extension (e.g., age--1.6.0--1.7.0.sql). # Excludes the upgrade template (y.y.y) and the synthetic stamped test file. -DATA = $(filter-out age--%-y.y.y.sql $(age_upgrade_test_sql),$(wildcard age--*--*.sql)) +DATA = $(filter-out age--%--y.y.y.sql $(age_upgrade_test_sql),$(wildcard age--*--*.sql)) +# ===== Regression test suite ===== # sorted in dependency order REGRESS = scan \ graphid \ @@ -203,10 +238,23 @@ REGRESS += drop srcdir=`pwd` ag_regress_dir = $(srcdir)/regress -REGRESS_OPTS = --load-extension=age --inputdir=$(ag_regress_dir) --outputdir=$(ag_regress_dir) --temp-instance=$(ag_regress_dir)/instance --port=61958 --encoding=UTF-8 --temp-config $(ag_regress_dir)/age_regression.conf +REGRESS_OPTS = --load-extension=age \ + --inputdir=$(ag_regress_dir) \ + --outputdir=$(ag_regress_dir) \ + --temp-instance=$(ag_regress_dir)/instance \ + --port=61958 \ + --encoding=UTF-8 \ + --temp-config $(ag_regress_dir)/age_regression.conf ag_regress_out = instance/ log/ results/ regression.* -EXTRA_CLEAN = $(addprefix $(ag_regress_dir)/, $(ag_regress_out)) src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/include/parser/cypher_kwlist_d.h $(all_age_sql) $(age_init_sql) $(age_upgrade_test_sql) $(ag_regress_dir)/age_upgrade_cleanup.sh +EXTRA_CLEAN = $(addprefix $(ag_regress_dir)/, $(ag_regress_out)) \ + src/backend/parser/cypher_gram.c \ + src/include/parser/cypher_gram_def.h \ + src/include/parser/cypher_kwlist_d.h \ + $(all_age_sql) \ + $(age_init_sql) \ + $(age_upgrade_test_sql) \ + $(ag_regress_dir)/age_upgrade_cleanup.sh GEN_KEYWORDLIST = $(PERL) -I ./tools/ ./tools/gen_keywordlist.pl GEN_KEYWORDLIST_DEPS = ./tools/gen_keywordlist.pl tools/PerfectHash.pm @@ -214,10 +262,13 @@ GEN_KEYWORDLIST_DEPS = ./tools/gen_keywordlist.pl tools/PerfectHash.pm ag_include_dir = $(srcdir)/src/include PG_CPPFLAGS = -I$(ag_include_dir) -I$(ag_include_dir)/parser +# ===== PGXS ===== PG_CONFIG ?= pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) +# ===== Build rules ===== + # 32-bit platform support: pass SIZEOF_DATUM=4 to enable (e.g., make SIZEOF_DATUM=4) # When SIZEOF_DATUM=4, PASSEDBYVALUE is stripped from graphid type for pass-by-reference. # If not specified, normal 64-bit behavior is used (PASSEDBYVALUE preserved). @@ -235,10 +286,11 @@ src/backend/parser/cypher_parser.o: src/backend/parser/cypher_gram.c src/include src/backend/parser/cypher_parser.bc: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/backend/parser/cypher_keywords.o: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/backend/parser/cypher_keywords.bc: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h +src/backend/parser/ag_scanner.c: FLEX_NO_BACKUP=yes # Build the default install SQL (age--.sql) from current HEAD's sql/sql_files. # This is what CREATE EXTENSION age installs — it contains ALL current functions. -# All 31 non-upgrade regression tests run against this complete SQL. +# Every non-upgrade regression test runs against this complete SQL. $(age_sql): $(SQLS) @echo "Building install SQL: $@ from HEAD" @cat $(SQLS) > $@ @@ -247,6 +299,12 @@ ifeq ($(SIZEOF_DATUM),4) @sed 's/^ PASSEDBYVALUE,$$/ -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@ endif +# ===== Upgrade regression-test support (2/2: rules + installcheck lifecycle) ===== +# +# Part 1/2 (variables) is above the PGXS include; the rules and target +# hooks below must follow the include. +# +# --- Synthetic SQL rules --- # Build synthetic "initial" version install SQL from the version-bump commit. # This represents the pre-upgrade state — the SQL at the time the version was # bumped in age.control. Used only by the upgrade test. @@ -266,9 +324,7 @@ $(age_upgrade_test_sql): $(AGE_UPGRADE_TEMPLATE) @sed -e "s/1\.X\.0/$(AGE_CURR_VER)/g" -e "s/y\.y\.y/$(AGE_CURR_VER)/g" $< > $@ endif -src/backend/parser/ag_scanner.c: FLEX_NO_BACKUP=yes - -# --- Upgrade test file lifecycle during installcheck --- +# --- installcheck lifecycle: stage synthetic files, then clean up --- # # Problem: The upgrade test needs age--.sql and age----.sql # in the PG extension directory for CREATE EXTENSION VERSION and ALTER @@ -288,7 +344,7 @@ SHAREDIR = $(shell $(PG_CONFIG) --sharedir) installcheck: export LC_COLLATE=C ifneq ($(AGE_HAS_UPGRADE_TEST),) .PHONY: _install_upgrade_test_files -_install_upgrade_test_files: $(age_init_sql) $(age_upgrade_test_sql) ## Build, install synthetic files, generate cleanup script +_install_upgrade_test_files: $(age_init_sql) $(age_upgrade_test_sql) # Build, install synthetic files, generate cleanup script @echo "Installing upgrade test files to $(SHAREDIR)/extension/" @$(INSTALL_DATA) $(age_init_sql) $(age_upgrade_test_sql) '$(SHAREDIR)/extension/' @printf '#!/bin/sh\nrm -f "$(SHAREDIR)/extension/$(age_init_sql)" "$(SHAREDIR)/extension/$(age_upgrade_test_sql)"\nrm -f "$(age_init_sql)" "$(age_upgrade_test_sql)" "$(ag_regress_dir)/age_upgrade_cleanup.sh"\n' > $(ag_regress_dir)/age_upgrade_cleanup.sh @@ -296,3 +352,60 @@ _install_upgrade_test_files: $(age_init_sql) $(age_upgrade_test_sql) ## Build, installcheck: _install_upgrade_test_files endif + +# ===== installcheck-existing: run tests against a running server ===== +# +# Runs the regression suite against an already-running PostgreSQL server +# instead of the private temp instance built by "make installcheck". +# +# "make installcheck" appends --temp-instance to REGRESS_OPTS, so it builds +# its own throwaway cluster and needs no running server. This target instead +# connects to the server selected by the standard libpq environment variables +# (PGHOST/PGPORT/PGUSER); PGDATABASE defaults to contrib_regression. Override +# any of them on the command line, e.g.: +# +# make installcheck-existing PGHOST=localhost PGPORT=5432 PGUSER=postgres +# +# pg_regress creates the database and loads the extension itself through +# --load-extension=age -- exactly as the temp-instance path does -- so no +# manual "CREATE EXTENSION" step is required. The connecting role must be +# allowed to CREATE DATABASE. +# +# This deliberately does NOT pass pg_regress --use-existing: that option skips +# database creation (which also disables --load-extension) and is only needed +# on clusters where the test role cannot CREATE DATABASE. For that narrow +# case, pre-create the database and extension and add --use-existing to +# EXTRA_REGRESS_OPTS. +# +# The upgrade test (age_upgrade) is excluded here: it installs synthetic +# extension files into the local $(SHAREDIR), which an existing or remote +# server would not see. Validate the upgrade path with "make installcheck". +# +# Locale note: locale-sensitive comparisons follow the existing server's own +# collation (fixed at its initdb time); the temp-instance locale flags do not +# apply to an already-running server. +PGDATABASE ?= contrib_regression +REGRESS_EXISTING = $(filter-out age_upgrade,$(REGRESS)) + +.PHONY: installcheck-existing +installcheck-existing: + $(pg_regress_installcheck) \ + --inputdir=$(ag_regress_dir) \ + --outputdir=$(ag_regress_dir) \ + --load-extension=age \ + $(if $(PGHOST),--host=$(PGHOST)) \ + $(if $(PGPORT),--port=$(PGPORT)) \ + $(if $(PGUSER),--user=$(PGUSER)) \ + --dbname=$(PGDATABASE) \ + $(REGRESS_EXISTING) + +# ===== Help ===== +.PHONY: help +help: + @echo "Apache AGE - common make targets:" + @echo " all Build the extension (default target)" + @echo " install Install the extension into the PostgreSQL tree" + @echo " installcheck Run the regression suite in a private temp instance" + @echo " installcheck-existing Run the regression suite against a running server" + @echo " clean Remove build artifacts" + @echo " help Show this message" From 01ea79de67b203b957658875f3ca0c4189a99207 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Sun, 21 Jun 2026 14:33:12 -0700 Subject: [PATCH 082/100] Make ag_catalog ownership and built-in resolution explicit (#2440) AGE places all of its objects in the ag_catalog schema. Make the assumptions around that schema explicit so installs and upgrades behave predictably regardless of how a database is provisioned: - Ownership-checked install: CREATE EXTENSION age installs into ag_catalog only when that schema does not already exist under a different owner, keeping ownership of AGE's catalog well-defined. - Deterministic name resolution: the pg_upgrade helper functions resolve built-ins from pg_catalog first and schema-qualify their format()/hashtext() calls, so their behavior does not depend on what else is defined in ag_catalog. - README note describing ag_catalog ownership and the install-time check. The upgrade script applies the same helper changes so existing installations get them on ALTER EXTENSION UPDATE. Adds an extension_security regression test covering the ownership check and the qualified-call / search_path properties. Assisted-by: GitHub Copilot (Claude Opus 4.8) modified: Makefile modified: README.md modified: age--1.7.0--y.y.y.sql new file: regress/expected/extension_security.out new file: regress/sql/extension_security.sql modified: sql/age_main.sql modified: sql/age_pg_upgrade.sql Resolved Conflicts: Makefile --- Makefile | 3 +- README.md | 10 +++ age--1.7.0--y.y.y.sql | 34 ++++++--- regress/expected/extension_security.out | 99 +++++++++++++++++++++++++ regress/sql/extension_security.sql | 82 ++++++++++++++++++++ sql/age_main.sql | 27 +++++++ sql/age_pg_upgrade.sql | 32 +++++--- 7 files changed, 265 insertions(+), 22 deletions(-) create mode 100644 regress/expected/extension_security.out create mode 100644 regress/sql/extension_security.sql diff --git a/Makefile b/Makefile index a4a93aaa1..b2d93ff4b 100644 --- a/Makefile +++ b/Makefile @@ -220,7 +220,8 @@ REGRESS = scan \ reserved_keyword_alias \ agtype_jsonb_cast \ containment_selectivity \ - subgraph + subgraph \ + extension_security ifneq ($(EXTRA_TESTS),) REGRESS += $(EXTRA_TESTS) diff --git a/README.md b/README.md index 819d9dcde..412b55e8d 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,16 @@ LOAD 'age'; SET search_path = ag_catalog, "$user", public; ``` +### Note on `ag_catalog` ownership + +AGE installs all of its objects into the `ag_catalog` schema. Install AGE +(`CREATE EXTENSION age`) **before** granting the `CREATE` privilege on the +database to other roles. A role that can create schemas could otherwise +pre-create `ag_catalog` and own it; `CREATE EXTENSION age` therefore refuses to +install when `ag_catalog` already exists and is owned by a different role. If you +hit that error, drop the stray schema (`DROP SCHEMA ag_catalog CASCADE`) or +transfer its ownership to the installing role, then retry. +

  Using AGE with Non-Autocommit Clients (psycopg, JDBC, etc.)

If you are using AGE from a database client that does **not** default to autocommit — most commonly `psycopg` v3 or JDBC — you must understand how PostgreSQL's transaction semantics apply to AGE's setup and DDL-like functions. Otherwise, you may see graphs or labels that appear to be created successfully, but are not visible from new connections. diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index 282eaa0f9..ad2ce20fd 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -41,7 +41,10 @@ CREATE FUNCTION ag_catalog.age_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql - SET search_path = ag_catalog, pg_catalog + -- Resolve built-in functions and operators from pg_catalog first so they + -- are not overridden by same-named objects defined in ag_catalog. The + -- ag_catalog objects referenced here are schema-qualified. + SET search_path = pg_catalog, ag_catalog AS $function$ DECLARE graph_count integer; @@ -108,7 +111,10 @@ COMMENT ON FUNCTION ag_catalog.age_prepare_pg_upgrade() IS CREATE FUNCTION ag_catalog.age_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql - SET search_path = ag_catalog, pg_catalog + -- Resolve built-in functions and operators from pg_catalog first so they + -- are not overridden by same-named objects defined in ag_catalog. The + -- ag_catalog objects referenced here are schema-qualified. + SET search_path = pg_catalog, ag_catalog AS $function$ DECLARE mapping_count integer; @@ -231,7 +237,7 @@ BEGIN -- and preserve original schema ownership. -- RAISE NOTICE 'Invalidating AGE caches...'; - PERFORM pg_catalog.pg_advisory_xact_lock(hashtext('age_finish_pg_upgrade')); + PERFORM pg_catalog.pg_advisory_xact_lock(pg_catalog.hashtext('age_finish_pg_upgrade')); DECLARE graph_rec RECORD; cache_invalidated boolean := false; @@ -245,8 +251,8 @@ BEGIN BEGIN -- Touch schema by changing owner to current_user then back to original -- This triggers cache invalidation without permanently changing ownership - EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); - EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); + EXECUTE pg_catalog.format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); + EXECUTE pg_catalog.format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); cache_invalidated := true; EXCEPTION WHEN insufficient_privilege THEN -- If we can't change ownership, skip this schema @@ -273,7 +279,10 @@ COMMENT ON FUNCTION ag_catalog.age_finish_pg_upgrade() IS CREATE FUNCTION ag_catalog.age_revert_pg_upgrade_changes() RETURNS void LANGUAGE plpgsql - SET search_path = ag_catalog, pg_catalog + -- Resolve built-in functions and operators from pg_catalog first so they + -- are not overridden by same-named objects defined in ag_catalog. The + -- ag_catalog objects referenced here are schema-qualified. + SET search_path = pg_catalog, ag_catalog AS $function$ BEGIN -- Check if namespace column is oid type (needs reverting) @@ -306,7 +315,7 @@ BEGIN -- Invalidate AGE's internal caches by touching each graph's namespace -- We use xact-level advisory lock and preserve original ownership -- - PERFORM pg_catalog.pg_advisory_xact_lock(hashtext('age_revert_pg_upgrade')); + PERFORM pg_catalog.pg_advisory_xact_lock(pg_catalog.hashtext('age_revert_pg_upgrade')); DECLARE graph_rec RECORD; BEGIN @@ -318,8 +327,8 @@ BEGIN LOOP BEGIN -- Touch schema by changing owner to current_user then back to original - EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); - EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); + EXECUTE pg_catalog.format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); + EXECUTE pg_catalog.format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); EXCEPTION WHEN insufficient_privilege THEN RAISE NOTICE 'Could not invalidate cache for schema % (insufficient privileges)', graph_rec.ns_name; END; @@ -345,7 +354,10 @@ CREATE FUNCTION ag_catalog.age_pg_upgrade_status() message text ) LANGUAGE plpgsql - SET search_path = ag_catalog, pg_catalog + -- Resolve built-in functions and operators from pg_catalog first so they + -- are not overridden by same-named objects defined in ag_catalog. The + -- ag_catalog objects referenced here are schema-qualified. + SET search_path = pg_catalog, ag_catalog AS $function$ DECLARE ns_type text; @@ -447,7 +459,7 @@ BEGIN AND t.tgname = '_age_cache_invalidate' ) THEN - EXECUTE format( + EXECUTE pg_catalog.format( 'CREATE TRIGGER _age_cache_invalidate ' 'AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ' 'ON %I.%I ' diff --git a/regress/expected/extension_security.out b/regress/expected/extension_security.out new file mode 100644 index 000000000..30241623a --- /dev/null +++ b/regress/expected/extension_security.out @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +-- +-- pg_upgrade helper functions resolve built-ins from pg_catalog first. +-- +-- Each helper must place pg_catalog ahead of ag_catalog in its search_path, so +-- that built-in functions and operators always resolve to pg_catalog and are +-- not overridden by same-named objects defined in ag_catalog. +-- +SELECT p.proname, + array_to_string(p.proconfig, ', ') AS proconfig +FROM pg_proc p +JOIN pg_namespace n ON n.oid = p.pronamespace +WHERE n.nspname = 'ag_catalog' + AND p.proname IN ('age_prepare_pg_upgrade', 'age_finish_pg_upgrade', + 'age_revert_pg_upgrade_changes', 'age_pg_upgrade_status') +ORDER BY p.proname; + proname | proconfig +-------------------------------+------------------------------------ + age_finish_pg_upgrade | search_path=pg_catalog, ag_catalog + age_pg_upgrade_status | search_path=pg_catalog, ag_catalog + age_prepare_pg_upgrade | search_path=pg_catalog, ag_catalog + age_revert_pg_upgrade_changes | search_path=pg_catalog, ag_catalog +(4 rows) + +-- +-- The helper bodies must not contain unqualified format()/hashtext() calls; +-- those built-ins are explicitly schema-qualified to pg_catalog. +-- +SELECT p.proname, + (p.prosrc ~ '[^.]\mformat\s*\(') AS has_unqualified_format, + (p.prosrc ~ '[^.]\mhashtext\s*\(') AS has_unqualified_hashtext +FROM pg_proc p +JOIN pg_namespace n ON n.oid = p.pronamespace +WHERE n.nspname = 'ag_catalog' + AND p.proname IN ('age_finish_pg_upgrade', 'age_revert_pg_upgrade_changes') +ORDER BY p.proname; + proname | has_unqualified_format | has_unqualified_hashtext +-------------------------------+------------------------+-------------------------- + age_finish_pg_upgrade | f | f + age_revert_pg_upgrade_changes | f | f +(2 rows) + +-- +-- Install-time ownership check: CREATE EXTENSION age installs into ag_catalog +-- only when that schema does not already exist under a different owner. The +-- check compares schema ownership against the installing role. Verify the +-- underlying detection both ways with a probe schema, without disturbing the +-- already-installed extension. +-- +CREATE ROLE age_probe_role NOLOGIN; +CREATE SCHEMA age_probe AUTHORIZATION age_probe_role; +-- A schema owned by a different role is detected as foreign-owned. +SELECT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace n + WHERE n.nspname = 'age_probe' + AND n.nspowner <> (SELECT r.oid FROM pg_catalog.pg_roles r + WHERE r.rolname = current_user) +) AS foreign_owner_detected; + foreign_owner_detected +------------------------ + t +(1 row) + +-- ag_catalog, owned by the current (installing) role here, is not flagged +-- (the check does not false-positive on a normal install). +SELECT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace n + WHERE n.nspname = 'ag_catalog' + AND n.nspowner <> (SELECT r.oid FROM pg_catalog.pg_roles r + WHERE r.rolname = current_user) +) AS installer_owned_flagged; + installer_owned_flagged +------------------------- + f +(1 row) + +DROP SCHEMA age_probe; +DROP ROLE age_probe_role; diff --git a/regress/sql/extension_security.sql b/regress/sql/extension_security.sql new file mode 100644 index 000000000..433283989 --- /dev/null +++ b/regress/sql/extension_security.sql @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +-- +-- pg_upgrade helper functions resolve built-ins from pg_catalog first. +-- +-- Each helper must place pg_catalog ahead of ag_catalog in its search_path, so +-- that built-in functions and operators always resolve to pg_catalog and are +-- not overridden by same-named objects defined in ag_catalog. +-- +SELECT p.proname, + array_to_string(p.proconfig, ', ') AS proconfig +FROM pg_proc p +JOIN pg_namespace n ON n.oid = p.pronamespace +WHERE n.nspname = 'ag_catalog' + AND p.proname IN ('age_prepare_pg_upgrade', 'age_finish_pg_upgrade', + 'age_revert_pg_upgrade_changes', 'age_pg_upgrade_status') +ORDER BY p.proname; + +-- +-- The helper bodies must not contain unqualified format()/hashtext() calls; +-- those built-ins are explicitly schema-qualified to pg_catalog. +-- +SELECT p.proname, + (p.prosrc ~ '[^.]\mformat\s*\(') AS has_unqualified_format, + (p.prosrc ~ '[^.]\mhashtext\s*\(') AS has_unqualified_hashtext +FROM pg_proc p +JOIN pg_namespace n ON n.oid = p.pronamespace +WHERE n.nspname = 'ag_catalog' + AND p.proname IN ('age_finish_pg_upgrade', 'age_revert_pg_upgrade_changes') +ORDER BY p.proname; + +-- +-- Install-time ownership check: CREATE EXTENSION age installs into ag_catalog +-- only when that schema does not already exist under a different owner. The +-- check compares schema ownership against the installing role. Verify the +-- underlying detection both ways with a probe schema, without disturbing the +-- already-installed extension. +-- +CREATE ROLE age_probe_role NOLOGIN; +CREATE SCHEMA age_probe AUTHORIZATION age_probe_role; + +-- A schema owned by a different role is detected as foreign-owned. +SELECT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace n + WHERE n.nspname = 'age_probe' + AND n.nspowner <> (SELECT r.oid FROM pg_catalog.pg_roles r + WHERE r.rolname = current_user) +) AS foreign_owner_detected; + +-- ag_catalog, owned by the current (installing) role here, is not flagged +-- (the check does not false-positive on a normal install). +SELECT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace n + WHERE n.nspname = 'ag_catalog' + AND n.nspowner <> (SELECT r.oid FROM pg_catalog.pg_roles r + WHERE r.rolname = current_user) +) AS installer_owned_flagged; + +DROP SCHEMA age_probe; +DROP ROLE age_probe_role; diff --git a/sql/age_main.sql b/sql/age_main.sql index 72f420002..233d0d23f 100644 --- a/sql/age_main.sql +++ b/sql/age_main.sql @@ -20,6 +20,33 @@ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION age" to load this file. \quit +-- +-- Ensure ag_catalog is created and owned by the installing role. +-- +-- CREATE EXTENSION places all of AGE's objects in ag_catalog. A normal install +-- creates that schema, owned by the installer. If ag_catalog already exists and +-- is owned by a different role, that role would retain control over the schema +-- that holds AGE's catalog objects. To keep ownership well-defined, refuse to +-- install into a pre-existing ag_catalog owned by another role. Ownership is +-- compared directly (not via role membership) so the check is exact even for a +-- superuser, who is otherwise considered a member of every role. +-- +DO $age_install_guard$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace n + WHERE n.nspname = 'ag_catalog' + AND n.nspowner <> (SELECT r.oid + FROM pg_catalog.pg_roles r + WHERE r.rolname = current_user) + ) THEN + RAISE EXCEPTION 'schema "ag_catalog" already exists and is not owned by the installing role "%"', current_user + USING HINT = 'Apache AGE will not install into a pre-existing ag_catalog owned by another role. Drop it (DROP SCHEMA ag_catalog CASCADE) or transfer its ownership to the installing role, then retry CREATE EXTENSION age.'; + END IF; +END +$age_install_guard$; + -- -- catalog tables -- diff --git a/sql/age_pg_upgrade.sql b/sql/age_pg_upgrade.sql index 42a06ecd6..68fbd1513 100644 --- a/sql/age_pg_upgrade.sql +++ b/sql/age_pg_upgrade.sql @@ -55,7 +55,10 @@ CREATE FUNCTION ag_catalog.age_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql - SET search_path = ag_catalog, pg_catalog + -- Resolve built-in functions and operators from pg_catalog first so they + -- are not overridden by same-named objects defined in ag_catalog. The + -- ag_catalog objects referenced here are schema-qualified. + SET search_path = pg_catalog, ag_catalog AS $function$ DECLARE graph_count integer; @@ -143,7 +146,10 @@ COMMENT ON FUNCTION ag_catalog.age_prepare_pg_upgrade() IS CREATE FUNCTION ag_catalog.age_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql - SET search_path = ag_catalog, pg_catalog + -- Resolve built-in functions and operators from pg_catalog first so they + -- are not overridden by same-named objects defined in ag_catalog. The + -- ag_catalog objects referenced here are schema-qualified. + SET search_path = pg_catalog, ag_catalog AS $function$ DECLARE mapping_count integer; @@ -266,7 +272,7 @@ BEGIN -- and preserve original schema ownership. -- RAISE NOTICE 'Invalidating AGE caches...'; - PERFORM pg_catalog.pg_advisory_xact_lock(hashtext('age_finish_pg_upgrade')); + PERFORM pg_catalog.pg_advisory_xact_lock(pg_catalog.hashtext('age_finish_pg_upgrade')); DECLARE graph_rec RECORD; cache_invalidated boolean := false; @@ -280,8 +286,8 @@ BEGIN BEGIN -- Touch schema by changing owner to current_user then back to original -- This triggers cache invalidation without permanently changing ownership - EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); - EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); + EXECUTE pg_catalog.format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); + EXECUTE pg_catalog.format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); cache_invalidated := true; EXCEPTION WHEN insufficient_privilege THEN -- If we can't change ownership, skip this schema @@ -330,7 +336,10 @@ COMMENT ON FUNCTION ag_catalog.age_finish_pg_upgrade() IS CREATE FUNCTION ag_catalog.age_revert_pg_upgrade_changes() RETURNS void LANGUAGE plpgsql - SET search_path = ag_catalog, pg_catalog + -- Resolve built-in functions and operators from pg_catalog first so they + -- are not overridden by same-named objects defined in ag_catalog. The + -- ag_catalog objects referenced here are schema-qualified. + SET search_path = pg_catalog, ag_catalog AS $function$ BEGIN -- Check if namespace column is oid type (needs reverting) @@ -363,7 +372,7 @@ BEGIN -- Invalidate AGE's internal caches by touching each graph's namespace -- We use xact-level advisory lock and preserve original ownership -- - PERFORM pg_catalog.pg_advisory_xact_lock(hashtext('age_revert_pg_upgrade')); + PERFORM pg_catalog.pg_advisory_xact_lock(pg_catalog.hashtext('age_revert_pg_upgrade')); DECLARE graph_rec RECORD; BEGIN @@ -375,8 +384,8 @@ BEGIN LOOP BEGIN -- Touch schema by changing owner to current_user then back to original - EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); - EXECUTE format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); + EXECUTE pg_catalog.format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, current_user); + EXECUTE pg_catalog.format('ALTER SCHEMA %I OWNER TO %I', graph_rec.ns_name, graph_rec.owner_name); EXCEPTION WHEN insufficient_privilege THEN RAISE NOTICE 'Could not invalidate cache for schema % (insufficient privileges)', graph_rec.ns_name; END; @@ -410,7 +419,10 @@ CREATE FUNCTION ag_catalog.age_pg_upgrade_status() message text ) LANGUAGE plpgsql - SET search_path = ag_catalog, pg_catalog + -- Resolve built-in functions and operators from pg_catalog first so they + -- are not overridden by same-named objects defined in ag_catalog. The + -- ag_catalog objects referenced here are schema-qualified. + SET search_path = pg_catalog, ag_catalog AS $function$ DECLARE ns_type text; From 3cc74a4f75252e5a9969f00bad68e3b333f01c5f Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Sun, 21 Jun 2026 14:34:37 -0700 Subject: [PATCH 083/100] cypher_with: add ORDER BY to non-deterministic RETURN queries (#2436) Several cypher_with regression queries RETURN multiple rows without an ORDER BY, so their row order depends on heap/scan order and can vary between runs, build types, and platforms. Add ORDER BY ASC to those queries so the expected output is stable. Ordering keys use id() (a single int64 that bypasses the locale-sensitive string comparison path and is reproducible from the test's deterministic setup order), or the projected path/scalar where that is what the query returns. Where the underlying vertex/edge was dropped by a WITH projection, its id is threaded through as an alias rather than reordering the projection. Full audit of cypher_with: all 23 multi-row result blocks were checked. After this change, every multi-row, non-EXPLAIN RETURN is deterministically ordered. The two remaining unordered multi-row blocks are left as-is: - "RETURN lbl" returns two identical "Person" rows, so order cannot drift; - the 13 EXPLAIN (VERBOSE, COSTS OFF) plan blocks emit a fixed serial plan (no parallel/gather nodes), so their row order is already deterministic. This is a test-only change (regress/sql/cypher_with.sql and regress/expected/cypher_with.out); no extension C code or SQL is modified. Row counts are unchanged (pure reordering). All 37 regression tests pass (installcheck) on PostgreSQL 18.3. Co-authored-by: GitHub Copilot modified: regress/expected/cypher_with.out modified: regress/sql/cypher_with.sql --- regress/expected/cypher_with.out | 25 ++++++++++++++++++------- regress/sql/cypher_with.sql | 13 ++++++++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/regress/expected/cypher_with.out b/regress/expected/cypher_with.out index 8864f026f..2fc330616 100644 --- a/regress/expected/cypher_with.out +++ b/regress/expected/cypher_with.out @@ -52,13 +52,14 @@ SELECT * FROM cypher('cypher_with', $$ MATCH (n)-[e]->(m) WITH n,e,m RETURN n,e,m + ORDER BY id(n) ASC, id(e) ASC, id(m) ASC $$) AS (N1 agtype, edge agtype, N2 agtype); n1 | edge | n2 --------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------- {"id": 281474976710657, "label": "", "properties": {"age": 36, "name": "Andres"}}::vertex | {"id": 844424930131969, "label": "BLOCKS", "end_id": 281474976710658, "start_id": 281474976710657, "properties": {}}::edge | {"id": 281474976710658, "label": "", "properties": {"age": 25, "name": "Caesar"}}::vertex - {"id": 281474976710659, "label": "", "properties": {"age": 55, "name": "Bossman"}}::vertex | {"id": 844424930131970, "label": "BLOCKS", "end_id": 281474976710660, "start_id": 281474976710659, "properties": {}}::edge | {"id": 281474976710660, "label": "", "properties": {"age": 35, "name": "David"}}::vertex {"id": 281474976710657, "label": "", "properties": {"age": 36, "name": "Andres"}}::vertex | {"id": 1125899906842625, "label": "KNOWS", "end_id": 281474976710659, "start_id": 281474976710657, "properties": {}}::edge | {"id": 281474976710659, "label": "", "properties": {"age": 55, "name": "Bossman"}}::vertex {"id": 281474976710658, "label": "", "properties": {"age": 25, "name": "Caesar"}}::vertex | {"id": 1125899906842626, "label": "KNOWS", "end_id": 281474976710661, "start_id": 281474976710658, "properties": {}}::edge | {"id": 281474976710661, "label": "", "properties": {"age": 37, "name": "George"}}::vertex + {"id": 281474976710659, "label": "", "properties": {"age": 55, "name": "Bossman"}}::vertex | {"id": 844424930131970, "label": "BLOCKS", "end_id": 281474976710660, "start_id": 281474976710659, "properties": {}}::edge | {"id": 281474976710660, "label": "", "properties": {"age": 35, "name": "David"}}::vertex {"id": 281474976710659, "label": "", "properties": {"age": 55, "name": "Bossman"}}::vertex | {"id": 1125899906842627, "label": "KNOWS", "end_id": 281474976710661, "start_id": 281474976710659, "properties": {}}::edge | {"id": 281474976710661, "label": "", "properties": {"age": 37, "name": "George"}}::vertex {"id": 281474976710660, "label": "", "properties": {"age": 35, "name": "David"}}::vertex | {"id": 1125899906842628, "label": "KNOWS", "end_id": 281474976710657, "start_id": 281474976710660, "properties": {}}::edge | {"id": 281474976710657, "label": "", "properties": {"age": 36, "name": "Andres"}}::vertex (6 rows) @@ -68,6 +69,7 @@ SELECT * FROM cypher('cypher_with', $$ MATCH (n)-[e]->(m) WITH n.name AS n1, e as edge, m.name as n2 RETURN n1,label(edge),n2 + ORDER BY id(edge) ASC $$) AS (start_node agtype,edge agtype, end_node agtype); start_node | edge | end_node ------------+----------+----------- @@ -83,13 +85,14 @@ SELECT * FROM cypher('cypher_with',$$ MATCH (person)-[r]->(otherPerson) WITH *, type(r) AS connectionType RETURN person.name, connectionType, otherPerson.name + ORDER BY id(person) ASC, id(otherPerson) ASC $$) AS (start_node agtype, connection agtype, end_node agtype); start_node | connection | end_node ------------+------------+----------- "Andres" | "BLOCKS" | "Caesar" - "Bossman" | "BLOCKS" | "David" "Andres" | "KNOWS" | "Bossman" "Caesar" | "KNOWS" | "George" + "Bossman" | "BLOCKS" | "David" "Bossman" | "KNOWS" | "George" "David" | "KNOWS" | "Andres" (6 rows) @@ -109,6 +112,7 @@ MATCH (george {name: 'George'})<-[]-(otherPerson) WITH otherPerson, toUpper(otherPerson.name) AS upperCaseName WHERE upperCaseName STARTS WITH 'C' RETURN otherPerson.name + ORDER BY id(otherPerson) ASC $$) as (name agtype); name ---------- @@ -120,6 +124,7 @@ SELECT * FROM cypher('cypher_with', $$ WITH otherPerson, count(*) AS foaf WHERE foaf > 1 RETURN otherPerson.name + ORDER BY id(otherPerson) ASC $$) as (name agtype); name ---------- @@ -131,15 +136,16 @@ SELECT * FROM cypher('cypher_with', $$ WITH p, length(p) AS path_length WHERE path_length > 1 RETURN p + ORDER BY p ASC $$) AS (pattern agtype); pattern -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - [{"id": 281474976710657, "label": "_ag_label_vertex", "properties": {"age": 36, "name": "Andres"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 281474976710659, "start_id": 281474976710657, "properties": {}}::edge, {"id": 281474976710659, "label": "_ag_label_vertex", "properties": {"age": 55, "name": "Bossman"}}::vertex, {"id": 1125899906842627, "label": "KNOWS", "end_id": 281474976710661, "start_id": 281474976710659, "properties": {}}::edge, {"id": 281474976710661, "label": "_ag_label_vertex", "properties": {"age": 37, "name": "George"}}::vertex]::path - [{"id": 281474976710657, "label": "_ag_label_vertex", "properties": {"age": 36, "name": "Andres"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 281474976710659, "start_id": 281474976710657, "properties": {}}::edge, {"id": 281474976710659, "label": "_ag_label_vertex", "properties": {"age": 55, "name": "Bossman"}}::vertex, {"id": 844424930131970, "label": "BLOCKS", "end_id": 281474976710660, "start_id": 281474976710659, "properties": {}}::edge, {"id": 281474976710660, "label": "_ag_label_vertex", "properties": {"age": 35, "name": "David"}}::vertex]::path [{"id": 281474976710657, "label": "_ag_label_vertex", "properties": {"age": 36, "name": "Andres"}}::vertex, {"id": 844424930131969, "label": "BLOCKS", "end_id": 281474976710658, "start_id": 281474976710657, "properties": {}}::edge, {"id": 281474976710658, "label": "_ag_label_vertex", "properties": {"age": 25, "name": "Caesar"}}::vertex, {"id": 1125899906842626, "label": "KNOWS", "end_id": 281474976710661, "start_id": 281474976710658, "properties": {}}::edge, {"id": 281474976710661, "label": "_ag_label_vertex", "properties": {"age": 37, "name": "George"}}::vertex]::path + [{"id": 281474976710657, "label": "_ag_label_vertex", "properties": {"age": 36, "name": "Andres"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 281474976710659, "start_id": 281474976710657, "properties": {}}::edge, {"id": 281474976710659, "label": "_ag_label_vertex", "properties": {"age": 55, "name": "Bossman"}}::vertex, {"id": 844424930131970, "label": "BLOCKS", "end_id": 281474976710660, "start_id": 281474976710659, "properties": {}}::edge, {"id": 281474976710660, "label": "_ag_label_vertex", "properties": {"age": 35, "name": "David"}}::vertex]::path + [{"id": 281474976710657, "label": "_ag_label_vertex", "properties": {"age": 36, "name": "Andres"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 281474976710659, "start_id": 281474976710657, "properties": {}}::edge, {"id": 281474976710659, "label": "_ag_label_vertex", "properties": {"age": 55, "name": "Bossman"}}::vertex, {"id": 1125899906842627, "label": "KNOWS", "end_id": 281474976710661, "start_id": 281474976710659, "properties": {}}::edge, {"id": 281474976710661, "label": "_ag_label_vertex", "properties": {"age": 37, "name": "George"}}::vertex]::path [{"id": 281474976710659, "label": "_ag_label_vertex", "properties": {"age": 55, "name": "Bossman"}}::vertex, {"id": 844424930131970, "label": "BLOCKS", "end_id": 281474976710660, "start_id": 281474976710659, "properties": {}}::edge, {"id": 281474976710660, "label": "_ag_label_vertex", "properties": {"age": 35, "name": "David"}}::vertex, {"id": 1125899906842628, "label": "KNOWS", "end_id": 281474976710657, "start_id": 281474976710660, "properties": {}}::edge, {"id": 281474976710657, "label": "_ag_label_vertex", "properties": {"age": 36, "name": "Andres"}}::vertex]::path - [{"id": 281474976710660, "label": "_ag_label_vertex", "properties": {"age": 35, "name": "David"}}::vertex, {"id": 1125899906842628, "label": "KNOWS", "end_id": 281474976710657, "start_id": 281474976710660, "properties": {}}::edge, {"id": 281474976710657, "label": "_ag_label_vertex", "properties": {"age": 36, "name": "Andres"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 281474976710659, "start_id": 281474976710657, "properties": {}}::edge, {"id": 281474976710659, "label": "_ag_label_vertex", "properties": {"age": 55, "name": "Bossman"}}::vertex]::path [{"id": 281474976710660, "label": "_ag_label_vertex", "properties": {"age": 35, "name": "David"}}::vertex, {"id": 1125899906842628, "label": "KNOWS", "end_id": 281474976710657, "start_id": 281474976710660, "properties": {}}::edge, {"id": 281474976710657, "label": "_ag_label_vertex", "properties": {"age": 36, "name": "Andres"}}::vertex, {"id": 844424930131969, "label": "BLOCKS", "end_id": 281474976710658, "start_id": 281474976710657, "properties": {}}::edge, {"id": 281474976710658, "label": "_ag_label_vertex", "properties": {"age": 25, "name": "Caesar"}}::vertex]::path + [{"id": 281474976710660, "label": "_ag_label_vertex", "properties": {"age": 35, "name": "David"}}::vertex, {"id": 1125899906842628, "label": "KNOWS", "end_id": 281474976710657, "start_id": 281474976710660, "properties": {}}::edge, {"id": 281474976710657, "label": "_ag_label_vertex", "properties": {"age": 36, "name": "Andres"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 281474976710659, "start_id": 281474976710657, "properties": {}}::edge, {"id": 281474976710659, "label": "_ag_label_vertex", "properties": {"age": 55, "name": "Bossman"}}::vertex]::path (6 rows) -- MATCH/WHERE with WITH/WHERE @@ -149,6 +155,7 @@ SELECT * FROM cypher('cypher_with', $$ WITH * WHERE m.name = 'Andres' RETURN m.name,label(e),b.name + ORDER BY id(m) ASC, id(e) ASC, id(b) ASC $$) AS (N1 agtype, edge agtype, N2 agtype); n1 | edge | n2 ----------+---------+----------- @@ -201,9 +208,10 @@ SELECT * FROM cypher('cypher_with', $$ MATCH (n)-[e]->(m) WITH n, e, m WHERE label(e) = 'KNOWS' - WITH n.name as n1, label(e) as edge, m.name as n2 + WITH id(e) AS eid, n.name as n1, label(e) as edge, m.name as n2 WHERE n1 = 'Andres' RETURN n1,edge,n2 + ORDER BY eid ASC $$) AS (N1 agtype, edge agtype, N2 agtype); n1 | edge | n2 ----------+---------+----------- @@ -217,6 +225,7 @@ SELECT * FROM cypher('cypher_with', $$ WITH x LIMIT 5 RETURN x + ORDER BY x ASC $$) as (name agtype); name ------ @@ -233,11 +242,12 @@ SELECT * FROM cypher('cypher_with', $$ WITH m as start_node, b as end_node WHERE end_node.name = 'George' RETURN id(start_node),start_node.name,id(end_node),end_node.name + ORDER BY id(start_node) ASC, id(end_node) ASC $$) AS (id1 agtype, name1 agtype, id2 agtype, name2 agtype); id1 | name1 | id2 | name2 -----------------+-----------+-----------------+---------- - 281474976710659 | "Bossman" | 281474976710661 | "George" 281474976710658 | "Caesar" | 281474976710661 | "George" + 281474976710659 | "Bossman" | 281474976710661 | "George" (2 rows) -- Expression item must be aliased. @@ -471,6 +481,7 @@ SELECT * FROM cypher('with_accessor_opt', $$ MATCH (n:Person) WITH n as m RETURN m + ORDER BY id(m) ASC $$) AS (n vertex); n --------------------------------------------------------------------- diff --git a/regress/sql/cypher_with.sql b/regress/sql/cypher_with.sql index 25e22b2a2..145356446 100644 --- a/regress/sql/cypher_with.sql +++ b/regress/sql/cypher_with.sql @@ -47,6 +47,7 @@ SELECT * FROM cypher('cypher_with', $$ MATCH (n)-[e]->(m) WITH n,e,m RETURN n,e,m + ORDER BY id(n) ASC, id(e) ASC, id(m) ASC $$) AS (N1 agtype, edge agtype, N2 agtype); -- WITH/AS @@ -55,12 +56,14 @@ SELECT * FROM cypher('cypher_with', $$ MATCH (n)-[e]->(m) WITH n.name AS n1, e as edge, m.name as n2 RETURN n1,label(edge),n2 + ORDER BY id(edge) ASC $$) AS (start_node agtype,edge agtype, end_node agtype); SELECT * FROM cypher('cypher_with',$$ MATCH (person)-[r]->(otherPerson) WITH *, type(r) AS connectionType RETURN person.name, connectionType, otherPerson.name + ORDER BY id(person) ASC, id(otherPerson) ASC $$) AS (start_node agtype, connection agtype, end_node agtype); SELECT * FROM cypher('cypher_with', $$ @@ -75,6 +78,7 @@ MATCH (george {name: 'George'})<-[]-(otherPerson) WITH otherPerson, toUpper(otherPerson.name) AS upperCaseName WHERE upperCaseName STARTS WITH 'C' RETURN otherPerson.name + ORDER BY id(otherPerson) ASC $$) as (name agtype); SELECT * FROM cypher('cypher_with', $$ @@ -82,6 +86,7 @@ SELECT * FROM cypher('cypher_with', $$ WITH otherPerson, count(*) AS foaf WHERE foaf > 1 RETURN otherPerson.name + ORDER BY id(otherPerson) ASC $$) as (name agtype); SELECT * FROM cypher('cypher_with', $$ @@ -89,6 +94,7 @@ SELECT * FROM cypher('cypher_with', $$ WITH p, length(p) AS path_length WHERE path_length > 1 RETURN p + ORDER BY p ASC $$) AS (pattern agtype); -- MATCH/WHERE with WITH/WHERE @@ -99,6 +105,7 @@ SELECT * FROM cypher('cypher_with', $$ WITH * WHERE m.name = 'Andres' RETURN m.name,label(e),b.name + ORDER BY id(m) ASC, id(e) ASC, id(b) ASC $$) AS (N1 agtype, edge agtype, N2 agtype); -- WITH/ORDER BY @@ -133,9 +140,10 @@ SELECT * FROM cypher('cypher_with', $$ MATCH (n)-[e]->(m) WITH n, e, m WHERE label(e) = 'KNOWS' - WITH n.name as n1, label(e) as edge, m.name as n2 + WITH id(e) AS eid, n.name as n1, label(e) as edge, m.name as n2 WHERE n1 = 'Andres' RETURN n1,edge,n2 + ORDER BY eid ASC $$) AS (N1 agtype, edge agtype, N2 agtype); SELECT * FROM cypher('cypher_with', $$ @@ -145,6 +153,7 @@ SELECT * FROM cypher('cypher_with', $$ WITH x LIMIT 5 RETURN x + ORDER BY x ASC $$) as (name agtype); SELECT * FROM cypher('cypher_with', $$ @@ -154,6 +163,7 @@ SELECT * FROM cypher('cypher_with', $$ WITH m as start_node, b as end_node WHERE end_node.name = 'George' RETURN id(start_node),start_node.name,id(end_node),end_node.name + ORDER BY id(start_node) ASC, id(end_node) ASC $$) AS (id1 agtype, name1 agtype, id2 agtype, name2 agtype); -- Expression item must be aliased. @@ -284,6 +294,7 @@ SELECT * FROM cypher('with_accessor_opt', $$ MATCH (n:Person) WITH n as m RETURN m + ORDER BY id(m) ASC $$) AS (n vertex); SELECT * FROM cypher('with_accessor_opt', $$ From 5abb02f8033db15d43a7dabce367155978294806 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Sun, 21 Jun 2026 14:35:36 -0700 Subject: [PATCH 084/100] Add shortest_path / all_shortest_paths SRFs (#2430) Add two C set-returning functions that compute unweighted (hop-count) shortest paths over the cached global graph adjacency via BFS, callable both at the SQL top level and inside a cypher() RETURN: - age_shortest_path(...) -> the single shortest path (0 or 1 rows) - age_all_shortest_paths(...) -> every shortest path, one per row The signature follows the natural Cypher argument order (graph, start, end, edge_types, direction, min_hops, max_hops), registered in sql/agtype_typecast.sql (install) and age--1.7.0--y.y.y.sql (upgrade). Unimplemented parameters fail loudly: multiple relationship types and a non-zero min_hops raise ERRCODE_FEATURE_NOT_SUPPORTED. A single edge type (string or one-element array) is honored, and a NULL endpoint yields no rows per Cypher null semantics (wrong-typed endpoints / NULL graph still error). To call the SRFs inside a cypher() RETURN, transform_cypher_return now sets query->hasTargetSRFs (it was the only results-producing clause that didn't, so the planner never added a ProjectSet node), and transform_FuncCall auto-prepends the graph name for snake_case shortest_path / all_shortest_paths. camelCase names are reserved for the future native grammar. Robustness: - BFS guards against non-existent endpoints (returns 0 rows instead of crashing) and honors CHECK_FOR_INTERRUPTS. - An unknown edge label now matches no edges instead of silently traversing all of them (get_label_relation returns InvalidOid). Adds the age_shortest_path regression test (directed/undirected, label filtering, parallel edges, self-loops, max_hops, the not-supported stubs, NULL and non-existent endpoint/graph guards). 38/38 installcheck pass. Co-authored-by: Copilot modified: Makefile modified: age--1.7.0--y.y.y.sql modified: sql/agtype_typecast.sql modified: src/backend/parser/cypher_clause.c modified: src/backend/parser/cypher_expr.c modified: src/backend/utils/adt/age_vle.c new file: regress/expected/age_shortest_path.out new file: regress/sql/age_shortest_path.sql --- Makefile | 1 + age--1.7.0--y.y.y.sql | 31 + regress/expected/age_shortest_path.out | 977 +++++++++++++++++++++++++ regress/sql/age_shortest_path.sql | 630 ++++++++++++++++ sql/agtype_typecast.sql | 31 + src/backend/parser/cypher_clause.c | 1 + src/backend/parser/cypher_expr.c | 13 +- src/backend/utils/adt/age_vle.c | 796 ++++++++++++++++++++ 8 files changed, 2475 insertions(+), 5 deletions(-) create mode 100644 regress/expected/age_shortest_path.out create mode 100644 regress/sql/age_shortest_path.sql diff --git a/Makefile b/Makefile index b2d93ff4b..41208ee02 100644 --- a/Makefile +++ b/Makefile @@ -201,6 +201,7 @@ REGRESS = scan \ cypher_delete \ cypher_with \ cypher_vle \ + age_shortest_path \ cypher_union \ cypher_call \ cypher_merge \ diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index ad2ce20fd..b40cde092 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -549,6 +549,37 @@ CALLED ON NULL INPUT PARALLEL UNSAFE AS 'MODULE_PATHNAME'; +-- Unweighted (hop-count) shortest path between two vertices, computed over the +-- cached global graph adjacency via BFS. Returns a single path (0 or 1 rows). +-- Argument order mirrors the Cypher shortestPath() pattern +-- (a)-[:type*min_hops..max_hops]->(b): +-- (graph_name, start, end, edge_types, direction, min_hops, max_hops) +CREATE FUNCTION ag_catalog.age_shortest_path(IN agtype, IN agtype, IN agtype, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL) + RETURNS SETOF agtype +LANGUAGE C +STABLE +CALLED ON NULL INPUT +PARALLEL UNSAFE +AS 'MODULE_PATHNAME'; + +-- All unweighted shortest paths between two vertices (one path per row). +-- Same argument order as age_shortest_path. +CREATE FUNCTION ag_catalog.age_all_shortest_paths(IN agtype, IN agtype, IN agtype, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL) + RETURNS SETOF agtype +LANGUAGE C +STABLE +CALLED ON NULL INPUT +PARALLEL UNSAFE +AS 'MODULE_PATHNAME'; + -- -- Composite types for vertex and edge -- diff --git a/regress/expected/age_shortest_path.out b/regress/expected/age_shortest_path.out new file mode 100644 index 000000000..7eb751d12 --- /dev/null +++ b/regress/expected/age_shortest_path.out @@ -0,0 +1,977 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +-- +-- age_shortest_path / age_all_shortest_paths +-- +SELECT * FROM create_graph('sp_graph'); +NOTICE: graph "sp_graph" has been created + create_graph +-------------- + +(1 row) + +-- Build a small deterministic graph: +-- +-- A +-- / \ +-- B C (A->B, A->C, B->D, C->D : two shortest A..D paths) +-- \ / +-- D +-- | +-- E (D->E : unique 3-hop path A..E) +-- +-- Z (isolated, unreachable) +-- +SELECT * FROM cypher('sp_graph', $$ + CREATE (a:Person {name: 'A'}), + (b:Person {name: 'B'}), + (c:Person {name: 'C'}), + (d:Person {name: 'D'}), + (e:Person {name: 'E'}), + (z:Person {name: 'Z'}), + (a)-[:KNOWS]->(b), + (a)-[:KNOWS]->(c), + (b)-[:KNOWS]->(d), + (c)-[:KNOWS]->(d), + (d)-[:KNOWS]->(e) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- materialize the global graph context +SELECT * FROM cypher('sp_graph', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + result +---------------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "Person", "in_degree": 0, "out_degree": 2, "self_loops": 0} + {"id": 844424930131970, "label": "Person", "in_degree": 1, "out_degree": 1, "self_loops": 0} + {"id": 844424930131971, "label": "Person", "in_degree": 1, "out_degree": 1, "self_loops": 0} + {"id": 844424930131972, "label": "Person", "in_degree": 2, "out_degree": 1, "self_loops": 0} + {"id": 844424930131973, "label": "Person", "in_degree": 1, "out_degree": 0, "self_loops": 0} + {"id": 844424930131974, "label": "Person", "in_degree": 0, "out_degree": 0, "self_loops": 0} +(6 rows) + +-- A -> D shortest path (length 2); expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)) +); + path_count +------------ + 1 +(1 row) + +-- all shortest A -> D; expected: 2 paths (A-B-D and A-C-D), each length 2 +SELECT path +FROM age_all_shortest_paths( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)) +) AS path +ORDER BY path; + path +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id": 844424930131969, "label": "Person", "properties": {"name": "A"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131970, "label": "Person", "properties": {"name": "B"}}::vertex, {"id": 1125899906842627, "label": "KNOWS", "end_id": 844424930131972, "start_id": 844424930131970, "properties": {}}::edge, {"id": 844424930131972, "label": "Person", "properties": {"name": "D"}}::vertex]::path + [{"id": 844424930131969, "label": "Person", "properties": {"name": "A"}}::vertex, {"id": 1125899906842626, "label": "KNOWS", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131971, "label": "Person", "properties": {"name": "C"}}::vertex, {"id": 1125899906842628, "label": "KNOWS", "end_id": 844424930131972, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131972, "label": "Person", "properties": {"name": "D"}}::vertex]::path +(2 rows) + +-- A -> E unique 3-hop path; expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'E'}) RETURN id(n) $$) AS (id agtype)) +); + path_count +------------ + 1 +(1 row) + +-- A -> E with max_hops = 2; expected: path_count = 0 (E is 3 hops away) +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'E'}) RETURN id(n) $$) AS (id agtype)), + NULL, NULL, NULL, 2::agtype +); + path_count +------------ + 0 +(1 row) + +-- zero-length path, start == end; expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)) +); + path_count +------------ + 1 +(1 row) + +-- unreachable vertex Z; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'Z'}) RETURN id(n) $$) AS (id agtype)) +); + path_count +------------ + 0 +(1 row) + +-- direction 'in': D -> A traversing edges backwards; expected: path_count = 2 +SELECT count(*) AS path_count +FROM age_all_shortest_paths( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL, '"in"'::agtype +); + path_count +------------ + 2 +(1 row) + +-- direction 'out': D -> A not reachable forwards; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL, '"out"'::agtype +); + path_count +------------ + 0 +(1 row) + +-- label filter 'KNOWS': A -> D still found; expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype +); + path_count +------------ + 1 +(1 row) + +-- error: invalid direction string; expected: ERROR (must be 'out', 'in', or 'any') +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + NULL, '"sideways"'::agtype +); +ERROR: direction argument must be one of 'out', 'in', or 'any' +-- error: start argument is neither a vertex nor an integer id; expected: ERROR +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + '"not_a_vertex"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)) +); +ERROR: start vertex argument must be a vertex or the integer id +-- +-- Non-existent endpoint guards. These must NOT crash the backend and must +-- return no rows (a path can only exist between vertices in the graph). +-- Previously, start == end on a non-existent vertex id was matched at BFS +-- depth 0 and path reconstruction dereferenced a missing vertex, crashing +-- the server. +-- +-- start == end on a non-existent integer id; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path('"sp_graph"'::agtype, 999999::agtype, 999999::agtype); + path_count +------------ + 0 +(1 row) + +-- existing start -> non-existent end; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + 999999::agtype +); + path_count +------------ + 0 +(1 row) + +-- non-existent start -> existing end; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + 999999::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)) +); + path_count +------------ + 0 +(1 row) + +-- all-shortest-paths with start == end non-existent; expected: 0 rows +SELECT count(*) AS path_count +FROM age_all_shortest_paths('"sp_graph"'::agtype, 999999::agtype, 999999::agtype); + path_count +------------ + 0 +(1 row) + +-- cleanup +SELECT * FROM drop_graph('sp_graph', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table sp_graph._ag_label_vertex +drop cascades to table sp_graph._ag_label_edge +drop cascades to table sp_graph."Person" +drop cascades to table sp_graph."KNOWS" +NOTICE: graph "sp_graph" has been dropped + drop_graph +------------ + +(1 row) + +-- +-- Empty graph: a graph that exists but has no vertices must return no rows +-- (and must not hang or crash) for any endpoint query. +-- +SELECT * FROM create_graph('sp_empty'); +NOTICE: graph "sp_empty" has been created + create_graph +-------------- + +(1 row) + +SELECT count(*) AS path_count +FROM age_shortest_path('"sp_empty"'::agtype, 0::agtype, 1::agtype); + path_count +------------ + 0 +(1 row) + +SELECT count(*) AS path_count +FROM age_all_shortest_paths('"sp_empty"'::agtype, 0::agtype, 0::agtype); + path_count +------------ + 0 +(1 row) + +SELECT * FROM drop_graph('sp_empty', true); +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table sp_empty._ag_label_vertex +drop cascades to table sp_empty._ag_label_edge +NOTICE: graph "sp_empty" has been dropped + drop_graph +------------ + +(1 row) + +-- +-- A large, programmatically generated graph (120 nodes) exercising long +-- shortest paths (length up to 20), high-multiplicity all-shortest-paths, +-- label filtering, and directed vs. undirected reachability. +-- +-- Nodes: (:N {id: 0..119}). Structures built on top of them: +-- +-- * Main chain 0 -> 1 -> ... -> 20 (unique 20-hop path) +-- * Alternate chain 0 -> 50 -> 51 -> ... -> 68 -> 20 +-- (a second, disjoint 20-hop path 0..20) +-- => all-shortest-paths 0..20 under KNOWS = 2 paths of length 20 +-- * 3x3 lattice on ids 70..78, id = 70 + 3*row + col, edges go right +-- (id->id+1) and down (id->id+3). Monotone 70..78 paths: +-- => all-shortest-paths 70..78 = C(4,2) = 6 paths of length 4 +-- * LIKES shortcut 0 -[:LIKES]-> 20 (1 hop; only visible when the edge +-- label filter is NOT restricted to KNOWS) +-- * Back-edge triangle 0 -> 96 -> 95 -> 0 +-- => directed 0->95 = 2 hops (0-96-95); undirected 0..95 = 1 hop +-- * Many unused ids (e.g. 119) remain isolated / unreachable. +-- +SELECT * FROM create_graph('sp_big'); +NOTICE: graph "sp_big" has been created + create_graph +-------------- + +(1 row) + +-- 120 vertices, ids 0..119 +SELECT * FROM cypher('sp_big', $$ + UNWIND range(0, 119) AS i CREATE (:N {id: i}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- main chain 0->1->...->20 (KNOWS) +SELECT * FROM cypher('sp_big', $$ + UNWIND range(0, 19) AS i + MATCH (a:N {id: i}), (b:N {id: i + 1}) + CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- alternate, disjoint 20-hop path 0->50->51->...->68->20 (KNOWS) +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 0}), (b:N {id: 50}) CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('sp_big', $$ + UNWIND range(50, 67) AS i + MATCH (a:N {id: i}), (b:N {id: i + 1}) + CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 68}), (b:N {id: 20}) CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- 3x3 lattice on ids 70..78: right edges (id -> id+1) +SELECT * FROM cypher('sp_big', $$ + UNWIND [0, 1, 2] AS r + UNWIND [0, 1] AS c + MATCH (a:N {id: 70 + 3 * r + c}), (b:N {id: 70 + 3 * r + c + 1}) + CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- 3x3 lattice: down edges (id -> id+3) +SELECT * FROM cypher('sp_big', $$ + UNWIND [0, 1] AS r + UNWIND [0, 1, 2] AS c + MATCH (a:N {id: 70 + 3 * r + c}), (b:N {id: 70 + 3 * (r + 1) + c}) + CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- back-edge triangle 0 -> 96 -> 95 -> 0 (KNOWS) +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 0}), (b:N {id: 96}) CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 96}), (b:N {id: 95}) CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + result +-------- +(0 rows) + +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 95}), (b:N {id: 0}) CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- labelled shortcut 0 -[:LIKES]-> 20 +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 0}), (b:N {id: 20}) CREATE (a)-[:LIKES]->(b) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- sanity: vertex count (also materializes the global context); expected: count = 120 +SELECT * FROM cypher('sp_big', $$ MATCH (n) RETURN count(n) $$) AS (n agtype); + n +----- + 120 +(1 row) + +-- all shortest 0 -> 20 under KNOWS (main chain + disjoint alternate); +-- expected: 2 paths, each exactly 20 hops +SELECT path +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 20}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype +) AS path +ORDER BY path; + path +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + [{"id": 844424930131969, "label": "N", "properties": {"id": 0}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131970, "label": "N", "properties": {"id": 1}}::vertex, {"id": 1125899906842626, "label": "KNOWS", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge, {"id": 844424930131971, "label": "N", "properties": {"id": 2}}::vertex, {"id": 1125899906842627, "label": "KNOWS", "end_id": 844424930131972, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131972, "label": "N", "properties": {"id": 3}}::vertex, {"id": 1125899906842628, "label": "KNOWS", "end_id": 844424930131973, "start_id": 844424930131972, "properties": {}}::edge, {"id": 844424930131973, "label": "N", "properties": {"id": 4}}::vertex, {"id": 1125899906842629, "label": "KNOWS", "end_id": 844424930131974, "start_id": 844424930131973, "properties": {}}::edge, {"id": 844424930131974, "label": "N", "properties": {"id": 5}}::vertex, {"id": 1125899906842630, "label": "KNOWS", "end_id": 844424930131975, "start_id": 844424930131974, "properties": {}}::edge, {"id": 844424930131975, "label": "N", "properties": {"id": 6}}::vertex, {"id": 1125899906842631, "label": "KNOWS", "end_id": 844424930131976, "start_id": 844424930131975, "properties": {}}::edge, {"id": 844424930131976, "label": "N", "properties": {"id": 7}}::vertex, {"id": 1125899906842632, "label": "KNOWS", "end_id": 844424930131977, "start_id": 844424930131976, "properties": {}}::edge, {"id": 844424930131977, "label": "N", "properties": {"id": 8}}::vertex, {"id": 1125899906842633, "label": "KNOWS", "end_id": 844424930131978, "start_id": 844424930131977, "properties": {}}::edge, {"id": 844424930131978, "label": "N", "properties": {"id": 9}}::vertex, {"id": 1125899906842634, "label": "KNOWS", "end_id": 844424930131979, "start_id": 844424930131978, "properties": {}}::edge, {"id": 844424930131979, "label": "N", "properties": {"id": 10}}::vertex, {"id": 1125899906842635, "label": "KNOWS", "end_id": 844424930131980, "start_id": 844424930131979, "properties": {}}::edge, {"id": 844424930131980, "label": "N", "properties": {"id": 11}}::vertex, {"id": 1125899906842636, "label": "KNOWS", "end_id": 844424930131981, "start_id": 844424930131980, "properties": {}}::edge, {"id": 844424930131981, "label": "N", "properties": {"id": 12}}::vertex, {"id": 1125899906842637, "label": "KNOWS", "end_id": 844424930131982, "start_id": 844424930131981, "properties": {}}::edge, {"id": 844424930131982, "label": "N", "properties": {"id": 13}}::vertex, {"id": 1125899906842638, "label": "KNOWS", "end_id": 844424930131983, "start_id": 844424930131982, "properties": {}}::edge, {"id": 844424930131983, "label": "N", "properties": {"id": 14}}::vertex, {"id": 1125899906842639, "label": "KNOWS", "end_id": 844424930131984, "start_id": 844424930131983, "properties": {}}::edge, {"id": 844424930131984, "label": "N", "properties": {"id": 15}}::vertex, {"id": 1125899906842640, "label": "KNOWS", "end_id": 844424930131985, "start_id": 844424930131984, "properties": {}}::edge, {"id": 844424930131985, "label": "N", "properties": {"id": 16}}::vertex, {"id": 1125899906842641, "label": "KNOWS", "end_id": 844424930131986, "start_id": 844424930131985, "properties": {}}::edge, {"id": 844424930131986, "label": "N", "properties": {"id": 17}}::vertex, {"id": 1125899906842642, "label": "KNOWS", "end_id": 844424930131987, "start_id": 844424930131986, "properties": {}}::edge, {"id": 844424930131987, "label": "N", "properties": {"id": 18}}::vertex, {"id": 1125899906842643, "label": "KNOWS", "end_id": 844424930131988, "start_id": 844424930131987, "properties": {}}::edge, {"id": 844424930131988, "label": "N", "properties": {"id": 19}}::vertex, {"id": 1125899906842644, "label": "KNOWS", "end_id": 844424930131989, "start_id": 844424930131988, "properties": {}}::edge, {"id": 844424930131989, "label": "N", "properties": {"id": 20}}::vertex]::path + [{"id": 844424930131969, "label": "N", "properties": {"id": 0}}::vertex, {"id": 1125899906842645, "label": "KNOWS", "end_id": 844424930132019, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930132019, "label": "N", "properties": {"id": 50}}::vertex, {"id": 1125899906842646, "label": "KNOWS", "end_id": 844424930132020, "start_id": 844424930132019, "properties": {}}::edge, {"id": 844424930132020, "label": "N", "properties": {"id": 51}}::vertex, {"id": 1125899906842647, "label": "KNOWS", "end_id": 844424930132021, "start_id": 844424930132020, "properties": {}}::edge, {"id": 844424930132021, "label": "N", "properties": {"id": 52}}::vertex, {"id": 1125899906842648, "label": "KNOWS", "end_id": 844424930132022, "start_id": 844424930132021, "properties": {}}::edge, {"id": 844424930132022, "label": "N", "properties": {"id": 53}}::vertex, {"id": 1125899906842649, "label": "KNOWS", "end_id": 844424930132023, "start_id": 844424930132022, "properties": {}}::edge, {"id": 844424930132023, "label": "N", "properties": {"id": 54}}::vertex, {"id": 1125899906842650, "label": "KNOWS", "end_id": 844424930132024, "start_id": 844424930132023, "properties": {}}::edge, {"id": 844424930132024, "label": "N", "properties": {"id": 55}}::vertex, {"id": 1125899906842651, "label": "KNOWS", "end_id": 844424930132025, "start_id": 844424930132024, "properties": {}}::edge, {"id": 844424930132025, "label": "N", "properties": {"id": 56}}::vertex, {"id": 1125899906842652, "label": "KNOWS", "end_id": 844424930132026, "start_id": 844424930132025, "properties": {}}::edge, {"id": 844424930132026, "label": "N", "properties": {"id": 57}}::vertex, {"id": 1125899906842653, "label": "KNOWS", "end_id": 844424930132027, "start_id": 844424930132026, "properties": {}}::edge, {"id": 844424930132027, "label": "N", "properties": {"id": 58}}::vertex, {"id": 1125899906842654, "label": "KNOWS", "end_id": 844424930132028, "start_id": 844424930132027, "properties": {}}::edge, {"id": 844424930132028, "label": "N", "properties": {"id": 59}}::vertex, {"id": 1125899906842655, "label": "KNOWS", "end_id": 844424930132029, "start_id": 844424930132028, "properties": {}}::edge, {"id": 844424930132029, "label": "N", "properties": {"id": 60}}::vertex, {"id": 1125899906842656, "label": "KNOWS", "end_id": 844424930132030, "start_id": 844424930132029, "properties": {}}::edge, {"id": 844424930132030, "label": "N", "properties": {"id": 61}}::vertex, {"id": 1125899906842657, "label": "KNOWS", "end_id": 844424930132031, "start_id": 844424930132030, "properties": {}}::edge, {"id": 844424930132031, "label": "N", "properties": {"id": 62}}::vertex, {"id": 1125899906842658, "label": "KNOWS", "end_id": 844424930132032, "start_id": 844424930132031, "properties": {}}::edge, {"id": 844424930132032, "label": "N", "properties": {"id": 63}}::vertex, {"id": 1125899906842659, "label": "KNOWS", "end_id": 844424930132033, "start_id": 844424930132032, "properties": {}}::edge, {"id": 844424930132033, "label": "N", "properties": {"id": 64}}::vertex, {"id": 1125899906842660, "label": "KNOWS", "end_id": 844424930132034, "start_id": 844424930132033, "properties": {}}::edge, {"id": 844424930132034, "label": "N", "properties": {"id": 65}}::vertex, {"id": 1125899906842661, "label": "KNOWS", "end_id": 844424930132035, "start_id": 844424930132034, "properties": {}}::edge, {"id": 844424930132035, "label": "N", "properties": {"id": 66}}::vertex, {"id": 1125899906842662, "label": "KNOWS", "end_id": 844424930132036, "start_id": 844424930132035, "properties": {}}::edge, {"id": 844424930132036, "label": "N", "properties": {"id": 67}}::vertex, {"id": 1125899906842663, "label": "KNOWS", "end_id": 844424930132037, "start_id": 844424930132036, "properties": {}}::edge, {"id": 844424930132037, "label": "N", "properties": {"id": 68}}::vertex, {"id": 1125899906842664, "label": "KNOWS", "end_id": 844424930131989, "start_id": 844424930132037, "properties": {}}::edge, {"id": 844424930131989, "label": "N", "properties": {"id": 20}}::vertex]::path +(2 rows) + +-- any label: the LIKES shortcut collapses 0 -> 20; expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 20}) RETURN id(n) $$) AS (id agtype)), + NULL, '"out"'::agtype +); + path_count +------------ + 1 +(1 row) + +-- all shortest 70 -> 78 across the 3x3 lattice; expected: path_count = 6 (C(4,2)) +SELECT count(*) AS path_count +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 70}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 78}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype +); + path_count +------------ + 6 +(1 row) + +-- the lattice paths listed; expected: 6 paths, each 4 hops +SELECT path +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 70}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 78}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype +) AS path +ORDER BY path; + path +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id": 844424930132039, "label": "N", "properties": {"id": 70}}::vertex, {"id": 1125899906842665, "label": "KNOWS", "end_id": 844424930132040, "start_id": 844424930132039, "properties": {}}::edge, {"id": 844424930132040, "label": "N", "properties": {"id": 71}}::vertex, {"id": 1125899906842666, "label": "KNOWS", "end_id": 844424930132041, "start_id": 844424930132040, "properties": {}}::edge, {"id": 844424930132041, "label": "N", "properties": {"id": 72}}::vertex, {"id": 1125899906842673, "label": "KNOWS", "end_id": 844424930132044, "start_id": 844424930132041, "properties": {}}::edge, {"id": 844424930132044, "label": "N", "properties": {"id": 75}}::vertex, {"id": 1125899906842676, "label": "KNOWS", "end_id": 844424930132047, "start_id": 844424930132044, "properties": {}}::edge, {"id": 844424930132047, "label": "N", "properties": {"id": 78}}::vertex]::path + [{"id": 844424930132039, "label": "N", "properties": {"id": 70}}::vertex, {"id": 1125899906842665, "label": "KNOWS", "end_id": 844424930132040, "start_id": 844424930132039, "properties": {}}::edge, {"id": 844424930132040, "label": "N", "properties": {"id": 71}}::vertex, {"id": 1125899906842672, "label": "KNOWS", "end_id": 844424930132043, "start_id": 844424930132040, "properties": {}}::edge, {"id": 844424930132043, "label": "N", "properties": {"id": 74}}::vertex, {"id": 1125899906842668, "label": "KNOWS", "end_id": 844424930132044, "start_id": 844424930132043, "properties": {}}::edge, {"id": 844424930132044, "label": "N", "properties": {"id": 75}}::vertex, {"id": 1125899906842676, "label": "KNOWS", "end_id": 844424930132047, "start_id": 844424930132044, "properties": {}}::edge, {"id": 844424930132047, "label": "N", "properties": {"id": 78}}::vertex]::path + [{"id": 844424930132039, "label": "N", "properties": {"id": 70}}::vertex, {"id": 1125899906842665, "label": "KNOWS", "end_id": 844424930132040, "start_id": 844424930132039, "properties": {}}::edge, {"id": 844424930132040, "label": "N", "properties": {"id": 71}}::vertex, {"id": 1125899906842672, "label": "KNOWS", "end_id": 844424930132043, "start_id": 844424930132040, "properties": {}}::edge, {"id": 844424930132043, "label": "N", "properties": {"id": 74}}::vertex, {"id": 1125899906842675, "label": "KNOWS", "end_id": 844424930132046, "start_id": 844424930132043, "properties": {}}::edge, {"id": 844424930132046, "label": "N", "properties": {"id": 77}}::vertex, {"id": 1125899906842670, "label": "KNOWS", "end_id": 844424930132047, "start_id": 844424930132046, "properties": {}}::edge, {"id": 844424930132047, "label": "N", "properties": {"id": 78}}::vertex]::path + [{"id": 844424930132039, "label": "N", "properties": {"id": 70}}::vertex, {"id": 1125899906842671, "label": "KNOWS", "end_id": 844424930132042, "start_id": 844424930132039, "properties": {}}::edge, {"id": 844424930132042, "label": "N", "properties": {"id": 73}}::vertex, {"id": 1125899906842667, "label": "KNOWS", "end_id": 844424930132043, "start_id": 844424930132042, "properties": {}}::edge, {"id": 844424930132043, "label": "N", "properties": {"id": 74}}::vertex, {"id": 1125899906842668, "label": "KNOWS", "end_id": 844424930132044, "start_id": 844424930132043, "properties": {}}::edge, {"id": 844424930132044, "label": "N", "properties": {"id": 75}}::vertex, {"id": 1125899906842676, "label": "KNOWS", "end_id": 844424930132047, "start_id": 844424930132044, "properties": {}}::edge, {"id": 844424930132047, "label": "N", "properties": {"id": 78}}::vertex]::path + [{"id": 844424930132039, "label": "N", "properties": {"id": 70}}::vertex, {"id": 1125899906842671, "label": "KNOWS", "end_id": 844424930132042, "start_id": 844424930132039, "properties": {}}::edge, {"id": 844424930132042, "label": "N", "properties": {"id": 73}}::vertex, {"id": 1125899906842667, "label": "KNOWS", "end_id": 844424930132043, "start_id": 844424930132042, "properties": {}}::edge, {"id": 844424930132043, "label": "N", "properties": {"id": 74}}::vertex, {"id": 1125899906842675, "label": "KNOWS", "end_id": 844424930132046, "start_id": 844424930132043, "properties": {}}::edge, {"id": 844424930132046, "label": "N", "properties": {"id": 77}}::vertex, {"id": 1125899906842670, "label": "KNOWS", "end_id": 844424930132047, "start_id": 844424930132046, "properties": {}}::edge, {"id": 844424930132047, "label": "N", "properties": {"id": 78}}::vertex]::path + [{"id": 844424930132039, "label": "N", "properties": {"id": 70}}::vertex, {"id": 1125899906842671, "label": "KNOWS", "end_id": 844424930132042, "start_id": 844424930132039, "properties": {}}::edge, {"id": 844424930132042, "label": "N", "properties": {"id": 73}}::vertex, {"id": 1125899906842674, "label": "KNOWS", "end_id": 844424930132045, "start_id": 844424930132042, "properties": {}}::edge, {"id": 844424930132045, "label": "N", "properties": {"id": 76}}::vertex, {"id": 1125899906842669, "label": "KNOWS", "end_id": 844424930132046, "start_id": 844424930132045, "properties": {}}::edge, {"id": 844424930132046, "label": "N", "properties": {"id": 77}}::vertex, {"id": 1125899906842670, "label": "KNOWS", "end_id": 844424930132047, "start_id": 844424930132046, "properties": {}}::edge, {"id": 844424930132047, "label": "N", "properties": {"id": 78}}::vertex]::path +(6 rows) + +-- max_hops = 19, one short of the 20-hop route; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 20}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, NULL, 19::agtype +); + path_count +------------ + 0 +(1 row) + +-- max_hops = 20 admits the full route; expected: path_count = 2 +SELECT count(*) AS path_count +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 20}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, NULL, 20::agtype +); + path_count +------------ + 2 +(1 row) + +-- DIRECTED out: 0 -> 95 must traverse 0->96->95; expected: 1 path (length 2) +SELECT path +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 95}) RETURN id(n) $$) AS (id agtype)), + NULL, '"out"'::agtype +) AS path +ORDER BY path; + path +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + [{"id": 844424930131969, "label": "N", "properties": {"id": 0}}::vertex, {"id": 1125899906842677, "label": "KNOWS", "end_id": 844424930132065, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930132065, "label": "N", "properties": {"id": 96}}::vertex, {"id": 1125899906842678, "label": "KNOWS", "end_id": 844424930132064, "start_id": 844424930132065, "properties": {}}::edge, {"id": 844424930132064, "label": "N", "properties": {"id": 95}}::vertex]::path +(1 row) + +-- UNDIRECTED: 0 .. 95 via the 95->0 back edge; expected: 1 path (length 1) +SELECT path +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 95}) RETURN id(n) $$) AS (id agtype)) +) AS path +ORDER BY path; + path +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id": 844424930131969, "label": "N", "properties": {"id": 0}}::vertex, {"id": 1125899906842679, "label": "KNOWS", "end_id": 844424930131969, "start_id": 844424930132064, "properties": {}}::edge, {"id": 844424930132064, "label": "N", "properties": {"id": 95}}::vertex]::path +(1 row) + +-- DIRECTED out: 78 -> 70 against lattice flow; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 78}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 70}) RETURN id(n) $$) AS (id agtype)), + NULL, '"out"'::agtype +); + path_count +------------ + 0 +(1 row) + +-- UNDIRECTED: 78 .. 70 reverses the lattice; expected: path_count = 6 +SELECT count(*) AS path_count +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 78}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 70}) RETURN id(n) $$) AS (id agtype)) +); + path_count +------------ + 6 +(1 row) + +-- isolated id 119 unreachable from 0; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 119}) RETURN id(n) $$) AS (id agtype)) +); + path_count +------------ + 0 +(1 row) + +-- zero-length path, start == end; expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)) +); + path_count +------------ + 1 +(1 row) + +-- cleanup +SELECT * FROM drop_graph('sp_big', true); +NOTICE: drop cascades to 5 other objects +DETAIL: drop cascades to table sp_big._ag_label_vertex +drop cascades to table sp_big._ag_label_edge +drop cascades to table sp_big."N" +drop cascades to table sp_big."KNOWS" +drop cascades to table sp_big."LIKES" +NOTICE: graph "sp_big" has been dropped + drop_graph +------------ + +(1 row) + +-- +-- Calling the age_* SRFs from inside cypher() (Tier 1). +-- +-- Because the functions are prefixed with age_, the cypher() parser resolves +-- the unqualified names 'shortest_path' and 'all_shortest_paths' to +-- ag_catalog.age_shortest_path / ag_catalog.age_all_shortest_paths, and the +-- graph name is auto-injected as the first argument (like vle/vertex_stats), +-- so callers pass only the bound endpoints. A whole vertex implicitly casts to +-- agtype, so the argument types resolve. The SRFs are set-returning and now +-- work in a cypher RETURN projection (ProjectSet), returning one row per path. +-- +SELECT * FROM create_graph('sp_cy'); +NOTICE: graph "sp_cy" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('sp_cy', $$ + CREATE (a:N {name: 'A'}), + (b:N {name: 'B'}), + (c:N {name: 'C'}), + (a)-[:KNOWS]->(b), + (b)-[:KNOWS]->(c) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- materialize the global graph context +SELECT * FROM cypher('sp_cy', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + result +----------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "N", "in_degree": 0, "out_degree": 1, "self_loops": 0} + {"id": 844424930131970, "label": "N", "in_degree": 1, "out_degree": 1, "self_loops": 0} + {"id": 844424930131971, "label": "N", "in_degree": 1, "out_degree": 0, "self_loops": 0} +(3 rows) + +-- shortest_path() inside a cypher RETURN; the graph name is auto-injected and +-- the bound vertices are passed; expected: 1 path A..C (length 2) +SELECT * FROM cypher('sp_cy', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN shortest_path(a, c) +$$) AS (path agtype); + path +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id": 844424930131969, "label": "N", "properties": {"name": "A"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "B"}}::vertex, {"id": 1125899906842626, "label": "KNOWS", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge, {"id": 844424930131971, "label": "N", "properties": {"name": "C"}}::vertex]::path +(1 row) + +-- all_shortest_paths() inside a cypher RETURN; expected: 1 path A..C (length 2) +SELECT * FROM cypher('sp_cy', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN all_shortest_paths(a, c) +$$) AS (path agtype); + path +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id": 844424930131969, "label": "N", "properties": {"name": "A"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "B"}}::vertex, {"id": 1125899906842626, "label": "KNOWS", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge, {"id": 844424930131971, "label": "N", "properties": {"name": "C"}}::vertex]::path +(1 row) + +-- in-cypher with an explicit edge-label filter; expected: 1 path A..C (length 2) +SELECT * FROM cypher('sp_cy', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN shortest_path(a, c, 'KNOWS') +$$) AS (path agtype); + path +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id": 844424930131969, "label": "N", "properties": {"name": "A"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "B"}}::vertex, {"id": 1125899906842626, "label": "KNOWS", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge, {"id": 844424930131971, "label": "N", "properties": {"name": "C"}}::vertex]::path +(1 row) + +-- still supported: call the SRF at the top level; expected: 1 path A..C (length 2) +SELECT path +FROM age_shortest_path( + '"sp_cy"'::agtype, + (SELECT id FROM cypher('sp_cy', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_cy', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)) +) AS path +ORDER BY path; + path +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id": 844424930131969, "label": "N", "properties": {"name": "A"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "B"}}::vertex, {"id": 1125899906842626, "label": "KNOWS", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge, {"id": 844424930131971, "label": "N", "properties": {"name": "C"}}::vertex]::path +(1 row) + +-- cleanup +SELECT * FROM drop_graph('sp_cy', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table sp_cy._ag_label_vertex +drop cascades to table sp_cy._ag_label_edge +drop cascades to table sp_cy."N" +drop cascades to table sp_cy."KNOWS" +NOTICE: graph "sp_cy" has been dropped + drop_graph +------------ + +(1 row) + +-- +-- Edge cases: parallel/multi-edges, self-loops, unknown edge labels, +-- max_hops boundaries (0 and negative), explicit 'any' direction, and +-- NULL / unknown-graph argument errors. +-- +SELECT * FROM create_graph('sp_edge'); +NOTICE: graph "sp_edge" has been created + create_graph +-------------- + +(1 row) + +-- A and B are connected by TWO parallel KNOWS edges plus one LIKES edge. +-- B->C is a single KNOWS edge. S has a self-loop. These exercise the +-- multi-predecessor (parallel edge) logic and the label filter. +SELECT * FROM cypher('sp_edge', $$ + CREATE (a:N {name: 'A'}), + (b:N {name: 'B'}), + (c:N {name: 'C'}), + (s:N {name: 'S'}), + (a)-[:KNOWS]->(b), + (a)-[:KNOWS]->(b), + (a)-[:LIKES]->(b), + (b)-[:KNOWS]->(c), + (s)-[:KNOWS]->(s) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- materialize the global graph context +SELECT * FROM cypher('sp_edge', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + result +----------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "N", "in_degree": 0, "out_degree": 3, "self_loops": 0} + {"id": 844424930131970, "label": "N", "in_degree": 3, "out_degree": 1, "self_loops": 0} + {"id": 844424930131971, "label": "N", "in_degree": 1, "out_degree": 0, "self_loops": 0} + {"id": 844424930131972, "label": "N", "in_degree": 1, "out_degree": 1, "self_loops": 1} +(4 rows) + +-- parallel edges: two distinct KNOWS edges A->B are two distinct shortest +-- paths; expected count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype); + count +------- + 2 +(1 row) + +-- no label filter: 2 KNOWS + 1 LIKES edge A->B are three distinct shortest +-- paths; expected count 3 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype, '"out"'::agtype); + count +------- + 3 +(1 row) + +-- single shortest path A->B picks exactly one of the parallel edges; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype))); + count +------- + 1 +(1 row) + +-- self-loop: a vertex with an edge to itself yields only the zero-length +-- path for start == end (the self-loop is never used); count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype))); + count +------- + 1 +(1 row) + +-- all_shortest_paths with start == end (existing vertex): one zero-length +-- path; count 1 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype))); + count +------- + 1 +(1 row) + +-- unknown relationship type matches no edges: A..C filtered by a label that +-- does not exist must return no path (NOT silently fall back to all edges); +-- count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"NOSUCHLABEL"'::agtype, '"out"'::agtype); + count +------- + 0 +(1 row) + +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"NOSUCHLABEL"'::agtype, '"out"'::agtype); + count +------- + 0 +(1 row) + +-- the zero-length (start == end) path has no edges, so an unknown label +-- still matches it; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype)), + '"NOSUCHLABEL"'::agtype, '"out"'::agtype); + count +------- + 1 +(1 row) + +-- existing label that does not connect the endpoints: LIKES only exists on +-- A->B, so A..C filtered by LIKES is unreachable; count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"LIKES"'::agtype, '"out"'::agtype); + count +------- + 0 +(1 row) + +-- max_hops = 0 with start == end: the zero-length path is still returned; +-- count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype, NULL::agtype, NULL::agtype, '0'::agtype); + count +------- + 1 +(1 row) + +-- max_hops = 0 with adjacent distinct endpoints: no path within zero hops; +-- count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype, NULL::agtype, NULL::agtype, '0'::agtype); + count +------- + 0 +(1 row) + +-- negative max_hops is treated as unbounded: A..C (length 2) is found; +-- count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype, NULL::agtype, NULL::agtype, '-1'::agtype); + count +------- + 1 +(1 row) + +-- explicit 'any' direction string (vs the default NULL == undirected); +-- two parallel KNOWS edges A->B give two shortest paths; count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"any"'::agtype); + count +------- + 2 +(1 row) + +-- NULL start (or end) vertex yields no rows (Cypher null semantics: a null +-- endpoint simply produces no match, it is not an error); count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + NULL::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype))); + count +------- + 0 +(1 row) + +-- NULL end vertex likewise yields no rows; count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype); + count +------- + 0 +(1 row) + +-- all_shortest_paths with a NULL endpoint also yields no rows; count 0 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype); + count +------- + 0 +(1 row) + +-- a single relationship type may be passed as a one-element array; expected: +-- same as the bare-string form, A..C under KNOWS (length 2); count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS"]'::agtype, '"out"'::agtype); + count +------- + 1 +(1 row) + +-- multiple relationship types are not yet supported; expected: ERROR +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS", "LIKES"]'::agtype, '"out"'::agtype); +ERROR: age_shortest_path: multiple relationship types are not yet supported +-- a non-zero minimum hop count is not yet supported; expected: ERROR +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); +ERROR: age_shortest_path: a minimum hop count is not yet supported +-- a minimum hop count of 0 is the default and is accepted; A..C (length 2); +-- count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 0::agtype); + count +------- + 1 +(1 row) + +-- a graph name that does not exist is an error +SELECT count(*) FROM age_shortest_path('"no_such_graph"'::agtype, '1'::agtype, '2'::agtype); +ERROR: schema "no_such_graph" does not exist +-- cleanup +SELECT * FROM drop_graph('sp_edge', true); +NOTICE: drop cascades to 5 other objects +DETAIL: drop cascades to table sp_edge._ag_label_vertex +drop cascades to table sp_edge._ag_label_edge +drop cascades to table sp_edge."N" +drop cascades to table sp_edge."KNOWS" +drop cascades to table sp_edge."LIKES" +NOTICE: graph "sp_edge" has been dropped + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/age_shortest_path.sql b/regress/sql/age_shortest_path.sql new file mode 100644 index 000000000..82b4d66bb --- /dev/null +++ b/regress/sql/age_shortest_path.sql @@ -0,0 +1,630 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +-- +-- age_shortest_path / age_all_shortest_paths +-- + +SELECT * FROM create_graph('sp_graph'); + +-- Build a small deterministic graph: +-- +-- A +-- / \ +-- B C (A->B, A->C, B->D, C->D : two shortest A..D paths) +-- \ / +-- D +-- | +-- E (D->E : unique 3-hop path A..E) +-- +-- Z (isolated, unreachable) +-- +SELECT * FROM cypher('sp_graph', $$ + CREATE (a:Person {name: 'A'}), + (b:Person {name: 'B'}), + (c:Person {name: 'C'}), + (d:Person {name: 'D'}), + (e:Person {name: 'E'}), + (z:Person {name: 'Z'}), + (a)-[:KNOWS]->(b), + (a)-[:KNOWS]->(c), + (b)-[:KNOWS]->(d), + (c)-[:KNOWS]->(d), + (d)-[:KNOWS]->(e) +$$) AS (result agtype); + +-- materialize the global graph context +SELECT * FROM cypher('sp_graph', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + +-- A -> D shortest path (length 2); expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)) +); + +-- all shortest A -> D; expected: 2 paths (A-B-D and A-C-D), each length 2 +SELECT path +FROM age_all_shortest_paths( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)) +) AS path +ORDER BY path; + +-- A -> E unique 3-hop path; expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'E'}) RETURN id(n) $$) AS (id agtype)) +); + +-- A -> E with max_hops = 2; expected: path_count = 0 (E is 3 hops away) +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'E'}) RETURN id(n) $$) AS (id agtype)), + NULL, NULL, NULL, 2::agtype +); + +-- zero-length path, start == end; expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)) +); + +-- unreachable vertex Z; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'Z'}) RETURN id(n) $$) AS (id agtype)) +); + +-- direction 'in': D -> A traversing edges backwards; expected: path_count = 2 +SELECT count(*) AS path_count +FROM age_all_shortest_paths( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL, '"in"'::agtype +); + +-- direction 'out': D -> A not reachable forwards; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL, '"out"'::agtype +); + +-- label filter 'KNOWS': A -> D still found; expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype +); + +-- error: invalid direction string; expected: ERROR (must be 'out', 'in', or 'any') +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + NULL, '"sideways"'::agtype +); + +-- error: start argument is neither a vertex nor an integer id; expected: ERROR +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + '"not_a_vertex"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)) +); + +-- +-- Non-existent endpoint guards. These must NOT crash the backend and must +-- return no rows (a path can only exist between vertices in the graph). +-- Previously, start == end on a non-existent vertex id was matched at BFS +-- depth 0 and path reconstruction dereferenced a missing vertex, crashing +-- the server. +-- + +-- start == end on a non-existent integer id; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path('"sp_graph"'::agtype, 999999::agtype, 999999::agtype); + +-- existing start -> non-existent end; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + 999999::agtype +); + +-- non-existent start -> existing end; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + 999999::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)) +); + +-- all-shortest-paths with start == end non-existent; expected: 0 rows +SELECT count(*) AS path_count +FROM age_all_shortest_paths('"sp_graph"'::agtype, 999999::agtype, 999999::agtype); + +-- cleanup +SELECT * FROM drop_graph('sp_graph', true); + + +-- +-- Empty graph: a graph that exists but has no vertices must return no rows +-- (and must not hang or crash) for any endpoint query. +-- +SELECT * FROM create_graph('sp_empty'); +SELECT count(*) AS path_count +FROM age_shortest_path('"sp_empty"'::agtype, 0::agtype, 1::agtype); +SELECT count(*) AS path_count +FROM age_all_shortest_paths('"sp_empty"'::agtype, 0::agtype, 0::agtype); +SELECT * FROM drop_graph('sp_empty', true); + + + +-- +-- A large, programmatically generated graph (120 nodes) exercising long +-- shortest paths (length up to 20), high-multiplicity all-shortest-paths, +-- label filtering, and directed vs. undirected reachability. +-- +-- Nodes: (:N {id: 0..119}). Structures built on top of them: +-- +-- * Main chain 0 -> 1 -> ... -> 20 (unique 20-hop path) +-- * Alternate chain 0 -> 50 -> 51 -> ... -> 68 -> 20 +-- (a second, disjoint 20-hop path 0..20) +-- => all-shortest-paths 0..20 under KNOWS = 2 paths of length 20 +-- * 3x3 lattice on ids 70..78, id = 70 + 3*row + col, edges go right +-- (id->id+1) and down (id->id+3). Monotone 70..78 paths: +-- => all-shortest-paths 70..78 = C(4,2) = 6 paths of length 4 +-- * LIKES shortcut 0 -[:LIKES]-> 20 (1 hop; only visible when the edge +-- label filter is NOT restricted to KNOWS) +-- * Back-edge triangle 0 -> 96 -> 95 -> 0 +-- => directed 0->95 = 2 hops (0-96-95); undirected 0..95 = 1 hop +-- * Many unused ids (e.g. 119) remain isolated / unreachable. +-- +SELECT * FROM create_graph('sp_big'); + +-- 120 vertices, ids 0..119 +SELECT * FROM cypher('sp_big', $$ + UNWIND range(0, 119) AS i CREATE (:N {id: i}) +$$) AS (result agtype); + +-- main chain 0->1->...->20 (KNOWS) +SELECT * FROM cypher('sp_big', $$ + UNWIND range(0, 19) AS i + MATCH (a:N {id: i}), (b:N {id: i + 1}) + CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + +-- alternate, disjoint 20-hop path 0->50->51->...->68->20 (KNOWS) +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 0}), (b:N {id: 50}) CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); +SELECT * FROM cypher('sp_big', $$ + UNWIND range(50, 67) AS i + MATCH (a:N {id: i}), (b:N {id: i + 1}) + CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 68}), (b:N {id: 20}) CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + +-- 3x3 lattice on ids 70..78: right edges (id -> id+1) +SELECT * FROM cypher('sp_big', $$ + UNWIND [0, 1, 2] AS r + UNWIND [0, 1] AS c + MATCH (a:N {id: 70 + 3 * r + c}), (b:N {id: 70 + 3 * r + c + 1}) + CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + +-- 3x3 lattice: down edges (id -> id+3) +SELECT * FROM cypher('sp_big', $$ + UNWIND [0, 1] AS r + UNWIND [0, 1, 2] AS c + MATCH (a:N {id: 70 + 3 * r + c}), (b:N {id: 70 + 3 * (r + 1) + c}) + CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + +-- back-edge triangle 0 -> 96 -> 95 -> 0 (KNOWS) +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 0}), (b:N {id: 96}) CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 96}), (b:N {id: 95}) CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 95}), (b:N {id: 0}) CREATE (a)-[:KNOWS]->(b) +$$) AS (result agtype); + +-- labelled shortcut 0 -[:LIKES]-> 20 +SELECT * FROM cypher('sp_big', $$ + MATCH (a:N {id: 0}), (b:N {id: 20}) CREATE (a)-[:LIKES]->(b) +$$) AS (result agtype); + +-- sanity: vertex count (also materializes the global context); expected: count = 120 +SELECT * FROM cypher('sp_big', $$ MATCH (n) RETURN count(n) $$) AS (n agtype); + +-- all shortest 0 -> 20 under KNOWS (main chain + disjoint alternate); +-- expected: 2 paths, each exactly 20 hops +SELECT path +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 20}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype +) AS path +ORDER BY path; + +-- any label: the LIKES shortcut collapses 0 -> 20; expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 20}) RETURN id(n) $$) AS (id agtype)), + NULL, '"out"'::agtype +); + +-- all shortest 70 -> 78 across the 3x3 lattice; expected: path_count = 6 (C(4,2)) +SELECT count(*) AS path_count +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 70}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 78}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype +); + +-- the lattice paths listed; expected: 6 paths, each 4 hops +SELECT path +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 70}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 78}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype +) AS path +ORDER BY path; + +-- max_hops = 19, one short of the 20-hop route; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 20}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, NULL, 19::agtype +); + +-- max_hops = 20 admits the full route; expected: path_count = 2 +SELECT count(*) AS path_count +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 20}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, NULL, 20::agtype +); + +-- DIRECTED out: 0 -> 95 must traverse 0->96->95; expected: 1 path (length 2) +SELECT path +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 95}) RETURN id(n) $$) AS (id agtype)), + NULL, '"out"'::agtype +) AS path +ORDER BY path; + +-- UNDIRECTED: 0 .. 95 via the 95->0 back edge; expected: 1 path (length 1) +SELECT path +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 95}) RETURN id(n) $$) AS (id agtype)) +) AS path +ORDER BY path; + +-- DIRECTED out: 78 -> 70 against lattice flow; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 78}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 70}) RETURN id(n) $$) AS (id agtype)), + NULL, '"out"'::agtype +); + +-- UNDIRECTED: 78 .. 70 reverses the lattice; expected: path_count = 6 +SELECT count(*) AS path_count +FROM age_all_shortest_paths( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 78}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 70}) RETURN id(n) $$) AS (id agtype)) +); + +-- isolated id 119 unreachable from 0; expected: path_count = 0 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 119}) RETURN id(n) $$) AS (id agtype)) +); + +-- zero-length path, start == end; expected: path_count = 1 +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_big"'::agtype, + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_big', $$ MATCH (n:N {id: 0}) RETURN id(n) $$) AS (id agtype)) +); + +-- cleanup +SELECT * FROM drop_graph('sp_big', true); + +-- +-- Calling the age_* SRFs from inside cypher() (Tier 1). +-- +-- Because the functions are prefixed with age_, the cypher() parser resolves +-- the unqualified names 'shortest_path' and 'all_shortest_paths' to +-- ag_catalog.age_shortest_path / ag_catalog.age_all_shortest_paths, and the +-- graph name is auto-injected as the first argument (like vle/vertex_stats), +-- so callers pass only the bound endpoints. A whole vertex implicitly casts to +-- agtype, so the argument types resolve. The SRFs are set-returning and now +-- work in a cypher RETURN projection (ProjectSet), returning one row per path. +-- +SELECT * FROM create_graph('sp_cy'); + +SELECT * FROM cypher('sp_cy', $$ + CREATE (a:N {name: 'A'}), + (b:N {name: 'B'}), + (c:N {name: 'C'}), + (a)-[:KNOWS]->(b), + (b)-[:KNOWS]->(c) +$$) AS (result agtype); + +-- materialize the global graph context +SELECT * FROM cypher('sp_cy', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + +-- shortest_path() inside a cypher RETURN; the graph name is auto-injected and +-- the bound vertices are passed; expected: 1 path A..C (length 2) +SELECT * FROM cypher('sp_cy', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN shortest_path(a, c) +$$) AS (path agtype); + +-- all_shortest_paths() inside a cypher RETURN; expected: 1 path A..C (length 2) +SELECT * FROM cypher('sp_cy', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN all_shortest_paths(a, c) +$$) AS (path agtype); + +-- in-cypher with an explicit edge-label filter; expected: 1 path A..C (length 2) +SELECT * FROM cypher('sp_cy', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN shortest_path(a, c, 'KNOWS') +$$) AS (path agtype); + +-- still supported: call the SRF at the top level; expected: 1 path A..C (length 2) +SELECT path +FROM age_shortest_path( + '"sp_cy"'::agtype, + (SELECT id FROM cypher('sp_cy', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_cy', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)) +) AS path +ORDER BY path; + +-- cleanup +SELECT * FROM drop_graph('sp_cy', true); + +-- +-- Edge cases: parallel/multi-edges, self-loops, unknown edge labels, +-- max_hops boundaries (0 and negative), explicit 'any' direction, and +-- NULL / unknown-graph argument errors. +-- +SELECT * FROM create_graph('sp_edge'); + +-- A and B are connected by TWO parallel KNOWS edges plus one LIKES edge. +-- B->C is a single KNOWS edge. S has a self-loop. These exercise the +-- multi-predecessor (parallel edge) logic and the label filter. +SELECT * FROM cypher('sp_edge', $$ + CREATE (a:N {name: 'A'}), + (b:N {name: 'B'}), + (c:N {name: 'C'}), + (s:N {name: 'S'}), + (a)-[:KNOWS]->(b), + (a)-[:KNOWS]->(b), + (a)-[:LIKES]->(b), + (b)-[:KNOWS]->(c), + (s)-[:KNOWS]->(s) +$$) AS (result agtype); + +-- materialize the global graph context +SELECT * FROM cypher('sp_edge', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + +-- parallel edges: two distinct KNOWS edges A->B are two distinct shortest +-- paths; expected count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype); + +-- no label filter: 2 KNOWS + 1 LIKES edge A->B are three distinct shortest +-- paths; expected count 3 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype, '"out"'::agtype); + +-- single shortest path A->B picks exactly one of the parallel edges; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype))); + +-- self-loop: a vertex with an edge to itself yields only the zero-length +-- path for start == end (the self-loop is never used); count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype))); + +-- all_shortest_paths with start == end (existing vertex): one zero-length +-- path; count 1 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype))); + +-- unknown relationship type matches no edges: A..C filtered by a label that +-- does not exist must return no path (NOT silently fall back to all edges); +-- count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"NOSUCHLABEL"'::agtype, '"out"'::agtype); + +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"NOSUCHLABEL"'::agtype, '"out"'::agtype); + +-- the zero-length (start == end) path has no edges, so an unknown label +-- still matches it; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'S'}) RETURN id(n) $$) AS (id agtype)), + '"NOSUCHLABEL"'::agtype, '"out"'::agtype); + +-- existing label that does not connect the endpoints: LIKES only exists on +-- A->B, so A..C filtered by LIKES is unreachable; count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"LIKES"'::agtype, '"out"'::agtype); + +-- max_hops = 0 with start == end: the zero-length path is still returned; +-- count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype, NULL::agtype, NULL::agtype, '0'::agtype); + +-- max_hops = 0 with adjacent distinct endpoints: no path within zero hops; +-- count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype, NULL::agtype, NULL::agtype, '0'::agtype); + +-- negative max_hops is treated as unbounded: A..C (length 2) is found; +-- count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype, NULL::agtype, NULL::agtype, '-1'::agtype); + +-- explicit 'any' direction string (vs the default NULL == undirected); +-- two parallel KNOWS edges A->B give two shortest paths; count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"any"'::agtype); + +-- NULL start (or end) vertex yields no rows (Cypher null semantics: a null +-- endpoint simply produces no match, it is not an error); count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + NULL::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype))); + +-- NULL end vertex likewise yields no rows; count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype); + +-- all_shortest_paths with a NULL endpoint also yields no rows; count 0 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype); + +-- a single relationship type may be passed as a one-element array; expected: +-- same as the bare-string form, A..C under KNOWS (length 2); count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS"]'::agtype, '"out"'::agtype); + +-- multiple relationship types are not yet supported; expected: ERROR +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS", "LIKES"]'::agtype, '"out"'::agtype); + +-- a non-zero minimum hop count is not yet supported; expected: ERROR +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); + +-- a minimum hop count of 0 is the default and is accepted; A..C (length 2); +-- count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 0::agtype); + +-- a graph name that does not exist is an error +SELECT count(*) FROM age_shortest_path('"no_such_graph"'::agtype, '1'::agtype, '2'::agtype); + +-- cleanup +SELECT * FROM drop_graph('sp_edge', true); diff --git a/sql/agtype_typecast.sql b/sql/agtype_typecast.sql index abca5e518..f12f215f6 100644 --- a/sql/agtype_typecast.sql +++ b/sql/agtype_typecast.sql @@ -98,6 +98,37 @@ CALLED ON NULL INPUT PARALLEL UNSAFE -- might be safe AS 'MODULE_PATHNAME'; +-- Unweighted (hop-count) shortest path between two vertices, computed over the +-- cached global graph adjacency via BFS. Returns a single path (0 or 1 rows). +-- Argument order mirrors the Cypher shortestPath() pattern +-- (a)-[:type*min_hops..max_hops]->(b): +-- (graph_name, start, end, edge_types, direction, min_hops, max_hops) +CREATE FUNCTION ag_catalog.age_shortest_path(IN agtype, IN agtype, IN agtype, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL) + RETURNS SETOF agtype +LANGUAGE C +STABLE +CALLED ON NULL INPUT +PARALLEL UNSAFE +AS 'MODULE_PATHNAME'; + +-- All unweighted shortest paths between two vertices (one path per row). +-- Same argument order as age_shortest_path. +CREATE FUNCTION ag_catalog.age_all_shortest_paths(IN agtype, IN agtype, IN agtype, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL, + IN agtype DEFAULT NULL) + RETURNS SETOF agtype +LANGUAGE C +STABLE +CALLED ON NULL INPUT +PARALLEL UNSAFE +AS 'MODULE_PATHNAME'; + -- function to build an edge for a VLE match CREATE FUNCTION ag_catalog.age_build_vle_match_edge(agtype, agtype) RETURNS agtype diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index dcd31b9df..5ac9dea65 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -2785,6 +2785,7 @@ static Query *transform_cypher_return(cypher_parsestate *cpstate, query->jointree = makeFromExpr(pstate->p_joinlist, NULL); query->hasAggs = pstate->p_hasAggs; query->hasSubLinks = pstate->p_hasSubLinks; + query->hasTargetSRFs = pstate->p_hasTargetSRFs; assign_query_collations(pstate, query); diff --git a/src/backend/parser/cypher_expr.c b/src/backend/parser/cypher_expr.c index e62f2c6e7..7e4f44600 100644 --- a/src/backend/parser/cypher_expr.c +++ b/src/backend/parser/cypher_expr.c @@ -2265,16 +2265,19 @@ static Node *transform_FuncCall(cypher_parsestate *cpstate, FuncCall *fn) fname = list_make2(makeString("ag_catalog"), makeString(ag_name)); /* - * Currently 3 functions need the graph name passed in as the first - * argument - in addition to the other arguments: startNode, endNode, - * and vle. So, check for those 3 functions here and that the arg list - * is not empty. Then prepend the graph name if necessary. + * Currently these functions need the graph name passed in as the + * first argument - in addition to the other arguments: startNode, + * endNode, vle, vertex_stats, shortest_path, and all_shortest_paths. + * So, check for those functions here and that the arg list is not + * empty. Then prepend the graph name if necessary. */ if ((list_length(targs) != 0) && (strcasecmp("startNode", name) == 0 || strcasecmp("endNode", name) == 0 || strcasecmp("vle", name) == 0 || - strcasecmp("vertex_stats", name) == 0)) + strcasecmp("vertex_stats", name) == 0 || + strcasecmp("shortest_path", name) == 0 || + strcasecmp("all_shortest_paths", name) == 0)) { char *graph_name = cpstate->graph_name; Datum d = string_to_agtype(graph_name); diff --git a/src/backend/utils/adt/age_vle.c b/src/backend/utils/adt/age_vle.c index 9e433b9e2..804d9e17e 100644 --- a/src/backend/utils/adt/age_vle.c +++ b/src/backend/utils/adt/age_vle.c @@ -65,6 +65,8 @@ #include "common/hashfn.h" #include "funcapi.h" +#include "miscadmin.h" +#include "nodes/pg_list.h" #include "utils/datum.h" #include "utils/lsyscache.h" @@ -1065,6 +1067,14 @@ static bool dfs_find_a_path_between(VLE_local_context *vlelctx) bool found = false; uint32 edge_hashvalue; + /* + * Allow this traversal to be cancelled (e.g. by a user Ctrl-C or a + * statement_timeout). On a large or densely connected graph this DFS + * can run for a long time, so we must yield to interrupt processing + * on every iteration. + */ + CHECK_FOR_INTERRUPTS(); + /* get an edge, but leave it on the stack for now */ edge_id = gid_stack_peek(edge_stack); /* @@ -1200,6 +1210,14 @@ static bool dfs_find_a_path_from(VLE_local_context *vlelctx) bool found = false; uint32 edge_hashvalue; + /* + * Allow this traversal to be cancelled (e.g. by a user Ctrl-C or a + * statement_timeout). On a large or densely connected graph this DFS + * can run for a long time, so we must yield to interrupt processing + * on every iteration. + */ + CHECK_FOR_INTERRUPTS(); + /* get an edge, but leave it on the stack for now */ edge_id = gid_stack_peek(edge_stack); /* @@ -2774,3 +2792,781 @@ Datum _ag_enforce_edge_uniqueness(PG_FUNCTION_ARGS) hash_destroy(exists_hash); PG_RETURN_BOOL(true); } + +/* + * --------------------------------------------------------------------------- + * Shortest path / all shortest paths + * --------------------------------------------------------------------------- + * + * Plain (non-grammar) set-returning functions that compute the unweighted + * (hop-count) shortest path between two vertices, built directly on top of the + * cached global graph (GRAPH_global_context) and its flat-array adjacency + * (VertexEdgeArray). These do NOT go through the VLE grammar/transform path; + * they are user-callable helpers: + * + * ag_catalog.age_shortest_path(graph, start, end + * [, edge_types [, direction [, min_hops [, max_hops]]]]) + * ag_catalog.age_all_shortest_paths(graph, start, end + * [, edge_types [, direction [, min_hops [, max_hops]]]]) + * + * Both perform a breadth-first search from the start vertex. age_shortest_path + * returns a single path (0 or 1 rows); age_all_shortest_paths returns every + * path whose length equals the minimum hop count (one row per path), by + * recording a predecessor multiset during the BFS and enumerating the + * resulting shortest-path DAG. + * + * Because BFS depth strictly increases, every emitted path is simple (no + * repeated vertex and therefore no repeated edge), satisfying openCypher + * edge-isomorphism for these fixed-length results. + */ + +/* Simple FIFO queue of graphids for the BFS frontier. */ +typedef struct sp_queue +{ + graphid *data; + int64 head; + int64 tail; + int64 cap; +} sp_queue; + +/* One predecessor edge on a shortest path (all-shortest-paths mode). */ +typedef struct sp_pred +{ + graphid edge; + graphid parent_vertex; +} sp_pred; + +/* Per-vertex BFS bookkeeping, keyed by vertex_id in the visited hashtable. */ +typedef struct sp_visit_entry +{ + graphid vertex_id; /* hash key — must be first */ + int64 depth; /* BFS depth from the source vertex */ + graphid parent_edge; /* single-path reconstruction */ + graphid parent_vertex; /* single-path reconstruction */ + List *preds; /* sp_pred * list for all-shortest-paths mode */ +} sp_visit_entry; + +/* Cross-call SRF state: the precomputed result paths streamed one per call. */ +typedef struct sp_srf_state +{ + Datum *paths; + int64 npaths; + int64 next; +} sp_srf_state; + +static void sp_queue_init(sp_queue *q) +{ + q->cap = 1024; + q->head = 0; + q->tail = 0; + q->data = palloc(sizeof(graphid) * q->cap); +} + +static void sp_queue_push(sp_queue *q, graphid v) +{ + if (q->tail == q->cap) + { + q->cap = q->cap * 2; + q->data = repalloc(q->data, sizeof(graphid) * q->cap); + } + q->data[q->tail] = v; + q->tail = q->tail + 1; +} + +static bool sp_queue_is_empty(sp_queue *q) +{ + return q->head == q->tail; +} + +static graphid sp_queue_pop(sp_queue *q) +{ + graphid v = q->data[q->head]; + + q->head = q->head + 1; + return v; +} + +/* Resolve a vertex argument (a vertex agtype or an integer id) to a graphid. */ +static graphid sp_agtype_to_graphid(agtype *agt, const char *argname) +{ + agtype_value *agtv = NULL; + + agtv = get_agtype_value("age_shortest_path", agt, AGTV_VERTEX, false); + + if (agtv != NULL && agtv->type == AGTV_VERTEX) + { + agtv = GET_AGTYPE_VALUE_OBJECT_VALUE(agtv, "id"); + } + else if (agtv == NULL || agtv->type != AGTV_INTEGER) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("%s argument must be a vertex or the integer id", + argname))); + } + + return agtv->val.int_value; +} + +/* Resolve the optional direction argument; NULL defaults to undirected. */ +static cypher_rel_dir sp_agtype_to_direction(agtype *agt) +{ + agtype_value *agtv = NULL; + char *s = NULL; + cypher_rel_dir dir = CYPHER_REL_DIR_NONE; + + if (agt == NULL) + { + return CYPHER_REL_DIR_NONE; + } + + agtv = get_agtype_value("age_shortest_path", agt, AGTV_STRING, true); + s = pnstrdup(agtv->val.string.val, agtv->val.string.len); + + if (pg_strcasecmp(s, "out") == 0) + { + dir = CYPHER_REL_DIR_RIGHT; + } + else if (pg_strcasecmp(s, "in") == 0) + { + dir = CYPHER_REL_DIR_LEFT; + } + else if (pg_strcasecmp(s, "any") == 0) + { + dir = CYPHER_REL_DIR_NONE; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("direction argument must be one of 'out', 'in', or 'any'"))); + } + + pfree_if_not_null(s); + return dir; +} + +/* + * Wrap an interleaved [vertex, edge, vertex, ... , vertex] graphid array in a + * VLE_path_container and materialize it as an AGTV_PATH agtype Datum. + */ +static Datum sp_build_path_datum(Oid graph_oid, graphid *alt, int64 alt_len) +{ + VLE_path_container *vpc = NULL; + graphid *arr = NULL; + agtype_value *agtv_path = NULL; + agtype *agt = NULL; + + vpc = create_VLE_path_container(alt_len); + vpc->graph_oid = graph_oid; + + arr = GET_GRAPHID_ARRAY_FROM_CONTAINER(vpc); + memcpy(arr, alt, sizeof(graphid) * alt_len); + + vpc->start_vid = alt[0]; + vpc->end_vid = alt[alt_len - 1]; + + agtv_path = build_path(vpc); + agt = agtype_value_to_agtype(agtv_path); + + return AGTYPE_P_GET_DATUM(agt); +} + +/* + * Breadth-first search from source toward target over the flat-array + * adjacency. Returns the visited hashtable; sets *out_found and (if found) + * *out_target_depth (the shortest hop count). In all-shortest-paths mode + * (collect_all) every shortest-path predecessor is recorded per vertex. + */ +static HTAB *sp_run_bfs(GRAPH_global_context *ggctx, graphid source, + graphid target, bool filter_edges, Oid edge_label_oid, + cypher_rel_dir dir, int64 max_hops, bool collect_all, + int64 *out_target_depth, bool *out_found) +{ + HASHCTL ctl; + HTAB *visited = NULL; + sp_queue q; + sp_visit_entry *se = NULL; + bool found = false; + int64 target_depth = -1; + bool dir_out = (dir == CYPHER_REL_DIR_RIGHT || dir == CYPHER_REL_DIR_NONE); + bool dir_in = (dir == CYPHER_REL_DIR_LEFT || dir == CYPHER_REL_DIR_NONE); + + /* visited hashtable: graphid -> sp_visit_entry */ + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(int64); + ctl.entrysize = sizeof(sp_visit_entry); + ctl.hash = graphid_hash; + visited = hash_create("age shortest path visited", 1024, &ctl, + HASH_ELEM | HASH_FUNCTION); + + /* + * A path can only exist between vertices that actually exist in the graph. + * If either endpoint is missing we are done: report "not found" and return + * the (empty) visited table. This guard is critical: without it a source + * that equals a non-existent target would be matched at depth 0 (see the + * "u == target" check below), and path reconstruction would then try to + * materialize a vertex that does not exist, dereferencing invalid memory + * and crashing the backend. + */ + if (get_vertex_entry(ggctx, source) == NULL || + get_vertex_entry(ggctx, target) == NULL) + { + *out_target_depth = -1; + *out_found = false; + return visited; + } + + sp_queue_init(&q); + + /* seed the frontier with the source vertex at depth 0 */ + se = (sp_visit_entry *) hash_search(visited, &source, HASH_ENTER, NULL); + se->vertex_id = source; + se->depth = 0; + se->parent_edge = 0; + se->parent_vertex = source; + se->preds = NIL; + sp_queue_push(&q, source); + + while (!sp_queue_is_empty(&q)) + { + graphid u = sp_queue_pop(&q); + sp_visit_entry *ue = NULL; + vertex_entry *ve = NULL; + int64 du = 0; + int pass = 0; + + /* + * Allow this search to be cancelled (e.g. by a user Ctrl-C or a + * statement_timeout). On a large graph the BFS frontier can grow very + * large, so we must yield to interrupt processing on every iteration. + */ + CHECK_FOR_INTERRUPTS(); + + ue = (sp_visit_entry *) hash_search(visited, &u, HASH_FIND, NULL); + du = ue->depth; + + /* target reached: record its (shortest) depth */ + if (u == target) + { + found = true; + if (target_depth < 0) + { + target_depth = du; + } + /* single-path mode: the first discovery is sufficient */ + if (!collect_all) + { + break; + } + } + + /* never expand at or beyond the shortest target depth */ + if (target_depth >= 0 && du >= target_depth) + { + continue; + } + + /* respect the optional upper hop bound */ + if (max_hops >= 0 && du >= max_hops) + { + continue; + } + + ve = get_vertex_entry(ggctx, u); + if (ve == NULL) + { + continue; + } + + /* pass 0 = outgoing edges, pass 1 = incoming edges */ + for (pass = 0; pass < 2; pass++) + { + VertexEdgeArray *edges = NULL; + int32 i = 0; + + if (pass == 0) + { + if (!dir_out) + { + continue; + } + edges = get_vertex_entry_edges_out_array(ve); + } + else + { + if (!dir_in) + { + continue; + } + edges = get_vertex_entry_edges_in_array(ve); + } + + if (edges == NULL || edges->array == NULL) + { + continue; + } + + for (i = 0; i < edges->size; i++) + { + graphid eid = edges->array[i]; + edge_entry *ee = NULL; + graphid v = 0; + sp_visit_entry *vse = NULL; + bool was_present = false; + + ee = get_edge_entry(ggctx, eid); + if (ee == NULL) + { + continue; + } + + /* + * Optional edge label filter. When a label filter is active + * we keep only edges whose label table oid matches. Note that + * a label name which does not exist in this graph resolves to + * InvalidOid; because no real edge can have an InvalidOid + * label table, every edge is then skipped and only the + * zero-length (start == end) path can match -- matching the + * openCypher semantics that an unknown relationship type + * matches no relationships. + */ + if (filter_edges && + get_edge_entry_label_table_oid(ee) != edge_label_oid) + { + continue; + } + + /* the neighbor depends on which side of the edge u is on */ + if (pass == 0) + { + v = get_edge_entry_end_vertex_id(ee); + } + else + { + v = get_edge_entry_start_vertex_id(ee); + } + + /* self loops never shorten a path to a different vertex */ + if (v == u) + { + continue; + } + + vse = (sp_visit_entry *) hash_search(visited, &v, HASH_ENTER, + &was_present); + if (!was_present) + { + vse->vertex_id = v; + vse->depth = du + 1; + vse->parent_edge = eid; + vse->parent_vertex = u; + vse->preds = NIL; + + if (collect_all) + { + sp_pred *p = palloc(sizeof(sp_pred)); + + p->edge = eid; + p->parent_vertex = u; + vse->preds = lappend(vse->preds, p); + } + + sp_queue_push(&q, v); + } + else if (collect_all && vse->depth == du + 1) + { + /* another equally-short predecessor of v */ + sp_pred *p = palloc(sizeof(sp_pred)); + + p->edge = eid; + p->parent_vertex = u; + vse->preds = lappend(vse->preds, p); + } + } + } + } + + *out_target_depth = target_depth; + *out_found = found; + return visited; +} + +/* + * Recursively enumerate every shortest path by walking the predecessor DAG + * from target back to source. Each completed path is appended to *out as a + * freshly allocated interleaved graphid array of length alt_len. + */ +static void sp_enumerate(HTAB *visited, graphid source, graphid cur, + graphid *alt, int64 alt_len, int64 pos, List **out) +{ + sp_visit_entry *e = NULL; + ListCell *lc = NULL; + + /* + * Enumerating every shortest path can be combinatorially expensive, so + * allow the user to cancel (Ctrl-C / statement_timeout) at each step. + */ + CHECK_FOR_INTERRUPTS(); + + alt[pos] = cur; + + if (cur == source) + { + /* a complete path only when we have consumed the whole array */ + if (pos == 0) + { + graphid *copy = palloc(sizeof(graphid) * alt_len); + + memcpy(copy, alt, sizeof(graphid) * alt_len); + *out = lappend(*out, copy); + } + return; + } + + e = (sp_visit_entry *) hash_search(visited, &cur, HASH_FIND, NULL); + if (e == NULL) + { + return; + } + + foreach(lc, e->preds) + { + sp_pred *p = (sp_pred *) lfirst(lc); + + alt[pos - 1] = p->edge; + sp_enumerate(visited, source, p->parent_vertex, alt, alt_len, pos - 2, + out); + } +} + +/* + * Resolve arguments, run the BFS, and materialize the result path(s) as an + * array of AGTV_PATH agtype Datums. Returns NULL with *out_count == 0 when no + * path exists. Caller must run in a context that survives the SRF. + */ +static Datum *sp_compute_paths(agtype *graph_name_agt, agtype *start_agt, + agtype *end_agt, agtype *label_agt, + agtype *dir_agt, agtype *minhops_agt, + agtype *maxhops_agt, bool collect_all, + int64 *out_count) +{ + agtype_value *agtv_temp = NULL; + char *graph_name = NULL; + Oid graph_oid = InvalidOid; + GRAPH_global_context *ggctx = NULL; + graphid source = 0; + graphid target = 0; + bool filter_edges = false; + Oid edge_label_oid = InvalidOid; + cypher_rel_dir dir = CYPHER_REL_DIR_NONE; + int64 max_hops = -1; + HTAB *visited = NULL; + int64 target_depth = -1; + bool found = false; + Datum *paths = NULL; + + *out_count = 0; + + /* the graph name is required */ + if (graph_name_agt == NULL) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("age_shortest_path: graph name cannot be NULL"))); + } + + agtv_temp = get_agtype_value("age_shortest_path", graph_name_agt, + AGTV_STRING, true); + graph_name = pnstrdup(agtv_temp->val.string.val, + agtv_temp->val.string.len); + graph_oid = get_graph_oid(graph_name); + + /* + * A NULL start or end vertex yields no rows, matching Cypher semantics + * where a null endpoint simply produces no match (it is not an error). + */ + if (start_agt == NULL || end_agt == NULL) + { + return NULL; + } + + source = sp_agtype_to_graphid(start_agt, "start vertex"); + target = sp_agtype_to_graphid(end_agt, "end vertex"); + + /* + * Optional edge type filter. A single relationship type may be supplied + * either as a bare string or as a one-element array. Multiple relationship + * types (an array with more than one element) are not yet supported. + */ + if (label_agt != NULL) + { + char *label_name = NULL; + + if (AGT_ROOT_IS_ARRAY(label_agt) && !AGT_ROOT_IS_SCALAR(label_agt)) + { + int nelems = AGT_ROOT_COUNT(label_agt); + + if (nelems > 1) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("age_shortest_path: multiple relationship types are not yet supported"))); + } + + if (nelems == 1) + { + agtv_temp = get_ith_agtype_value_from_container( + &label_agt->root, 0); + if (agtv_temp->type != AGTV_STRING) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("age_shortest_path: relationship type must be a string"))); + } + if (agtv_temp->val.string.len != 0) + { + label_name = pnstrdup(agtv_temp->val.string.val, + agtv_temp->val.string.len); + edge_label_oid = get_label_relation(label_name, graph_oid); + filter_edges = true; + } + } + } + else + { + agtv_temp = get_agtype_value("age_shortest_path", label_agt, + AGTV_STRING, true); + if (agtv_temp->val.string.len != 0) + { + label_name = pnstrdup(agtv_temp->val.string.val, + agtv_temp->val.string.len); + edge_label_oid = get_label_relation(label_name, graph_oid); + filter_edges = true; + } + } + } + + /* optional direction (defaults to undirected) */ + dir = sp_agtype_to_direction(dir_agt); + + /* + * Optional minimum hop count. A genuine minimum-length constraint needs a + * different search than plain BFS, so for now only the default (NULL or 0) + * is accepted; any other value is rejected loudly. + */ + if (minhops_agt != NULL) + { + int64 min_hops = 0; + + agtv_temp = get_agtype_value("age_shortest_path", minhops_agt, + AGTV_INTEGER, true); + min_hops = agtv_temp->val.int_value; + if (min_hops != 0) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("age_shortest_path: a minimum hop count is not yet supported"))); + } + } + + /* optional upper hop bound (NULL or negative means unbounded) */ + if (maxhops_agt != NULL) + { + agtv_temp = get_agtype_value("age_shortest_path", maxhops_agt, + AGTV_INTEGER, true); + max_hops = agtv_temp->val.int_value; + if (max_hops < 0) + { + max_hops = -1; + } + } + + /* build / fetch the global graph cache for this graph */ + ggctx = manage_GRAPH_global_contexts(graph_name, graph_oid); + if (ggctx == NULL) + { + return NULL; + } + + /* run the breadth-first search */ + visited = sp_run_bfs(ggctx, source, target, filter_edges, edge_label_oid, + dir, max_hops, collect_all, &target_depth, &found); + + if (!found) + { + hash_destroy(visited); + return NULL; + } + + if (!collect_all) + { + /* reconstruct the single shortest path from the parent pointers */ + int64 alt_len = (2 * target_depth) + 1; + graphid *alt = palloc(sizeof(graphid) * alt_len); + int64 pos = alt_len - 1; + graphid cur = target; + + alt[pos] = cur; + pos = pos - 1; + while (cur != source) + { + sp_visit_entry *e = NULL; + + e = (sp_visit_entry *) hash_search(visited, &cur, HASH_FIND, NULL); + alt[pos] = e->parent_edge; + pos = pos - 1; + alt[pos] = e->parent_vertex; + pos = pos - 1; + cur = e->parent_vertex; + } + + paths = palloc(sizeof(Datum)); + paths[0] = sp_build_path_datum(graph_oid, alt, alt_len); + *out_count = 1; + } + else + { + /* enumerate every equal-length shortest path */ + int64 alt_len = (2 * target_depth) + 1; + graphid *alt = palloc(sizeof(graphid) * alt_len); + List *arrays = NIL; + ListCell *lc = NULL; + int64 n = 0; + int64 idx = 0; + + sp_enumerate(visited, source, target, alt, alt_len, alt_len - 1, + &arrays); + + n = list_length(arrays); + paths = palloc(sizeof(Datum) * (n > 0 ? n : 1)); + foreach(lc, arrays) + { + graphid *a = (graphid *) lfirst(lc); + + paths[idx] = sp_build_path_datum(graph_oid, a, alt_len); + idx = idx + 1; + } + *out_count = n; + } + + hash_destroy(visited); + return paths; +} + +/* + * Shared SRF driver for age_shortest_path / age_all_shortest_paths. The first + * call computes every result path up front and stores them; subsequent calls + * stream them one per row. + */ +static Datum sp_srf_impl(FunctionCallInfo fcinfo, bool collect_all) +{ + FuncCallContext *funcctx = NULL; + sp_srf_state *state = NULL; + + if (SRF_IS_FIRSTCALL()) + { + MemoryContext oldctx; + agtype *a_graph = NULL; + agtype *a_start = NULL; + agtype *a_end = NULL; + agtype *a_label = NULL; + agtype *a_dir = NULL; + agtype *a_min = NULL; + agtype *a_max = NULL; + + funcctx = SRF_FIRSTCALL_INIT(); + oldctx = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + /* + * Argument order mirrors the Cypher shortestPath() pattern + * (a)-[:type*min_hops..max_hops]->(b): + * 0 graph, 1 start, 2 end, 3 edge_types, 4 direction, + * 5 min_hops, 6 max_hops + */ + a_graph = PG_ARGISNULL(0) ? NULL : AG_GET_ARG_AGTYPE_P(0); + a_start = PG_ARGISNULL(1) ? NULL : AG_GET_ARG_AGTYPE_P(1); + a_end = PG_ARGISNULL(2) ? NULL : AG_GET_ARG_AGTYPE_P(2); + a_label = PG_ARGISNULL(3) ? NULL : AG_GET_ARG_AGTYPE_P(3); + a_dir = PG_ARGISNULL(4) ? NULL : AG_GET_ARG_AGTYPE_P(4); + a_min = PG_ARGISNULL(5) ? NULL : AG_GET_ARG_AGTYPE_P(5); + a_max = PG_ARGISNULL(6) ? NULL : AG_GET_ARG_AGTYPE_P(6); + + /* treat an explicit agtype null the same as a SQL NULL */ + if (a_start != NULL && is_agtype_null(a_start)) + { + a_start = NULL; + } + if (a_end != NULL && is_agtype_null(a_end)) + { + a_end = NULL; + } + if (a_label != NULL && is_agtype_null(a_label)) + { + a_label = NULL; + } + if (a_dir != NULL && is_agtype_null(a_dir)) + { + a_dir = NULL; + } + if (a_min != NULL && is_agtype_null(a_min)) + { + a_min = NULL; + } + if (a_max != NULL && is_agtype_null(a_max)) + { + a_max = NULL; + } + + state = palloc0(sizeof(sp_srf_state)); + state->next = 0; + state->paths = sp_compute_paths(a_graph, a_start, a_end, a_label, + a_dir, a_min, a_max, collect_all, + &state->npaths); + funcctx->user_fctx = state; + + MemoryContextSwitchTo(oldctx); + } + + funcctx = SRF_PERCALL_SETUP(); + state = (sp_srf_state *) funcctx->user_fctx; + + if (state->next < state->npaths) + { + Datum d = state->paths[state->next]; + + state->next = state->next + 1; + SRF_RETURN_NEXT(funcctx, d); + } + + SRF_RETURN_DONE(funcctx); +} + +/* + * age_shortest_path(graph_name, start, end [, edge_types [, direction + * [, min_hops [, max_hops]]]]) -> SETOF agtype + * + * Returns the single unweighted shortest path (as an AGTV_PATH) between the + * start and end vertices, or no rows if unreachable. + */ +PG_FUNCTION_INFO_V1(age_shortest_path); + +Datum age_shortest_path(PG_FUNCTION_ARGS) +{ + return sp_srf_impl(fcinfo, false); +} + +/* + * age_all_shortest_paths(graph_name, start, end [, edge_types [, direction + * [, min_hops [, max_hops]]]]) -> SETOF agtype + * + * Returns every unweighted shortest path (one AGTV_PATH per row) between the + * start and end vertices, i.e. all paths whose length equals the minimum hop + * count, or no rows if unreachable. + */ +PG_FUNCTION_INFO_V1(age_all_shortest_paths); + +Datum age_all_shortest_paths(PG_FUNCTION_ARGS) +{ + return sp_srf_impl(fcinfo, true); +} From 2bc8e95f00193e33a0ba8cf8839d7a97f65beeb8 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Mon, 22 Jun 2026 12:34:31 -0400 Subject: [PATCH 085/100] Support pattern expressions as boolean expressions (#2360) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Support pattern expressions in WHERE clause via GLR parser (issue #1577) Enable bare graph patterns as boolean expressions in WHERE clauses: MATCH (a:Person), (b:Person) WHERE (a)-[:KNOWS]->(b) -- now valid, equivalent to EXISTS(...) RETURN a.name, b.name Previously, this required wrapping in EXISTS(): WHERE EXISTS((a)-[:KNOWS]->(b)) The bare pattern syntax is standard openCypher and is used extensively in Neo4j. Its absence was the most frequently cited migration blocker. Implementation approach: - Switch the Cypher parser from LALR(1) to Bison GLR mode. GLR handles the inherent ambiguity between parenthesized expressions '(' expr ')' and graph path nodes '(' var_name label_opt props ')' by forking the parse stack and discarding the failing path. - Add anonymous_path as an expr_atom alternative with %dprec 1 (lower priority than expression path at %dprec 2). The action wraps the pattern in a cypher_sub_pattern + EXISTS SubLink, reusing the same transform_cypher_sub_pattern() machinery as explicit EXISTS(). - Extract make_exists_pattern_sublink() helper shared by both EXISTS(pattern) and bare pattern rules. - Fix YYLLOC_DEFAULT to use YYRHSLOC() for GLR compatibility. - %dprec annotations on expr_var/var_name_opt resolve the reduce/reduce conflict between expression variables and pattern node variables. Conflict budget: 7 shift/reduce (path extension vs arithmetic on -/<), 3 reduce/reduce (expr_var vs var_name_opt on )/}/=). All are expected and handled correctly by GLR forking + %dprec disambiguation. All 32 regression tests pass (31 existing + 1 new). New pattern_expression test covers: bare patterns, NOT patterns, labeled nodes, AND/OR combinations, left-directed patterns, anonymous nodes, multi-hop patterns, EXISTS() backward compatibility, and non-pattern expression regression checks. Co-Authored-By: Claude Opus 4.6 (1M context) * Address Copilot review: comment placement, %expect docs, test wording 1. Move "Helper function to create an ExplainStmt node" comment from above make_exists_pattern_sublink() to above make_explain_stmt() where it belongs. 2. Add block comment documenting the %expect/%expect-rr conflict budget: 7 S/R from path vs arithmetic on - and <, 3 R/R from expr_var vs var_name_opt on ) } =. 3. Clarify test comment: "Regular expressions" -> "Regular (non-pattern) expressions" to avoid confusion with regex. Regression test: pattern_expression OK. Co-Authored-By: Claude Opus 4.6 (1M context) * Address Copilot round 3: broaden scope, remove %expect fragility - Pattern expressions are now accepted anywhere an expr is valid (RETURN, WITH, SET, CASE, boolean combinations), not only WHERE. This matches openCypher semantics and documents the broader surface area that was already implicitly enabled by adding anonymous_path to expr_atom. Added regression tests for each new context: RETURN projection (bare and AS-aliased), mixed with other projections, CASE WHEN, boolean AND/OR combinators, SET to persist a computed boolean property, and WITH ... WHERE pipeline. - Remove the hardcoded `%expect 7` / `%expect-rr 3` conflict budget from cypher_gram.y. The exact conflict counts can drift across Bison versions and distros, which would break builds even though the grammar is correct (GLR handles the conflicts at runtime via fork + %dprec). Instead, pass -Wno-conflicts-sr / -Wno-conflicts-rr via BISONFLAGS in the Makefile so the build stays clean without binding us to a specific Bison release. Kept a block comment in the grammar explaining why GLR conflicts are expected and how they resolve. Co-Authored-By: Claude Opus 4.6 (1M context) * Address jrgemignani review: keep -Werror, restore %expect budget Reverts the broad `-Werror` drop and the no-%expect approach from the prior round on jrgemignani's request. The earlier framing — that conflict counts drift across Bison versions, so %expect is fragile — overcorrected: it removed the only build-time alarm bell for unintended new conflicts. Makefile: keep -Werror so any unexpected Bison warning (unused rules, undeclared types, etc.) still fails the build; downgrade only the two conflict categories to plain warnings via -Wno-error=conflicts-sr -Wno-error=conflicts-rr. pgxs auto-adds -Wno-deprecated, so existing %name-prefix= / %pure-parser directives remain non-erroring. cypher_gram.y: add `%expect 7` and `%expect-rr 3` matching the Bison 3.8.2 totals. Bison treats %expect as exact-match, not as a ceiling — any deviation fails the build and forces an audit of the new conflicts. Comment updated to reflect that future Bison versions reporting different counts should bump the numbers explicitly with a version note in the commit message, rather than removing the directive. No grammar or runtime change. Cassert installcheck 34/34 AGE tests green. * Add follow-up regression coverage for pattern expressions (#2360) Addresses the non-blocking test-coverage follow-ups from the review: pattern expressions in additional contexts opened up by allowing anonymous_path as an expr_atom. New cases (all verified against a PG18 build): - Single-node pattern on a bound variable (a:Label). Documented as an EXISTS existence check, NOT an openCypher label predicate: a matching label is always true, and a non-matching label hits AGE's pre-existing "multiple labels for variable" restriction (captured as expected error). - Pattern expressions inside list and map literals. - Pattern expressions as function arguments: collect() shows correct per-row booleans; count() counts all rows (non-null bool) -- documented so the value is not mistaken for a bug. - Pattern expression in OPTIONAL MATCH ... WHERE (null-preserving). - EXISTS() and a bare pattern combined in one predicate. make installcheck: 33/33 green. --------- Co-authored-by: Claude Opus 4.6 (1M context) --- Makefile | 17 +- regress/expected/pattern_expression.out | 457 ++++++++++++++++++++++++ regress/sql/pattern_expression.sql | 305 ++++++++++++++++ src/backend/parser/cypher_gram.y | 101 ++++-- 4 files changed, 859 insertions(+), 21 deletions(-) create mode 100644 regress/expected/pattern_expression.out create mode 100644 regress/sql/pattern_expression.sql diff --git a/Makefile b/Makefile index 41208ee02..b0059213c 100644 --- a/Makefile +++ b/Makefile @@ -215,6 +215,7 @@ REGRESS = scan \ jsonb_operators \ list_comprehension \ predicate_functions \ + pattern_expression \ map_projection \ direct_field_access \ security \ @@ -282,7 +283,21 @@ src/include/parser/cypher_kwlist_d.h: src/include/parser/cypher_kwlist.h $(GEN_K src/include/parser/cypher_gram_def.h: src/backend/parser/cypher_gram.c -src/backend/parser/cypher_gram.c: BISONFLAGS += --defines=src/include/parser/cypher_gram_def.h -Werror +# +# The Cypher grammar uses GLR mode with a number of inherent shift/reduce +# and reduce/reduce conflicts arising from the ambiguity between +# parenthesized expressions and graph patterns (both start with '('). +# GLR handles these correctly at runtime by forking at the conflict +# point; %dprec annotations resolve cases where both forks succeed. +# +# We keep -Werror so any unexpected Bison warning (unused rules, undeclared +# types, etc.) still fails the build; we downgrade only the two conflict +# categories to plain warnings via -Wno-error=. The exact conflict totals +# are pinned by %expect / %expect-rr in cypher_gram.y, which Bison treats +# as exact-match: any deviation fails the build and forces an audit of +# the new conflicts. +# +src/backend/parser/cypher_gram.c: BISONFLAGS += --defines=src/include/parser/cypher_gram_def.h -Werror -Wno-error=conflicts-sr -Wno-error=conflicts-rr src/backend/parser/cypher_parser.o: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/backend/parser/cypher_parser.bc: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h diff --git a/regress/expected/pattern_expression.out b/regress/expected/pattern_expression.out new file mode 100644 index 000000000..0494d49b9 --- /dev/null +++ b/regress/expected/pattern_expression.out @@ -0,0 +1,457 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +SELECT create_graph('pattern_expr'); +NOTICE: graph "pattern_expr" has been created + create_graph +-------------- + +(1 row) + +-- +-- Setup test data +-- +SELECT * FROM cypher('pattern_expr', $$ + CREATE (alice:Person {name: 'Alice'})-[:KNOWS]->(bob:Person {name: 'Bob'}), + (alice)-[:WORKS_WITH]->(charlie:Person {name: 'Charlie'}), + (dave:Person {name: 'Dave'}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- +-- Basic pattern expression in WHERE +-- +-- Bare pattern: (a)-[:REL]->(b) +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE (a)-[:KNOWS]->(b) + RETURN a.name, b.name + ORDER BY a.name, b.name +$$) AS (a agtype, b agtype); + a | b +---------+------- + "Alice" | "Bob" +(1 row) + +-- +-- NOT pattern expression +-- +-- Find people who don't KNOW anyone +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WHERE NOT (a)-[:KNOWS]->(:Person) + RETURN a.name + ORDER BY a.name +$$) AS (result agtype); + result +----------- + "Bob" + "Charlie" + "Dave" +(3 rows) + +-- +-- Pattern with labeled first node +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE (a:Person)-[:KNOWS]->(b) + RETURN a.name, b.name + ORDER BY a.name +$$) AS (a agtype, b agtype); + a | b +---------+------- + "Alice" | "Bob" +(1 row) + +-- +-- Pattern combined with AND +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE (a)-[:KNOWS]->(b) AND a.name = 'Alice' + RETURN a.name, b.name +$$) AS (a agtype, b agtype); + a | b +---------+------- + "Alice" | "Bob" +(1 row) + +-- +-- Pattern combined with OR +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE (a)-[:KNOWS]->(b) OR (a)-[:WORKS_WITH]->(b) + RETURN a.name, b.name + ORDER BY a.name, b.name +$$) AS (a agtype, b agtype); + a | b +---------+----------- + "Alice" | "Bob" + "Alice" | "Charlie" +(2 rows) + +-- +-- Left-directed pattern +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE (a)<-[:KNOWS]-(b) + RETURN a.name, b.name + ORDER BY a.name +$$) AS (a agtype, b agtype); + a | b +-------+--------- + "Bob" | "Alice" +(1 row) + +-- +-- Pattern with anonymous nodes +-- +-- Find anyone who has any outgoing KNOWS relationship +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WHERE (a)-[:KNOWS]->() + RETURN a.name + ORDER BY a.name +$$) AS (result agtype); + result +--------- + "Alice" +(1 row) + +-- +-- Multiple relationship pattern +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (c:Person) + WHERE (a)-[:KNOWS]->()-[:WORKS_WITH]->(c) + RETURN a.name, c.name + ORDER BY a.name +$$) AS (a agtype, c agtype); + a | c +---+--- +(0 rows) + +-- +-- Existing EXISTS() syntax still works (backward compatibility) +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE EXISTS((a)-[:KNOWS]->(b)) + RETURN a.name, b.name + ORDER BY a.name +$$) AS (a agtype, b agtype); + a | b +---------+------- + "Alice" | "Bob" +(1 row) + +-- +-- Pattern expression produces same results as EXISTS() +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WHERE (a)-[:KNOWS]->(:Person) + RETURN a.name + ORDER BY a.name +$$) AS (result agtype); + result +--------- + "Alice" +(1 row) + +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WHERE EXISTS((a)-[:KNOWS]->(:Person)) + RETURN a.name + ORDER BY a.name +$$) AS (result agtype); + result +--------- + "Alice" +(1 row) + +-- +-- Regular (non-pattern) expressions still work (no regression) +-- +SELECT * FROM cypher('pattern_expr', $$ + RETURN (1 + 2) +$$) AS (result agtype); + result +-------- + 3 +(1 row) + +SELECT * FROM cypher('pattern_expr', $$ + MATCH (n:Person) + WHERE n.name = 'Alice' + RETURN (n.name) +$$) AS (result agtype); + result +--------- + "Alice" +(1 row) + +-- +-- Pattern expressions in RETURN (boolean projection) +-- +-- Each person gets a column showing whether they know someone +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, (a)-[:KNOWS]->(:Person) AS knows_someone + ORDER BY a.name +$$) AS (name agtype, knows_someone agtype); + name | knows_someone +-----------+--------------- + "Alice" | true + "Bob" | false + "Charlie" | false + "Dave" | false +(4 rows) + +-- Mix pattern expression with other projections +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, (a)-[:KNOWS]->(:Person), (a)-[:WORKS_WITH]->(:Person) + ORDER BY a.name +$$) AS (name agtype, knows agtype, works_with agtype); + name | knows | works_with +-----------+-------+------------ + "Alice" | true | true + "Bob" | false | false + "Charlie" | false | false + "Dave" | false | false +(4 rows) + +-- +-- Pattern expressions in CASE WHEN +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, + CASE WHEN (a)-[:KNOWS]->(:Person) THEN 'social' + ELSE 'loner' + END + ORDER BY a.name +$$) AS (name agtype, kind agtype); + name | kind +-----------+---------- + "Alice" | "social" + "Bob" | "loner" + "Charlie" | "loner" + "Dave" | "loner" +(4 rows) + +-- +-- Pattern expressions combined with boolean operators in RETURN +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, + (a)-[:KNOWS]->(:Person) AND (a)-[:WORKS_WITH]->(:Person) AS has_both, + (a)-[:KNOWS]->(:Person) OR (a)-[:WORKS_WITH]->(:Person) AS has_either + ORDER BY a.name +$$) AS (name agtype, has_both agtype, has_either agtype); + name | has_both | has_either +-----------+----------+------------ + "Alice" | true | true + "Bob" | false | false + "Charlie" | false | false + "Dave" | false | false +(4 rows) + +-- +-- Pattern expression in SET (store boolean as property) +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + SET a.is_social = (a)-[:KNOWS]->(:Person) + RETURN a.name, a.is_social + ORDER BY a.name +$$) AS (name agtype, is_social agtype); + name | is_social +-----------+----------- + "Alice" | true + "Bob" | false + "Charlie" | false + "Dave" | false +(4 rows) + +-- +-- Pattern expression in WITH (carry boolean through pipeline) +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WITH a.name AS name, (a)-[:KNOWS]->(:Person) AS knows + WHERE knows + RETURN name + ORDER BY name +$$) AS (result agtype); + result +--------- + "Alice" +(1 row) + +-- +-- Follow-up coverage (review #2360): pattern expressions in additional +-- expression contexts opened up by allowing anonymous_path as an expr_atom. +-- +-- +-- Single-node pattern on an already-bound variable: (a:Label) +-- +-- NOTE: this is an EXISTS existence check on the bound variable, NOT an +-- openCypher label predicate. A matching label is therefore always true +-- (the variable is already bound), and a *different* label is rejected by +-- AGE's pre-existing "multiple labels for variable" restriction rather than +-- evaluating to false. Both behaviours are captured here so any future change +-- to single-node-pattern semantics is caught by this test. +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, (a:Person) + ORDER BY a.name +$$) AS (name agtype, is_person agtype); + name | is_person +-----------+----------- + "Alice" | true + "Bob" | true + "Charlie" | true + "Dave" | true +(4 rows) + +-- A non-matching label errors (pre-existing limitation, not a regression) +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, (a:Animal) + ORDER BY a.name +$$) AS (name agtype, is_animal agtype); +ERROR: multiple labels for variable 'a' are not supported +LINE 3: RETURN a.name, (a:Animal) + ^ +-- +-- Pattern expressions inside a list literal +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, [(a)-[:KNOWS]->(:Person), (a)-[:WORKS_WITH]->(:Person)] + ORDER BY a.name +$$) AS (name agtype, flags agtype); + name | flags +-----------+---------------- + "Alice" | [true, true] + "Bob" | [false, false] + "Charlie" | [false, false] + "Dave" | [false, false] +(4 rows) + +-- +-- Pattern expressions inside a map literal +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, {knows: (a)-[:KNOWS]->(:Person), works: (a)-[:WORKS_WITH]->(:Person)} + ORDER BY a.name +$$) AS (name agtype, m agtype); + name | m +-----------+---------------------------------- + "Alice" | {"knows": true, "works": true} + "Bob" | {"knows": false, "works": false} + "Charlie" | {"knows": false, "works": false} + "Dave" | {"knows": false, "works": false} +(4 rows) + +-- +-- Pattern expressions as function arguments +-- +-- collect() shows the per-row boolean values are correct (ORDER BY before +-- the aggregate so the collected order is deterministic across scan plans). +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WITH a ORDER BY a.name + RETURN collect((a)-[:KNOWS]->(:Person)) +$$) AS (vals agtype); + vals +----------------------------- + [true, false, false, false] +(1 row) + +-- count() counts non-null values; a boolean (including false) is non-null, +-- so this counts every row rather than only the matching ones. This is the +-- expected SQL aggregate semantics, documented here so the value is not +-- mistaken for a bug. +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN count((a)-[:KNOWS]->(:Person)) +$$) AS (c agtype); + c +--- + 4 +(1 row) + +-- +-- Pattern expression in OPTIONAL MATCH ... WHERE (null-preserving) +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + OPTIONAL MATCH (b:Person) WHERE (a)-[:KNOWS]->(b) + RETURN a.name, b.name + ORDER BY a.name, b.name +$$) AS (a agtype, b agtype); + a | b +-----------+------- + "Alice" | "Bob" + "Bob" | + "Charlie" | + "Dave" | +(4 rows) + +-- +-- EXISTS() and a bare pattern combined in a single predicate +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WHERE EXISTS((a)-[:KNOWS]->(:Person)) AND (a)-[:WORKS_WITH]->(:Person) + RETURN a.name + ORDER BY a.name +$$) AS (name agtype); + name +--------- + "Alice" +(1 row) + +-- +-- Cleanup +-- +SELECT * FROM drop_graph('pattern_expr', true); +NOTICE: drop cascades to 5 other objects +DETAIL: drop cascades to table pattern_expr._ag_label_vertex +drop cascades to table pattern_expr._ag_label_edge +drop cascades to table pattern_expr."Person" +drop cascades to table pattern_expr."KNOWS" +drop cascades to table pattern_expr."WORKS_WITH" +NOTICE: graph "pattern_expr" has been dropped + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/pattern_expression.sql b/regress/sql/pattern_expression.sql new file mode 100644 index 000000000..fff8476e5 --- /dev/null +++ b/regress/sql/pattern_expression.sql @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +SELECT create_graph('pattern_expr'); + +-- +-- Setup test data +-- +SELECT * FROM cypher('pattern_expr', $$ + CREATE (alice:Person {name: 'Alice'})-[:KNOWS]->(bob:Person {name: 'Bob'}), + (alice)-[:WORKS_WITH]->(charlie:Person {name: 'Charlie'}), + (dave:Person {name: 'Dave'}) +$$) AS (result agtype); + +-- +-- Basic pattern expression in WHERE +-- +-- Bare pattern: (a)-[:REL]->(b) +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE (a)-[:KNOWS]->(b) + RETURN a.name, b.name + ORDER BY a.name, b.name +$$) AS (a agtype, b agtype); + +-- +-- NOT pattern expression +-- +-- Find people who don't KNOW anyone +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WHERE NOT (a)-[:KNOWS]->(:Person) + RETURN a.name + ORDER BY a.name +$$) AS (result agtype); + +-- +-- Pattern with labeled first node +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE (a:Person)-[:KNOWS]->(b) + RETURN a.name, b.name + ORDER BY a.name +$$) AS (a agtype, b agtype); + +-- +-- Pattern combined with AND +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE (a)-[:KNOWS]->(b) AND a.name = 'Alice' + RETURN a.name, b.name +$$) AS (a agtype, b agtype); + +-- +-- Pattern combined with OR +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE (a)-[:KNOWS]->(b) OR (a)-[:WORKS_WITH]->(b) + RETURN a.name, b.name + ORDER BY a.name, b.name +$$) AS (a agtype, b agtype); + +-- +-- Left-directed pattern +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE (a)<-[:KNOWS]-(b) + RETURN a.name, b.name + ORDER BY a.name +$$) AS (a agtype, b agtype); + +-- +-- Pattern with anonymous nodes +-- +-- Find anyone who has any outgoing KNOWS relationship +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WHERE (a)-[:KNOWS]->() + RETURN a.name + ORDER BY a.name +$$) AS (result agtype); + +-- +-- Multiple relationship pattern +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (c:Person) + WHERE (a)-[:KNOWS]->()-[:WORKS_WITH]->(c) + RETURN a.name, c.name + ORDER BY a.name +$$) AS (a agtype, c agtype); + +-- +-- Existing EXISTS() syntax still works (backward compatibility) +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person), (b:Person) + WHERE EXISTS((a)-[:KNOWS]->(b)) + RETURN a.name, b.name + ORDER BY a.name +$$) AS (a agtype, b agtype); + +-- +-- Pattern expression produces same results as EXISTS() +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WHERE (a)-[:KNOWS]->(:Person) + RETURN a.name + ORDER BY a.name +$$) AS (result agtype); + +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WHERE EXISTS((a)-[:KNOWS]->(:Person)) + RETURN a.name + ORDER BY a.name +$$) AS (result agtype); + +-- +-- Regular (non-pattern) expressions still work (no regression) +-- +SELECT * FROM cypher('pattern_expr', $$ + RETURN (1 + 2) +$$) AS (result agtype); + +SELECT * FROM cypher('pattern_expr', $$ + MATCH (n:Person) + WHERE n.name = 'Alice' + RETURN (n.name) +$$) AS (result agtype); + +-- +-- Pattern expressions in RETURN (boolean projection) +-- +-- Each person gets a column showing whether they know someone +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, (a)-[:KNOWS]->(:Person) AS knows_someone + ORDER BY a.name +$$) AS (name agtype, knows_someone agtype); + +-- Mix pattern expression with other projections +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, (a)-[:KNOWS]->(:Person), (a)-[:WORKS_WITH]->(:Person) + ORDER BY a.name +$$) AS (name agtype, knows agtype, works_with agtype); + +-- +-- Pattern expressions in CASE WHEN +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, + CASE WHEN (a)-[:KNOWS]->(:Person) THEN 'social' + ELSE 'loner' + END + ORDER BY a.name +$$) AS (name agtype, kind agtype); + +-- +-- Pattern expressions combined with boolean operators in RETURN +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, + (a)-[:KNOWS]->(:Person) AND (a)-[:WORKS_WITH]->(:Person) AS has_both, + (a)-[:KNOWS]->(:Person) OR (a)-[:WORKS_WITH]->(:Person) AS has_either + ORDER BY a.name +$$) AS (name agtype, has_both agtype, has_either agtype); + +-- +-- Pattern expression in SET (store boolean as property) +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + SET a.is_social = (a)-[:KNOWS]->(:Person) + RETURN a.name, a.is_social + ORDER BY a.name +$$) AS (name agtype, is_social agtype); + +-- +-- Pattern expression in WITH (carry boolean through pipeline) +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WITH a.name AS name, (a)-[:KNOWS]->(:Person) AS knows + WHERE knows + RETURN name + ORDER BY name +$$) AS (result agtype); + +-- +-- Follow-up coverage (review #2360): pattern expressions in additional +-- expression contexts opened up by allowing anonymous_path as an expr_atom. +-- + +-- +-- Single-node pattern on an already-bound variable: (a:Label) +-- +-- NOTE: this is an EXISTS existence check on the bound variable, NOT an +-- openCypher label predicate. A matching label is therefore always true +-- (the variable is already bound), and a *different* label is rejected by +-- AGE's pre-existing "multiple labels for variable" restriction rather than +-- evaluating to false. Both behaviours are captured here so any future change +-- to single-node-pattern semantics is caught by this test. +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, (a:Person) + ORDER BY a.name +$$) AS (name agtype, is_person agtype); + +-- A non-matching label errors (pre-existing limitation, not a regression) +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, (a:Animal) + ORDER BY a.name +$$) AS (name agtype, is_animal agtype); + +-- +-- Pattern expressions inside a list literal +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, [(a)-[:KNOWS]->(:Person), (a)-[:WORKS_WITH]->(:Person)] + ORDER BY a.name +$$) AS (name agtype, flags agtype); + +-- +-- Pattern expressions inside a map literal +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN a.name, {knows: (a)-[:KNOWS]->(:Person), works: (a)-[:WORKS_WITH]->(:Person)} + ORDER BY a.name +$$) AS (name agtype, m agtype); + +-- +-- Pattern expressions as function arguments +-- +-- collect() shows the per-row boolean values are correct (ORDER BY before +-- the aggregate so the collected order is deterministic across scan plans). +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WITH a ORDER BY a.name + RETURN collect((a)-[:KNOWS]->(:Person)) +$$) AS (vals agtype); + +-- count() counts non-null values; a boolean (including false) is non-null, +-- so this counts every row rather than only the matching ones. This is the +-- expected SQL aggregate semantics, documented here so the value is not +-- mistaken for a bug. +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + RETURN count((a)-[:KNOWS]->(:Person)) +$$) AS (c agtype); + +-- +-- Pattern expression in OPTIONAL MATCH ... WHERE (null-preserving) +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + OPTIONAL MATCH (b:Person) WHERE (a)-[:KNOWS]->(b) + RETURN a.name, b.name + ORDER BY a.name, b.name +$$) AS (a agtype, b agtype); + +-- +-- EXISTS() and a bare pattern combined in a single predicate +-- +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a:Person) + WHERE EXISTS((a)-[:KNOWS]->(:Person)) AND (a)-[:WORKS_WITH]->(:Person) + RETURN a.name + ORDER BY a.name +$$) AS (name agtype); + +-- +-- Cleanup +-- +SELECT * FROM drop_graph('pattern_expr', true); diff --git a/src/backend/parser/cypher_gram.y b/src/backend/parser/cypher_gram.y index c614e1dbe..b23fa705a 100644 --- a/src/backend/parser/cypher_gram.y +++ b/src/backend/parser/cypher_gram.y @@ -34,7 +34,7 @@ do \ { \ if ((n) > 0) \ - current = (rhs)[1]; \ + current = YYRHSLOC(rhs, 1); \ else \ current = -1; \ } while (0) @@ -49,6 +49,43 @@ %locations %name-prefix="cypher_yy" %pure-parser +/* + * GLR mode handles the ambiguity between parenthesized expressions and + * graph patterns. For example, WHERE (a)-[:KNOWS]->(b) starts with (a) + * which is valid as both an expression and a path_node. The parser forks + * at the conflict point and discards the failing path. %dprec annotations + * on expr_var/var_name_opt and '(' expr ')'/anonymous_path resolve cases + * where both paths succeed (bare (a) prefers the expression interpretation). + */ +%glr-parser +/* + * GLR conflicts are expected and correct for this grammar. They arise + * from the inherent ambiguity between parenthesized expressions and + * graph patterns: the shift/reduce conflicts on '-', '<', '{', + * PARAMETER and ')' all come from path extension vs. arithmetic or + * parenthesized-expression alternatives after a leading '(', and the + * reduce/reduce conflicts on ')', '}' and '=' come from the overlap + * between expr_var and var_name_opt. GLR handles all of these by + * forking at the conflict point and discarding the failing alternative; + * %dprec annotations on expr_var/var_name_opt and '(' expr ')' / + * anonymous_path resolve cases where both forks succeed (bare (a) + * prefers the expression interpretation). + * + * The %expect / %expect-rr counts below match the Bison-reported totals + * (7 SR / 3 RR on Bison 3.8.2). Bison treats %expect as exact, not as + * a ceiling: any deviation up or down fails the build. That is the + * alarm bell — if a grammar change moves either count, the build stops + * and the conflicts must be audited to confirm they remain the inherent + * '(' expr ')' vs anonymous_path ambiguities (resolved by %dprec at + * runtime) rather than an unintended new ambiguity. The Makefile + * downgrades -Wconflicts-sr / -Wconflicts-rr from errors to warnings + * (-Wno-error=conflicts-{sr,rr}) so %expect, not the warning category, + * controls the build-fail threshold. If a future Bison version reports + * different counts for the same grammar, update these numbers and note + * the version in the commit message. + */ +%expect 7 +%expect-rr 3 %lex-param {ag_scanner_t scanner} %parse-param {ag_scanner_t scanner} @@ -292,6 +329,9 @@ static Node *build_predicate_function_node(cypher_predicate_function_kind kind, Node *var, Node *expr, Node *where, int location); +/* pattern expression helper */ +static Node *make_exists_pattern_sublink(Node *pattern, int location); + /* helper functions */ static ExplainStmt *make_explain_stmt(List *options); static void validate_return_item_aliases(List *items, ag_scanner_t scanner); @@ -1876,21 +1916,7 @@ expr_func_subexpr: } | EXISTS '(' anonymous_path ')' { - cypher_sub_pattern *sub; - SubLink *n; - - sub = make_ag_node(cypher_sub_pattern); - sub->kind = CSP_EXISTS; - sub->pattern = list_make1($3); - - n = makeNode(SubLink); - n->subLinkType = EXISTS_SUBLINK; - n->subLinkId = 0; - n->testexpr = NULL; - n->operName = NIL; - n->subselect = (Node *) sub; - n->location = @1; - $$ = (Node *)node_to_agtype((Node *)n, "boolean", @1); + $$ = make_exists_pattern_sublink($3, @1); } | EXISTS '(' property_value ')' { @@ -2026,7 +2052,7 @@ expr_atom: $$ = (Node *)n; } - | '(' expr ')' + | '(' expr ')' %dprec 2 { Node *n = $2; @@ -2037,6 +2063,17 @@ expr_atom: } $$ = n; } + | anonymous_path %dprec 1 + { + /* + * Bare pattern in expression context is semantically + * equivalent to EXISTS(pattern). Example: + * WHERE (a)-[:KNOWS]->(b) + * becomes + * WHERE EXISTS((a)-[:KNOWS]->(b)) + */ + $$ = make_exists_pattern_sublink($1, @1); + } | expr_case | expr_var | expr_func @@ -2288,7 +2325,7 @@ expr_case_default: ; expr_var: - var_name + var_name %dprec 2 { ColumnRef *n; @@ -2374,11 +2411,11 @@ var_name_alias: ; var_name_opt: - /* empty */ + /* empty */ %dprec 1 { $$ = NULL; } - | var_name + | var_name %dprec 1 ; label_name: @@ -3585,6 +3622,30 @@ static Node *build_predicate_function_node(cypher_predicate_function_kind kind, } } +/* + * Wrap a graph pattern in an EXISTS SubLink. Used by both + * EXISTS(pattern) syntax and bare pattern expressions in WHERE. + */ +static Node *make_exists_pattern_sublink(Node *pattern, int location) +{ + cypher_sub_pattern *sub; + SubLink *n; + + sub = make_ag_node(cypher_sub_pattern); + sub->kind = CSP_EXISTS; + sub->pattern = list_make1(pattern); + + n = makeNode(SubLink); + n->subLinkType = EXISTS_SUBLINK; + n->subLinkId = 0; + n->testexpr = NULL; + n->operName = NIL; + n->subselect = (Node *) sub; + n->location = location; + + return (Node *)node_to_agtype((Node *)n, "boolean", location); +} + /* Helper function to create an ExplainStmt node */ static ExplainStmt *make_explain_stmt(List *options) { From 92e48e9b907f491c92db487790c1624dba23b812 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Fri, 26 Jun 2026 15:54:54 -0700 Subject: [PATCH 086/100] Add reduce() list folding function (#2435) Implement the openCypher reduce(acc = init, var IN list | body) expression, which folds an arbitrary expression over a list, threading an accumulator across the elements in list order. This closes a long-standing gap (reduce() was previously unsupported) and works both at the SQL top level and inside a cypher() RETURN/WHERE. Implementation -------------- reduce() is desugared, at transform time, into a correlated scalar subquery over a new ordered aggregate rather than a new executor node, so no PostgreSQL core changes are required: CASE WHEN list IS NULL THEN NULL ELSE COALESCE((SELECT ag_catalog.age_reduce( , ''::text, r.elem ORDER BY r.ord) FROM unnest() WITH ORDINALITY AS r(elem, ord)), ) END - A new cypher_reduce extensible node carries the accumulator/element names and the init/list/body expressions (grammar production, keyword, and the copy/out/read serialization plumbing). - The fold body is transformed against a throwaway two-column agtype namespace, its accumulator and element Var references are rewritten to PARAM_EXEC params 0 and 1, and it is serialized with nodeToString() into a text argument. - age_reduce_transfn (a custom agtype aggregate transition function) deserializes and compiles the body once per group with ExecInitExpr, then evaluates it per element with ExecEvalExpr, rebinding the two params. The body is normalized to agtype at transform time so a boolean or other non-agtype result cannot be misread as a by-reference Datum. Semantics --------- - List order is preserved (unnest WITH ORDINALITY + aggregate ORDER BY). - An empty list yields the initial value; a NULL list yields NULL. - The list and initial value may reference outer-query variables (e.g. reduce(total = 0, n IN nodes(p) | total + n.age)); the body may reference only the accumulator and element. - Arithmetic, string, list-building, boolean/comparison (AND/OR/=/>), CASE, and element property-access bodies are all supported. - Outer-variable, query-parameter, nested-reduce, and aggregate references inside the body raise a clean ERRCODE_FEATURE_NOT_SUPPORTED error. - reduce is registered as a safe keyword so it remains usable as a property or map key, preserving backward compatibility. Tests ----- Adds the age_reduce regression test (registered in the install SQL and the upgrade template so age_upgrade passes), covering: arithmetic/product/string folds; order sensitivity; empty/NULL list; NULL element and NULL init; list-building and CASE bodies; boolean and comparison bodies; element property access; multiple and nested (in list/init) reduce(); reduce() in boolean expressions, WHERE, and list comprehensions; folds over collected nodes and node list properties; the not-supported rejections; and reduce as a map key. Following reviewer feedback, three further semantics-coverage gaps are pinned directly so the mechanisms that make the aggregate desugaring correct are exercised by tests rather than only correct by inspection: - A fold body that produces null mid-fold and then recovers: the agtype 'null' running state is a readable value, so a later element folds back out of it (distinct from "null propagates to a null result", which was already covered). - An empty list with a NULL initial value: COALESCE(, init) yields NULL, kept distinct from a body that legitimately folds to agtype 'null', which must not be resurrected to the initial value. - A type error and a runtime division-by-zero error in the body: both abort cleanly out of the standalone per-element evaluator rather than corrupting the running aggregate state. All multi-row results are ordered. 42/42 installcheck pass. Future work ----------- The body restriction (accumulator and element only) is a property of the standalone expression evaluation and can be relaxed without core changes: - Allow loop-invariant outer-variable and cypher $parameter references in the body by capturing them as additional eager aggregate arguments bound to extra param slots. - Support a nested reduce() inside the body via an SPI-based evaluation fallback for subquery-bearing bodies. Aggregates inside the body remain intentionally unsupported, matching the openCypher specification. Co-Authored-By: Claude Opus 4.8 (1M context) modified: Makefile modified: age--1.7.0--y.y.y.sql new file: regress/expected/age_reduce.out new file: regress/sql/age_reduce.sql modified: sql/age_aggregate.sql modified: src/backend/nodes/ag_nodes.c modified: src/backend/nodes/cypher_copyfuncs.c modified: src/backend/nodes/cypher_outfuncs.c modified: src/backend/nodes/cypher_readfuncs.c modified: src/backend/parser/cypher_analyze.c modified: src/backend/parser/cypher_clause.c modified: src/backend/parser/cypher_gram.y modified: src/backend/utils/adt/agtype.c modified: src/include/nodes/ag_nodes.h modified: src/include/nodes/cypher_copyfuncs.h modified: src/include/nodes/cypher_nodes.h modified: src/include/nodes/cypher_outfuncs.h modified: src/include/nodes/cypher_readfuncs.h modified: src/include/parser/cypher_kwlist.h --- Makefile | 1 + age--1.7.0--y.y.y.sql | 22 ++ regress/expected/age_reduce.out | 538 +++++++++++++++++++++++++++ regress/sql/age_reduce.sql | 350 +++++++++++++++++ sql/age_aggregate.sql | 22 ++ src/backend/nodes/ag_nodes.c | 6 +- src/backend/nodes/cypher_copyfuncs.c | 12 + src/backend/nodes/cypher_outfuncs.c | 12 + src/backend/nodes/cypher_readfuncs.c | 14 + src/backend/parser/cypher_analyze.c | 19 + src/backend/parser/cypher_clause.c | 421 +++++++++++++++++++++ src/backend/parser/cypher_gram.y | 92 ++++- src/backend/utils/adt/agtype.c | 165 ++++++++ src/include/nodes/ag_nodes.h | 4 +- src/include/nodes/cypher_copyfuncs.h | 4 + src/include/nodes/cypher_nodes.h | 18 + src/include/nodes/cypher_outfuncs.h | 1 + src/include/nodes/cypher_readfuncs.h | 3 + src/include/parser/cypher_kwlist.h | 1 + 19 files changed, 1701 insertions(+), 4 deletions(-) create mode 100644 regress/expected/age_reduce.out create mode 100644 regress/sql/age_reduce.sql diff --git a/Makefile b/Makefile index b0059213c..21c7f81f2 100644 --- a/Makefile +++ b/Makefile @@ -216,6 +216,7 @@ REGRESS = scan \ list_comprehension \ predicate_functions \ pattern_expression \ + age_reduce \ map_projection \ direct_field_access \ security \ diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index b40cde092..ec909d2bb 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -1100,3 +1100,25 @@ $function$; COMMENT ON FUNCTION ag_catalog.create_subgraph(name, name, text, text) IS 'Materializes a new persistent graph as the induced subgraph of from_graph selected by a Cypher node predicate (on n) and relationship predicate (on r); ''*'' keeps all. An edge is kept only if its predicate holds and both endpoints are kept. Returns (node_count, relationship_count).'; + +-- +-- reduce(acc = init, var IN list | body) fold support +-- +-- Transition function for the age_reduce aggregate. The fold body is compiled +-- by transform_cypher_reduce() with the accumulator and element rewritten to +-- PARAM_EXEC params 0 and 1 and serialized into the text argument; the +-- transition evaluates it for each element in list order. It must be callable +-- with a NULL transition state (no initcond), so it is intentionally not STRICT. +CREATE FUNCTION ag_catalog.age_reduce_transfn(agtype, agtype, text, agtype) + RETURNS agtype + LANGUAGE c +PARALLEL UNSAFE +AS 'MODULE_PATHNAME'; + +-- aggregate definition for reduce(); direct arguments are +-- (init, serialized-body, element), with the element fed ORDER BY ordinality. +CREATE AGGREGATE ag_catalog.age_reduce(agtype, text, agtype) +( + stype = agtype, + sfunc = ag_catalog.age_reduce_transfn +); diff --git a/regress/expected/age_reduce.out b/regress/expected/age_reduce.out new file mode 100644 index 000000000..8a198965a --- /dev/null +++ b/regress/expected/age_reduce.out @@ -0,0 +1,538 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +SELECT create_graph('reduce'); +NOTICE: graph "reduce" has been created + create_graph +-------------- + +(1 row) + +-- +-- Basic folds +-- +-- sum +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x) +$$) AS (result agtype); + result +-------- + 6 +(1 row) + +-- sum of a longer list +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] | s + x) +$$) AS (result agtype); + result +-------- + 55 +(1 row) + +-- product (factorial) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(p = 1, x IN [1, 2, 3, 4, 5] | p * x) +$$) AS (result agtype); + result +-------- + 120 +(1 row) + +-- non-zero initial accumulator +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 100, x IN [1, 2, 3] | s + x) +$$) AS (result agtype); + result +-------- + 106 +(1 row) + +-- single element +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [42] | s + x) +$$) AS (result agtype); + result +-------- + 42 +(1 row) + +-- +-- List order is significant +-- +-- left-associative subtraction: ((((0-1)-2)-3)-4) = -10 +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3, 4] | s - x) +$$) AS (result agtype); + result +-------- + -10 +(1 row) + +-- forward string concatenation +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = '', x IN ['a', 'b', 'c'] | s + x) +$$) AS (result agtype); + result +-------- + "abc" +(1 row) + +-- reverse string concatenation (element before accumulator) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = '', x IN ['a', 'b', 'c'] | x + s) +$$) AS (result agtype); + result +-------- + "cba" +(1 row) + +-- +-- Empty and NULL list semantics +-- +-- empty list returns the initial value +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [] | s + x) +$$) AS (result agtype); + result +-------- + 0 +(1 row) + +-- empty list returns the initial value (non-zero) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 999, x IN [] | s + x) +$$) AS (result agtype); + result +-------- + 999 +(1 row) + +-- NULL list returns NULL +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN null | s + x) +$$) AS (result agtype); + result +-------- + +(1 row) + +-- empty list with a NULL initial value yields NULL: the list is empty (not +-- null) so the fold runs over zero rows, and COALESCE(, init) is +-- COALESCE(NULL, NULL) -> NULL. (Distinct from a NULL *list*, which the outer +-- CASE short-circuits to NULL, and from a non-empty list with a NULL init, +-- which seeds the accumulator with agtype 'null'.) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = null, x IN [] | s + x) +$$) AS (result agtype); + result +-------- + +(1 row) + +-- +-- NULL handling within the fold +-- +-- a NULL element propagates through arithmetic to NULL +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, null, 3] | s + x) +$$) AS (result agtype); + result +-------- + null +(1 row) + +-- NULL initial value +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = null, x IN [1, 2, 3] | s) +$$) AS (result agtype); + result +-------- + null +(1 row) + +-- a body that always evaluates to null yields null, NOT the initial value: +-- every step stores agtype 'null' as the running state, so the final state is +-- a real agtype 'null' and the empty-list COALESCE(..., init) guard must not +-- resurrect the initial value here (the load-bearing fold-to-null vs empty-list +-- distinction) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 7, x IN [1, 2, 3] | null) +$$) AS (result agtype); + result +-------- + null +(1 row) + +-- the accumulator legitimately becomes null mid-fold and the body climbs back +-- out of it: element 2 sets the accumulator to null, element 3 produces a fresh +-- non-null value, and element 4 reads that recovered state (999 + 4), proving a +-- null intermediate state does not poison the rest of the fold +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3, 4] | + CASE WHEN x = 2 THEN null + WHEN x = 3 THEN 999 + ELSE s + x END) +$$) AS (result agtype); + result +-------- + 1003 +(1 row) + +-- +-- Errors raised from the fold body propagate cleanly +-- +-- a type error in the body (agtype number + map) aborts the statement rather +-- than corrupting the running aggregate state or crashing the backend +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2] | s + {a: 1}) +$$) AS (result agtype); +ERROR: invalid left operand for agtype concatenation +-- a runtime arithmetic error in the body (division by zero) likewise aborts +-- the fold; the error surfaces from the standalone per-element evaluator +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 1, x IN [1, 0, 2] | s / x) +$$) AS (result agtype); +ERROR: division by zero +-- +-- Building a list with the accumulator +-- +-- collect squares +SELECT * FROM cypher('reduce', $$ + RETURN reduce(acc = [], x IN [1, 2, 3] | acc + [x * x]) +$$) AS (result agtype); + result +----------- + [1, 4, 9] +(1 row) + +-- +-- A conditional body (CASE) +-- +-- sum of even elements only +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3, 4, 5, 6] | CASE WHEN x % 2 = 0 THEN s + x ELSE s END) +$$) AS (result agtype); + result +-------- + 12 +(1 row) + +-- +-- Boolean and comparison fold bodies +-- +-- the body evaluates to a boolean, which is normalized to an agtype boolean +-- (a boolean accumulator is a real Cypher use case for "all"/"any" style folds) +-- logical AND fold: all true? +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = true, x IN [true, true, false] | s AND x) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- logical OR fold: any true? +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = false, x IN [false, true, false] | s OR x) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- a comparison body +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = false, x IN [1, 2, 3] | x = 2) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- "does any element equal 2?" (search fold) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(found = false, x IN [1, 2, 3] | found OR x = 2) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- "are all elements positive?" (using a comparison inside the fold) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = true, x IN [1, 2, 3] | s AND x > 0) +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = true, x IN [1, -2, 3] | s AND x > 0) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- +-- Property access on the element variable +-- +-- sum a field across a list of maps +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [{n: 1}, {n: 2}, {n: 3}] | s + x.n) +$$) AS (result agtype); + result +-------- + 6 +(1 row) + +-- concatenate a string field across a list of maps +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = '', x IN [{w: 'a'}, {w: 'b'}, {w: 'c'}] | s + x.w) +$$) AS (result agtype); + result +-------- + "abc" +(1 row) + +-- +-- Multiple reduce() in one expression +-- +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x) + reduce(p = 1, y IN [2, 3] | p * y) +$$) AS (result agtype); + result +-------- + 12 +(1 row) + +-- +-- reduce() in a boolean expression +-- +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x) > 5 + AND reduce(p = 1, y IN [2, 3] | p * y) < 10 +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- +-- reduce() nested in the list or initial value of another reduce() +-- +-- nesting is allowed in the list and the initial value (both evaluated in the +-- outer context) even though it is rejected inside the fold body. +-- nested reduce() in the list +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [reduce(a = 0, y IN [1, 2, 3] | a + y), 10] | s + x) +$$) AS (result agtype); + result +-------- + 16 +(1 row) + +-- nested reduce() in the initial value +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = reduce(a = 0, y IN [1, 2, 3] | a + y), x IN [10, 20] | s + x) +$$) AS (result agtype); + result +-------- + 36 +(1 row) + +-- +-- reduce() over a correlated (per-row) list +-- +SELECT * FROM cypher('reduce', $$ + UNWIND [[1, 2], [3, 4, 5], []] AS arr + RETURN reduce(s = 0, x IN arr | s + x) AS total + ORDER BY total +$$) AS (result agtype); + result +-------- + 0 + 3 + 12 +(3 rows) + +-- +-- reduce() with the list and initial value bound in an outer clause +-- +SELECT * FROM cypher('reduce', $$ + WITH [10, 20, 30] AS ns + RETURN reduce(t = 0, n IN ns | t + n) +$$) AS (result agtype); + result +-------- + 60 +(1 row) + +-- the initial value may reference an outer variable (correlation is allowed +-- in the init and the list, only not in the body) +SELECT * FROM cypher('reduce', $$ + WITH 5 AS base + RETURN reduce(s = base, x IN [1, 2, 3] | s + x) +$$) AS (result agtype); + result +-------- + 11 +(1 row) + +-- +-- reduce() nested inside a list comprehension +-- +SELECT * FROM cypher('reduce', $$ + RETURN [v IN [1, 2, 3] | reduce(s = 0, x IN [v, v, v] | s + x)] +$$) AS (result agtype); + result +----------- + [3, 6, 9] +(1 row) + +-- +-- reduce() in a WHERE clause +-- +SELECT * FROM cypher('reduce', $$ + UNWIND [[1, 2, 3], [1, 1], [10]] AS l + WITH l WHERE reduce(s = 0, x IN l | s + x) > 3 + RETURN l + ORDER BY l +$$) AS (result agtype); + result +----------- + [1, 2, 3] + [10] +(2 rows) + +-- +-- reduce() over graph data (the canonical Cypher example) +-- +SELECT * FROM cypher('reduce', $$ + CREATE (:person {name: 'Alice', age: 38}), + (:person {name: 'Bob', age: 25}), + (:person {name: 'Daniel', age: 54}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- sum the ages of all person nodes +SELECT * FROM cypher('reduce', $$ + MATCH (p:person) + WITH collect(p) AS people + RETURN reduce(total = 0, n IN people | total + n.age) +$$) AS (result agtype); + result +-------- + 117 +(1 row) + +-- +-- reduce() over a graph node's list property +-- +SELECT * FROM cypher('reduce', $$ + CREATE (:bag {name: 'low', vals: [1, 2, 3]}), + (:bag {name: 'mid', vals: [5, 5, 5]}), + (:bag {name: 'high', vals: [10, 20, 30]}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- filter nodes by a reduce() over their list property +SELECT * FROM cypher('reduce', $$ + MATCH (u:bag) WHERE reduce(s = 0, x IN u.vals | s + x) > 10 + RETURN u.name + ORDER BY u.name +$$) AS (result agtype); + result +-------- + "high" + "mid" +(2 rows) + +-- compute a reduce() value per node and order by it +SELECT * FROM cypher('reduce', $$ + MATCH (u:bag) + RETURN u.name AS name, reduce(s = 0, x IN u.vals | s + x) AS total + ORDER BY total +$$) AS (name agtype, total agtype); + name | total +--------+------- + "low" | 6 + "mid" | 15 + "high" | 60 +(3 rows) + +-- +-- Not-yet-supported constructs raise a clean feature error +-- +-- an outer variable referenced in the body +SELECT * FROM cypher('reduce', $$ + WITH 5 AS w + RETURN reduce(s = 0, x IN [1, 2] | s + x + w) +$$) AS (result agtype); +ERROR: a reduce() expression may only reference its accumulator and element variables +LINE 1: SELECT * FROM cypher('reduce', $$ + ^ +-- a nested reduce() in the body +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2] | s + reduce(t = 0, y IN [x] | t + y)) +$$) AS (result agtype); +ERROR: subqueries (including a nested reduce()) are not supported in a reduce() expression +LINE 1: SELECT * FROM cypher('reduce', $$ + ^ +-- an aggregate function in the body +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2] | s + count(x)) +$$) AS (result agtype); +ERROR: aggregate functions are not supported in a reduce() expression +LINE 1: SELECT * FROM cypher('reduce', $$ + ^ +-- +-- "reduce" as a property key name (safe_keywords backward compatibility): +-- because reduce() introduced a reserved keyword, confirm the word is still +-- usable as a map key, the same way any/none/single are. +-- +SELECT * FROM cypher('reduce', $$ + RETURN {reduce: 1, any: 2, none: 3} +$$) AS (result agtype); + result +------------------------------------ + {"any": 2, "none": 3, "reduce": 1} +(1 row) + +-- +-- Cleanup +-- +SELECT * FROM drop_graph('reduce', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table reduce._ag_label_vertex +drop cascades to table reduce._ag_label_edge +drop cascades to table reduce.person +drop cascades to table reduce.bag +NOTICE: graph "reduce" has been dropped + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/age_reduce.sql b/regress/sql/age_reduce.sql new file mode 100644 index 000000000..cf1261010 --- /dev/null +++ b/regress/sql/age_reduce.sql @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +SELECT create_graph('reduce'); + +-- +-- Basic folds +-- +-- sum +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x) +$$) AS (result agtype); + +-- sum of a longer list +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] | s + x) +$$) AS (result agtype); + +-- product (factorial) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(p = 1, x IN [1, 2, 3, 4, 5] | p * x) +$$) AS (result agtype); + +-- non-zero initial accumulator +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 100, x IN [1, 2, 3] | s + x) +$$) AS (result agtype); + +-- single element +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [42] | s + x) +$$) AS (result agtype); + +-- +-- List order is significant +-- +-- left-associative subtraction: ((((0-1)-2)-3)-4) = -10 +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3, 4] | s - x) +$$) AS (result agtype); + +-- forward string concatenation +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = '', x IN ['a', 'b', 'c'] | s + x) +$$) AS (result agtype); + +-- reverse string concatenation (element before accumulator) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = '', x IN ['a', 'b', 'c'] | x + s) +$$) AS (result agtype); + +-- +-- Empty and NULL list semantics +-- +-- empty list returns the initial value +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [] | s + x) +$$) AS (result agtype); + +-- empty list returns the initial value (non-zero) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 999, x IN [] | s + x) +$$) AS (result agtype); + +-- NULL list returns NULL +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN null | s + x) +$$) AS (result agtype); + +-- empty list with a NULL initial value yields NULL: the list is empty (not +-- null) so the fold runs over zero rows, and COALESCE(, init) is +-- COALESCE(NULL, NULL) -> NULL. (Distinct from a NULL *list*, which the outer +-- CASE short-circuits to NULL, and from a non-empty list with a NULL init, +-- which seeds the accumulator with agtype 'null'.) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = null, x IN [] | s + x) +$$) AS (result agtype); + +-- +-- NULL handling within the fold +-- +-- a NULL element propagates through arithmetic to NULL +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, null, 3] | s + x) +$$) AS (result agtype); + +-- NULL initial value +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = null, x IN [1, 2, 3] | s) +$$) AS (result agtype); + +-- a body that always evaluates to null yields null, NOT the initial value: +-- every step stores agtype 'null' as the running state, so the final state is +-- a real agtype 'null' and the empty-list COALESCE(..., init) guard must not +-- resurrect the initial value here (the load-bearing fold-to-null vs empty-list +-- distinction) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 7, x IN [1, 2, 3] | null) +$$) AS (result agtype); + +-- the accumulator legitimately becomes null mid-fold and the body climbs back +-- out of it: element 2 sets the accumulator to null, element 3 produces a fresh +-- non-null value, and element 4 reads that recovered state (999 + 4), proving a +-- null intermediate state does not poison the rest of the fold +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3, 4] | + CASE WHEN x = 2 THEN null + WHEN x = 3 THEN 999 + ELSE s + x END) +$$) AS (result agtype); + +-- +-- Errors raised from the fold body propagate cleanly +-- +-- a type error in the body (agtype number + map) aborts the statement rather +-- than corrupting the running aggregate state or crashing the backend +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2] | s + {a: 1}) +$$) AS (result agtype); + +-- a runtime arithmetic error in the body (division by zero) likewise aborts +-- the fold; the error surfaces from the standalone per-element evaluator +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 1, x IN [1, 0, 2] | s / x) +$$) AS (result agtype); + +-- +-- Building a list with the accumulator +-- +-- collect squares +SELECT * FROM cypher('reduce', $$ + RETURN reduce(acc = [], x IN [1, 2, 3] | acc + [x * x]) +$$) AS (result agtype); + +-- +-- A conditional body (CASE) +-- +-- sum of even elements only +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3, 4, 5, 6] | CASE WHEN x % 2 = 0 THEN s + x ELSE s END) +$$) AS (result agtype); + +-- +-- Boolean and comparison fold bodies +-- +-- the body evaluates to a boolean, which is normalized to an agtype boolean +-- (a boolean accumulator is a real Cypher use case for "all"/"any" style folds) +-- logical AND fold: all true? +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = true, x IN [true, true, false] | s AND x) +$$) AS (result agtype); + +-- logical OR fold: any true? +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = false, x IN [false, true, false] | s OR x) +$$) AS (result agtype); + +-- a comparison body +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = false, x IN [1, 2, 3] | x = 2) +$$) AS (result agtype); + +-- "does any element equal 2?" (search fold) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(found = false, x IN [1, 2, 3] | found OR x = 2) +$$) AS (result agtype); + +-- "are all elements positive?" (using a comparison inside the fold) +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = true, x IN [1, 2, 3] | s AND x > 0) +$$) AS (result agtype); + +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = true, x IN [1, -2, 3] | s AND x > 0) +$$) AS (result agtype); + +-- +-- Property access on the element variable +-- +-- sum a field across a list of maps +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [{n: 1}, {n: 2}, {n: 3}] | s + x.n) +$$) AS (result agtype); + +-- concatenate a string field across a list of maps +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = '', x IN [{w: 'a'}, {w: 'b'}, {w: 'c'}] | s + x.w) +$$) AS (result agtype); + +-- +-- Multiple reduce() in one expression +-- +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x) + reduce(p = 1, y IN [2, 3] | p * y) +$$) AS (result agtype); + +-- +-- reduce() in a boolean expression +-- +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x) > 5 + AND reduce(p = 1, y IN [2, 3] | p * y) < 10 +$$) AS (result agtype); + +-- +-- reduce() nested in the list or initial value of another reduce() +-- +-- nesting is allowed in the list and the initial value (both evaluated in the +-- outer context) even though it is rejected inside the fold body. +-- nested reduce() in the list +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [reduce(a = 0, y IN [1, 2, 3] | a + y), 10] | s + x) +$$) AS (result agtype); + +-- nested reduce() in the initial value +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = reduce(a = 0, y IN [1, 2, 3] | a + y), x IN [10, 20] | s + x) +$$) AS (result agtype); + +-- +-- reduce() over a correlated (per-row) list +-- +SELECT * FROM cypher('reduce', $$ + UNWIND [[1, 2], [3, 4, 5], []] AS arr + RETURN reduce(s = 0, x IN arr | s + x) AS total + ORDER BY total +$$) AS (result agtype); + +-- +-- reduce() with the list and initial value bound in an outer clause +-- +SELECT * FROM cypher('reduce', $$ + WITH [10, 20, 30] AS ns + RETURN reduce(t = 0, n IN ns | t + n) +$$) AS (result agtype); + +-- the initial value may reference an outer variable (correlation is allowed +-- in the init and the list, only not in the body) +SELECT * FROM cypher('reduce', $$ + WITH 5 AS base + RETURN reduce(s = base, x IN [1, 2, 3] | s + x) +$$) AS (result agtype); + +-- +-- reduce() nested inside a list comprehension +-- +SELECT * FROM cypher('reduce', $$ + RETURN [v IN [1, 2, 3] | reduce(s = 0, x IN [v, v, v] | s + x)] +$$) AS (result agtype); + +-- +-- reduce() in a WHERE clause +-- +SELECT * FROM cypher('reduce', $$ + UNWIND [[1, 2, 3], [1, 1], [10]] AS l + WITH l WHERE reduce(s = 0, x IN l | s + x) > 3 + RETURN l + ORDER BY l +$$) AS (result agtype); + +-- +-- reduce() over graph data (the canonical Cypher example) +-- +SELECT * FROM cypher('reduce', $$ + CREATE (:person {name: 'Alice', age: 38}), + (:person {name: 'Bob', age: 25}), + (:person {name: 'Daniel', age: 54}) +$$) AS (result agtype); + +-- sum the ages of all person nodes +SELECT * FROM cypher('reduce', $$ + MATCH (p:person) + WITH collect(p) AS people + RETURN reduce(total = 0, n IN people | total + n.age) +$$) AS (result agtype); + +-- +-- reduce() over a graph node's list property +-- +SELECT * FROM cypher('reduce', $$ + CREATE (:bag {name: 'low', vals: [1, 2, 3]}), + (:bag {name: 'mid', vals: [5, 5, 5]}), + (:bag {name: 'high', vals: [10, 20, 30]}) +$$) AS (result agtype); + +-- filter nodes by a reduce() over their list property +SELECT * FROM cypher('reduce', $$ + MATCH (u:bag) WHERE reduce(s = 0, x IN u.vals | s + x) > 10 + RETURN u.name + ORDER BY u.name +$$) AS (result agtype); + +-- compute a reduce() value per node and order by it +SELECT * FROM cypher('reduce', $$ + MATCH (u:bag) + RETURN u.name AS name, reduce(s = 0, x IN u.vals | s + x) AS total + ORDER BY total +$$) AS (name agtype, total agtype); + +-- +-- Not-yet-supported constructs raise a clean feature error +-- +-- an outer variable referenced in the body +SELECT * FROM cypher('reduce', $$ + WITH 5 AS w + RETURN reduce(s = 0, x IN [1, 2] | s + x + w) +$$) AS (result agtype); + +-- a nested reduce() in the body +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2] | s + reduce(t = 0, y IN [x] | t + y)) +$$) AS (result agtype); + +-- an aggregate function in the body +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2] | s + count(x)) +$$) AS (result agtype); + +-- +-- "reduce" as a property key name (safe_keywords backward compatibility): +-- because reduce() introduced a reserved keyword, confirm the word is still +-- usable as a map key, the same way any/none/single are. +-- +SELECT * FROM cypher('reduce', $$ + RETURN {reduce: 1, any: 2, none: 3} +$$) AS (result agtype); + +-- +-- Cleanup +-- +SELECT * FROM drop_graph('reduce', true); diff --git a/sql/age_aggregate.sql b/sql/age_aggregate.sql index a8ea425cc..fb258e5c5 100644 --- a/sql/age_aggregate.sql +++ b/sql/age_aggregate.sql @@ -216,3 +216,25 @@ CREATE AGGREGATE ag_catalog.age_collect(variadic "any") finalfunc = ag_catalog.age_collect_aggfinalfn, parallel = safe ); + +-- +-- reduce(acc = init, var IN list | body) fold support +-- +-- Transition function for the age_reduce aggregate. The fold body is compiled +-- by transform_cypher_reduce() with the accumulator and element rewritten to +-- PARAM_EXEC params 0 and 1 and serialized into the text argument; the +-- transition evaluates it for each element in list order. It must be callable +-- with a NULL transition state (no initcond), so it is intentionally not STRICT. +CREATE FUNCTION ag_catalog.age_reduce_transfn(agtype, agtype, text, agtype) + RETURNS agtype + LANGUAGE c +PARALLEL UNSAFE +AS 'MODULE_PATHNAME'; + +-- aggregate definition for reduce(); direct arguments are +-- (init, serialized-body, element), with the element fed ORDER BY ordinality. +CREATE AGGREGATE ag_catalog.age_reduce(agtype, text, agtype) +( + stype = agtype, + sfunc = ag_catalog.age_reduce_transfn +); diff --git a/src/backend/nodes/ag_nodes.c b/src/backend/nodes/ag_nodes.c index bd78549ca..ac659e1b6 100644 --- a/src/backend/nodes/ag_nodes.c +++ b/src/backend/nodes/ag_nodes.c @@ -65,7 +65,8 @@ const char *node_names[] = { "cypher_delete_information", "cypher_delete_item", "cypher_merge_information", - "cypher_predicate_function" + "cypher_predicate_function", + "cypher_reduce" }; /* @@ -134,7 +135,8 @@ const ExtensibleNodeMethods node_methods[] = { DEFINE_NODE_METHODS_EXTENDED(cypher_delete_information), DEFINE_NODE_METHODS_EXTENDED(cypher_delete_item), DEFINE_NODE_METHODS_EXTENDED(cypher_merge_information), - DEFINE_NODE_METHODS_EXTENDED(cypher_predicate_function) + DEFINE_NODE_METHODS_EXTENDED(cypher_predicate_function), + DEFINE_NODE_METHODS_EXTENDED(cypher_reduce) }; static bool equal_ag_node(const ExtensibleNode *a, const ExtensibleNode *b) diff --git a/src/backend/nodes/cypher_copyfuncs.c b/src/backend/nodes/cypher_copyfuncs.c index 283096ca7..549218759 100644 --- a/src/backend/nodes/cypher_copyfuncs.c +++ b/src/backend/nodes/cypher_copyfuncs.c @@ -185,3 +185,15 @@ void copy_cypher_predicate_function(ExtensibleNode *newnode, COPY_NODE_FIELD(expr); COPY_NODE_FIELD(where); } + +/* copy function for cypher_reduce */ +void copy_cypher_reduce(ExtensibleNode *newnode, const ExtensibleNode *from) +{ + COPY_LOCALS(cypher_reduce); + + COPY_STRING_FIELD(acc_varname); + COPY_NODE_FIELD(init_expr); + COPY_STRING_FIELD(elem_varname); + COPY_NODE_FIELD(list_expr); + COPY_NODE_FIELD(body_expr); +} diff --git a/src/backend/nodes/cypher_outfuncs.c b/src/backend/nodes/cypher_outfuncs.c index 84d32a8f8..4a35be02f 100644 --- a/src/backend/nodes/cypher_outfuncs.c +++ b/src/backend/nodes/cypher_outfuncs.c @@ -200,6 +200,18 @@ void out_cypher_predicate_function(StringInfo str, const ExtensibleNode *node) WRITE_NODE_FIELD(where); } +/* serialization function for the cypher_reduce ExtensibleNode. */ +void out_cypher_reduce(StringInfo str, const ExtensibleNode *node) +{ + DEFINE_AG_NODE(cypher_reduce); + + WRITE_STRING_FIELD(acc_varname); + WRITE_NODE_FIELD(init_expr); + WRITE_STRING_FIELD(elem_varname); + WRITE_NODE_FIELD(list_expr); + WRITE_NODE_FIELD(body_expr); +} + /* serialization function for the cypher_merge ExtensibleNode. */ void out_cypher_merge(StringInfo str, const ExtensibleNode *node) { diff --git a/src/backend/nodes/cypher_readfuncs.c b/src/backend/nodes/cypher_readfuncs.c index 1e7e0ef82..a9a2ffabd 100644 --- a/src/backend/nodes/cypher_readfuncs.c +++ b/src/backend/nodes/cypher_readfuncs.c @@ -329,3 +329,17 @@ void read_cypher_predicate_function(struct ExtensibleNode *node) READ_NODE_FIELD(expr); READ_NODE_FIELD(where); } + +/* + * Deserialize a string representing the cypher_reduce data structure. + */ +void read_cypher_reduce(struct ExtensibleNode *node) +{ + READ_LOCALS(cypher_reduce); + + READ_STRING_FIELD(acc_varname); + READ_NODE_FIELD(init_expr); + READ_STRING_FIELD(elem_varname); + READ_NODE_FIELD(list_expr); + READ_NODE_FIELD(body_expr); +} diff --git a/src/backend/parser/cypher_analyze.c b/src/backend/parser/cypher_analyze.c index 5b72f332e..5dd53dcd0 100644 --- a/src/backend/parser/cypher_analyze.c +++ b/src/backend/parser/cypher_analyze.c @@ -865,6 +865,25 @@ bool cypher_raw_expr_tree_walker_impl(Node *node, return true; } } + else if (is_ag_node(node, cypher_reduce)) + { + cypher_reduce *rd = (cypher_reduce *)node; + + if (WALK(rd->init_expr)) + { + return true; + } + + if (WALK(rd->list_expr)) + { + return true; + } + + if (WALK(rd->body_expr)) + { + return true; + } + } /* Add more node types here as needed */ else { diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 5ac9dea65..a3a1a5044 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -50,6 +50,7 @@ #include "parser/cypher_expr.h" #include "parser/cypher_item.h" #include "parser/cypher_parse_agg.h" +#include "utils/agtype.h" #include "parser/cypher_transform_entity.h" #include "utils/ag_cache.h" #include "utils/ag_func.h" @@ -315,6 +316,10 @@ static Query *transform_cypher_list_comprehension(cypher_parsestate *cpstate, static Query *transform_cypher_predicate_function(cypher_parsestate *cpstate, cypher_clause *clause); +/* reduce */ +static Query *transform_cypher_reduce(cypher_parsestate *cpstate, + cypher_clause *clause); + /* merge */ static Query *transform_cypher_merge(cypher_parsestate *cpstate, cypher_clause *clause); @@ -579,6 +584,10 @@ Query *transform_cypher_clause(cypher_parsestate *cpstate, { result = transform_cypher_predicate_function(cpstate, clause); } + else if (is_ag_node(self, cypher_reduce)) + { + result = transform_cypher_reduce(cpstate, clause); + } else { ereport(ERROR, (errmsg_internal("unexpected Node for cypher_clause"))); @@ -2016,6 +2025,418 @@ static Query *transform_cypher_predicate_function(cypher_parsestate *cpstate, } } +/* + * Mutator context for rewriting the fold body's accumulator/element Vars + * (columns 1 and 2 of the throwaway namespace RTE) into PARAM_EXEC params. + */ +typedef struct reduce_var_param_context +{ + int varno; /* rangetable index of the dummy (acc, elem) RTE */ +} reduce_var_param_context; + +/* + * Rewrite Var(varno, 1) -> Param(PARAM_EXEC, 0) [accumulator] and + * Var(varno, 2) -> Param(PARAM_EXEC, 1) [element] in the transformed fold + * body, so the body can be evaluated standalone inside age_reduce_transfn + * with the two params rebound for every element. + */ +static Node *reduce_var_to_param_mutator(Node *node, void *context) +{ + reduce_var_param_context *ctx = (reduce_var_param_context *) context; + + if (node == NULL) + { + return NULL; + } + + if (IsA(node, Var)) + { + Var *var = (Var *) node; + + /* + * Only the dummy (acc, elem) RTE at this level is rewritten. The + * varlevelsup == 0 check is essential: an outer-query RTE can share + * the same varno (each parse state's range table is numbered from 1), + * so without it a correlated outer reference at attno 1/2 would be + * silently rewritten into the accumulator/element param. Outer Vars + * are instead left in place and rejected by reduce_body_check_walker. + */ + if (var->varno == ctx->varno && var->varlevelsup == 0 && + (var->varattno == 1 || var->varattno == 2)) + { + Param *param = makeNode(Param); + + param->paramkind = PARAM_EXEC; + param->paramid = var->varattno - 1; + param->paramtype = AGTYPEOID; + param->paramtypmod = -1; + param->paramcollid = InvalidOid; + param->location = -1; + + return (Node *) param; + } + } + + return expression_tree_mutator(node, reduce_var_to_param_mutator, context); +} + +/* + * Build a throwaway subquery "SELECT NULL::agtype AS , NULL::agtype AS + * " used only to give the fold body a namespace in which the accumulator + * and element variables resolve to agtype columns. Those references are later + * rewritten to PARAM_EXEC params and the subquery is discarded. + */ +static Query *make_reduce_var_subquery(char *acc_name, char *elem_name) +{ + Query *subquery = makeNode(Query); + Const *acc_const; + Const *elem_const; + TargetEntry *acc_te; + TargetEntry *elem_te; + + acc_const = makeConst(AGTYPEOID, -1, InvalidOid, -1, (Datum) 0, true, false); + elem_const = makeConst(AGTYPEOID, -1, InvalidOid, -1, (Datum) 0, true, false); + + acc_te = makeTargetEntry((Expr *) acc_const, 1, acc_name, false); + elem_te = makeTargetEntry((Expr *) elem_const, 2, elem_name, false); + + subquery->commandType = CMD_SELECT; + subquery->targetList = list_make2(acc_te, elem_te); + subquery->jointree = makeFromExpr(NIL, NULL); + subquery->rtable = NIL; + subquery->rteperminfos = NIL; + + return subquery; +} + +/* + * Validate a transformed-and-mutated reduce() fold body. After + * reduce_var_to_param_mutator() has replaced the accumulator and element with + * PARAM_EXEC params 0 and 1, a valid body is a pure expression over those two + * params: it must contain no other Vars (outer-query references), no other + * params, and no aggregates or subqueries, because the body is evaluated + * standalone (ExecEvalExpr) inside age_reduce_transfn with only those two + * param slots bound. + */ +static bool reduce_body_check_walker(Node *node, void *context) +{ + if (node == NULL) + { + return false; + } + + if (IsA(node, Var)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("a reduce() expression may only reference its accumulator and element variables"))); + } + + if (IsA(node, Param)) + { + Param *param = (Param *) node; + + if (param->paramkind != PARAM_EXEC || + (param->paramid != 0 && param->paramid != 1)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("a reduce() expression may not reference query parameters"))); + } + } + + if (IsA(node, Aggref) || IsA(node, GroupingFunc) || IsA(node, WindowFunc)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("aggregate functions are not supported in a reduce() expression"))); + } + + if (IsA(node, SubLink)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("subqueries (including a nested reduce()) are not supported in a reduce() expression"))); + } + + return expression_tree_walker(node, reduce_body_check_walker, context); +} + +/* + * Transform a cypher_reduce node into a query tree. + * + * reduce(acc = init, var IN list | body) is rewritten into a scalar subquery + * over the age_reduce aggregate, with the list unnested WITH ORDINALITY and the + * aggregate ordered by that ordinality so the fold runs in list order: + * + * SELECT ag_catalog.age_reduce(, ''::text, + * r.elem ORDER BY r.ord) + * FROM unnest() WITH ORDINALITY AS r(elem, ord) + * + * The fold body is transformed separately with the accumulator and element + * rewritten to PARAM_EXEC params 0 and 1, serialized into the text argument, + * and evaluated per element inside age_reduce_transfn. + * + * The null/empty-list guard + * (CASE WHEN list IS NULL THEN NULL ELSE COALESCE(, init) END) is built + * at the grammar level in build_reduce_node(). + */ +static Query *transform_cypher_reduce(cypher_parsestate *cpstate, + cypher_clause *clause) +{ + cypher_reduce *reduce = (cypher_reduce *) clause->self; + Query *query; + Query *var_subquery; + cypher_parsestate *body_cpstate; + ParseState *body_pstate; + ParseNamespaceItem *body_pnsi; + Node *body_node; + char *body_serialized; + reduce_var_param_context mutator_ctx; + cypher_parsestate *child_cpstate; + ParseState *child_pstate; + FuncCall *unnest_fc; + RangeFunction *rf; + RangeTblEntry *rte = NULL; + int rtindex = 0; + List *namespace = NULL; + Node *from_item; + Node *init_node; + Node *elem_var; + Var *ord_var; + TargetEntry *ord_te; + SortGroupClause *sortcl; + Oid sort_ltop; + Oid sort_eqop; + bool sort_hashable; + Const *body_const; + Aggref *agg; + Oid agg_oid; + Oid agg_argtypes[3]; + TargetEntry *result_te; + + /* + * 1. Resolve the fold body's accumulator and element variables against a + * throwaway 2-column agtype subquery, rewrite those Vars to PARAM_EXEC + * params, validate it is a pure expression over those params, and + * serialize the body for age_reduce_transfn. + */ + body_cpstate = make_cypher_parsestate(cpstate); + body_pstate = (ParseState *) body_cpstate; + + var_subquery = make_reduce_var_subquery(reduce->acc_varname, + reduce->elem_varname); + body_pnsi = addRangeTableEntryForSubquery(body_pstate, var_subquery, + makeAlias("reduce_vars", NIL), + false, true); + addNSItemToQuery(body_pstate, body_pnsi, false, true, true); + + body_node = transform_cypher_expr(body_cpstate, reduce->body_expr, + EXPR_KIND_SELECT_TARGET); + + /* + * The accumulator is always an agtype value (the aggregate's stype is + * agtype). A fold body can legitimately produce a non-agtype scalar -- for + * example "s AND x" or "x = 2" yield a boolean -- so normalize the body to + * agtype here. Without this the transition function would treat a by-value + * Datum (e.g. bool) as a by-reference varlena and crash. A boolean is + * wrapped in ag_catalog.bool_to_agtype() (AGE registers no implicit + * boolean-to-agtype cast); any other non-agtype type is coerced through + * the normal cast machinery, which raises a clean error if impossible. + */ + if (exprType(body_node) != AGTYPEOID) + { + if (exprType(body_node) == BOOLOID) + { + Oid bool_to_agtype_oid = get_ag_func_oid("bool_to_agtype", 1, + BOOLOID); + + body_node = (Node *) makeFuncExpr(bool_to_agtype_oid, AGTYPEOID, + list_make1(body_node), + InvalidOid, InvalidOid, + COERCE_EXPLICIT_CALL); + } + else + { + body_node = coerce_to_common_type(body_pstate, body_node, + AGTYPEOID, "reduce"); + } + } + + mutator_ctx.varno = body_pnsi->p_rtindex; + body_node = reduce_var_to_param_mutator(body_node, &mutator_ctx); + + reduce_body_check_walker(body_node, NULL); + + body_serialized = nodeToString(body_node); + + free_cypher_parsestate(body_cpstate); + + /* + * 2. Build the outer aggregate query: + * SELECT age_reduce(, ''::text, r.elem ORDER BY r.ord) + * FROM unnest() WITH ORDINALITY AS r(elem, ord) + */ + query = makeNode(Query); + query->commandType = CMD_SELECT; + + child_cpstate = make_cypher_parsestate(cpstate); + child_pstate = (ParseState *) child_cpstate; + + unnest_fc = makeFuncCall(list_make1(makeString("unnest")), + list_make1(reduce->list_expr), + COERCE_SQL_SYNTAX, -1); + rf = makeNode(RangeFunction); + rf->lateral = false; + rf->ordinality = true; + rf->is_rowsfrom = false; + rf->functions = list_make1(list_make2((Node *) unnest_fc, NIL)); + rf->alias = makeAlias("reduce_src", + list_make2(makeString(reduce->elem_varname), + makeString("reduce_ordinality"))); + rf->coldeflist = NIL; + + from_item = transform_from_clause_item(child_cpstate, (Node *) rf, + &rte, &rtindex, &namespace); + checkNameSpaceConflicts(child_pstate, child_pstate->p_namespace, namespace); + child_pstate->p_joinlist = lappend(child_pstate->p_joinlist, from_item); + child_pstate->p_namespace = list_concat(child_pstate->p_namespace, + namespace); + setNamespaceLateralState(child_pstate->p_namespace, false, true); + + /* arguments to age_reduce: init, serialized body text, element column */ + init_node = transform_cypher_expr(child_cpstate, reduce->init_expr, + EXPR_KIND_SELECT_TARGET); + elem_var = colNameToVar(child_pstate, reduce->elem_varname, false, -1); + body_const = makeConst(TEXTOID, -1, InvalidOid, -1, + CStringGetTextDatum(body_serialized), false, false); + + /* the WITH ORDINALITY column (bigint), used only to order the fold */ + ord_var = makeVar(rtindex, 2, INT8OID, -1, InvalidOid, 0); + get_sort_group_operators(INT8OID, true, true, false, + &sort_ltop, &sort_eqop, NULL, &sort_hashable); + + ord_te = makeTargetEntry((Expr *) ord_var, 4, NULL, true); + ord_te->ressortgroupref = 1; + + sortcl = makeNode(SortGroupClause); + sortcl->tleSortGroupRef = 1; + sortcl->eqop = sort_eqop; + sortcl->sortop = sort_ltop; + sortcl->reverse_sort = false; + sortcl->nulls_first = false; + sortcl->hashable = sort_hashable; + + /* + * Evaluate exactly once per reduce() instead of once per element. + * A regular aggregate argument is evaluated by the executor for every + * input row, but age_reduce_transfn only reads the init argument on the + * first transition (when the running state is still NULL). Re-evaluating + * an expensive init wastes work, and a volatile init would fire its side + * effects once per element. + * + * Rows are fed to the aggregate in ascending ordinality order, so the + * first transition is always the row with ordinality 1. Wrapping init in + * CASE WHEN reduce_ordinality = 1 THEN ELSE NULL::agtype END + * computes on exactly that row (CASE only evaluates the matching + * branch's result) and passes a NULL init -- which the transition + * function ignores -- on every other row. The empty-list case is handled + * separately by the COALESCE(..., init) guard in build_reduce_node(). + */ + { + OpExpr *ord_is_first; + Const *one_const; + CaseWhen *init_when; + CaseExpr *init_case; + Const *null_init; + + one_const = makeConst(INT8OID, -1, InvalidOid, sizeof(int64), + Int64GetDatum(1), false, true); + + ord_is_first = makeNode(OpExpr); + ord_is_first->opno = sort_eqop; /* int8 equality */ + ord_is_first->opfuncid = get_opcode(sort_eqop); + ord_is_first->opresulttype = BOOLOID; + ord_is_first->opretset = false; + ord_is_first->opcollid = InvalidOid; + ord_is_first->inputcollid = InvalidOid; + ord_is_first->args = list_make2(copyObject(ord_var), one_const); + ord_is_first->location = -1; + + null_init = makeConst(AGTYPEOID, -1, InvalidOid, -1, (Datum) 0, + true, false); + + init_when = makeNode(CaseWhen); + init_when->expr = (Expr *) ord_is_first; + init_when->result = (Expr *) init_node; + init_when->location = -1; + + init_case = makeNode(CaseExpr); + init_case->casetype = AGTYPEOID; + init_case->casecollid = InvalidOid; + init_case->arg = NULL; + init_case->args = list_make1(init_when); + init_case->defresult = (Expr *) null_init; + init_case->location = -1; + + init_node = (Node *) init_case; + } + + /* look up the age_reduce(agtype, text, agtype) aggregate */ + agg_argtypes[0] = AGTYPEOID; + agg_argtypes[1] = TEXTOID; + agg_argtypes[2] = AGTYPEOID; + agg_oid = LookupFuncName(list_make2(makeString("ag_catalog"), + makeString("age_reduce")), + 3, agg_argtypes, false); + + agg = makeNode(Aggref); + agg->aggfnoid = agg_oid; + agg->aggtype = AGTYPEOID; + agg->aggcollid = InvalidOid; + agg->inputcollid = InvalidOid; + agg->aggtranstype = InvalidOid; /* filled by the planner */ + agg->aggargtypes = list_make3_oid(AGTYPEOID, TEXTOID, AGTYPEOID); + agg->aggdirectargs = NIL; + agg->args = list_make4(makeTargetEntry((Expr *) init_node, 1, NULL, false), + makeTargetEntry((Expr *) body_const, 2, NULL, false), + makeTargetEntry((Expr *) elem_var, 3, NULL, false), + ord_te); + agg->aggorder = list_make1(sortcl); + agg->aggdistinct = NIL; + agg->aggfilter = NULL; + agg->aggstar = false; + agg->aggvariadic = false; + agg->aggkind = AGGKIND_NORMAL; + agg->aggpresorted = false; + agg->agglevelsup = 0; + agg->aggsplit = AGGSPLIT_SIMPLE; + agg->aggno = -1; + agg->aggtransno = -1; + agg->location = -1; + + child_pstate->p_hasAggs = true; + + result_te = makeTargetEntry((Expr *) agg, + (AttrNumber) child_pstate->p_next_resno++, + "reduce", false); + + query->targetList = list_make1(result_te); + query->jointree = makeFromExpr(child_pstate->p_joinlist, NULL); + query->rtable = child_pstate->p_rtable; + query->rteperminfos = child_pstate->p_rteperminfos; + query->hasAggs = true; + query->hasSubLinks = child_pstate->p_hasSubLinks; + query->hasTargetSRFs = child_pstate->p_hasTargetSRFs; + + assign_query_collations(child_pstate, query); + parse_check_aggregates(child_pstate, query); + + free_cypher_parsestate(child_cpstate); + + return query; +} + /* * Iterate through the list of items to delete and extract the variable name. * Then find the resno that the variable name belongs to. diff --git a/src/backend/parser/cypher_gram.y b/src/backend/parser/cypher_gram.y index b23fa705a..83d69c83b 100644 --- a/src/backend/parser/cypher_gram.y +++ b/src/backend/parser/cypher_gram.y @@ -131,7 +131,7 @@ MATCH MERGE NONE NOT NULL_P ON OPERATOR OPTIONAL OR ORDER - REMOVE RETURN + REDUCE REMOVE RETURN SET SINGLE SKIP STARTS THEN TRUE_P UNION UNWIND @@ -332,6 +332,11 @@ static Node *build_predicate_function_node(cypher_predicate_function_kind kind, /* pattern expression helper */ static Node *make_exists_pattern_sublink(Node *pattern, int location); +/* reduce(acc = init, var IN list | body) */ +static Node *build_reduce_node(char *acc_varname, Node *init_expr, + char *elem_varname, Node *list_expr, + Node *body_expr, int location); + /* helper functions */ static ExplainStmt *make_explain_stmt(List *options); static void validate_return_item_aliases(List *items, ag_scanner_t scanner); @@ -1963,6 +1968,10 @@ expr_func_subexpr: { $$ = build_predicate_function_node(CPFK_SINGLE, $3, $5, $7, @1); } + | REDUCE '(' var_name '=' expr ',' var_name IN expr '|' expr ')' + { + $$ = build_reduce_node($3, $5, $7, $9, $11, @1); + } ; expr_subquery: @@ -2561,6 +2570,7 @@ safe_keywords: | OPTIONAL { $$ = KEYWORD_STRDUP($1); } | OR { $$ = KEYWORD_STRDUP($1); } | ORDER { $$ = KEYWORD_STRDUP($1); } + | REDUCE { $$ = KEYWORD_STRDUP($1); } | REMOVE { $$ = KEYWORD_STRDUP($1); } | RETURN { $$ = KEYWORD_STRDUP($1); } | SET { $$ = KEYWORD_STRDUP($1); } @@ -3646,6 +3656,86 @@ static Node *make_exists_pattern_sublink(Node *pattern, int location) return (Node *)node_to_agtype((Node *)n, "boolean", location); } +/* + * Helper function to build a reduce() grammar node. + * + * Follows the openCypher syntax: + * reduce(acc = init, var IN list | body) + * + * The accumulator `acc` is seeded with `init` and threaded across the + * elements of `list` (bound to `var`) in list order, with `body` producing + * the next accumulator value at each step. The result is the final + * accumulator value, or `init` when the list is empty. + * + * The reduce node is wrapped in an EXPR_SUBLINK (scalar subquery) whose + * subselect is the cypher_reduce node; transform_cypher_reduce() in + * cypher_clause.c rewrites it into a correlated scalar subquery over an + * ordered aggregate. + * + * The whole thing is then wrapped so the openCypher null/empty-list semantics + * hold without the transform layer having to special-case them: + * + * CASE WHEN list IS NULL THEN NULL + * ELSE COALESCE((reduce subquery), init) END + * + * A NULL list yields NULL; an empty list yields `init` (the aggregate runs + * over zero rows and returns SQL NULL, which COALESCE replaces with init); + * a non-empty list yields the fold result. The list and init grammar nodes + * are shared between the reduce node and this guard, which is safe because + * AGE's expression transformer builds new nodes rather than mutating in place. + */ +static Node *build_reduce_node(char *acc_varname, Node *init_expr, + char *elem_varname, Node *list_expr, + Node *body_expr, int location) +{ + SubLink *sub; + cypher_reduce *reduce_node = NULL; + CoalesceExpr *coalesce; + NullTest *null_test; + CaseWhen *case_when; + CaseExpr *guard; + + reduce_node = make_ag_node(cypher_reduce); + reduce_node->acc_varname = acc_varname; + reduce_node->init_expr = init_expr; + reduce_node->elem_varname = elem_varname; + reduce_node->list_expr = list_expr; + reduce_node->body_expr = body_expr; + + sub = makeNode(SubLink); + sub->subLinkId = 0; + sub->testexpr = NULL; + sub->operName = NIL; + sub->subselect = (Node *) reduce_node; + sub->location = location; + sub->subLinkType = EXPR_SUBLINK; + + /* COALESCE((reduce subquery), init) -- empty list falls back to init */ + coalesce = makeNode(CoalesceExpr); + coalesce->args = list_make2((Node *) sub, init_expr); + coalesce->location = location; + + /* CASE WHEN list IS NULL THEN NULL ELSE END */ + null_test = makeNode(NullTest); + null_test->arg = (Expr *) list_expr; + null_test->nulltesttype = IS_NULL; + null_test->argisrow = false; + null_test->location = location; + + case_when = makeNode(CaseWhen); + case_when->expr = (Expr *) null_test; + case_when->result = (Expr *) make_null_const(location); + case_when->location = location; + + guard = makeNode(CaseExpr); + guard->arg = NULL; + guard->args = list_make1(case_when); + guard->defresult = (Expr *) coalesce; + guard->location = location; + + return (Node *) guard; +} + /* Helper function to create an ExplainStmt node */ static ExplainStmt *make_explain_stmt(List *options) { diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index cfff8138f..bf69bf1fa 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -47,8 +47,11 @@ #include "miscadmin.h" #include "parser/parse_coerce.h" #include "nodes/nodes.h" +#include "nodes/nodeFuncs.h" +#include "executor/executor.h" #include "utils/acl.h" #include "utils/builtins.h" +#include "utils/datum.h" #include "executor/cypher_utils.h" #include "utils/float.h" #include "utils/lsyscache.h" @@ -11575,6 +11578,168 @@ Datum age_float8_stddev_pop_aggfinalfn(PG_FUNCTION_ARGS) PG_RETURN_POINTER(agtype_value_to_agtype(&agtv_float)); } +/* + * Per-aggregate-group evaluation state for reduce(). Caches the compiled + * fold-body expression and a standalone ExprContext whose PARAM_EXEC slots + * (0 = accumulator, 1 = current element) are rebound on every element. + */ +typedef struct reduce_eval_ctx +{ + ExprState *body_state; /* compiled fold-body expression */ + ExprContext *econtext; /* eval context carrying the param slots */ + ParamExecData *params; /* [0] = accumulator, [1] = current element */ +} reduce_eval_ctx; + +/* Build an agtype 'null' Datum (a real agtype value, not a SQL NULL). */ +static Datum reduce_agtype_null(void) +{ + agtype_value agtv; + + agtv.type = AGTV_NULL; + return AGTYPE_P_GET_DATUM(agtype_value_to_agtype(&agtv)); +} + +/* + * age_reduce_transfn(state agtype, init agtype, body text, element agtype) + * + * Transition function for the age_reduce aggregate that implements the Cypher + * reduce(acc = init, var IN list | body) fold. The fold body is compiled by + * transform_cypher_reduce() with the accumulator and element rewritten to + * PARAM_EXEC params 0 and 1, then serialized into the `body` text argument. + * + * On the first element of a group the accumulator is seeded from `init` + * (the running state is NULL because the aggregate uses no initcond); on + * every element the body is evaluated with the params rebound, and the result + * becomes the next accumulator state. + * + * The accumulator and element are normalized to a non-NULL agtype 'null' + * before evaluation so that (a) the fold body sees agtype values and Cypher + * null semantics apply, and (b) the running state is never a SQL NULL, which + * keeps PG_ARGISNULL(0) a reliable "first element of the group" signal even + * when the fold legitimately produces null. + */ +PG_FUNCTION_INFO_V1(age_reduce_transfn); + +Datum age_reduce_transfn(PG_FUNCTION_ARGS) +{ + MemoryContext aggcontext; + MemoryContext oldctx; + reduce_eval_ctx *rc; + Datum acc; + Datum element; + Datum result; + bool result_isnull; + + if (!AggCheckCallContext(fcinfo, &aggcontext)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("age_reduce_transfn called in a non-aggregate context"))); + } + + /* the fold can run over a large list; stay responsive to cancellation */ + CHECK_FOR_INTERRUPTS(); + + /* + * One-time per-FmgrInfo setup: deserialize and compile the fold body, and + * build the standalone ExprContext plus its two PARAM_EXEC slots. The body + * text is a query constant, so caching the compiled state across groups is + * correct. + */ + rc = (reduce_eval_ctx *) fcinfo->flinfo->fn_extra; + if (rc == NULL) + { + text *body_txt; + char *body_str; + Node *body_node; + + if (PG_ARGISNULL(2)) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("age_reduce: missing fold expression"))); + } + + oldctx = MemoryContextSwitchTo(fcinfo->flinfo->fn_mcxt); + rc = (reduce_eval_ctx *) palloc0(sizeof(reduce_eval_ctx)); + body_txt = PG_GETARG_TEXT_PP(2); + body_str = text_to_cstring(body_txt); + body_node = (Node *) stringToNode(body_str); + + /* + * age_reduce() is SQL-callable, so the serialized body argument is + * not guaranteed to have come from transform_cypher_reduce(). The + * running state is stored as an agtype varlena (the datumCopy() below + * uses typbyval=false, typlen=-1), so a body that evaluates to a + * by-value type (e.g. a bare boolean or integer) would have its Datum + * misread as a pointer and could crash the backend. Reject any body + * whose result type is not agtype. transform_cypher_reduce() always + * normalizes the fold body to agtype, so a planner-generated reduce() + * is never rejected here. + */ + if (exprType(body_node) != AGTYPEOID) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("age_reduce: fold expression must return agtype"))); + } + + rc->body_state = ExecInitExpr((Expr *) body_node, NULL); + rc->econtext = CreateStandaloneExprContext(); + rc->params = (ParamExecData *) palloc0(sizeof(ParamExecData) * 2); + rc->econtext->ecxt_param_exec_vals = rc->params; + fcinfo->flinfo->fn_extra = rc; + MemoryContextSwitchTo(oldctx); + } + + /* + * Seed the accumulator. The aggregate declares no initcond, so on the + * first element the running state (arg 0) is NULL and we use `init` + * (arg 1); thereafter the accumulator is the prior state. A NULL init is + * normalized to agtype 'null'. + */ + if (PG_ARGISNULL(0)) + { + acc = PG_ARGISNULL(1) ? reduce_agtype_null() : PG_GETARG_DATUM(1); + } + else + { + acc = PG_GETARG_DATUM(0); + } + + /* a NULL element is likewise normalized to agtype 'null' */ + element = PG_ARGISNULL(3) ? reduce_agtype_null() : PG_GETARG_DATUM(3); + + /* bind PARAM_EXEC 0 = accumulator, 1 = current element */ + rc->params[0].value = acc; + rc->params[0].isnull = false; + rc->params[0].execPlan = NULL; + rc->params[1].value = element; + rc->params[1].isnull = false; + rc->params[1].execPlan = NULL; + + /* evaluate the fold body for this element */ + ResetExprContext(rc->econtext); + result = ExecEvalExpr(rc->body_state, rc->econtext, &result_isnull); + + /* + * Never let the running state become a SQL NULL: a null fold result is + * stored as agtype 'null' so the next element is not mistaken for the + * first one (see PG_ARGISNULL(0) above). + */ + if (result_isnull) + { + result = reduce_agtype_null(); + } + + /* the new state must survive in the aggregate context across elements */ + oldctx = MemoryContextSwitchTo(aggcontext); + result = datumCopy(result, false, -1); + MemoryContextSwitchTo(oldctx); + + PG_RETURN_DATUM(result); +} + PG_FUNCTION_INFO_V1(age_agtype_larger_aggtransfn); Datum age_agtype_larger_aggtransfn(PG_FUNCTION_ARGS) diff --git a/src/include/nodes/ag_nodes.h b/src/include/nodes/ag_nodes.h index 47c55041b..57b2deabc 100644 --- a/src/include/nodes/ag_nodes.h +++ b/src/include/nodes/ag_nodes.h @@ -77,7 +77,9 @@ typedef enum ag_node_tag cypher_delete_item_t, cypher_merge_information_t, /* predicate functions */ - cypher_predicate_function_t + cypher_predicate_function_t, + /* reduce */ + cypher_reduce_t } ag_node_tag; extern const char *node_names[]; diff --git a/src/include/nodes/cypher_copyfuncs.h b/src/include/nodes/cypher_copyfuncs.h index e770cebe2..fcad598b7 100644 --- a/src/include/nodes/cypher_copyfuncs.h +++ b/src/include/nodes/cypher_copyfuncs.h @@ -56,4 +56,8 @@ void copy_cypher_merge_information(ExtensibleNode *newnode, /* predicate function data structure */ void copy_cypher_predicate_function(ExtensibleNode *newnode, const ExtensibleNode *from); + +/* reduce data structure */ +void copy_cypher_reduce(ExtensibleNode *newnode, + const ExtensibleNode *from); #endif diff --git a/src/include/nodes/cypher_nodes.h b/src/include/nodes/cypher_nodes.h index 3433bebb0..5efbe95f7 100644 --- a/src/include/nodes/cypher_nodes.h +++ b/src/include/nodes/cypher_nodes.h @@ -247,6 +247,24 @@ typedef struct cypher_predicate_function Node *where; /* the predicate to test */ } cypher_predicate_function; +/* + * reduce(acc = init, var IN list | body) + * + * Folds `body` over `list`, threading an accumulator `acc` (seeded with + * `init`) across the elements in list order, binding each element to `var`. + * Transformed into a correlated scalar subquery over an ordered aggregate + * by transform_cypher_reduce() in cypher_clause.c. + */ +typedef struct cypher_reduce +{ + ExtensibleNode extensible; + char *acc_varname; /* accumulator variable name */ + Node *init_expr; /* initial accumulator value */ + char *elem_varname; /* per-element variable name */ + Node *list_expr; /* the list to fold over */ + Node *body_expr; /* the fold expression evaluated per element */ +} cypher_reduce; + typedef enum cypher_map_projection_element_type { PROPERTY_SELECTOR = 0, /* map_var { .key } */ diff --git a/src/include/nodes/cypher_outfuncs.h b/src/include/nodes/cypher_outfuncs.h index 55285bdba..fc2a830b9 100644 --- a/src/include/nodes/cypher_outfuncs.h +++ b/src/include/nodes/cypher_outfuncs.h @@ -50,6 +50,7 @@ void out_cypher_map_projection(StringInfo str, const ExtensibleNode *node); void out_cypher_list(StringInfo str, const ExtensibleNode *node); void out_cypher_list_comprehension(StringInfo str, const ExtensibleNode *node); void out_cypher_predicate_function(StringInfo str, const ExtensibleNode *node); +void out_cypher_reduce(StringInfo str, const ExtensibleNode *node); /* comparison expression */ void out_cypher_comparison_aexpr(StringInfo str, const ExtensibleNode *node); diff --git a/src/include/nodes/cypher_readfuncs.h b/src/include/nodes/cypher_readfuncs.h index 9202ba511..2f27f91cd 100644 --- a/src/include/nodes/cypher_readfuncs.h +++ b/src/include/nodes/cypher_readfuncs.h @@ -54,4 +54,7 @@ void read_cypher_merge_information(struct ExtensibleNode *node); /* predicate function data structure */ void read_cypher_predicate_function(struct ExtensibleNode *node); +/* reduce data structure */ +void read_cypher_reduce(struct ExtensibleNode *node); + #endif diff --git a/src/include/parser/cypher_kwlist.h b/src/include/parser/cypher_kwlist.h index 44ac09452..909b5d272 100644 --- a/src/include/parser/cypher_kwlist.h +++ b/src/include/parser/cypher_kwlist.h @@ -36,6 +36,7 @@ PG_KEYWORD("operator", OPERATOR, RESERVED_KEYWORD) PG_KEYWORD("optional", OPTIONAL, RESERVED_KEYWORD) PG_KEYWORD("or", OR, RESERVED_KEYWORD) PG_KEYWORD("order", ORDER, RESERVED_KEYWORD) +PG_KEYWORD("reduce", REDUCE, RESERVED_KEYWORD) PG_KEYWORD("remove", REMOVE, RESERVED_KEYWORD) PG_KEYWORD("return", RETURN, RESERVED_KEYWORD) PG_KEYWORD("set", SET, RESERVED_KEYWORD) From ce073ef0c796c7ede90e5c9db24de297564eacce Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Mon, 29 Jun 2026 06:47:55 -0700 Subject: [PATCH 087/100] resolve subgraph staging sequences via regclass (#2446) The vertex/edge staging copies in create_subgraph() generated new graphids with nextval(%L), which binds the sequence as a string literal and invokes the nextval(text) overload. That re-resolves the schema-qualified sequence name on each call. Cast the literal to regclass (nextval(%L::regclass)) so the sequence is resolved once to its OID, matching how AGE defines its label id defaults (nextval('schema.seq'::regclass)). Applied to both the vertex and edge staging queries, in sql/age_subgraph.sql and the identical body in the age--1.7.0--y.y.y.sql upgrade template so the upgrade-path catalog comparison still matches. Behavior is unchanged; all 38 regression tests pass against PostgreSQL 18. Addresses Copilot review feedback on apache/age#2441. Co-authored-by: GitHub Copilot (Claude Opus 4.8) <[email protected]> modified: age--1.7.0--y.y.y.sql modified: sql/age_subgraph.sql --- age--1.7.0--y.y.y.sql | 4 ++-- sql/age_subgraph.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index ec909d2bb..6dc8f707f 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -1013,7 +1013,7 @@ BEGIN EXECUTE format( 'CREATE TEMP TABLE _ag_sg_vstage ON COMMIT DROP AS ' 'SELECT t.id AS old_id, ' - ' ag_catalog._graphid(%s, nextval(%L)) AS new_id, ' + ' ag_catalog._graphid(%s, nextval(%L::regclass)) AS new_id, ' ' t.properties AS props ' 'FROM ONLY %s t ' 'WHERE EXISTS (SELECT 1 FROM _ag_sg_kept_v k WHERE k.gid = t.id)', @@ -1076,7 +1076,7 @@ BEGIN DROP TABLE IF EXISTS _ag_sg_estage; EXECUTE format( 'CREATE TEMP TABLE _ag_sg_estage ON COMMIT DROP AS ' - 'SELECT ag_catalog._graphid(%s, nextval(%L)) AS new_id, ' + 'SELECT ag_catalog._graphid(%s, nextval(%L::regclass)) AS new_id, ' ' vs.new_id AS new_start, ve.new_id AS new_end, ' ' x.properties AS props ' 'FROM ONLY %s x ' diff --git a/sql/age_subgraph.sql b/sql/age_subgraph.sql index 960790ded..0d7e3648d 100644 --- a/sql/age_subgraph.sql +++ b/sql/age_subgraph.sql @@ -205,7 +205,7 @@ BEGIN EXECUTE format( 'CREATE TEMP TABLE _ag_sg_vstage ON COMMIT DROP AS ' 'SELECT t.id AS old_id, ' - ' ag_catalog._graphid(%s, nextval(%L)) AS new_id, ' + ' ag_catalog._graphid(%s, nextval(%L::regclass)) AS new_id, ' ' t.properties AS props ' 'FROM ONLY %s t ' 'WHERE EXISTS (SELECT 1 FROM _ag_sg_kept_v k WHERE k.gid = t.id)', @@ -268,7 +268,7 @@ BEGIN DROP TABLE IF EXISTS _ag_sg_estage; EXECUTE format( 'CREATE TEMP TABLE _ag_sg_estage ON COMMIT DROP AS ' - 'SELECT ag_catalog._graphid(%s, nextval(%L)) AS new_id, ' + 'SELECT ag_catalog._graphid(%s, nextval(%L::regclass)) AS new_id, ' ' vs.new_id AS new_start, ve.new_id AS new_end, ' ' x.properties AS props ' 'FROM ONLY %s x ' From 08da2cedca49cd3735743107e38876197b762425 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Mon, 29 Jun 2026 06:48:05 -0700 Subject: [PATCH 088/100] Support relationship-type filters and a minimum hop count (#2442) Support relationship-type filters and a minimum hop count in shortest_path SRFs age_shortest_path / age_all_shortest_paths gain two related capabilities, both following openCypher / Neo4j semantics. Relationship-type filtering: the edge_types argument now accepts an array of types; an edge matches when its label is any one of the requested types. A bare string or a one-element array keeps the single-type behaviour, an empty string/array or NULL means no filter, and an unknown type matches nothing. sp_run_bfs takes an Oid set rather than a single oid, and sp_compute_paths resolves the argument into that set. Minimum hop count: the new min_hops argument is a lower bound on the path length. When it does not exceed the true shortest distance it imposes no constraint, so the normal BFS shortest-path result is returned. When it exceeds the shortest distance, BFS cannot produce a qualifying path, so the search falls back to the variable-length-edge depth-first engine (sp_minhops_fallback), which enumerates edge-distinct paths (relationship-uniqueness / trail semantics) and returns the shortest path(s) whose length is at least min_hops. This regime permits revisiting a vertex and closed walks back to the start, but never reusing an edge. A private memory context bounds the search and a cost guard caps the number of examined paths, raising PROGRAM_LIMIT_EXCEEDED (with a hint to bound the search with a maximum hop count) when the cap is exceeded. The hard regime combined with multiple relationship types is unsupported, because the VLE engine matches a single label; that case raises FEATURE_NOT_SUPPORTED. Regression coverage spans single- and multi-type filters, directed and undirected reachability, multiplicity of equal-length paths, max_hops bounds, NULL and non-existent endpoints, and both min_hops regimes, including a vertex-revisiting longer path (sp_revisit) and a closed-walk cycle back to the start (sp_tri). The in-cypher() Tier 1 call forms are exercised as well. Review feedback addressed: 1. Error messages now report the function actually called. age_shortest_path and age_all_shortest_paths share their argument-resolution helpers, which hard-coded an "age_shortest_path" prefix regardless of the caller; the caller's name is now threaded through so each function reports its own (this also corrects a mislabeled multi-type min_hops error). A new regression case (sp_errname) pins the behaviour for both functions. 2. age_all_shortest_paths now bounds the number of materialized result paths. The shortest-path DAG can contain exponentially many equal-length paths, all built up front before the first row streams; enumeration is capped at SP_MAX_RESULT_PATHS (1,000,000), raising PROGRAM_LIMIT_EXCEEDED with a hint to narrow the search, mirroring the existing min-hops candidate cap. 3. The BFS search state (visited table, frontier queue, predecessor multiset, and intermediate path arrays) now lives in a private scratch memory context that is deleted once the surviving result Datums are built in the SRF context, rather than persisting in multi_call_memory_ctx for the life of the SRF. This bounds peak memory to the result set plus one search and matches the pattern sp_minhops_fallback already used. 4. A second review round tightened memory hygiene and reporting: the pnstrdup'd relationship-type name is freed once resolved to an oid (it was retained for the life of the SRF) in both the array and scalar cases; the invalid-direction error now carries the called function's name like the other argument errors; the min-hops fallback's private context is renamed to a caller-neutral "age shortest path minhops" (it is shared by both SRFs); and the multi-type label-filter comment is corrected to note that an unknown type merely contributes no matches -- known types in the same set still match, and only an all-unknown set leaves just the zero-length path. 41/41 installcheck. Co-authored-by: Copilot modified: regress/expected/age_shortest_path.out modified: regress/sql/age_shortest_path.sql modified: src/backend/utils/adt/age_vle.c --- regress/expected/age_shortest_path.out | 762 ++++++++++++++++++++++++- regress/sql/age_shortest_path.sql | 487 +++++++++++++++- src/backend/utils/adt/age_vle.c | 445 +++++++++++++-- 3 files changed, 1627 insertions(+), 67 deletions(-) diff --git a/regress/expected/age_shortest_path.out b/regress/expected/age_shortest_path.out index 7eb751d12..f7f8f1300 100644 --- a/regress/expected/age_shortest_path.out +++ b/regress/expected/age_shortest_path.out @@ -158,6 +158,20 @@ FROM age_all_shortest_paths( 2 (1 row) +-- single shortest path with direction 'in': D -> A backwards; expected: +-- path_count = 1 (the single-path variant picks one of the two routes) +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL, '"in"'::agtype +); + path_count +------------ + 1 +(1 row) + -- direction 'out': D -> A not reachable forwards; expected: path_count = 0 SELECT count(*) AS path_count FROM age_shortest_path( @@ -192,7 +206,7 @@ FROM age_shortest_path( (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), NULL, '"sideways"'::agtype ); -ERROR: direction argument must be one of 'out', 'in', or 'any' +ERROR: age_shortest_path: direction argument must be one of 'out', 'in', or 'any' -- error: start argument is neither a vertex nor an integer id; expected: ERROR SELECT count(*) AS path_count FROM age_shortest_path( @@ -201,6 +215,15 @@ FROM age_shortest_path( (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)) ); ERROR: start vertex argument must be a vertex or the integer id +-- error: end argument is neither a vertex nor an integer id; expected: ERROR +-- (symmetric to the start-vertex check above) +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"not_a_vertex"'::agtype +); +ERROR: end vertex argument must be a vertex or the integer id -- -- Non-existent endpoint guards. These must NOT crash the backend and must -- return no rows (a path can only exist between vertices in the graph). @@ -932,20 +955,113 @@ SELECT count(*) FROM age_shortest_path( 1 (1 row) --- multiple relationship types are not yet supported; expected: ERROR +-- multiple relationship types: an array of types matches an edge whose type +-- is any one of them. A..C single shortest under {KNOWS, LIKES} (length 2); +-- count 1 SELECT count(*) FROM age_shortest_path( '"sp_edge"'::agtype, (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), '["KNOWS", "LIKES"]'::agtype, '"out"'::agtype); -ERROR: age_shortest_path: multiple relationship types are not yet supported --- a non-zero minimum hop count is not yet supported; expected: ERROR + count +------- + 1 +(1 row) + +-- all shortest A..C under {KNOWS, LIKES}: three A->B edges (2 KNOWS + 1 LIKES) +-- each extend by the single B->C KNOWS edge; count 3 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS", "LIKES"]'::agtype, '"out"'::agtype); + count +------- + 3 +(1 row) + +-- all shortest A..B under {KNOWS, LIKES}: the two parallel KNOWS edges plus +-- the one LIKES edge are three distinct one-hop paths; count 3 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS", "LIKES"]'::agtype, '"any"'::agtype); + count +------- + 3 +(1 row) + +-- a multi-type array containing an unknown type ignores the unknown member: +-- {NOSUCHLABEL, KNOWS} still finds A..C via KNOWS; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["NOSUCHLABEL", "KNOWS"]'::agtype, '"out"'::agtype); + count +------- + 1 +(1 row) + +-- a multi-type array of only types that do not connect the endpoints: +-- {LIKES} reaches B but B..C has no LIKES edge, so A..C is unreachable; +-- count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["LIKES", "NOSUCHLABEL"]'::agtype, '"out"'::agtype); + count +------- + 0 +(1 row) + +-- an empty relationship-type array imposes no filter (same as NULL): A..C +-- (length 2) is found; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '[]'::agtype, '"out"'::agtype); + count +------- + 1 +(1 row) + +-- a non-string element in the relationship-type array is an error +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS", 7]'::agtype, '"out"'::agtype); +ERROR: age_shortest_path: relationship type must be a string +-- a minimum hop count that does not exceed the shortest distance imposes no +-- extra constraint; A..C via KNOWS has length 2, so min_hops=2 is accepted and +-- returns the length-2 path; count 1 SELECT count(*) FROM age_shortest_path( '"sp_edge"'::agtype, (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); -ERROR: age_shortest_path: a minimum hop count is not yet supported + count +------- + 1 +(1 row) + +-- a minimum hop count greater than the shortest distance falls back to the +-- exhaustive (VLE) search; A..C in this DAG has no length-3 path (every longer +-- route would have to reuse an edge), so min_hops=3 yields no rows; count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 3::agtype); + count +------- + 0 +(1 row) + -- a minimum hop count of 0 is the default and is accepted; A..C (length 2); -- count 1 SELECT count(*) FROM age_shortest_path( @@ -961,6 +1077,39 @@ SELECT count(*) FROM age_shortest_path( -- a graph name that does not exist is an error SELECT count(*) FROM age_shortest_path('"no_such_graph"'::agtype, '1'::agtype, '2'::agtype); ERROR: schema "no_such_graph" does not exist +-- a NULL graph name is an error (the graph name is required, unlike the +-- endpoints which accept NULL as "no match") +SELECT count(*) FROM age_shortest_path( + NULL::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype))); +ERROR: age_shortest_path: graph name cannot be NULL +-- a non-integer max_hops is an error (the hop bounds must be integers) +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, NULL::agtype, '"not_an_int"'::agtype); +ERROR: age_shortest_path: agtype argument of wrong type +-- a non-integer min_hops is an error (symmetric to max_hops above) +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, '"not_an_int"'::agtype); +ERROR: age_shortest_path: agtype argument of wrong type +-- a negative min_hops is clamped to 0 (no constraint), so A..C (length 2) is +-- still found; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, '-3'::agtype); + count +------- + 1 +(1 row) + -- cleanup SELECT * FROM drop_graph('sp_edge', true); NOTICE: drop cascades to 5 other objects @@ -975,3 +1124,606 @@ NOTICE: graph "sp_edge" has been dropped (1 row) +-- +-- Calling shortest_path / all_shortest_paths from inside cypher() (Tier 1) +-- WITH a relationship-type filter -- both a single type (bare string) and +-- multiple types (a list literal). The graph name is auto-injected, so the +-- in-cypher call passes only the bound endpoints and the type filter. +-- +-- Graph: A and B are joined by two parallel KNOWS edges plus one LIKES edge; +-- B->C is a single KNOWS edge. This lets the all_shortest_paths variant return +-- more than one path so the multiplicity is visible. +-- +SELECT * FROM create_graph('sp_cy_lbl'); +NOTICE: graph "sp_cy_lbl" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('sp_cy_lbl', $$ + CREATE (a:N {name: 'A'}), + (b:N {name: 'B'}), + (c:N {name: 'C'}), + (a)-[:KNOWS]->(b), + (a)-[:KNOWS]->(b), + (a)-[:LIKES]->(b), + (b)-[:KNOWS]->(c) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- materialize the global graph context +SELECT * FROM cypher('sp_cy_lbl', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + result +----------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "N", "in_degree": 0, "out_degree": 3, "self_loops": 0} + {"id": 844424930131970, "label": "N", "in_degree": 3, "out_degree": 1, "self_loops": 0} + {"id": 844424930131971, "label": "N", "in_degree": 1, "out_degree": 0, "self_loops": 0} +(3 rows) + +-- shortest_path() in-cypher with a single relationship type; A..C via KNOWS +-- (length 2); expected: 1 path +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN shortest_path(a, c, 'KNOWS') +$$) AS (path agtype); + count +------- + 1 +(1 row) + +-- all_shortest_paths() in-cypher with a single relationship type; the two +-- parallel KNOWS edges A->B make two distinct shortest A..C paths; expected: 2 +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN all_shortest_paths(a, c, 'KNOWS') +$$) AS (path agtype); + count +------- + 2 +(1 row) + +-- shortest_path() in-cypher with multiple relationship types passed as a list +-- literal; A..C under {KNOWS, LIKES} (length 2); expected: 1 path +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN shortest_path(a, c, ['KNOWS', 'LIKES']) +$$) AS (path agtype); + count +------- + 1 +(1 row) + +-- all_shortest_paths() in-cypher with multiple relationship types; the three +-- A->B edges (2 KNOWS + 1 LIKES) each extend by the single B->C KNOWS edge, +-- giving three distinct shortest A..C paths; expected: 3 +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN all_shortest_paths(a, c, ['KNOWS', 'LIKES']) +$$) AS (path agtype); + count +------- + 3 +(1 row) + +-- all_shortest_paths() in-cypher, multiple types, adjacent endpoints: the two +-- parallel KNOWS edges plus the one LIKES edge are three one-hop A..B paths; +-- expected: 3 +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (b:N {name:'B'}) + RETURN all_shortest_paths(a, b, ['KNOWS', 'LIKES']) +$$) AS (path agtype); + count +------- + 3 +(1 row) + +-- multiple types where only one connects the endpoints: {LIKES} reaches B but +-- B->C has no LIKES edge, so A..C is unreachable in-cypher; expected: 0 +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN all_shortest_paths(a, c, ['LIKES']) +$$) AS (path agtype); + count +------- + 0 +(1 row) + +-- cleanup +SELECT * FROM drop_graph('sp_cy_lbl', true); +NOTICE: drop cascades to 5 other objects +DETAIL: drop cascades to table sp_cy_lbl._ag_label_vertex +drop cascades to table sp_cy_lbl._ag_label_edge +drop cascades to table sp_cy_lbl."N" +drop cascades to table sp_cy_lbl."KNOWS" +drop cascades to table sp_cy_lbl."LIKES" +NOTICE: graph "sp_cy_lbl" has been dropped + drop_graph +------------ + +(1 row) + +-- +-- Minimum hop count fallback (Tier: VLE exhaustive search). When the requested +-- minimum hop count exceeds the true shortest distance, the BFS shortest-path +-- cannot satisfy it (it needs longer paths), so the implementation falls back +-- to the variable-length-edge depth-first engine and returns the shortest +-- path(s) whose length is at least min_hops. +-- +-- Graph: A reaches C directly (length 1) and also via two distinct +-- intermediate vertices B1 and B2 (length 2 each): +-- A->C, A->B1->C, A->B2->C +-- +SELECT * FROM create_graph('sp_min'); +NOTICE: graph "sp_min" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('sp_min', $$ + CREATE (a:N {name: 'A'}), + (c:N {name: 'C'}), + (b1:N {name: 'B1'}), + (b2:N {name: 'B2'}), + (a)-[:KNOWS]->(c), + (a)-[:KNOWS]->(b1), + (b1)-[:KNOWS]->(c), + (a)-[:KNOWS]->(b2), + (b2)-[:KNOWS]->(c) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- materialize the global graph context +SELECT * FROM cypher('sp_min', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + result +----------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "N", "in_degree": 0, "out_degree": 3, "self_loops": 0} + {"id": 844424930131970, "label": "N", "in_degree": 3, "out_degree": 0, "self_loops": 0} + {"id": 844424930131971, "label": "N", "in_degree": 1, "out_degree": 1, "self_loops": 0} + {"id": 844424930131972, "label": "N", "in_degree": 1, "out_degree": 1, "self_loops": 0} +(4 rows) + +-- baseline: the shortest A..C is the direct length-1 edge; count 1 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype); + count +------- + 1 +(1 row) + +-- min_hops=2 excludes the direct edge and falls back to the exhaustive search; +-- the two length-2 routes A->B1->C and A->B2->C are the shortest qualifying +-- paths; all_shortest_paths returns both; count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); + count +------- + 2 +(1 row) + +-- single shortest_path with min_hops=2 picks exactly one of the two length-2 +-- routes; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); + count +------- + 1 +(1 row) + +-- the qualifying length-2 paths materialize correctly; all_shortest_paths +-- returns the full, order-stable set (the single shortest_path variant would +-- return an arbitrary one of the two equal-length routes, which is not a +-- deterministic choice), so both A->B1->C and A->B2->C are listed +SELECT path FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype) AS path +ORDER BY path; + path +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id": 844424930131969, "label": "N", "properties": {"name": "A"}}::vertex, {"id": 1125899906842626, "label": "KNOWS", "end_id": 844424930131971, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131971, "label": "N", "properties": {"name": "B1"}}::vertex, {"id": 1125899906842627, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "C"}}::vertex]::path + [{"id": 844424930131969, "label": "N", "properties": {"name": "A"}}::vertex, {"id": 1125899906842628, "label": "KNOWS", "end_id": 844424930131972, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131972, "label": "N", "properties": {"name": "B2"}}::vertex, {"id": 1125899906842629, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131972, "properties": {}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "C"}}::vertex]::path +(2 rows) + +-- min_hops=2 with a matching max_hops=2 returns the same two length-2 paths; +-- count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype, 2::agtype); + count +------- + 2 +(1 row) + +-- min_hops greater than max_hops is unsatisfiable; count 0 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype, 1::agtype); + count +------- + 0 +(1 row) + +-- min_hops=3 has no qualifying path (a length-3 A..C would have to reuse an +-- edge, which relationship-uniqueness forbids); count 0 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 3::agtype); + count +------- + 0 +(1 row) + +-- no edge-type filter also reaches the fallback; the two length-2 routes are +-- returned; count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype, '"out"'::agtype, 2::agtype); + count +------- + 2 +(1 row) + +-- the fallback honours direction: traversing edges backwards (C..A, 'in') with +-- min_hops=2 also returns the two length-2 routes; count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"in"'::agtype, 2::agtype); + count +------- + 2 +(1 row) + +-- the fallback respects direction: there is no forward C..A path, so 'out' +-- with min_hops=2 returns nothing; count 0 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); + count +------- + 0 +(1 row) + +-- an unknown relationship type in the fallback matches no edges, so even the +-- shortest qualifying path cannot be formed; count 0 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"NOSUCHLABEL"'::agtype, '"out"'::agtype, 2::agtype); + count +------- + 0 +(1 row) + +-- a minimum hop count greater than the shortest distance combined with +-- multiple relationship types is not supported (the VLE engine matches a +-- single label only); expected: ERROR +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS", "LIKES"]'::agtype, '"out"'::agtype, 2::agtype); +ERROR: age_all_shortest_paths: a minimum hop count greater than the shortest path length is not supported with multiple relationship types +-- cleanup +SELECT * FROM drop_graph('sp_min', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table sp_min._ag_label_vertex +drop cascades to table sp_min._ag_label_edge +drop cascades to table sp_min."N" +drop cascades to table sp_min."KNOWS" +NOTICE: graph "sp_min" has been dropped + drop_graph +------------ + +(1 row) + +-- +-- Minimum hop count fallback with a VERTEX-REVISITING longer path. This is the +-- defining behaviour of the exhaustive-search regime: relationship-uniqueness +-- (Cypher trail semantics) forbids reusing an EDGE but permits revisiting a +-- VERTEX, so a qualifying path longer than the shortest distance may loop back +-- through an already-seen node. +-- +-- Graph: A -> B -> C, C -> B (back edge), B -> D +-- A..D shortest distance = 2 (A->B->D) +-- there is no edge-distinct length-3 A..D path +-- the only edge-distinct length-4 A..D path is A->B->C->B->D, which +-- revisits vertex B but uses each of the four edges exactly once +-- +SELECT * FROM create_graph('sp_revisit'); +NOTICE: graph "sp_revisit" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('sp_revisit', $$ + CREATE (a:N {name: 'A'}), + (b:N {name: 'B'}), + (c:N {name: 'C'}), + (d:N {name: 'D'}), + (a)-[:KNOWS]->(b), + (b)-[:KNOWS]->(c), + (c)-[:KNOWS]->(b), + (b)-[:KNOWS]->(d) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- materialize the global graph context +SELECT * FROM cypher('sp_revisit', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + result +----------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "N", "in_degree": 0, "out_degree": 1, "self_loops": 0} + {"id": 844424930131970, "label": "N", "in_degree": 2, "out_degree": 2, "self_loops": 0} + {"id": 844424930131971, "label": "N", "in_degree": 1, "out_degree": 1, "self_loops": 0} + {"id": 844424930131972, "label": "N", "in_degree": 1, "out_degree": 0, "self_loops": 0} +(4 rows) + +-- min_hops=2 equals the shortest distance, so the easy (BFS) regime returns the +-- direct A->B->D route; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_revisit"'::agtype, + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); + count +------- + 1 +(1 row) + +-- min_hops is a lower bound, not an exact length: with min_hops=3 there is no +-- length-3 edge-distinct path, so the search returns the next-shortest +-- qualifying path, the length-4 route A->B->C->B->D; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_revisit"'::agtype, + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 3::agtype); + count +------- + 1 +(1 row) + +-- min_hops=4 is satisfied only by the vertex-revisiting A->B->C->B->D path; +-- count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_revisit"'::agtype, + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 4::agtype); + count +------- + 1 +(1 row) + +-- the length-4 path is unique, so its materialized form is deterministic; it +-- visits B twice (B appears at positions 2 and 4) yet repeats no edge +SELECT path FROM age_shortest_path( + '"sp_revisit"'::agtype, + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 4::agtype) AS path +ORDER BY path; + path +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id": 844424930131969, "label": "N", "properties": {"name": "A"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "B"}}::vertex, {"id": 1125899906842626, "label": "KNOWS", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge, {"id": 844424930131971, "label": "N", "properties": {"name": "C"}}::vertex, {"id": 1125899906842627, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "B"}}::vertex, {"id": 1125899906842628, "label": "KNOWS", "end_id": 844424930131972, "start_id": 844424930131970, "properties": {}}::edge, {"id": 844424930131972, "label": "N", "properties": {"name": "D"}}::vertex]::path +(1 row) + +-- min_hops=5 exhausts the four edges without an edge-distinct path; count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_revisit"'::agtype, + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 5::agtype); + count +------- + 0 +(1 row) + +-- cleanup +SELECT * FROM drop_graph('sp_revisit', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table sp_revisit._ag_label_vertex +drop cascades to table sp_revisit._ag_label_edge +drop cascades to table sp_revisit."N" +drop cascades to table sp_revisit."KNOWS" +NOTICE: graph "sp_revisit" has been dropped + drop_graph +------------ + +(1 row) + +-- +-- Minimum hop count fallback with a CLOSED WALK (start == end through a cycle). +-- When start and end are the same vertex the shortest distance is 0, so any +-- positive min_hops forces the exhaustive search to look for a cycle that +-- returns to the start using edge-distinct steps. +-- +-- Graph: a single directed triangle A -> B -> C -> A +-- A..A shortest distance = 0 (the zero-length path) +-- the only edge-distinct closed walk is the length-3 triangle A->B->C->A +-- +SELECT * FROM create_graph('sp_tri'); +NOTICE: graph "sp_tri" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('sp_tri', $$ + CREATE (a:N {name: 'A'}), + (b:N {name: 'B'}), + (c:N {name: 'C'}), + (a)-[:KNOWS]->(b), + (b)-[:KNOWS]->(c), + (c)-[:KNOWS]->(a) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- materialize the global graph context +SELECT * FROM cypher('sp_tri', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + result +----------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "N", "in_degree": 1, "out_degree": 1, "self_loops": 0} + {"id": 844424930131970, "label": "N", "in_degree": 1, "out_degree": 1, "self_loops": 0} + {"id": 844424930131971, "label": "N", "in_degree": 1, "out_degree": 1, "self_loops": 0} +(3 rows) + +-- no min_hops: start == end yields the zero-length path; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_tri"'::agtype, + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype); + count +------- + 1 +(1 row) + +-- min_hops=3 forces the exhaustive search to find the closed triangle walk +-- A->B->C->A; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_tri"'::agtype, + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 3::agtype); + count +------- + 1 +(1 row) + +-- the closed walk is unique, so its materialized form is deterministic; it +-- starts and ends at A and traverses each triangle edge once +SELECT path FROM age_shortest_path( + '"sp_tri"'::agtype, + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 3::agtype) AS path +ORDER BY path; + path +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + [{"id": 844424930131969, "label": "N", "properties": {"name": "A"}}::vertex, {"id": 1125899906842625, "label": "KNOWS", "end_id": 844424930131970, "start_id": 844424930131969, "properties": {}}::edge, {"id": 844424930131970, "label": "N", "properties": {"name": "B"}}::vertex, {"id": 1125899906842626, "label": "KNOWS", "end_id": 844424930131971, "start_id": 844424930131970, "properties": {}}::edge, {"id": 844424930131971, "label": "N", "properties": {"name": "C"}}::vertex, {"id": 1125899906842627, "label": "KNOWS", "end_id": 844424930131969, "start_id": 844424930131971, "properties": {}}::edge, {"id": 844424930131969, "label": "N", "properties": {"name": "A"}}::vertex]::path +(1 row) + +-- min_hops=4 cannot be met without reusing an edge of the three-edge triangle; +-- count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_tri"'::agtype, + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 4::agtype); + count +------- + 0 +(1 row) + +-- cleanup +SELECT * FROM drop_graph('sp_tri', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table sp_tri._ag_label_vertex +drop cascades to table sp_tri._ag_label_edge +drop cascades to table sp_tri."N" +drop cascades to table sp_tri."KNOWS" +NOTICE: graph "sp_tri" has been dropped + drop_graph +------------ + +(1 row) + +-- +-- Error messages report the actual SRF that was called. age_shortest_path and +-- age_all_shortest_paths share their argument-resolution helpers; these cases +-- confirm each reports its own name in the error text rather than a single +-- hard-coded "age_shortest_path" prefix. +-- +SELECT * FROM create_graph('sp_errname'); +NOTICE: graph "sp_errname" has been created + create_graph +-------------- + +(1 row) + +SELECT * FROM cypher('sp_errname', $$ + CREATE (a:N {name: 'A'})-[:KNOWS]->(b:N {name: 'B'}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- materialize the global graph context +SELECT * FROM cypher('sp_errname', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + result +----------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "N", "in_degree": 0, "out_degree": 1, "self_loops": 0} + {"id": 844424930131970, "label": "N", "in_degree": 1, "out_degree": 0, "self_loops": 0} +(2 rows) + +-- a NULL graph name errors with the called function's name as the prefix +SELECT count(*) FROM age_shortest_path(NULL::agtype, 0::agtype, 1::agtype); +ERROR: age_shortest_path: graph name cannot be NULL +SELECT count(*) FROM age_all_shortest_paths(NULL::agtype, 0::agtype, 1::agtype); +ERROR: age_all_shortest_paths: graph name cannot be NULL +-- a non-string relationship type errors with the called function's name as the +-- prefix (the array element 1 is an integer, not a string) +SELECT count(*) FROM age_shortest_path( + '"sp_errname"'::agtype, + (SELECT id FROM cypher('sp_errname', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_errname', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + '[1]'::agtype); +ERROR: age_shortest_path: relationship type must be a string +SELECT count(*) FROM age_all_shortest_paths( + '"sp_errname"'::agtype, + (SELECT id FROM cypher('sp_errname', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_errname', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + '[1]'::agtype); +ERROR: age_all_shortest_paths: relationship type must be a string +-- cleanup +SELECT * FROM drop_graph('sp_errname', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table sp_errname._ag_label_vertex +drop cascades to table sp_errname._ag_label_edge +drop cascades to table sp_errname."N" +drop cascades to table sp_errname."KNOWS" +NOTICE: graph "sp_errname" has been dropped + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/age_shortest_path.sql b/regress/sql/age_shortest_path.sql index 82b4d66bb..5bf86d364 100644 --- a/regress/sql/age_shortest_path.sql +++ b/regress/sql/age_shortest_path.sql @@ -115,6 +115,16 @@ FROM age_all_shortest_paths( NULL, '"in"'::agtype ); +-- single shortest path with direction 'in': D -> A backwards; expected: +-- path_count = 1 (the single-path variant picks one of the two routes) +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + NULL, '"in"'::agtype +); + -- direction 'out': D -> A not reachable forwards; expected: path_count = 0 SELECT count(*) AS path_count FROM age_shortest_path( @@ -150,6 +160,15 @@ FROM age_shortest_path( (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)) ); +-- error: end argument is neither a vertex nor an integer id; expected: ERROR +-- (symmetric to the start-vertex check above) +SELECT count(*) AS path_count +FROM age_shortest_path( + '"sp_graph"'::agtype, + (SELECT id FROM cypher('sp_graph', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"not_a_vertex"'::agtype +); + -- -- Non-existent endpoint guards. These must NOT crash the backend and must -- return no rows (a path can only exist between vertices in the graph). @@ -601,20 +620,81 @@ SELECT count(*) FROM age_shortest_path( (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), '["KNOWS"]'::agtype, '"out"'::agtype); --- multiple relationship types are not yet supported; expected: ERROR +-- multiple relationship types: an array of types matches an edge whose type +-- is any one of them. A..C single shortest under {KNOWS, LIKES} (length 2); +-- count 1 SELECT count(*) FROM age_shortest_path( '"sp_edge"'::agtype, (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), '["KNOWS", "LIKES"]'::agtype, '"out"'::agtype); --- a non-zero minimum hop count is not yet supported; expected: ERROR +-- all shortest A..C under {KNOWS, LIKES}: three A->B edges (2 KNOWS + 1 LIKES) +-- each extend by the single B->C KNOWS edge; count 3 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS", "LIKES"]'::agtype, '"out"'::agtype); + +-- all shortest A..B under {KNOWS, LIKES}: the two parallel KNOWS edges plus +-- the one LIKES edge are three distinct one-hop paths; count 3 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS", "LIKES"]'::agtype, '"any"'::agtype); + +-- a multi-type array containing an unknown type ignores the unknown member: +-- {NOSUCHLABEL, KNOWS} still finds A..C via KNOWS; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["NOSUCHLABEL", "KNOWS"]'::agtype, '"out"'::agtype); + +-- a multi-type array of only types that do not connect the endpoints: +-- {LIKES} reaches B but B..C has no LIKES edge, so A..C is unreachable; +-- count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["LIKES", "NOSUCHLABEL"]'::agtype, '"out"'::agtype); + +-- an empty relationship-type array imposes no filter (same as NULL): A..C +-- (length 2) is found; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '[]'::agtype, '"out"'::agtype); + +-- a non-string element in the relationship-type array is an error +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS", 7]'::agtype, '"out"'::agtype); + +-- a minimum hop count that does not exceed the shortest distance imposes no +-- extra constraint; A..C via KNOWS has length 2, so min_hops=2 is accepted and +-- returns the length-2 path; count 1 SELECT count(*) FROM age_shortest_path( '"sp_edge"'::agtype, (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); +-- a minimum hop count greater than the shortest distance falls back to the +-- exhaustive (VLE) search; A..C in this DAG has no length-3 path (every longer +-- route would have to reuse an edge), so min_hops=3 yields no rows; count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 3::agtype); + -- a minimum hop count of 0 is the default and is accepted; A..C (length 2); -- count 1 SELECT count(*) FROM age_shortest_path( @@ -626,5 +706,408 @@ SELECT count(*) FROM age_shortest_path( -- a graph name that does not exist is an error SELECT count(*) FROM age_shortest_path('"no_such_graph"'::agtype, '1'::agtype, '2'::agtype); +-- a NULL graph name is an error (the graph name is required, unlike the +-- endpoints which accept NULL as "no match") +SELECT count(*) FROM age_shortest_path( + NULL::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype))); + +-- a non-integer max_hops is an error (the hop bounds must be integers) +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, NULL::agtype, '"not_an_int"'::agtype); + +-- a non-integer min_hops is an error (symmetric to max_hops above) +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, '"not_an_int"'::agtype); + +-- a negative min_hops is clamped to 0 (no constraint), so A..C (length 2) is +-- still found; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_edge"'::agtype, + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_edge', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, '-3'::agtype); + -- cleanup SELECT * FROM drop_graph('sp_edge', true); + +-- +-- Calling shortest_path / all_shortest_paths from inside cypher() (Tier 1) +-- WITH a relationship-type filter -- both a single type (bare string) and +-- multiple types (a list literal). The graph name is auto-injected, so the +-- in-cypher call passes only the bound endpoints and the type filter. +-- +-- Graph: A and B are joined by two parallel KNOWS edges plus one LIKES edge; +-- B->C is a single KNOWS edge. This lets the all_shortest_paths variant return +-- more than one path so the multiplicity is visible. +-- +SELECT * FROM create_graph('sp_cy_lbl'); + +SELECT * FROM cypher('sp_cy_lbl', $$ + CREATE (a:N {name: 'A'}), + (b:N {name: 'B'}), + (c:N {name: 'C'}), + (a)-[:KNOWS]->(b), + (a)-[:KNOWS]->(b), + (a)-[:LIKES]->(b), + (b)-[:KNOWS]->(c) +$$) AS (result agtype); + +-- materialize the global graph context +SELECT * FROM cypher('sp_cy_lbl', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + +-- shortest_path() in-cypher with a single relationship type; A..C via KNOWS +-- (length 2); expected: 1 path +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN shortest_path(a, c, 'KNOWS') +$$) AS (path agtype); + +-- all_shortest_paths() in-cypher with a single relationship type; the two +-- parallel KNOWS edges A->B make two distinct shortest A..C paths; expected: 2 +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN all_shortest_paths(a, c, 'KNOWS') +$$) AS (path agtype); + +-- shortest_path() in-cypher with multiple relationship types passed as a list +-- literal; A..C under {KNOWS, LIKES} (length 2); expected: 1 path +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN shortest_path(a, c, ['KNOWS', 'LIKES']) +$$) AS (path agtype); + +-- all_shortest_paths() in-cypher with multiple relationship types; the three +-- A->B edges (2 KNOWS + 1 LIKES) each extend by the single B->C KNOWS edge, +-- giving three distinct shortest A..C paths; expected: 3 +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN all_shortest_paths(a, c, ['KNOWS', 'LIKES']) +$$) AS (path agtype); + +-- all_shortest_paths() in-cypher, multiple types, adjacent endpoints: the two +-- parallel KNOWS edges plus the one LIKES edge are three one-hop A..B paths; +-- expected: 3 +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (b:N {name:'B'}) + RETURN all_shortest_paths(a, b, ['KNOWS', 'LIKES']) +$$) AS (path agtype); + +-- multiple types where only one connects the endpoints: {LIKES} reaches B but +-- B->C has no LIKES edge, so A..C is unreachable in-cypher; expected: 0 +SELECT count(*) FROM cypher('sp_cy_lbl', $$ + MATCH (a:N {name:'A'}), (c:N {name:'C'}) + RETURN all_shortest_paths(a, c, ['LIKES']) +$$) AS (path agtype); + +-- cleanup +SELECT * FROM drop_graph('sp_cy_lbl', true); + +-- +-- Minimum hop count fallback (Tier: VLE exhaustive search). When the requested +-- minimum hop count exceeds the true shortest distance, the BFS shortest-path +-- cannot satisfy it (it needs longer paths), so the implementation falls back +-- to the variable-length-edge depth-first engine and returns the shortest +-- path(s) whose length is at least min_hops. +-- +-- Graph: A reaches C directly (length 1) and also via two distinct +-- intermediate vertices B1 and B2 (length 2 each): +-- A->C, A->B1->C, A->B2->C +-- +SELECT * FROM create_graph('sp_min'); + +SELECT * FROM cypher('sp_min', $$ + CREATE (a:N {name: 'A'}), + (c:N {name: 'C'}), + (b1:N {name: 'B1'}), + (b2:N {name: 'B2'}), + (a)-[:KNOWS]->(c), + (a)-[:KNOWS]->(b1), + (b1)-[:KNOWS]->(c), + (a)-[:KNOWS]->(b2), + (b2)-[:KNOWS]->(c) +$$) AS (result agtype); + +-- materialize the global graph context +SELECT * FROM cypher('sp_min', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + +-- baseline: the shortest A..C is the direct length-1 edge; count 1 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype); + +-- min_hops=2 excludes the direct edge and falls back to the exhaustive search; +-- the two length-2 routes A->B1->C and A->B2->C are the shortest qualifying +-- paths; all_shortest_paths returns both; count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); + +-- single shortest_path with min_hops=2 picks exactly one of the two length-2 +-- routes; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); + +-- the qualifying length-2 paths materialize correctly; all_shortest_paths +-- returns the full, order-stable set (the single shortest_path variant would +-- return an arbitrary one of the two equal-length routes, which is not a +-- deterministic choice), so both A->B1->C and A->B2->C are listed +SELECT path FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype) AS path +ORDER BY path; + +-- min_hops=2 with a matching max_hops=2 returns the same two length-2 paths; +-- count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype, 2::agtype); + +-- min_hops greater than max_hops is unsatisfiable; count 0 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype, 1::agtype); + +-- min_hops=3 has no qualifying path (a length-3 A..C would have to reuse an +-- edge, which relationship-uniqueness forbids); count 0 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 3::agtype); + +-- no edge-type filter also reaches the fallback; the two length-2 routes are +-- returned; count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + NULL::agtype, '"out"'::agtype, 2::agtype); + +-- the fallback honours direction: traversing edges backwards (C..A, 'in') with +-- min_hops=2 also returns the two length-2 routes; count 2 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"in"'::agtype, 2::agtype); + +-- the fallback respects direction: there is no forward C..A path, so 'out' +-- with min_hops=2 returns nothing; count 0 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); + +-- an unknown relationship type in the fallback matches no edges, so even the +-- shortest qualifying path cannot be formed; count 0 +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '"NOSUCHLABEL"'::agtype, '"out"'::agtype, 2::agtype); + +-- a minimum hop count greater than the shortest distance combined with +-- multiple relationship types is not supported (the VLE engine matches a +-- single label only); expected: ERROR +SELECT count(*) FROM age_all_shortest_paths( + '"sp_min"'::agtype, + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_min', $$ MATCH (n {name:'C'}) RETURN id(n) $$) AS (id agtype)), + '["KNOWS", "LIKES"]'::agtype, '"out"'::agtype, 2::agtype); + +-- cleanup +SELECT * FROM drop_graph('sp_min', true); + +-- +-- Minimum hop count fallback with a VERTEX-REVISITING longer path. This is the +-- defining behaviour of the exhaustive-search regime: relationship-uniqueness +-- (Cypher trail semantics) forbids reusing an EDGE but permits revisiting a +-- VERTEX, so a qualifying path longer than the shortest distance may loop back +-- through an already-seen node. +-- +-- Graph: A -> B -> C, C -> B (back edge), B -> D +-- A..D shortest distance = 2 (A->B->D) +-- there is no edge-distinct length-3 A..D path +-- the only edge-distinct length-4 A..D path is A->B->C->B->D, which +-- revisits vertex B but uses each of the four edges exactly once +-- +SELECT * FROM create_graph('sp_revisit'); + +SELECT * FROM cypher('sp_revisit', $$ + CREATE (a:N {name: 'A'}), + (b:N {name: 'B'}), + (c:N {name: 'C'}), + (d:N {name: 'D'}), + (a)-[:KNOWS]->(b), + (b)-[:KNOWS]->(c), + (c)-[:KNOWS]->(b), + (b)-[:KNOWS]->(d) +$$) AS (result agtype); + +-- materialize the global graph context +SELECT * FROM cypher('sp_revisit', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + +-- min_hops=2 equals the shortest distance, so the easy (BFS) regime returns the +-- direct A->B->D route; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_revisit"'::agtype, + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 2::agtype); + +-- min_hops is a lower bound, not an exact length: with min_hops=3 there is no +-- length-3 edge-distinct path, so the search returns the next-shortest +-- qualifying path, the length-4 route A->B->C->B->D; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_revisit"'::agtype, + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 3::agtype); + +-- min_hops=4 is satisfied only by the vertex-revisiting A->B->C->B->D path; +-- count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_revisit"'::agtype, + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 4::agtype); + +-- the length-4 path is unique, so its materialized form is deterministic; it +-- visits B twice (B appears at positions 2 and 4) yet repeats no edge +SELECT path FROM age_shortest_path( + '"sp_revisit"'::agtype, + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 4::agtype) AS path +ORDER BY path; + +-- min_hops=5 exhausts the four edges without an edge-distinct path; count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_revisit"'::agtype, + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_revisit', $$ MATCH (n {name:'D'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 5::agtype); + +-- cleanup +SELECT * FROM drop_graph('sp_revisit', true); + +-- +-- Minimum hop count fallback with a CLOSED WALK (start == end through a cycle). +-- When start and end are the same vertex the shortest distance is 0, so any +-- positive min_hops forces the exhaustive search to look for a cycle that +-- returns to the start using edge-distinct steps. +-- +-- Graph: a single directed triangle A -> B -> C -> A +-- A..A shortest distance = 0 (the zero-length path) +-- the only edge-distinct closed walk is the length-3 triangle A->B->C->A +-- +SELECT * FROM create_graph('sp_tri'); + +SELECT * FROM cypher('sp_tri', $$ + CREATE (a:N {name: 'A'}), + (b:N {name: 'B'}), + (c:N {name: 'C'}), + (a)-[:KNOWS]->(b), + (b)-[:KNOWS]->(c), + (c)-[:KNOWS]->(a) +$$) AS (result agtype); + +-- materialize the global graph context +SELECT * FROM cypher('sp_tri', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + +-- no min_hops: start == end yields the zero-length path; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_tri"'::agtype, + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype); + +-- min_hops=3 forces the exhaustive search to find the closed triangle walk +-- A->B->C->A; count 1 +SELECT count(*) FROM age_shortest_path( + '"sp_tri"'::agtype, + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 3::agtype); + +-- the closed walk is unique, so its materialized form is deterministic; it +-- starts and ends at A and traverses each triangle edge once +SELECT path FROM age_shortest_path( + '"sp_tri"'::agtype, + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 3::agtype) AS path +ORDER BY path; + +-- min_hops=4 cannot be met without reusing an edge of the three-edge triangle; +-- count 0 +SELECT count(*) FROM age_shortest_path( + '"sp_tri"'::agtype, + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_tri', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + '"KNOWS"'::agtype, '"out"'::agtype, 4::agtype); + +-- cleanup +SELECT * FROM drop_graph('sp_tri', true); + +-- +-- Error messages report the actual SRF that was called. age_shortest_path and +-- age_all_shortest_paths share their argument-resolution helpers; these cases +-- confirm each reports its own name in the error text rather than a single +-- hard-coded "age_shortest_path" prefix. +-- +SELECT * FROM create_graph('sp_errname'); + +SELECT * FROM cypher('sp_errname', $$ + CREATE (a:N {name: 'A'})-[:KNOWS]->(b:N {name: 'B'}) +$$) AS (result agtype); + +-- materialize the global graph context +SELECT * FROM cypher('sp_errname', $$ MATCH (u) RETURN vertex_stats(u) ORDER BY id(u) $$) + AS (result agtype); + +-- a NULL graph name errors with the called function's name as the prefix +SELECT count(*) FROM age_shortest_path(NULL::agtype, 0::agtype, 1::agtype); +SELECT count(*) FROM age_all_shortest_paths(NULL::agtype, 0::agtype, 1::agtype); + +-- a non-string relationship type errors with the called function's name as the +-- prefix (the array element 1 is an integer, not a string) +SELECT count(*) FROM age_shortest_path( + '"sp_errname"'::agtype, + (SELECT id FROM cypher('sp_errname', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_errname', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + '[1]'::agtype); +SELECT count(*) FROM age_all_shortest_paths( + '"sp_errname"'::agtype, + (SELECT id FROM cypher('sp_errname', $$ MATCH (n {name:'A'}) RETURN id(n) $$) AS (id agtype)), + (SELECT id FROM cypher('sp_errname', $$ MATCH (n {name:'B'}) RETURN id(n) $$) AS (id agtype)), + '[1]'::agtype); + +-- cleanup +SELECT * FROM drop_graph('sp_errname', true); diff --git a/src/backend/utils/adt/age_vle.c b/src/backend/utils/adt/age_vle.c index 804d9e17e..cb036b154 100644 --- a/src/backend/utils/adt/age_vle.c +++ b/src/backend/utils/adt/age_vle.c @@ -2887,11 +2887,12 @@ static graphid sp_queue_pop(sp_queue *q) } /* Resolve a vertex argument (a vertex agtype or an integer id) to a graphid. */ -static graphid sp_agtype_to_graphid(agtype *agt, const char *argname) +static graphid sp_agtype_to_graphid(agtype *agt, char *fname, + const char *argname) { agtype_value *agtv = NULL; - agtv = get_agtype_value("age_shortest_path", agt, AGTV_VERTEX, false); + agtv = get_agtype_value(fname, agt, AGTV_VERTEX, false); if (agtv != NULL && agtv->type == AGTV_VERTEX) { @@ -2909,7 +2910,7 @@ static graphid sp_agtype_to_graphid(agtype *agt, const char *argname) } /* Resolve the optional direction argument; NULL defaults to undirected. */ -static cypher_rel_dir sp_agtype_to_direction(agtype *agt) +static cypher_rel_dir sp_agtype_to_direction(agtype *agt, char *fname) { agtype_value *agtv = NULL; char *s = NULL; @@ -2920,7 +2921,7 @@ static cypher_rel_dir sp_agtype_to_direction(agtype *agt) return CYPHER_REL_DIR_NONE; } - agtv = get_agtype_value("age_shortest_path", agt, AGTV_STRING, true); + agtv = get_agtype_value(fname, agt, AGTV_STRING, true); s = pnstrdup(agtv->val.string.val, agtv->val.string.len); if (pg_strcasecmp(s, "out") == 0) @@ -2939,7 +2940,8 @@ static cypher_rel_dir sp_agtype_to_direction(agtype *agt) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("direction argument must be one of 'out', 'in', or 'any'"))); + errmsg("%s: direction argument must be one of 'out', 'in', or 'any'", + fname))); } pfree_if_not_null(s); @@ -2979,7 +2981,7 @@ static Datum sp_build_path_datum(Oid graph_oid, graphid *alt, int64 alt_len) * (collect_all) every shortest-path predecessor is recorded per vertex. */ static HTAB *sp_run_bfs(GRAPH_global_context *ggctx, graphid source, - graphid target, bool filter_edges, Oid edge_label_oid, + graphid target, Oid *label_oids, int n_label_oids, cypher_rel_dir dir, int64 max_hops, bool collect_all, int64 *out_target_depth, bool *out_found) { @@ -3123,18 +3125,35 @@ static HTAB *sp_run_bfs(GRAPH_global_context *ggctx, graphid source, /* * Optional edge label filter. When a label filter is active - * we keep only edges whose label table oid matches. Note that - * a label name which does not exist in this graph resolves to - * InvalidOid; because no real edge can have an InvalidOid - * label table, every edge is then skipped and only the - * zero-length (start == end) path can match -- matching the - * openCypher semantics that an unknown relationship type - * matches no relationships. + * (n_label_oids > 0) we keep only edges whose label table oid + * is one of the requested relationship types. A requested type + * that does not exist in this graph resolves to InvalidOid; + * since no real edge can have an InvalidOid label table, such a + * type contributes no matches and simply drops out of the set, + * while edges of any of the other (known) requested types still + * match. Only when every requested type is unknown does the + * filter match no edges, leaving just the zero-length + * (start == end) path -- matching the openCypher semantics that + * an unknown relationship type matches no relationships. */ - if (filter_edges && - get_edge_entry_label_table_oid(ee) != edge_label_oid) + if (n_label_oids > 0) { - continue; + Oid ee_label_oid = get_edge_entry_label_table_oid(ee); + bool label_match = false; + int li = 0; + + for (li = 0; li < n_label_oids; li++) + { + if (label_oids[li] == ee_label_oid) + { + label_match = true; + break; + } + } + if (!label_match) + { + continue; + } } /* the neighbor depends on which side of the edge u is on */ @@ -3192,13 +3211,25 @@ static HTAB *sp_run_bfs(GRAPH_global_context *ggctx, graphid source, return visited; } +/* + * Maximum number of result paths age_all_shortest_paths will materialize + * before raising an error. The shortest-path DAG can contain exponentially + * many equal-length paths (grid-like or multi-edge graphs), and they are all + * built up front in the SRF's memory context, so this is a backstop against + * unbounded memory growth. CHECK_FOR_INTERRUPTS() in sp_enumerate still allows + * cancellation, but a fast explosion can outrun a statement_timeout. + */ +#define SP_MAX_RESULT_PATHS 1000000 + /* * Recursively enumerate every shortest path by walking the predecessor DAG * from target back to source. Each completed path is appended to *out as a - * freshly allocated interleaved graphid array of length alt_len. + * freshly allocated interleaved graphid array of length alt_len. The running + * total is capped at SP_MAX_RESULT_PATHS to bound peak memory. */ static void sp_enumerate(HTAB *visited, graphid source, graphid cur, - graphid *alt, int64 alt_len, int64 pos, List **out) + graphid *alt, int64 alt_len, int64 pos, + char *fname, List **out) { sp_visit_entry *e = NULL; ListCell *lc = NULL; @@ -3220,6 +3251,20 @@ static void sp_enumerate(HTAB *visited, graphid source, graphid cur, memcpy(copy, alt, sizeof(graphid) * alt_len); *out = lappend(*out, copy); + + /* + * Bound the number of materialized paths. Without a ceiling, a + * combinatorial shortest-path DAG could exhaust memory before the + * first row is returned. + */ + if (list_length(*out) > SP_MAX_RESULT_PATHS) + { + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("%s: shortest path count exceeded %d", + fname, SP_MAX_RESULT_PATHS), + errhint("Narrow the search with a relationship type or a maximum hop count, or use age_shortest_path for a single path."))); + } } return; } @@ -3236,10 +3281,193 @@ static void sp_enumerate(HTAB *visited, graphid source, graphid cur, alt[pos - 1] = p->edge; sp_enumerate(visited, source, p->parent_vertex, alt, alt_len, pos - 2, - out); + fname, out); } } +/* + * Maximum number of distinct paths the minimum-hop fallback will enumerate + * before giving up. The exhaustive DFS used for a minimum hop count greater + * than the shortest distance can explode on dense graphs, so this acts as a + * safety valve alongside CHECK_FOR_INTERRUPTS()/statement_timeout in the DFS. + */ +#define SP_MINHOPS_MAX_PATHS 1000000 + +/* + * Fallback for the "minimum hop count greater than the shortest distance" + * regime, which plain BFS cannot satisfy (it requires longer, vertex-revisiting + * paths under relationship-uniqueness). This reuses the VLE depth-first engine + * directly: it builds a VLE_local_context by hand (no fcinfo), enumerates every + * path whose length is within [min_hops, max_hops], and keeps only those of the + * smallest qualifying length. For shortest_path one such path is returned; for + * all_shortest_paths every tie at that length is returned. Returns NULL with + * *out_count == 0 when no qualifying path exists. + * + * The VLE engine matches a single edge label oid only, so a multi-type filter + * is rejected by the caller before reaching here. A single label_oid of + * InvalidOid means "any edge label". + */ +static Datum *sp_minhops_fallback(GRAPH_global_context *ggctx, Oid graph_oid, + const char *graph_name, char *fname, + graphid source, graphid target, Oid label_oid, + cypher_rel_dir dir, int64 min_hops, + int64 max_hops, bool collect_all, + int64 *out_count) +{ + MemoryContext oldctx = CurrentMemoryContext; + MemoryContext tmpctx = NULL; + VLE_local_context *vlelctx = NULL; + agtype_value av_empty; + agtype *empty_constraint = NULL; + List *best = NIL; + ListCell *lc = NULL; + int64 best_len = PG_INT64_MAX; + int64 examined = 0; + int64 result_len = 0; + int64 n = 0; + int64 idx = 0; + Datum *paths = NULL; + + *out_count = 0; + + /* do the VLE enumeration in a private context we can throw away at the end */ + tmpctx = AllocSetContextCreate(oldctx, "age shortest path minhops", + ALLOCSET_DEFAULT_SIZES); + MemoryContextSwitchTo(tmpctx); + + /* an empty property constraint object: every edge satisfies it */ + av_empty.type = AGTV_OBJECT; + av_empty.val.object.num_pairs = 0; + av_empty.val.object.pairs = NULL; + empty_constraint = agtype_value_to_agtype(&av_empty); + + /* build the VLE local context by hand (no fcinfo, no caching) */ + vlelctx = palloc0(sizeof(VLE_local_context)); + vlelctx->graph_name = pnstrdup(graph_name, strlen(graph_name)); + vlelctx->graph_oid = graph_oid; + vlelctx->ggctx = ggctx; + vlelctx->path_function = VLE_FUNCTION_PATHS_BETWEEN; + vlelctx->next_vertex = NULL; + vlelctx->vsid = source; + vlelctx->veid = target; + vlelctx->edge_property_constraint = empty_constraint; + vlelctx->edge_property_constraint_datum = + AGTYPE_P_GET_DATUM(empty_constraint); + vlelctx->edge_property_constraint_hash = + datum_image_hash(vlelctx->edge_property_constraint_datum, false, -1); + vlelctx->edge_label_name = NULL; + vlelctx->edge_label_name_oid = label_oid; + vlelctx->lidx = (min_hops > 0) ? min_hops : 1; + if (max_hops < 0) + { + vlelctx->uidx_infinite = true; + vlelctx->uidx = 0; + } + else + { + vlelctx->uidx_infinite = false; + vlelctx->uidx = max_hops; + } + vlelctx->edge_direction = dir; + vlelctx->use_cache = false; + vlelctx->vle_grammar_node_id = 0; + vlelctx->next = NULL; + vlelctx->is_dirty = true; + + create_VLE_local_state_hashtable(vlelctx); + vlelctx->dfs_vertex_stack = new_gid_stack(); + vlelctx->dfs_edge_stack = new_gid_stack(); + vlelctx->dfs_path_stack = new_gid_stack(); + load_initial_dfs_stacks(vlelctx); + + /* + * Enumerate qualifying paths, keeping only those of the smallest length + * seen. The DFS yields paths in no particular length order, so a strictly + * shorter path resets the kept set. + */ + while (dfs_find_a_path_between(vlelctx)) + { + int64 hops = gid_stack_size(vlelctx->dfs_path_stack); + bool take = false; + bool reset = false; + + examined = examined + 1; + if (examined > SP_MINHOPS_MAX_PATHS) + { + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("%s: minimum hop count search exceeded %d candidate paths", + fname, SP_MINHOPS_MAX_PATHS), + errhint("Provide a maximum hop count to bound the search."))); + } + + if (hops < best_len) + { + take = true; + reset = true; + } + else if (hops == best_len && collect_all) + { + take = true; + } + + if (take) + { + VLE_path_container *vpc = NULL; + graphid *garr = NULL; + int64 arrlen = 0; + + vpc = build_VLE_path_container(vlelctx); + garr = GET_GRAPHID_ARRAY_FROM_CONTAINER(vpc); + arrlen = vpc->graphid_array_size; + + /* copy the path into the surviving context and record it */ + MemoryContextSwitchTo(oldctx); + if (reset) + { + list_free_deep(best); + best = NIL; + best_len = hops; + } + { + graphid *copy = palloc(sizeof(graphid) * arrlen); + + memcpy(copy, garr, sizeof(graphid) * arrlen); + best = lappend(best, copy); + } + MemoryContextSwitchTo(tmpctx); + + pfree(vpc); + } + } + + /* tear down the VLE engine state, then drop the whole scratch context */ + free_VLE_local_context(vlelctx); + MemoryContextSwitchTo(oldctx); + MemoryContextDelete(tmpctx); + + n = list_length(best); + if (n == 0) + { + return NULL; + } + + /* every kept path has the same (minimum qualifying) length */ + result_len = (2 * best_len) + 1; + paths = palloc(sizeof(Datum) * n); + foreach(lc, best) + { + graphid *a = (graphid *) lfirst(lc); + + paths[idx] = sp_build_path_datum(graph_oid, a, result_len); + idx = idx + 1; + } + + list_free_deep(best); + *out_count = n; + return paths; +} + /* * Resolve arguments, run the BFS, and materialize the result path(s) as an * array of AGTV_PATH agtype Datums. Returns NULL with *out_count == 0 when no @@ -3248,8 +3476,8 @@ static void sp_enumerate(HTAB *visited, graphid source, graphid cur, static Datum *sp_compute_paths(agtype *graph_name_agt, agtype *start_agt, agtype *end_agt, agtype *label_agt, agtype *dir_agt, agtype *minhops_agt, - agtype *maxhops_agt, bool collect_all, - int64 *out_count) + agtype *maxhops_agt, char *fname, + bool collect_all, int64 *out_count) { agtype_value *agtv_temp = NULL; char *graph_name = NULL; @@ -3257,14 +3485,17 @@ static Datum *sp_compute_paths(agtype *graph_name_agt, agtype *start_agt, GRAPH_global_context *ggctx = NULL; graphid source = 0; graphid target = 0; - bool filter_edges = false; - Oid edge_label_oid = InvalidOid; + Oid *label_oids = NULL; + int n_label_oids = 0; cypher_rel_dir dir = CYPHER_REL_DIR_NONE; + int64 min_hops = 0; int64 max_hops = -1; HTAB *visited = NULL; int64 target_depth = -1; bool found = false; Datum *paths = NULL; + MemoryContext oldctx = CurrentMemoryContext; + MemoryContext scratch = NULL; *out_count = 0; @@ -3273,10 +3504,10 @@ static Datum *sp_compute_paths(agtype *graph_name_agt, agtype *start_agt, { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("age_shortest_path: graph name cannot be NULL"))); + errmsg("%s: graph name cannot be NULL", fname))); } - agtv_temp = get_agtype_value("age_shortest_path", graph_name_agt, + agtv_temp = get_agtype_value(fname, graph_name_agt, AGTV_STRING, true); graph_name = pnstrdup(agtv_temp->val.string.val, agtv_temp->val.string.len); @@ -3288,16 +3519,21 @@ static Datum *sp_compute_paths(agtype *graph_name_agt, agtype *start_agt, */ if (start_agt == NULL || end_agt == NULL) { + pfree_if_not_null(graph_name); return NULL; } - source = sp_agtype_to_graphid(start_agt, "start vertex"); - target = sp_agtype_to_graphid(end_agt, "end vertex"); + source = sp_agtype_to_graphid(start_agt, fname, "start vertex"); + target = sp_agtype_to_graphid(end_agt, fname, "end vertex"); /* - * Optional edge type filter. A single relationship type may be supplied - * either as a bare string or as a one-element array. Multiple relationship - * types (an array with more than one element) are not yet supported. + * Optional edge type filter. A relationship type may be supplied as a + * bare string, or one or more types may be supplied as an array of + * strings. Each (non-empty) type name is resolved to its edge label table + * oid; an edge is kept when its label oid is one of the requested set. An + * empty string, an empty array, or NULL means no filter (every edge is + * traversed). An unknown type resolves to InvalidOid and so matches no + * edges. */ if (label_agt != NULL) { @@ -3306,74 +3542,84 @@ static Datum *sp_compute_paths(agtype *graph_name_agt, agtype *start_agt, if (AGT_ROOT_IS_ARRAY(label_agt) && !AGT_ROOT_IS_SCALAR(label_agt)) { int nelems = AGT_ROOT_COUNT(label_agt); + int i = 0; - if (nelems > 1) + if (nelems > 0) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("age_shortest_path: multiple relationship types are not yet supported"))); + label_oids = palloc(sizeof(Oid) * nelems); } - if (nelems == 1) + for (i = 0; i < nelems; i++) { agtv_temp = get_ith_agtype_value_from_container( - &label_agt->root, 0); + &label_agt->root, i); if (agtv_temp->type != AGTV_STRING) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("age_shortest_path: relationship type must be a string"))); + errmsg("%s: relationship type must be a string", + fname))); } + /* skip empty type names; they impose no constraint */ if (agtv_temp->val.string.len != 0) { label_name = pnstrdup(agtv_temp->val.string.val, agtv_temp->val.string.len); - edge_label_oid = get_label_relation(label_name, graph_oid); - filter_edges = true; + label_oids[n_label_oids] = + get_label_relation(label_name, graph_oid); + n_label_oids = n_label_oids + 1; + + /* the resolved oid is all we keep; free the type name */ + pfree(label_name); + label_name = NULL; } } } else { - agtv_temp = get_agtype_value("age_shortest_path", label_agt, + agtv_temp = get_agtype_value(fname, label_agt, AGTV_STRING, true); if (agtv_temp->val.string.len != 0) { label_name = pnstrdup(agtv_temp->val.string.val, agtv_temp->val.string.len); - edge_label_oid = get_label_relation(label_name, graph_oid); - filter_edges = true; + label_oids = palloc(sizeof(Oid)); + label_oids[0] = get_label_relation(label_name, graph_oid); + n_label_oids = 1; + + /* the resolved oid is all we keep; free the type name */ + pfree(label_name); + label_name = NULL; } } } /* optional direction (defaults to undirected) */ - dir = sp_agtype_to_direction(dir_agt); + dir = sp_agtype_to_direction(dir_agt, fname); /* - * Optional minimum hop count. A genuine minimum-length constraint needs a - * different search than plain BFS, so for now only the default (NULL or 0) - * is accepted; any other value is rejected loudly. + * Optional minimum hop count (NULL or negative means none). A minimum + * that does not exceed the true shortest distance imposes no additional + * constraint, so it is handled directly by the BFS result below. A + * minimum greater than the shortest distance requires enumerating longer, + * vertex-revisiting paths, which plain BFS cannot do; that case falls + * back to the VLE depth-first engine after the search (see below). */ if (minhops_agt != NULL) { - int64 min_hops = 0; - - agtv_temp = get_agtype_value("age_shortest_path", minhops_agt, + agtv_temp = get_agtype_value(fname, minhops_agt, AGTV_INTEGER, true); min_hops = agtv_temp->val.int_value; - if (min_hops != 0) + if (min_hops < 0) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("age_shortest_path: a minimum hop count is not yet supported"))); + min_hops = 0; } } /* optional upper hop bound (NULL or negative means unbounded) */ if (maxhops_agt != NULL) { - agtv_temp = get_agtype_value("age_shortest_path", maxhops_agt, + agtv_temp = get_agtype_value(fname, maxhops_agt, AGTV_INTEGER, true); max_hops = agtv_temp->val.int_value; if (max_hops < 0) @@ -3386,19 +3632,87 @@ static Datum *sp_compute_paths(agtype *graph_name_agt, agtype *start_agt, ggctx = manage_GRAPH_global_contexts(graph_name, graph_oid); if (ggctx == NULL) { + pfree_if_not_null(graph_name); + pfree_if_not_null(label_oids); return NULL; } + /* + * Run the search and reconstruct the result path(s) in a private scratch + * context. The BFS bookkeeping (visited table, frontier queue, predecessor + * multiset) and the intermediate path arrays are only needed while we + * compute; the surviving result Datums are built in the caller's + * (SRF-lifetime) context and copied out before the scratch context is + * deleted. This bounds peak memory to the result set plus one search, + * rather than retaining the whole search state for the life of the SRF. + */ + scratch = AllocSetContextCreate(oldctx, "age shortest path scratch", + ALLOCSET_DEFAULT_SIZES); + MemoryContextSwitchTo(scratch); + /* run the breadth-first search */ - visited = sp_run_bfs(ggctx, source, target, filter_edges, edge_label_oid, + visited = sp_run_bfs(ggctx, source, target, label_oids, n_label_oids, dir, max_hops, collect_all, &target_depth, &found); if (!found) { - hash_destroy(visited); + MemoryContextSwitchTo(oldctx); + MemoryContextDelete(scratch); + pfree_if_not_null(graph_name); + pfree_if_not_null(label_oids); return NULL; } + /* + * A minimum hop count greater than the true shortest distance can only be + * satisfied by longer, vertex-revisiting paths (Neo4j's exhaustive search + * regime). Plain BFS cannot produce those, so fall back to the VLE + * depth-first engine for that case. When min_hops <= target_depth the + * bound imposes no additional constraint and the shortest path(s) already + * found are returned unchanged. + * + * The VLE engine matches a single edge label only, so a multi-type filter + * combined with this regime is still unsupported. + */ + if (min_hops > 0 && target_depth < min_hops) + { + Oid fallback_label_oid = InvalidOid; + + /* the BFS scratch is no longer needed; the fallback uses its own */ + MemoryContextSwitchTo(oldctx); + MemoryContextDelete(scratch); + + if (n_label_oids > 1) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("%s: a minimum hop count greater than the shortest path length is not supported with multiple relationship types", + fname))); + } + + if (n_label_oids == 1) + { + fallback_label_oid = label_oids[0]; + } + + /* + * The fallback duplicates graph_name internally and only needs the + * resolved label oid, so the temporaries are freed here once its + * result is captured rather than retained for the SRF's lifetime. + */ + { + Datum *fb_paths; + + fb_paths = sp_minhops_fallback(ggctx, graph_oid, graph_name, fname, + source, target, fallback_label_oid, + dir, min_hops, max_hops, collect_all, + out_count); + pfree_if_not_null(graph_name); + pfree_if_not_null(label_oids); + return fb_paths; + } + } + if (!collect_all) { /* reconstruct the single shortest path from the parent pointers */ @@ -3421,6 +3735,8 @@ static Datum *sp_compute_paths(agtype *graph_name_agt, agtype *start_agt, cur = e->parent_vertex; } + /* build the surviving result Datum in the caller's context */ + MemoryContextSwitchTo(oldctx); paths = palloc(sizeof(Datum)); paths[0] = sp_build_path_datum(graph_oid, alt, alt_len); *out_count = 1; @@ -3436,9 +3752,12 @@ static Datum *sp_compute_paths(agtype *graph_name_agt, agtype *start_agt, int64 idx = 0; sp_enumerate(visited, source, target, alt, alt_len, alt_len - 1, - &arrays); + fname, &arrays); n = list_length(arrays); + + /* build the surviving result Datums in the caller's context */ + MemoryContextSwitchTo(oldctx); paths = palloc(sizeof(Datum) * (n > 0 ? n : 1)); foreach(lc, arrays) { @@ -3450,7 +3769,11 @@ static Datum *sp_compute_paths(agtype *graph_name_agt, agtype *start_agt, *out_count = n; } - hash_destroy(visited); + /* results are copied out; drop the BFS/enumeration scratch */ + MemoryContextSwitchTo(oldctx); + MemoryContextDelete(scratch); + pfree_if_not_null(graph_name); + pfree_if_not_null(label_oids); return paths; } @@ -3521,8 +3844,10 @@ static Datum sp_srf_impl(FunctionCallInfo fcinfo, bool collect_all) state = palloc0(sizeof(sp_srf_state)); state->next = 0; state->paths = sp_compute_paths(a_graph, a_start, a_end, a_label, - a_dir, a_min, a_max, collect_all, - &state->npaths); + a_dir, a_min, a_max, + collect_all ? "age_all_shortest_paths" + : "age_shortest_path", + collect_all, &state->npaths); funcctx->user_fctx = state; MemoryContextSwitchTo(oldctx); From fe48740d8597710e54b195176de332f7e95910e1 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Mon, 29 Jun 2026 12:57:18 -0400 Subject: [PATCH 089/100] Fix single-node labeled pattern expressions not filtering by label (#2443) (#2444) A single-node labeled pattern used as a boolean expression -- e.g. `WHERE (a:Person)`, `WHERE EXISTS((a:Person))` -- was accepted but did not test the bound vertex's label. It desugars to an EXISTS sub-pattern, and make_path_join_quals() returned early for vertex-only patterns (list_length(entities) < 3), emitting no quals. With no edge to carry a correlation, the sub-pattern referenced nothing from the enclosing query, so the planner produced an uncorrelated one-time InitPlan that was trivially true whenever any vertex of that label existed -- the predicate matched every outer row. Emit an explicit label-id filter for a vertex-only pattern whose vertex carries a non-default label and whose variable is declared in an ancestor parse state (i.e. a correlated reference). make_qual() builds a name-based id reference that resolves to the outer variable, so the filter both correlates the sub-pattern to that variable and enforces the label. Freshly scanned, non-correlated vertices (no ancestor binding) are untouched, so plain MATCH (a:Person) and "does any X exist" EXISTS checks are unaffected. Add regression coverage in pattern_expression: WHERE (a:Person), WHERE NOT (a:Person), and EXISTS((a:Company)) against a graph with a non-Person vertex. All 41 regression tests pass. --- regress/expected/pattern_expression.out | 70 ++++++++++++++++++++++--- regress/sql/pattern_expression.sql | 49 ++++++++++++++--- src/backend/parser/cypher_clause.c | 50 +++++++++++++++++- 3 files changed, 154 insertions(+), 15 deletions(-) diff --git a/regress/expected/pattern_expression.out b/regress/expected/pattern_expression.out index 0494d49b9..93a02e3fa 100644 --- a/regress/expected/pattern_expression.out +++ b/regress/expected/pattern_expression.out @@ -320,12 +320,15 @@ $$) AS (result agtype); -- -- Single-node pattern on an already-bound variable: (a:Label) -- --- NOTE: this is an EXISTS existence check on the bound variable, NOT an --- openCypher label predicate. A matching label is therefore always true --- (the variable is already bound), and a *different* label is rejected by --- AGE's pre-existing "multiple labels for variable" restriction rather than --- evaluating to false. Both behaviours are captured here so any future change --- to single-node-pattern semantics is caught by this test. +-- NOTE: as of #2443 a single-node labeled pattern is a correlated label +-- predicate -- in WHERE / EXISTS it tests whether the bound vertex actually +-- has the label (see the WHERE (a:Person) / EXISTS((a:Company)) cases in the +-- #2443 section below). Here the variable is already bound to the SAME label, +-- so the predicate is trivially true (the label matches). A *different* label +-- on an already-bound variable is still rejected by AGE's pre-existing +-- "multiple labels for variable" restriction rather than evaluating to false; +-- that is an orthogonal limitation, captured here so any future change to +-- single-node-pattern semantics is caught by this test. SELECT * FROM cypher('pattern_expr', $$ MATCH (a:Person) RETURN a.name, (a:Person) @@ -439,16 +442,69 @@ $$) AS (name agtype); "Alice" (1 row) +-- +-- Single-node labeled pattern as a boolean (#2443) +-- +-- A bound vertex carrying a label, e.g. (a:Person), must test that vertex's +-- label rather than be trivially true. Add a non-Person vertex so the filter +-- is observable (every other vertex in this graph is a :Person). +SELECT * FROM cypher('pattern_expr', $$ + CREATE (:Company {name: 'Acme'}) +$$) AS (result agtype); + result +-------- +(0 rows) + +-- bare single-node label predicate in WHERE: only the :Person vertices +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a) + WHERE (a:Person) + RETURN a.name + ORDER BY a.name +$$) AS (name agtype); + name +----------- + "Alice" + "Bob" + "Charlie" + "Dave" +(4 rows) + +-- negated: only the non-Person vertex +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a) + WHERE NOT (a:Person) + RETURN a.name + ORDER BY a.name +$$) AS (name agtype); + name +-------- + "Acme" +(1 row) + +-- EXISTS() form of a single-node label predicate +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a) + WHERE EXISTS((a:Company)) + RETURN a.name + ORDER BY a.name +$$) AS (name agtype); + name +-------- + "Acme" +(1 row) + -- -- Cleanup -- SELECT * FROM drop_graph('pattern_expr', true); -NOTICE: drop cascades to 5 other objects +NOTICE: drop cascades to 6 other objects DETAIL: drop cascades to table pattern_expr._ag_label_vertex drop cascades to table pattern_expr._ag_label_edge drop cascades to table pattern_expr."Person" drop cascades to table pattern_expr."KNOWS" drop cascades to table pattern_expr."WORKS_WITH" +drop cascades to table pattern_expr."Company" NOTICE: graph "pattern_expr" has been dropped drop_graph ------------ diff --git a/regress/sql/pattern_expression.sql b/regress/sql/pattern_expression.sql index fff8476e5..9ded819ef 100644 --- a/regress/sql/pattern_expression.sql +++ b/regress/sql/pattern_expression.sql @@ -222,12 +222,15 @@ $$) AS (result agtype); -- -- Single-node pattern on an already-bound variable: (a:Label) -- --- NOTE: this is an EXISTS existence check on the bound variable, NOT an --- openCypher label predicate. A matching label is therefore always true --- (the variable is already bound), and a *different* label is rejected by --- AGE's pre-existing "multiple labels for variable" restriction rather than --- evaluating to false. Both behaviours are captured here so any future change --- to single-node-pattern semantics is caught by this test. +-- NOTE: as of #2443 a single-node labeled pattern is a correlated label +-- predicate -- in WHERE / EXISTS it tests whether the bound vertex actually +-- has the label (see the WHERE (a:Person) / EXISTS((a:Company)) cases in the +-- #2443 section below). Here the variable is already bound to the SAME label, +-- so the predicate is trivially true (the label matches). A *different* label +-- on an already-bound variable is still rejected by AGE's pre-existing +-- "multiple labels for variable" restriction rather than evaluating to false; +-- that is an orthogonal limitation, captured here so any future change to +-- single-node-pattern semantics is caught by this test. SELECT * FROM cypher('pattern_expr', $$ MATCH (a:Person) RETURN a.name, (a:Person) @@ -299,6 +302,40 @@ SELECT * FROM cypher('pattern_expr', $$ ORDER BY a.name $$) AS (name agtype); +-- +-- Single-node labeled pattern as a boolean (#2443) +-- +-- A bound vertex carrying a label, e.g. (a:Person), must test that vertex's +-- label rather than be trivially true. Add a non-Person vertex so the filter +-- is observable (every other vertex in this graph is a :Person). +SELECT * FROM cypher('pattern_expr', $$ + CREATE (:Company {name: 'Acme'}) +$$) AS (result agtype); + +-- bare single-node label predicate in WHERE: only the :Person vertices +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a) + WHERE (a:Person) + RETURN a.name + ORDER BY a.name +$$) AS (name agtype); + +-- negated: only the non-Person vertex +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a) + WHERE NOT (a:Person) + RETURN a.name + ORDER BY a.name +$$) AS (name agtype); + +-- EXISTS() form of a single-node label predicate +SELECT * FROM cypher('pattern_expr', $$ + MATCH (a) + WHERE EXISTS((a:Company)) + RETURN a.name + ORDER BY a.name +$$) AS (name agtype); + -- -- Cleanup -- diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index a3a1a5044..6582ff8d1 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -5873,10 +5873,56 @@ static List *make_path_join_quals(cypher_parsestate *cpstate, List *entities) List *quals = NIL; List *join_quals; - /* for vertex only queries, there is no work to do */ + /* + * Vertex-only patterns have no edges, so the edge-driven correlation and + * label-filter logic below never runs. That is correct for a freshly + * scanned vertex -- its label comes from its label-table scan. But a + * vertex that refers to a variable from an ENCLOSING query -- e.g. the + * (a:Person) in MATCH (a) WHERE (a:Person) / EXISTS((a:Person)) -- is not + * scanned from its label table here. Without an explicit filter such a + * sub-pattern is uncorrelated and trivially true (the label is never + * tested). If the vertex carries a non-default label and its variable + * exists in an ancestor parse state, emit a label-id filter: make_qual + * builds a name-based id reference that resolves to the outer variable, + * which both correlates the sub-pattern to it and enforces the label. + */ if (list_length(entities) < 3) { - return NIL; + cypher_parsestate *parent_cpstate = + (cypher_parsestate *) cpstate->pstate.parentParseState; + ListCell *vlc; + + if (parent_cpstate != NULL) + { + foreach (vlc, entities) + { + transform_entity *ent = lfirst(vlc); + char *label; + char *name; + + if (ent->type != ENT_VERTEX) + { + continue; + } + + label = ent->entity.node->label; + name = ent->entity.node->name; + + if (label != NULL && !IS_DEFAULT_LABEL_VERTEX(label) && + name != NULL && + find_variable(parent_cpstate, name) != NULL) + { + Node *id_field = make_qual(cpstate, ent, "id"); + + quals = lappend(quals, + filter_vertices_on_label_id(cpstate, + id_field, + label)); + } + } + } + + return quals; } lc = list_head(entities); From 54668e1b50aa22e76b8b522ef4531134c80c7ca2 Mon Sep 17 00:00:00 2001 From: Prashant Chinnam <5108573+crprashant@users.noreply.github.com> Date: Mon, 29 Jun 2026 10:16:57 -0700 Subject: [PATCH 090/100] Preserve null-valued keys in map literals (#2391) (#2412) * Preserve null-valued keys in map literals (#2391) Map literals such as RETURN {a: null} previously dropped keys whose values were null, producing {} instead of {"a": null}. This diverged from the openCypher / Neo4j semantics where map literals preserve every key the user wrote, including those bound to null. Root cause: cypher_map.keep_null defaulted to false (zero-initialised), so the grammar-produced node fed agtype_build_map_nonull, which strips null entries. Call sites that legitimately need strip-null semantics (CREATE node/edge property maps and SET = assignments) already set keep_null=false explicitly, and the MATCH pattern path sets it to true explicitly. Flipping the grammar default to true therefore only affects the cases that were buggy (bare map expressions and nested map values), and leaves CREATE/SET behaviour unchanged. Two preexisting tests encoded the old buggy output and are updated: expr.out (bare RETURN maps now keep the null value) and agtype.out (a nested map inside an orderability test no longer drops its null entry, shifting one row in the ORDER BY result). Dedicated regression coverage for #2391 is added to regress/sql/expr.sql. * Address review: move 'End of tests' marker and drop order claim - Move 'End of tests' to after the Issue 2391 test block so it marks the actual end of the file. - Remove 'in order' from the mixed-values comment since map key ordering is not guaranteed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Update accessor EXPLAIN expected output for null-preserving map literals Map literals now build with agtype_build_map (keep_null) instead of agtype_build_map_nonull, so the accessor-optimization EXPLAIN plan in expr.out must reflect the new function name. Also strip a stray trailing CRLF on the last line of expr.sql/expr.out that leaked a carriage return into the regression output. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add nested-map write coverage and clarify top-level strip wording A reviewer noted the keep_null=true default reaches further than the PR summary stated: CREATE / SET = only override keep_null=false on the top-level property map, not recursively. A nested map value is its own cypher_map node, so it now inherits the new default and preserves its null-valued keys (e.g. CREATE (n {a: {b: null}}) stores a -> {"b": null}). - regress/sql/expr.sql, regress/expected/expr.out: pin the nested case under a write with CREATE (n:Nested {a: {b: null}}), MATCH ... SET n = {a: {b: null}}, and a MATCH verify. - src/backend/parser/cypher_gram.y: clarify the map: rule comment to state the CREATE / SET override is top-level only and nested map values keep the null-preserving default. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- regress/expected/agtype.out | 4 +- regress/expected/expr.out | 148 +++++++++++++++++++++++++++++-- regress/sql/expr.sql | 54 +++++++++++ src/backend/parser/cypher_gram.y | 10 +++ 4 files changed, 205 insertions(+), 11 deletions(-) diff --git a/regress/expected/agtype.out b/regress/expected/agtype.out index b2f3b83e1..0f4775b88 100644 --- a/regress/expected/agtype.out +++ b/regress/expected/agtype.out @@ -2207,8 +2207,8 @@ SELECT * FROM cypher('orderability_graph', $$ MATCH (n) RETURN n ORDER BY n.prop {"id": 844424930131981, "label": "vertex", "properties": {"prop": [{"id": 0, "label": "v", "properties": {"i": 0}}::vertex, {"id": 2, "label": "e", "end_id": 1, "start_id": 0, "properties": {"i": 0}}::edge, {"id": 1, "label": "v", "properties": {"i": 0}}::vertex]::path}}::vertex {"id": 844424930131980, "label": "vertex", "properties": {"prop": {"id": 2, "label": "e", "end_id": 1, "start_id": 0, "properties": {"i": 0}}::edge}}::vertex {"id": 844424930131979, "label": "vertex", "properties": {"prop": {"id": 0, "label": "v", "properties": {"i": 0}}::vertex}}::vertex - {"id": 844424930131978, "label": "vertex", "properties": {"prop": {"bool": true}}}::vertex {"id": 844424930131977, "label": "vertex", "properties": {"prop": {"i": 0, "bool": true}}}::vertex + {"id": 844424930131978, "label": "vertex", "properties": {"prop": {"i": null, "bool": true}}}::vertex {"id": 844424930131975, "label": "vertex", "properties": {"prop": [1, 2, 3]}}::vertex {"id": 844424930131976, "label": "vertex", "properties": {"prop": [1, 2, 3, 4, 5]}}::vertex {"id": 844424930131973, "label": "vertex", "properties": {"prop": "string"}}::vertex @@ -2230,8 +2230,8 @@ SELECT * FROM cypher('orderability_graph', $$ MATCH (n) RETURN n ORDER BY n.prop {"id": 844424930131973, "label": "vertex", "properties": {"prop": "string"}}::vertex {"id": 844424930131976, "label": "vertex", "properties": {"prop": [1, 2, 3, 4, 5]}}::vertex {"id": 844424930131975, "label": "vertex", "properties": {"prop": [1, 2, 3]}}::vertex + {"id": 844424930131978, "label": "vertex", "properties": {"prop": {"i": null, "bool": true}}}::vertex {"id": 844424930131977, "label": "vertex", "properties": {"prop": {"i": 0, "bool": true}}}::vertex - {"id": 844424930131978, "label": "vertex", "properties": {"prop": {"bool": true}}}::vertex {"id": 844424930131979, "label": "vertex", "properties": {"prop": {"id": 0, "label": "v", "properties": {"i": 0}}::vertex}}::vertex {"id": 844424930131980, "label": "vertex", "properties": {"prop": {"id": 2, "label": "e", "end_id": 1, "start_id": 0, "properties": {"i": 0}}::edge}}::vertex {"id": 844424930131981, "label": "vertex", "properties": {"prop": [{"id": 0, "label": "v", "properties": {"i": 0}}::vertex, {"id": 2, "label": "e", "end_id": 1, "start_id": 0, "properties": {"i": 0}}::edge, {"id": 1, "label": "v", "properties": {"i": 0}}::vertex]::path}}::vertex diff --git a/regress/expected/expr.out b/regress/expected/expr.out index 3c80bcae5..d6b9ac155 100644 --- a/regress/expected/expr.out +++ b/regress/expected/expr.out @@ -40,18 +40,18 @@ SELECT * FROM cypher('expr', $$RETURN {}$$) AS r(c agtype); SELECT * FROM cypher('expr', $$ RETURN {s: 's', i: 1, f: 1.0, b: true, z: null} $$) AS r(c agtype); - c ------------------------------------------ - {"b": true, "f": 1.0, "i": 1, "s": "s"} + c +---------------------------------------------------- + {"b": true, "f": 1.0, "i": 1, "s": "s", "z": null} (1 row) -- nested maps SELECT * FROM cypher('expr', $$ RETURN {s: {s: 's'}, t: {i: 1, e: {f: 1.0}, s: {a: {b: true}}}, z: null} $$) AS r(c agtype); - c ----------------------------------------------------------------------------- - {"s": {"s": "s"}, "t": {"e": {"f": 1.0}, "i": 1, "s": {"a": {"b": true}}}} + c +--------------------------------------------------------------------------------------- + {"s": {"s": "s"}, "t": {"e": {"f": 1.0}, "i": 1, "s": {"a": {"b": true}}}, "z": null} (1 row) -- @@ -9866,10 +9866,10 @@ SELECT * FROM cypher('accessor_opt', $$ MATCH (n:Person) RETURN {id: id(n), name: n.name} $$) AS (plan agtype); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------ Seq Scan on accessor_opt."Person" n - Output: agtype_build_map_nonull('id'::text, n.id, 'name'::text, agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype])) + Output: agtype_build_map('id'::text, n.id, 'name'::text, agtype_access_operator(VARIADIC ARRAY[n.properties, '"name"'::agtype])) (2 rows) SELECT * FROM cypher('accessor_opt', $$ @@ -10558,6 +10558,136 @@ NOTICE: graph "list" has been dropped (1 row) +-- +-- Issue 2391 - map literals must preserve keys whose values are null +-- +SELECT create_graph('issue_2391'); +NOTICE: graph "issue_2391" has been created + create_graph +-------------- + +(1 row) + +-- single-key null +SELECT * FROM cypher('issue_2391', $$ + RETURN {a: null} AS m +$$) AS (m agtype); + m +------------- + {"a": null} +(1 row) + +-- multiple null values +SELECT * FROM cypher('issue_2391', $$ + RETURN {companyName: null, sinceYear: null} AS m +$$) AS (m agtype); + m +------------------------------------------ + {"sinceYear": null, "companyName": null} +(1 row) + +-- keys() must see the null-valued key +SELECT * FROM cypher('issue_2391', $$ + RETURN keys({a: null}) AS ks +$$) AS (ks agtype); + ks +------- + ["a"] +(1 row) + +-- coalesce passes a non-null map (map itself is not null) through +SELECT * FROM cypher('issue_2391', $$ + RETURN coalesce({a: null}, null) AS m +$$) AS (m agtype); + m +------------- + {"a": null} +(1 row) + +-- nested map values inside an expression also preserve nulls +SELECT * FROM cypher('issue_2391', $$ + RETURN {outer: {inner: null, kept: 1}} AS m +$$) AS (m agtype); + m +--------------------------------------- + {"outer": {"kept": 1, "inner": null}} +(1 row) + +-- mixed non-null and null values are all preserved +SELECT * FROM cypher('issue_2391', $$ + RETURN {a: 1, b: null, c: 'x'} AS m +$$) AS (m agtype); + m +------------------------------- + {"a": 1, "b": null, "c": "x"} +(1 row) + +-- control: empty map is still empty +SELECT * FROM cypher('issue_2391', $$ + RETURN {} AS m +$$) AS (m agtype); + m +---- + {} +(1 row) + +-- control: CREATE must still strip top-level null properties so +-- setting a property to null removes it from storage +SELECT * FROM cypher('issue_2391', $$ + CREATE (n:Item {keep: 1, drop: null}) RETURN n +$$) AS (n agtype); + n +----------------------------------------------------------------------------- + {"id": 844424930131969, "label": "Item", "properties": {"keep": 1}}::vertex +(1 row) + +SELECT * FROM cypher('issue_2391', $$ + MATCH (n:Item) RETURN n +$$) AS (n agtype); + n +----------------------------------------------------------------------------- + {"id": 844424930131969, "label": "Item", "properties": {"keep": 1}}::vertex +(1 row) + +-- nested map values under a write (CREATE / SET =) are preserved: the +-- top-level property map is null-stripped, but a nested map literal is +-- its own node and keeps its null-valued keys +SELECT * FROM cypher('issue_2391', $$ + CREATE (n:Nested {a: {b: null}}) RETURN n +$$) AS (n agtype); + n +--------------------------------------------------------------------------------------- + {"id": 1125899906842625, "label": "Nested", "properties": {"a": {"b": null}}}::vertex +(1 row) + +SELECT * FROM cypher('issue_2391', $$ + MATCH (n:Nested) SET n = {a: {b: null}} RETURN n +$$) AS (n agtype); + n +--------------------------------------------------------------------------------------- + {"id": 1125899906842625, "label": "Nested", "properties": {"a": {"b": null}}}::vertex +(1 row) + +SELECT * FROM cypher('issue_2391', $$ + MATCH (n:Nested) RETURN n +$$) AS (n agtype); + n +--------------------------------------------------------------------------------------- + {"id": 1125899906842625, "label": "Nested", "properties": {"a": {"b": null}}}::vertex +(1 row) + +SELECT * FROM drop_graph('issue_2391', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table issue_2391._ag_label_vertex +drop cascades to table issue_2391._ag_label_edge +drop cascades to table issue_2391."Item" +drop cascades to table issue_2391."Nested" +NOTICE: graph "issue_2391" has been dropped + drop_graph +------------ + +(1 row) + -- -- End of tests -- diff --git a/regress/sql/expr.sql b/regress/sql/expr.sql index b951a2367..251349cef 100644 --- a/regress/sql/expr.sql +++ b/regress/sql/expr.sql @@ -4136,6 +4136,60 @@ SELECT * FROM drop_graph('regex', true); SELECT * FROM drop_graph('keys', true); SELECT * FROM drop_graph('list', true); +-- +-- Issue 2391 - map literals must preserve keys whose values are null +-- +SELECT create_graph('issue_2391'); +-- single-key null +SELECT * FROM cypher('issue_2391', $$ + RETURN {a: null} AS m +$$) AS (m agtype); +-- multiple null values +SELECT * FROM cypher('issue_2391', $$ + RETURN {companyName: null, sinceYear: null} AS m +$$) AS (m agtype); +-- keys() must see the null-valued key +SELECT * FROM cypher('issue_2391', $$ + RETURN keys({a: null}) AS ks +$$) AS (ks agtype); +-- coalesce passes a non-null map (map itself is not null) through +SELECT * FROM cypher('issue_2391', $$ + RETURN coalesce({a: null}, null) AS m +$$) AS (m agtype); +-- nested map values inside an expression also preserve nulls +SELECT * FROM cypher('issue_2391', $$ + RETURN {outer: {inner: null, kept: 1}} AS m +$$) AS (m agtype); +-- mixed non-null and null values are all preserved +SELECT * FROM cypher('issue_2391', $$ + RETURN {a: 1, b: null, c: 'x'} AS m +$$) AS (m agtype); +-- control: empty map is still empty +SELECT * FROM cypher('issue_2391', $$ + RETURN {} AS m +$$) AS (m agtype); +-- control: CREATE must still strip top-level null properties so +-- setting a property to null removes it from storage +SELECT * FROM cypher('issue_2391', $$ + CREATE (n:Item {keep: 1, drop: null}) RETURN n +$$) AS (n agtype); +SELECT * FROM cypher('issue_2391', $$ + MATCH (n:Item) RETURN n +$$) AS (n agtype); +-- nested map values under a write (CREATE / SET =) are preserved: the +-- top-level property map is null-stripped, but a nested map literal is +-- its own node and keeps its null-valued keys +SELECT * FROM cypher('issue_2391', $$ + CREATE (n:Nested {a: {b: null}}) RETURN n +$$) AS (n agtype); +SELECT * FROM cypher('issue_2391', $$ + MATCH (n:Nested) SET n = {a: {b: null}} RETURN n +$$) AS (n agtype); +SELECT * FROM cypher('issue_2391', $$ + MATCH (n:Nested) RETURN n +$$) AS (n agtype); +SELECT * FROM drop_graph('issue_2391', true); + -- -- End of tests -- diff --git a/src/backend/parser/cypher_gram.y b/src/backend/parser/cypher_gram.y index 83d69c83b..ddd62d7ee 100644 --- a/src/backend/parser/cypher_gram.y +++ b/src/backend/parser/cypher_gram.y @@ -2127,6 +2127,16 @@ map: n = make_ag_node(cypher_map); n->keyvals = $2; + /* + * By default, a Cypher map literal preserves keys whose + * values are null (openCypher / Neo4j semantics: e.g. + * RETURN {a: null} yields {a: null}, not {}). CREATE and + * SET = override this to false on the top-level property + * map in cypher_clause.c so null properties are stripped + * on write; a nested map value is its own node and keeps + * this default, preserving its null-valued keys. + */ + n->keep_null = true; $$ = (Node *)n; } From 513767136a6e6660f6549c9c4b46f9524de1aef9 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Wed, 1 Jul 2026 19:01:42 -0700 Subject: [PATCH 091/100] Fix Node.js driver CI build broken by @types/node drift (#2452) The Node.js driver CI (npm install -> npm run build -> tsc) failed with parser errors in node_modules/@types/node/ffi.d.ts (TS1139/TS1005/TS1109/ TS1128). package-lock.json is gitignored, so CI resolves dependencies purely from package.json. @types/node was only pulled transitively via a wildcard range (@types/pg and jest depend on @types/node@*), so a fresh install grabbed the latest (26.x). That version uses `const` type parameters (a TypeScript 5.0 feature) in ffi.d.ts, which typescript@4.9 cannot parse. skipLibCheck does not suppress these parser-level errors. The runtime Node version is unrelated: @types/node is resolved from the npm dependency graph, not the Node.js runtime. Fix: - Add a bounded direct devDependency "@types/node": "^20.19.0" so a fresh install constrains the typings to the Node 20 LTS line, which is compatible with typescript@4.9 and keeps the toolchain consistent (eslint 7 / typescript-eslint 4 / TS 4.9 / Node 20 typings). - Pin CI to Node 20 (setup-node@v4, node-version: 20) for reproducibility and to match the pinned typings; replaces the deprecated setup-node@v3 and floating node-version: latest. Verified: a clean, no-lockfile install (matching CI) now resolves @types/node@20.19.43 and tsc builds successfully. Co-authored-by: Copilot modified: .github/workflows/nodejs-driver.yaml modified: drivers/nodejs/package.json --- .github/workflows/nodejs-driver.yaml | 4 ++-- drivers/nodejs/package.json | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nodejs-driver.yaml b/.github/workflows/nodejs-driver.yaml index 3d9e07023..20135ef13 100644 --- a/.github/workflows/nodejs-driver.yaml +++ b/.github/workflows/nodejs-driver.yaml @@ -22,9 +22,9 @@ jobs: run: docker compose up -d - name: Set up Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: - node-version: latest + node-version: 20 - name: Install dependencies run: npm install diff --git a/drivers/nodejs/package.json b/drivers/nodejs/package.json index 15c2371f4..d17aa3b32 100644 --- a/drivers/nodejs/package.json +++ b/drivers/nodejs/package.json @@ -34,6 +34,7 @@ }, "devDependencies": { "@types/jest": "^29.5.14", + "@types/node": "^20.19.0", "@types/pg": "^7.14.10", "@typescript-eslint/eslint-plugin": "^4.22.1", "@typescript-eslint/parser": "^4.22.1", From fafdaa1c0ef5f4ab8ff09482f20a382b845dc9e5 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Thu, 2 Jul 2026 10:39:37 -0400 Subject: [PATCH 092/100] Fix stack overflow and precision loss in toFloatList() conversion (#2451) toFloatList()'s AGTV_FLOAT branch formatted each element with sprintf(buffer, "%f", ...) into a fixed 64-byte stack buffer and then re-parsed the string back into a float. This had two defects: 1. Stack overflow. "%f" prints the full integer part with no width limit, so a large magnitude overflows the 64-byte buffer. The value is query-reachable: RETURN toFloatList([1.0e308]) needs ~317 bytes (309 integer digits + ".000000") and smashes the stack. This is the issue reported in #2410. 2. Precision loss. "%f" emits only 6 fractional digits, so the format-and-reparse round trip was lossy -- toFloatList([0.123456789]) returned 0.123457. The element is already a float8, so the whole format/reparse step is unnecessary. Assign elem->val.float_value directly. This removes the stack buffer entirely (no magic buffer size to justify) and fixes both the overflow and the precision loss at once. Also harden toStringList(): its "%.*g"/"%ld" conversions use bounded formats and were never overflow-prone, but switch them from sprintf to snprintf as defensive depth. Add regression coverage to regress/sql/expr.sql for both the large magnitude case (no overflow) and precision preservation. This reimplements the fix originally proposed by David Christensen in #2410, whose report identified the sprintf overflow. Co-authored-by: David Christensen --- regress/expected/expr.out | 20 ++++++++++++++++++++ regress/sql/expr.sql | 10 ++++++++++ src/backend/utils/adt/agtype.c | 19 +++++++++++-------- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/regress/expected/expr.out b/regress/expected/expr.out index d6b9ac155..806a6f65c 100644 --- a/regress/expected/expr.out +++ b/regress/expected/expr.out @@ -3548,6 +3548,26 @@ $$) AS (toFloatList agtype); [1.20002] (1 row) +-- large magnitudes must not overflow the conversion (regression: unbounded +-- sprintf into a fixed stack buffer overflowed for values like 1.0e308) +SELECT * FROM cypher('expr', $$ + RETURN toFloatList([1.0e308, -1.0e308]) +$$) AS (toFloatList agtype); + tofloatlist +------------------- + [1e+308, -1e+308] +(1 row) + +-- precision must be preserved (regression: "%f" format truncated to 6 digits, +-- so 0.123456789 came back as 0.123457) +SELECT * FROM cypher('expr', $$ + RETURN toFloatList([0.123456789]) +$$) AS (toFloatList agtype); + tofloatlist +--------------- + [0.123456789] +(1 row) + -- should return null SELECT * FROM cypher('expr', $$ RETURN toFloatList(['true']) diff --git a/regress/sql/expr.sql b/regress/sql/expr.sql index 251349cef..d4d900a1c 100644 --- a/regress/sql/expr.sql +++ b/regress/sql/expr.sql @@ -1520,6 +1520,16 @@ $$) AS (toFloatList agtype); SELECT * FROM cypher('expr', $$ RETURN toFloatList([1.20002]) $$) AS (toFloatList agtype); +-- large magnitudes must not overflow the conversion (regression: unbounded +-- sprintf into a fixed stack buffer overflowed for values like 1.0e308) +SELECT * FROM cypher('expr', $$ + RETURN toFloatList([1.0e308, -1.0e308]) +$$) AS (toFloatList agtype); +-- precision must be preserved (regression: "%f" format truncated to 6 digits, +-- so 0.123456789 came back as 0.123457) +SELECT * FROM cypher('expr', $$ + RETURN toFloatList([0.123456789]) +$$) AS (toFloatList agtype); -- should return null SELECT * FROM cypher('expr', $$ RETURN toFloatList(['true']) diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index bf69bf1fa..0e1a7963f 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -7099,8 +7099,6 @@ Datum age_tofloatlist(PG_FUNCTION_ARGS) int count; int i; bool is_valid = false; - float8 float_num; - char buffer[64]; /* check for null */ if (PG_ARGISNULL(0)) @@ -7160,11 +7158,16 @@ Datum age_tofloatlist(PG_FUNCTION_ARGS) case AGTV_FLOAT: + /* + * The element is already a float8, so assign it directly. The + * previous approach formatted it to a string with sprintf() and + * re-parsed it: that both overflowed a fixed 64-byte stack buffer + * for large magnitudes (e.g. 1.0e308 needs ~317 chars) and lost + * precision, since "%f" truncates to 6 fractional digits. Direct + * assignment avoids both problems. + */ float_elem.type = AGTV_FLOAT; - float_num = elem->val.float_value; - sprintf(buffer, "%f", float_num); - string = buffer; - float_elem.val.float_value = float8in_internal_null(string, NULL, "double precision", string, &is_valid); + float_elem.val.float_value = elem->val.float_value; agis_result.res = push_agtype_value(&agis_result.parse_state, WAGT_ELEM, &float_elem); break; @@ -8146,7 +8149,7 @@ Datum age_tostringlist(PG_FUNCTION_ARGS) case AGTV_FLOAT: - sprintf(buffer, "%.*g", DBL_DIG, elem->val.float_value); + snprintf(buffer, sizeof(buffer), "%.*g", DBL_DIG, elem->val.float_value); string_elem.val.string.val = pstrdup(buffer); string_elem.val.string.len = strlen(buffer); @@ -8157,7 +8160,7 @@ Datum age_tostringlist(PG_FUNCTION_ARGS) case AGTV_INTEGER: - sprintf(buffer, "%ld", elem->val.int_value); + snprintf(buffer, sizeof(buffer), "%ld", elem->val.int_value); string_elem.val.string.val = pstrdup(buffer); string_elem.val.string.len = strlen(buffer); From 7bfeb24cff957e443cdc91b79a76c6401e6f8428 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Thu, 2 Jul 2026 09:58:56 -0700 Subject: [PATCH 093/100] Support outer references in reduce() fold bodies (#2448) Allow a reduce(acc = init, var IN list | body) fold body to reference loop-invariant values from the enclosing query -- outer-query variables and cypher() parameters -- in addition to the accumulator and element. These were previously rejected with ERRCODE_FEATURE_NOT_SUPPORTED. How it works ------------ The fold body is still compiled to a standalone expression evaluated by age_reduce_transfn, so an outer reference (which cannot be evaluated there) is captured at transform time and supplied as a value: - After the accumulator and element are rewritten to PARAM_EXEC params 0 and 1, transform_cypher_reduce() walks the body and replaces each loop-invariant outer reference -- one that references an outer Var or a cypher() $parameter but not the accumulator/element -- with a new PARAM_EXEC param 2, 3, ... in body order. Capture is at leaf granularity: only the bare outer value is hoisted out of the body, while the operators, function calls and CASE/AND/OR/coalesce branches around it stay in the serialized body. Because a captured value becomes an aggregate argument that the executor evaluates eagerly and unconditionally, hoisting a whole computed subtree (for example "1/z" under a never-taken CASE branch) would defeat the fold's short-circuiting; capturing only the leaf keeps evaluation under the body's own control flow. The one exception is an outer reference that is not itself agtype-typed -- most commonly the graphid inside a graph vertex/edge variable -- whose smallest enclosing agtype-typed subtree is captured whole, since it cannot stand alone as an agtype[] extra. - The captured expressions are passed to the aggregate as a trailing agtype[] argument; age_reduce(agtype, text, agtype, agtype[]) and its transition function gain this argument. - age_reduce_transfn sizes its param array to 2 + the number of captures and binds the captured values to params 2.. on every row. Because the captures are evaluated in the outer query context as ordinary aggregate arguments, a correlated capture is re-evaluated per group, so an outer value that varies per row (for example under UNWIND) is folded with the correct value. Each capture slot is rebound on every row, and the trailing extras argument is read only when the aggregate actually passes it (PG_NARGS), keeping the transition safe under direct age_reduce() SQL calls and an older 4-argument signature. This keeps the no-core-patch design: the body is still a serialized standalone expression, and the only new machinery is the captured-value plumbing. Still rejected -------------- Subqueries in the body (including a nested reduce()) and aggregate functions remain unsupported and raise a clean ERRCODE_FEATURE_NOT_SUPPORTED error: a subquery cannot be planned as a plain aggregate argument, and an aggregate in a per-element fold is undefined per the openCypher specification. Tests ----- age_reduce gains an "Outer references in the fold body" section covering a plain outer variable, an outer variable used as a multiplier, two distinct outer variables, a property of an outer graph variable, the same outer variable referenced more than once, a property of an outer map, a subexpression that mixes an outer reference with the element (only the loop-invariant part is captured), an outer reference inside a CASE branch of the body, a NULL outer value propagating through the fold, multiple captures mixing a NULL and a non-NULL outer value, an outer variable that changes per row (captured per group), and a cypher() parameter supplied via a prepared statement. A "Short-circuit evaluation is preserved for outer references in the body" section verifies that a guarded outer sub-expression is not evaluated on a branch that is not taken: a never-taken CASE THEN branch, a never-taken CASE ELSE branch, an OR and an AND that short-circuit, and a coalesce -- each of which would divide by zero if the outer "1/w" were hoisted into an eagerly evaluated aggregate argument -- plus a guarded branch that is taken and evaluates its outer division normally. The previously-rejected outer-variable case is moved out of the not-supported section, which now covers a nested reduce() (any subquery in the body is unsupported) and an aggregate in the body. The same change also broadens the base reduce() coverage with value-type folds (a float accumulator, negative numbers, a map accumulator passed through unchanged, and list elements indexed in the body), function calls in the fold body (a scalar function over the element and the list itself produced by a function), reduce() composed with surrounding expressions (consumed by another function and used in a comparison), and syntax-error checks for each required piece of the form -- the "= init", ", var IN list", and "| body" clauses, plus a rejected qualified iterator variable. 42/42 installcheck pass. Co-authored-by: Copilot modified: age--1.7.0--y.y.y.sql modified: regress/expected/age_reduce.out modified: regress/sql/age_reduce.sql modified: sql/age_aggregate.sql modified: src/backend/parser/cypher_clause.c modified: src/backend/utils/adt/agtype.c --- age--1.7.0--y.y.y.sql | 14 +- regress/expected/age_reduce.out | 326 ++++++++++++++++++++++++++- regress/sql/age_reduce.sql | 206 ++++++++++++++++- sql/age_aggregate.sql | 14 +- src/backend/parser/cypher_clause.c | 349 +++++++++++++++++++++++++++-- src/backend/utils/adt/agtype.c | 94 +++++++- 6 files changed, 952 insertions(+), 51 deletions(-) diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--y.y.y.sql index 6dc8f707f..fd1f28160 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--y.y.y.sql @@ -1107,17 +1107,21 @@ COMMENT ON FUNCTION ag_catalog.create_subgraph(name, name, text, text) IS -- Transition function for the age_reduce aggregate. The fold body is compiled -- by transform_cypher_reduce() with the accumulator and element rewritten to -- PARAM_EXEC params 0 and 1 and serialized into the text argument; the --- transition evaluates it for each element in list order. It must be callable --- with a NULL transition state (no initcond), so it is intentionally not STRICT. -CREATE FUNCTION ag_catalog.age_reduce_transfn(agtype, agtype, text, agtype) +-- transition evaluates it for each element in list order. The trailing +-- agtype[] argument carries the loop-invariant outer values (outer-query +-- variables and cypher() parameters) referenced by the body, bound to +-- PARAM_EXEC params 2, 3, ... It must be callable with a NULL transition state +-- (no initcond), so it is intentionally not STRICT. +CREATE FUNCTION ag_catalog.age_reduce_transfn(agtype, agtype, text, agtype, agtype[]) RETURNS agtype LANGUAGE c PARALLEL UNSAFE AS 'MODULE_PATHNAME'; -- aggregate definition for reduce(); direct arguments are --- (init, serialized-body, element), with the element fed ORDER BY ordinality. -CREATE AGGREGATE ag_catalog.age_reduce(agtype, text, agtype) +-- (init, serialized-body, element, captured-outer-values), with the element +-- fed ORDER BY ordinality. +CREATE AGGREGATE ag_catalog.age_reduce(agtype, text, agtype, agtype[]) ( stype = agtype, sfunc = ag_catalog.age_reduce_transfn diff --git a/regress/expected/age_reduce.out b/regress/expected/age_reduce.out index 8a198965a..5f3bf29f8 100644 --- a/regress/expected/age_reduce.out +++ b/regress/expected/age_reduce.out @@ -222,6 +222,87 @@ $$) AS (result agtype); [1, 4, 9] (1 row) +-- +-- Value types in the fold +-- +-- a float accumulator and float elements +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0.0, x IN [1.5, 2.5, 3.0] | s + x) +$$) AS (result agtype); + result +-------- + 7.0 +(1 row) + +-- negative numbers +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [-1, -2, -3] | s + x) +$$) AS (result agtype); + result +-------- + -6 +(1 row) + +-- a map accumulator passed through unchanged +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = {n: 0}, x IN [1, 2, 3] | s) +$$) AS (result agtype); + result +---------- + {"n": 0} +(1 row) + +-- elements that are themselves lists, indexed in the body +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [[1, 2], [3, 4], [5, 6]] | s + x[0]) +$$) AS (result agtype); + result +-------- + 9 +(1 row) + +-- +-- Function calls in the fold body +-- +-- a scalar function applied to the element +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN ['a', 'bb', 'ccc'] | s + size(x)) +$$) AS (result agtype); + result +-------- + 6 +(1 row) + +-- the list itself produced by a function +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN range(1, 5) | s + x) +$$) AS (result agtype); + result +-------- + 15 +(1 row) + +-- +-- Composing reduce() with surrounding expressions +-- +-- the reduce() result consumed by another function +SELECT * FROM cypher('reduce', $$ + RETURN size(reduce(s = [], x IN [1, 2, 3, 4] | s + [x])) +$$) AS (result agtype); + result +-------- + 4 +(1 row) + +-- the reduce() result used in a comparison +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x) = 6 +$$) AS (result agtype); + result +-------- + true +(1 row) + -- -- A conditional body (CASE) -- @@ -484,17 +565,197 @@ $$) AS (name agtype, total agtype); (3 rows) -- --- Not-yet-supported constructs raise a clean feature error +-- Outer references in the fold body -- --- an outer variable referenced in the body +-- The body may reference loop-invariant values from the enclosing query: an +-- outer variable, a property of an outer variable, or a cypher() parameter. +-- a plain outer variable in the body SELECT * FROM cypher('reduce', $$ WITH 5 AS w - RETURN reduce(s = 0, x IN [1, 2] | s + x + w) + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x + w) $$) AS (result agtype); -ERROR: a reduce() expression may only reference its accumulator and element variables -LINE 1: SELECT * FROM cypher('reduce', $$ - ^ --- a nested reduce() in the body + result +-------- + 21 +(1 row) + +-- an outer variable used as a multiplier +SELECT * FROM cypher('reduce', $$ + WITH 3 AS factor + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x * factor) +$$) AS (result agtype); + result +-------- + 18 +(1 row) + +-- two distinct outer variables in the body +SELECT * FROM cypher('reduce', $$ + WITH 2 AS a, 100 AS b + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x * a + b) +$$) AS (result agtype); + result +-------- + 312 +(1 row) + +-- a property of an outer (graph) variable in the body +SELECT * FROM cypher('reduce', $$ + MATCH (u:bag) WHERE u.name = 'mid' + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x + u.vals[0]) +$$) AS (result agtype); + result +-------- + 21 +(1 row) + +-- the same outer variable referenced more than once in the body +SELECT * FROM cypher('reduce', $$ + WITH 7 AS k + RETURN reduce(s = 0, x IN [1, 2, 3] | s + k + k) +$$) AS (result agtype); + result +-------- + 42 +(1 row) + +-- a property of an outer map referenced in the body +SELECT * FROM cypher('reduce', $$ + WITH {factor: 10} AS m + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x * m.factor) +$$) AS (result agtype); + result +-------- + 60 +(1 row) + +-- a subexpression that mixes an outer reference with the element: only the +-- loop-invariant part (the outer list) is captured, the element index is not +SELECT * FROM cypher('reduce', $$ + WITH [10, 20, 30] AS lookup + RETURN reduce(s = 0, x IN [1, 2, 3] | s + lookup[x - 1]) +$$) AS (result agtype); + result +-------- + 60 +(1 row) + +-- an outer reference inside a CASE branch of the body is captured +SELECT * FROM cypher('reduce', $$ + WITH 10 AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | CASE WHEN x % 2 = 0 THEN s + w ELSE s + x END) +$$) AS (result agtype); + result +-------- + 14 +(1 row) + +-- a NULL outer value propagates through the fold +SELECT * FROM cypher('reduce', $$ + WITH null AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x + w) +$$) AS (result agtype); + result +-------- + null +(1 row) + +-- multiple outer captures with a mix of NULL and non-NULL: each is bound to its +-- own slot (the non-NULL multiplier is bound and the NULL still propagates) +SELECT * FROM cypher('reduce', $$ + WITH 3 AS a, null AS b + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x * a + b) +$$) AS (result agtype); + result +-------- + null +(1 row) + +-- an outer variable that changes per row is captured per group +SELECT * FROM cypher('reduce', $$ + UNWIND [1, 2, 3] AS m + RETURN reduce(s = 0, x IN [1, 2, 3, 4] | s + x * m) AS total + ORDER BY total +$$) AS (result agtype); + result +-------- + 10 + 20 + 30 +(3 rows) + +-- +-- Short-circuit evaluation is preserved for outer references in the body +-- +-- Only the outer leaf is captured; operators and CASE/AND/OR branches stay in +-- the body, so a guarded outer sub-expression is not evaluated on a branch +-- that is not taken. Each case below would divide by zero if the whole "1/w" +-- were hoisted into an eagerly evaluated aggregate argument instead. +-- the THEN branch is never taken, so "1/w" is not evaluated (expect 6) +SELECT * FROM cypher('reduce', $$ + WITH 0 AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | CASE WHEN false THEN s + 1/w ELSE s + x END) +$$) AS (result agtype); + result +-------- + 6 +(1 row) + +-- the ELSE branch is never taken, so "1/w" is not evaluated (expect 6) +SELECT * FROM cypher('reduce', $$ + WITH 0 AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | CASE WHEN true THEN s + x ELSE s + 1/w END) +$$) AS (result agtype); + result +-------- + 6 +(1 row) + +-- OR short-circuits once "w = 0" is true, so "1/w > 0" is not evaluated +SELECT * FROM cypher('reduce', $$ + WITH 0 AS w + RETURN reduce(s = true, x IN [1, 2, 3] | s AND (w = 0 OR 1/w > 0)) +$$) AS (result agtype); + result +-------- + true +(1 row) + +-- AND short-circuits once "w <> 0" is false, so "1/w > 0" is not evaluated +SELECT * FROM cypher('reduce', $$ + WITH 0 AS w + RETURN reduce(s = true, x IN [1, 2, 3] | s AND (w <> 0 AND 1/w > 0)) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- coalesce short-circuits: "1/w" is not evaluated when arg 1 is non-null +SELECT * FROM cypher('reduce', $$ + WITH 0 AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | s + coalesce(w, 1/w)) +$$) AS (result agtype); + result +-------- + 0 +(1 row) + +-- when the guarded branch is taken, the outer sub-expression is evaluated +-- normally (division by a non-zero outer value): x = 2 -> s + 10/2 (expect 9) +SELECT * FROM cypher('reduce', $$ + WITH 2 AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | CASE WHEN x % 2 = 0 THEN s + 10/w ELSE s + x END) +$$) AS (result agtype); + result +-------- + 9 +(1 row) + +-- +-- Not-yet-supported constructs raise a clean feature error +-- +-- a nested reduce() in the body (any subquery in the body is unsupported) SELECT * FROM cypher('reduce', $$ RETURN reduce(s = 0, x IN [1, 2] | s + reduce(t = 0, y IN [x] | t + y)) $$) AS (result agtype); @@ -509,6 +770,57 @@ ERROR: aggregate functions are not supported in a reduce() expression LINE 1: SELECT * FROM cypher('reduce', $$ ^ -- +-- Syntax errors: each required piece of the reduce() form is enforced +-- +-- missing "= init" +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s, x IN [1, 2] | s + x) +$$) AS (result agtype); +ERROR: syntax error at or near "," +LINE 2: RETURN reduce(s, x IN [1, 2] | s + x) + ^ +-- missing ", var IN list" +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0 | s) +$$) AS (result agtype); +ERROR: syntax error at or near "|" +LINE 2: RETURN reduce(s = 0 | s) + ^ +-- missing "| body" +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2]) +$$) AS (result agtype); +ERROR: syntax error at or near ")" +LINE 2: RETURN reduce(s = 0, x IN [1, 2]) + ^ +-- a qualified iterator variable is not allowed +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x.y IN [1, 2] | s) +$$) AS (result agtype); +ERROR: syntax error at or near "." +LINE 2: RETURN reduce(s = 0, x.y IN [1, 2] | s) + ^ +-- +-- cypher() parameter referenced in the fold body (via a prepared statement) +-- +PREPARE reduce_param(agtype) AS + SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x + $p) + $$, $1) AS (result agtype); +EXECUTE reduce_param('{"p": 10}'); + result +-------- + 36 +(1 row) + +EXECUTE reduce_param('{"p": 100}'); + result +-------- + 306 +(1 row) + +DEALLOCATE reduce_param; +-- -- "reduce" as a property key name (safe_keywords backward compatibility): -- because reduce() introduced a reserved keyword, confirm the word is still -- usable as a map key, the same way any/none/single are. diff --git a/regress/sql/age_reduce.sql b/regress/sql/age_reduce.sql index cf1261010..fbe324cd1 100644 --- a/regress/sql/age_reduce.sql +++ b/regress/sql/age_reduce.sql @@ -151,6 +151,55 @@ SELECT * FROM cypher('reduce', $$ RETURN reduce(acc = [], x IN [1, 2, 3] | acc + [x * x]) $$) AS (result agtype); +-- +-- Value types in the fold +-- +-- a float accumulator and float elements +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0.0, x IN [1.5, 2.5, 3.0] | s + x) +$$) AS (result agtype); + +-- negative numbers +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [-1, -2, -3] | s + x) +$$) AS (result agtype); + +-- a map accumulator passed through unchanged +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = {n: 0}, x IN [1, 2, 3] | s) +$$) AS (result agtype); + +-- elements that are themselves lists, indexed in the body +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [[1, 2], [3, 4], [5, 6]] | s + x[0]) +$$) AS (result agtype); + +-- +-- Function calls in the fold body +-- +-- a scalar function applied to the element +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN ['a', 'bb', 'ccc'] | s + size(x)) +$$) AS (result agtype); + +-- the list itself produced by a function +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN range(1, 5) | s + x) +$$) AS (result agtype); + +-- +-- Composing reduce() with surrounding expressions +-- +-- the reduce() result consumed by another function +SELECT * FROM cypher('reduce', $$ + RETURN size(reduce(s = [], x IN [1, 2, 3, 4] | s + [x])) +$$) AS (result agtype); + +-- the reduce() result used in a comparison +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x) = 6 +$$) AS (result agtype); + -- -- A conditional body (CASE) -- @@ -317,15 +366,127 @@ SELECT * FROM cypher('reduce', $$ $$) AS (name agtype, total agtype); -- --- Not-yet-supported constructs raise a clean feature error +-- Outer references in the fold body -- --- an outer variable referenced in the body +-- The body may reference loop-invariant values from the enclosing query: an +-- outer variable, a property of an outer variable, or a cypher() parameter. +-- a plain outer variable in the body SELECT * FROM cypher('reduce', $$ WITH 5 AS w - RETURN reduce(s = 0, x IN [1, 2] | s + x + w) + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x + w) +$$) AS (result agtype); + +-- an outer variable used as a multiplier +SELECT * FROM cypher('reduce', $$ + WITH 3 AS factor + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x * factor) +$$) AS (result agtype); + +-- two distinct outer variables in the body +SELECT * FROM cypher('reduce', $$ + WITH 2 AS a, 100 AS b + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x * a + b) +$$) AS (result agtype); + +-- a property of an outer (graph) variable in the body +SELECT * FROM cypher('reduce', $$ + MATCH (u:bag) WHERE u.name = 'mid' + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x + u.vals[0]) $$) AS (result agtype); --- a nested reduce() in the body +-- the same outer variable referenced more than once in the body +SELECT * FROM cypher('reduce', $$ + WITH 7 AS k + RETURN reduce(s = 0, x IN [1, 2, 3] | s + k + k) +$$) AS (result agtype); + +-- a property of an outer map referenced in the body +SELECT * FROM cypher('reduce', $$ + WITH {factor: 10} AS m + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x * m.factor) +$$) AS (result agtype); + +-- a subexpression that mixes an outer reference with the element: only the +-- loop-invariant part (the outer list) is captured, the element index is not +SELECT * FROM cypher('reduce', $$ + WITH [10, 20, 30] AS lookup + RETURN reduce(s = 0, x IN [1, 2, 3] | s + lookup[x - 1]) +$$) AS (result agtype); + +-- an outer reference inside a CASE branch of the body is captured +SELECT * FROM cypher('reduce', $$ + WITH 10 AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | CASE WHEN x % 2 = 0 THEN s + w ELSE s + x END) +$$) AS (result agtype); + +-- a NULL outer value propagates through the fold +SELECT * FROM cypher('reduce', $$ + WITH null AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x + w) +$$) AS (result agtype); + +-- multiple outer captures with a mix of NULL and non-NULL: each is bound to its +-- own slot (the non-NULL multiplier is bound and the NULL still propagates) +SELECT * FROM cypher('reduce', $$ + WITH 3 AS a, null AS b + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x * a + b) +$$) AS (result agtype); + +-- an outer variable that changes per row is captured per group +SELECT * FROM cypher('reduce', $$ + UNWIND [1, 2, 3] AS m + RETURN reduce(s = 0, x IN [1, 2, 3, 4] | s + x * m) AS total + ORDER BY total +$$) AS (result agtype); + +-- +-- Short-circuit evaluation is preserved for outer references in the body +-- +-- Only the outer leaf is captured; operators and CASE/AND/OR branches stay in +-- the body, so a guarded outer sub-expression is not evaluated on a branch +-- that is not taken. Each case below would divide by zero if the whole "1/w" +-- were hoisted into an eagerly evaluated aggregate argument instead. +-- the THEN branch is never taken, so "1/w" is not evaluated (expect 6) +SELECT * FROM cypher('reduce', $$ + WITH 0 AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | CASE WHEN false THEN s + 1/w ELSE s + x END) +$$) AS (result agtype); + +-- the ELSE branch is never taken, so "1/w" is not evaluated (expect 6) +SELECT * FROM cypher('reduce', $$ + WITH 0 AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | CASE WHEN true THEN s + x ELSE s + 1/w END) +$$) AS (result agtype); + +-- OR short-circuits once "w = 0" is true, so "1/w > 0" is not evaluated +SELECT * FROM cypher('reduce', $$ + WITH 0 AS w + RETURN reduce(s = true, x IN [1, 2, 3] | s AND (w = 0 OR 1/w > 0)) +$$) AS (result agtype); + +-- AND short-circuits once "w <> 0" is false, so "1/w > 0" is not evaluated +SELECT * FROM cypher('reduce', $$ + WITH 0 AS w + RETURN reduce(s = true, x IN [1, 2, 3] | s AND (w <> 0 AND 1/w > 0)) +$$) AS (result agtype); + +-- coalesce short-circuits: "1/w" is not evaluated when arg 1 is non-null +SELECT * FROM cypher('reduce', $$ + WITH 0 AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | s + coalesce(w, 1/w)) +$$) AS (result agtype); + +-- when the guarded branch is taken, the outer sub-expression is evaluated +-- normally (division by a non-zero outer value): x = 2 -> s + 10/2 (expect 9) +SELECT * FROM cypher('reduce', $$ + WITH 2 AS w + RETURN reduce(s = 0, x IN [1, 2, 3] | CASE WHEN x % 2 = 0 THEN s + 10/w ELSE s + x END) +$$) AS (result agtype); + +-- +-- Not-yet-supported constructs raise a clean feature error +-- +-- a nested reduce() in the body (any subquery in the body is unsupported) SELECT * FROM cypher('reduce', $$ RETURN reduce(s = 0, x IN [1, 2] | s + reduce(t = 0, y IN [x] | t + y)) $$) AS (result agtype); @@ -335,6 +496,43 @@ SELECT * FROM cypher('reduce', $$ RETURN reduce(s = 0, x IN [1, 2] | s + count(x)) $$) AS (result agtype); +-- +-- Syntax errors: each required piece of the reduce() form is enforced +-- +-- missing "= init" +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s, x IN [1, 2] | s + x) +$$) AS (result agtype); + +-- missing ", var IN list" +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0 | s) +$$) AS (result agtype); + +-- missing "| body" +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2]) +$$) AS (result agtype); + +-- a qualified iterator variable is not allowed +SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x.y IN [1, 2] | s) +$$) AS (result agtype); + +-- +-- cypher() parameter referenced in the fold body (via a prepared statement) +-- +PREPARE reduce_param(agtype) AS + SELECT * FROM cypher('reduce', $$ + RETURN reduce(s = 0, x IN [1, 2, 3] | s + x + $p) + $$, $1) AS (result agtype); + +EXECUTE reduce_param('{"p": 10}'); + +EXECUTE reduce_param('{"p": 100}'); + +DEALLOCATE reduce_param; + -- -- "reduce" as a property key name (safe_keywords backward compatibility): -- because reduce() introduced a reserved keyword, confirm the word is still diff --git a/sql/age_aggregate.sql b/sql/age_aggregate.sql index fb258e5c5..9ad715683 100644 --- a/sql/age_aggregate.sql +++ b/sql/age_aggregate.sql @@ -223,17 +223,21 @@ CREATE AGGREGATE ag_catalog.age_collect(variadic "any") -- Transition function for the age_reduce aggregate. The fold body is compiled -- by transform_cypher_reduce() with the accumulator and element rewritten to -- PARAM_EXEC params 0 and 1 and serialized into the text argument; the --- transition evaluates it for each element in list order. It must be callable --- with a NULL transition state (no initcond), so it is intentionally not STRICT. -CREATE FUNCTION ag_catalog.age_reduce_transfn(agtype, agtype, text, agtype) +-- transition evaluates it for each element in list order. The trailing +-- agtype[] argument carries the loop-invariant outer values (outer-query +-- variables and cypher() parameters) referenced by the body, bound to +-- PARAM_EXEC params 2, 3, ... It must be callable with a NULL transition state +-- (no initcond), so it is intentionally not STRICT. +CREATE FUNCTION ag_catalog.age_reduce_transfn(agtype, agtype, text, agtype, agtype[]) RETURNS agtype LANGUAGE c PARALLEL UNSAFE AS 'MODULE_PATHNAME'; -- aggregate definition for reduce(); direct arguments are --- (init, serialized-body, element), with the element fed ORDER BY ordinality. -CREATE AGGREGATE ag_catalog.age_reduce(agtype, text, agtype) +-- (init, serialized-body, element, captured-outer-values), with the element +-- fed ORDER BY ordinality. +CREATE AGGREGATE ag_catalog.age_reduce(agtype, text, agtype, agtype[]) ( stype = agtype, sfunc = ag_catalog.age_reduce_transfn diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 6582ff8d1..72370ba06 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -2110,15 +2110,60 @@ static Query *make_reduce_var_subquery(char *acc_name, char *elem_name) } /* - * Validate a transformed-and-mutated reduce() fold body. After - * reduce_var_to_param_mutator() has replaced the accumulator and element with - * PARAM_EXEC params 0 and 1, a valid body is a pure expression over those two - * params: it must contain no other Vars (outer-query references), no other - * params, and no aggregates or subqueries, because the body is evaluated - * standalone (ExecEvalExpr) inside age_reduce_transfn with only those two - * param slots bound. + * Walker: true if the subtree references the reduce() accumulator or element, + * i.e. it contains PARAM_EXEC param 0 or 1 (assigned by + * reduce_var_to_param_mutator). Such a subtree changes per element and cannot + * be captured as a loop-invariant outer value. */ -static bool reduce_body_check_walker(Node *node, void *context) +static bool reduce_expr_has_acc_elem(Node *node, void *context) +{ + if (node == NULL) + { + return false; + } + + if (IsA(node, Param)) + { + Param *param = (Param *) node; + + if (param->paramkind == PARAM_EXEC && + (param->paramid == 0 || param->paramid == 1)) + { + return true; + } + } + + return expression_tree_walker(node, reduce_expr_has_acc_elem, context); +} + +/* + * Walker: true if the subtree contains an aggregate, grouping, or window + * function. Such a node cannot be evaluated standalone and must not be folded + * into a captured outer value (it would become an illegal nested aggregate). + */ +static bool reduce_expr_has_aggregate(Node *node, void *context) +{ + if (node == NULL) + { + return false; + } + + if (IsA(node, Aggref) || IsA(node, GroupingFunc) || IsA(node, WindowFunc)) + { + return true; + } + + return expression_tree_walker(node, reduce_expr_has_aggregate, context); +} + +/* + * Walker: true if the subtree references anything that cannot be evaluated + * standalone -- an outer-query Var or a non-PARAM_EXEC parameter (e.g. a + * cypher() $parameter, which transforms to agtype_access_operator over a + * PARAM_EXTERN). Such a subtree must be captured and supplied to the fold via + * the extras array. A subtree of only constants does not need capturing. + */ +static bool reduce_expr_needs_capture(Node *node, void *context) { if (node == NULL) { @@ -2127,24 +2172,150 @@ static bool reduce_body_check_walker(Node *node, void *context) if (IsA(node, Var)) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("a reduce() expression may only reference its accumulator and element variables"))); + return true; } if (IsA(node, Param)) { Param *param = (Param *) node; - if (param->paramkind != PARAM_EXEC || - (param->paramid != 0 && param->paramid != 1)) + if (param->paramkind != PARAM_EXEC) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("a reduce() expression may not reference query parameters"))); + return true; + } + } + + return expression_tree_walker(node, reduce_expr_needs_capture, context); +} + +/* + * Walker: true if the subtree contains a subquery (SubLink). A captured outer + * value is supplied to the aggregate as a plain expression argument, which the + * standalone fold evaluator cannot plan, so a subtree containing a subquery is + * never captured -- it falls through to the explicit rejection instead. + */ +static bool reduce_expr_has_sublink(Node *node, void *context) +{ + if (node == NULL) + { + return false; + } + + if (IsA(node, SubLink)) + { + return true; + } + + return expression_tree_walker(node, reduce_expr_has_sublink, context); +} + +/* + * Walker: true if the subtree contains an outer reference that is not itself + * agtype-typed -- a non-agtype Var, or a non-agtype non-PARAM_EXEC Param. The + * common case is the graphid component of a graph vertex/edge variable: a + * pattern variable expands to a builder over its underlying columns, one of + * which is a graphid Var. Such a value cannot stand alone as an agtype[] extra, + * so its smallest enclosing agtype-typed subtree is captured whole rather than + * being decomposed to leaves. + */ +static bool reduce_expr_has_nonagtype_outer(Node *node, void *context) +{ + if (node == NULL) + { + return false; + } + + if (IsA(node, Var)) + { + if (((Var *) node)->vartype != AGTYPEOID) + { + return true; + } + } + + if (IsA(node, Param)) + { + Param *param = (Param *) node; + + if (param->paramkind != PARAM_EXEC && param->paramtype != AGTYPEOID) + { + return true; } } + return expression_tree_walker(node, reduce_expr_has_nonagtype_outer, + context); +} + +/* + * Mutator context for capturing loop-invariant outer references in a reduce() + * fold body. Each captured subtree is assigned the next PARAM_EXEC id (starting + * at 2, after the accumulator and element) and collected, in id order, so the + * caller can supply the values to age_reduce_transfn through the extras array. + */ +typedef struct reduce_capture_context +{ + int next_slot; /* next PARAM_EXEC id to assign (starts at 2) */ + List *captured; /* captured outer-reference exprs, in slot order */ +} reduce_capture_context; + +/* + * Capture the loop-invariant outer references in a reduce() fold body. + * + * After reduce_var_to_param_mutator() has rewritten the accumulator and + * element to PARAM_EXEC params 0 and 1, the remaining outer references (outer- + * query variables and cypher() $parameters) are replaced by new PARAM_EXEC + * params 2, 3, ... and collected in slot order. Each captured value is + * loop-invariant within a fold, so the executor evaluates it once per row in + * the outer query context (as an aggregate argument) and binds it to its slot; + * the body expression is then evaluated per element by the fold. + * + * Capture is as fine-grained as the agtype[] extras argument allows, because a + * captured value becomes an aggregate argument that the executor evaluates + * eagerly and unconditionally. Hoisting a whole computed subtree out of the + * body would defeat short-circuiting: in + * reduce(s = 0, x IN [1] | CASE WHEN false THEN s + 1/z ELSE s END) + * capturing "1/z" would divide by zero even though the WHEN branch is never + * taken. So when every outer reference in a loop-invariant subtree is itself + * agtype-typed, the mutator recurses and captures only the bare leaves (here + * "z"), leaving the operators and CASE/AND/OR branches in the body under the + * fold's own control flow. A leaf read cannot raise an error, so evaluating it + * eagerly is safe; the only cost is re-evaluating a loop-invariant + * sub-computation per element, which is always correct. + * + * The exception is an outer reference that is not agtype-typed and so cannot be + * an agtype[] extra on its own -- most commonly the graphid inside a graph + * vertex/edge variable, which expands to a builder over its underlying columns. + * Such a subtree cannot be decomposed to agtype leaves, so its smallest + * enclosing agtype-typed subtree is captured whole (for example the scalar + * value of "u.vals[0]"). A property read like that cannot raise an error + * either, so eager evaluation is still safe. + * + * Aggregates and subqueries (including a nested reduce()) are rejected + * outright: an aggregate is undefined inside a per-element fold, and a subquery + * cannot be supplied as a plain aggregate argument or evaluated standalone. + */ +static Node *reduce_capture_mutator(Node *node, void *context) +{ + reduce_capture_context *ctx = (reduce_capture_context *) context; + + if (node == NULL) + { + return NULL; + } + + /* + * Container / support nodes that expression_tree_mutator hands us are not + * themselves typed expressions (calling exprType on them errors), so just + * recurse into them. For an agtype scalar fold body these are List nodes + * (argument lists) and CaseWhen nodes (CASE branches). + */ + if (IsA(node, List) || IsA(node, CaseWhen)) + { + return expression_tree_mutator(node, reduce_capture_mutator, context); + } + + /* an aggregate in the fold body is never supported */ if (IsA(node, Aggref) || IsA(node, GroupingFunc) || IsA(node, WindowFunc)) { ereport(ERROR, @@ -2152,6 +2323,62 @@ static bool reduce_body_check_walker(Node *node, void *context) errmsg("aggregate functions are not supported in a reduce() expression"))); } + /* + * A loop-invariant, agtype-typed subtree that references an outer value and + * embeds no aggregate or subquery is a capture candidate. The exprType + * guard is evaluated first and the accumulator/element and subquery tests + * short-circuit the rest, so the walkers are never run on a non-agtype + * wrapper or descended into a nested reduce()'s subquery -- both of which + * would otherwise trip the tree walker on a node it cannot type. + */ + if (exprType(node) == AGTYPEOID && + !reduce_expr_has_acc_elem(node, NULL) && + !reduce_expr_has_aggregate(node, NULL) && + !reduce_expr_has_sublink(node, NULL) && + reduce_expr_needs_capture(node, NULL)) + { + /* + * Capture the whole subtree when it is a bare outer leaf (a Var or a + * cypher() $parameter Param) or the smallest enclosing agtype-typed + * wrapper of a non-agtype outer reference -- most commonly the graphid + * of a graph vertex/edge variable, which cannot stand alone as an + * agtype[] extra. Such a value is a plain read that cannot raise an + * error, so evaluating it eagerly as an aggregate argument is safe. + */ + if (IsA(node, Var) || + (IsA(node, Param) && ((Param *) node)->paramkind != PARAM_EXEC) || + reduce_expr_has_nonagtype_outer(node, NULL)) + { + Param *param = makeNode(Param); + + param->paramkind = PARAM_EXEC; + param->paramid = ctx->next_slot++; + param->paramtype = AGTYPEOID; + param->paramtypmod = -1; + param->paramcollid = InvalidOid; + param->location = -1; + + ctx->captured = lappend(ctx->captured, copyObject(node)); + + return (Node *) param; + } + + /* + * Otherwise every outer reference in the subtree is agtype-typed and + * the node itself is a computation (an operator, function call, or CASE + * result). Recurse to capture those outer leaves individually and leave + * the computation in the body, so the fold's own control flow -- not an + * eagerly evaluated aggregate argument -- decides whether it runs. This + * is what preserves CASE/AND/OR short-circuiting. + */ + return expression_tree_mutator(node, reduce_capture_mutator, context); + } + + /* + * A subquery in the body (for example a nested reduce()) is never captured + * -- the capture test above excludes it -- and cannot be evaluated + * standalone by the fold; reject it. + */ if (IsA(node, SubLink)) { ereport(ERROR, @@ -2159,6 +2386,43 @@ static bool reduce_body_check_walker(Node *node, void *context) errmsg("subqueries (including a nested reduce()) are not supported in a reduce() expression"))); } + return expression_tree_mutator(node, reduce_capture_mutator, context); +} + +/* + * Safety net run after reduce_capture_mutator(). A valid body now references + * only PARAM_EXEC params (0/1 for the accumulator and element, 2.. for the + * captured outer values) and constants. Any remaining Var or non-PARAM_EXEC + * parameter is an outer reference that could not be captured (for example a + * non-agtype-typed one); reject it cleanly rather than letting it reach the + * standalone evaluator. + */ +static bool reduce_body_check_walker(Node *node, void *context) +{ + if (node == NULL) + { + return false; + } + + if (IsA(node, Var)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("a reduce() expression references a value that cannot be used in the fold body"))); + } + + if (IsA(node, Param)) + { + Param *param = (Param *) node; + + if (param->paramkind != PARAM_EXEC) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("a reduce() expression references a value that cannot be used in the fold body"))); + } + } + return expression_tree_walker(node, reduce_body_check_walker, context); } @@ -2170,12 +2434,17 @@ static bool reduce_body_check_walker(Node *node, void *context) * aggregate ordered by that ordinality so the fold runs in list order: * * SELECT ag_catalog.age_reduce(, ''::text, - * r.elem ORDER BY r.ord) + * r.elem, + * ORDER BY r.ord) * FROM unnest() WITH ORDINALITY AS r(elem, ord) * * The fold body is transformed separately with the accumulator and element * rewritten to PARAM_EXEC params 0 and 1, serialized into the text argument, - * and evaluated per element inside age_reduce_transfn. + * and evaluated per element inside age_reduce_transfn. Loop-invariant outer + * references in the body (outer-query variables and cypher() parameters) are + * captured as PARAM_EXEC params 2.. and passed through the trailing agtype[] + * argument so the body can use values from the enclosing query; correlated + * captures are re-evaluated per group. * * The null/empty-list guard * (CASE WHEN list IS NULL THEN NULL ELSE COALESCE(, init) END) is built @@ -2193,6 +2462,7 @@ static Query *transform_cypher_reduce(cypher_parsestate *cpstate, Node *body_node; char *body_serialized; reduce_var_param_context mutator_ctx; + reduce_capture_context capture_ctx; cypher_parsestate *child_cpstate; ParseState *child_pstate; FuncCall *unnest_fc; @@ -2210,9 +2480,11 @@ static Query *transform_cypher_reduce(cypher_parsestate *cpstate, Oid sort_eqop; bool sort_hashable; Const *body_const; + ArrayExpr *extras_arr; + List *extras_exprs = NIL; Aggref *agg; Oid agg_oid; - Oid agg_argtypes[3]; + Oid agg_argtypes[4]; TargetEntry *result_te; /* @@ -2266,6 +2538,18 @@ static Query *transform_cypher_reduce(cypher_parsestate *cpstate, mutator_ctx.varno = body_pnsi->p_rtindex; body_node = reduce_var_to_param_mutator(body_node, &mutator_ctx); + /* + * Capture loop-invariant outer references (outer-query variables and + * cypher() parameters) in the body as PARAM_EXEC params 2.. and collect + * them in slot order; their values are supplied to the fold through the + * extras array argument built below. + */ + capture_ctx.next_slot = 2; + capture_ctx.captured = NIL; + body_node = reduce_capture_mutator(body_node, &capture_ctx); + extras_exprs = capture_ctx.captured; + + /* reject anything in the body that could not be captured or evaluated */ reduce_body_check_walker(body_node, NULL); body_serialized = nodeToString(body_node); @@ -2316,7 +2600,7 @@ static Query *transform_cypher_reduce(cypher_parsestate *cpstate, get_sort_group_operators(INT8OID, true, true, false, &sort_ltop, &sort_eqop, NULL, &sort_hashable); - ord_te = makeTargetEntry((Expr *) ord_var, 4, NULL, true); + ord_te = makeTargetEntry((Expr *) ord_var, 5, NULL, true); ord_te->ressortgroupref = 1; sortcl = makeNode(SortGroupClause); @@ -2382,13 +2666,28 @@ static Query *transform_cypher_reduce(cypher_parsestate *cpstate, init_node = (Node *) init_case; } - /* look up the age_reduce(agtype, text, agtype) aggregate */ + /* + * The captured loop-invariant outer values (outer-query variables and + * cypher() parameters referenced by the body) are passed to the aggregate + * as an agtype[] argument, in the same order their PARAM_EXEC params 2, 3, + * ... were assigned. When the body references nothing outside the + * accumulator and element this is an empty array. + */ + extras_arr = makeNode(ArrayExpr); + extras_arr->array_typeid = AGTYPEARRAYOID; + extras_arr->element_typeid = AGTYPEOID; + extras_arr->elements = extras_exprs; + extras_arr->multidims = false; + extras_arr->location = -1; + + /* look up the age_reduce(agtype, text, agtype, agtype[]) aggregate */ agg_argtypes[0] = AGTYPEOID; agg_argtypes[1] = TEXTOID; agg_argtypes[2] = AGTYPEOID; + agg_argtypes[3] = AGTYPEARRAYOID; agg_oid = LookupFuncName(list_make2(makeString("ag_catalog"), makeString("age_reduce")), - 3, agg_argtypes, false); + 4, agg_argtypes, false); agg = makeNode(Aggref); agg->aggfnoid = agg_oid; @@ -2396,11 +2695,13 @@ static Query *transform_cypher_reduce(cypher_parsestate *cpstate, agg->aggcollid = InvalidOid; agg->inputcollid = InvalidOid; agg->aggtranstype = InvalidOid; /* filled by the planner */ - agg->aggargtypes = list_make3_oid(AGTYPEOID, TEXTOID, AGTYPEOID); + agg->aggargtypes = list_make4_oid(AGTYPEOID, TEXTOID, AGTYPEOID, + AGTYPEARRAYOID); agg->aggdirectargs = NIL; - agg->args = list_make4(makeTargetEntry((Expr *) init_node, 1, NULL, false), + agg->args = list_make5(makeTargetEntry((Expr *) init_node, 1, NULL, false), makeTargetEntry((Expr *) body_const, 2, NULL, false), makeTargetEntry((Expr *) elem_var, 3, NULL, false), + makeTargetEntry((Expr *) extras_arr, 4, NULL, false), ord_te); agg->aggorder = list_make1(sortcl); agg->aggdistinct = NIL; diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index 0e1a7963f..10d95ee43 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -11584,13 +11584,16 @@ Datum age_float8_stddev_pop_aggfinalfn(PG_FUNCTION_ARGS) /* * Per-aggregate-group evaluation state for reduce(). Caches the compiled * fold-body expression and a standalone ExprContext whose PARAM_EXEC slots - * (0 = accumulator, 1 = current element) are rebound on every element. + * are rebound on every element. Slot 0 = accumulator, slot 1 = current + * element, and slots 2 .. nparams-1 = captured loop-invariant outer values + * (outer-query variables and cypher() parameters referenced by the body). */ typedef struct reduce_eval_ctx { ExprState *body_state; /* compiled fold-body expression */ ExprContext *econtext; /* eval context carrying the param slots */ - ParamExecData *params; /* [0] = accumulator, [1] = current element */ + ParamExecData *params; /* [0]=accumulator, [1]=element, [2..]=outer refs */ + int nparams; /* total param slots = 2 + number of captures */ } reduce_eval_ctx; /* Build an agtype 'null' Datum (a real agtype value, not a SQL NULL). */ @@ -11603,12 +11606,17 @@ static Datum reduce_agtype_null(void) } /* - * age_reduce_transfn(state agtype, init agtype, body text, element agtype) + * age_reduce_transfn(state agtype, init agtype, body text, element agtype, + * extras agtype[]) * * Transition function for the age_reduce aggregate that implements the Cypher * reduce(acc = init, var IN list | body) fold. The fold body is compiled by * transform_cypher_reduce() with the accumulator and element rewritten to * PARAM_EXEC params 0 and 1, then serialized into the `body` text argument. + * Any loop-invariant outer-query variable or cypher() parameter referenced by + * the body is captured into the `extras` agtype array and rewritten to a + * PARAM_EXEC param 2, 3, ... in body order; those slots are bound from the + * array here. * * On the first element of a group the accumulator is seeded from `init` * (the running state is NULL because the aggregate uses no initcond); on @@ -11655,6 +11663,7 @@ Datum age_reduce_transfn(PG_FUNCTION_ARGS) text *body_txt; char *body_str; Node *body_node; + int n_extras = 0; if (PG_ARGISNULL(2)) { @@ -11663,6 +11672,25 @@ Datum age_reduce_transfn(PG_FUNCTION_ARGS) errmsg("age_reduce: missing fold expression"))); } + /* + * The number of captured outer values is fixed for this aggregate + * call (the body's structure does not change between rows), so it is + * read once here to size the param array. Their values are bound per + * row below because a correlated capture changes between groups. + * + * The PG_NARGS() guard lets the function tolerate being reached + * through an older 4-argument aggregate definition (for example a + * stale catalog paired with a newer age.so): a missing extras + * argument is simply treated as zero captures. + */ + if (PG_NARGS() > 4 && !PG_ARGISNULL(4)) + { + ArrayType *extras_arr = PG_GETARG_ARRAYTYPE_P(4); + + n_extras = ArrayGetNItems(ARR_NDIM(extras_arr), + ARR_DIMS(extras_arr)); + } + oldctx = MemoryContextSwitchTo(fcinfo->flinfo->fn_mcxt); rc = (reduce_eval_ctx *) palloc0(sizeof(reduce_eval_ctx)); body_txt = PG_GETARG_TEXT_PP(2); @@ -11689,7 +11717,9 @@ Datum age_reduce_transfn(PG_FUNCTION_ARGS) rc->body_state = ExecInitExpr((Expr *) body_node, NULL); rc->econtext = CreateStandaloneExprContext(); - rc->params = (ParamExecData *) palloc0(sizeof(ParamExecData) * 2); + rc->nparams = 2 + n_extras; + rc->params = (ParamExecData *) palloc0(sizeof(ParamExecData) * + rc->nparams); rc->econtext->ecxt_param_exec_vals = rc->params; fcinfo->flinfo->fn_extra = rc; MemoryContextSwitchTo(oldctx); @@ -11713,6 +11743,9 @@ Datum age_reduce_transfn(PG_FUNCTION_ARGS) /* a NULL element is likewise normalized to agtype 'null' */ element = PG_ARGISNULL(3) ? reduce_agtype_null() : PG_GETARG_DATUM(3); + /* evaluate the fold body for this element */ + ResetExprContext(rc->econtext); + /* bind PARAM_EXEC 0 = accumulator, 1 = current element */ rc->params[0].value = acc; rc->params[0].isnull = false; @@ -11721,8 +11754,57 @@ Datum age_reduce_transfn(PG_FUNCTION_ARGS) rc->params[1].isnull = false; rc->params[1].execPlan = NULL; - /* evaluate the fold body for this element */ - ResetExprContext(rc->econtext); + /* + * Bind the captured loop-invariant outer values to params 2 .. The values + * are pulled from the extras array every row because correlated captures + * differ between groups; the per-row deconstruction is done in the + * econtext's per-tuple memory (reset above) so it does not leak. A NULL + * array element is normalized to agtype 'null' like the accumulator and + * element. + * + * Every slot 2 .. nparams-1 is rebound on every row, so a slot never + * retains a value from a previous row -- which, after the per-tuple reset + * above, would be a dangling pointer. If the extras array supplies fewer + * values than there are capture slots (only reachable through a direct SQL + * call with a varying-length array), the unsupplied slots are filled with + * agtype 'null'. The PG_NARGS() guard keeps the arg-4 access safe under an + * older 4-argument signature. + */ + if (rc->nparams > 2 && PG_NARGS() > 4 && !PG_ARGISNULL(4)) + { + ArrayType *extras_arr = PG_GETARG_ARRAYTYPE_P(4); + Oid elemtype = ARR_ELEMTYPE(extras_arr); + int16 typlen; + bool typbyval; + char typalign; + Datum *ex_vals; + bool *ex_nulls; + int ex_n; + int i; + MemoryContext per_tuple = rc->econtext->ecxt_per_tuple_memory; + MemoryContext save = MemoryContextSwitchTo(per_tuple); + + get_typlenbyvalalign(elemtype, &typlen, &typbyval, &typalign); + deconstruct_array(extras_arr, elemtype, typlen, typbyval, typalign, + &ex_vals, &ex_nulls, &ex_n); + + for (i = 0; (2 + i) < rc->nparams; i++) + { + if (i < ex_n && !ex_nulls[i]) + { + rc->params[2 + i].value = ex_vals[i]; + } + else + { + rc->params[2 + i].value = reduce_agtype_null(); + } + rc->params[2 + i].isnull = false; + rc->params[2 + i].execPlan = NULL; + } + + MemoryContextSwitchTo(save); + } + result = ExecEvalExpr(rc->body_state, rc->econtext, &result_isnull); /* From 09fce2ced3e93bd76e237211001ad332e44c4e8b Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Thu, 2 Jul 2026 13:00:47 -0400 Subject: [PATCH 094/100] Fix segfault and out-of-bounds reads in file loaders on malformed rows (#2453) * Fix segfault and out-of-bounds reads in file loaders on malformed rows load_edges_from_file() and load_labels_from_file() build their COPY parser with only format=csv and header=false, so COPY uses its default comma delimiter. A file delimited by anything else (or a malformed row) then parses with an unexpected column count, and the loaders indexed the parsed fields without validating that count: - process_edge_row() reads the four fixed fields fields[0..3] unconditionally. A non-comma-delimited edge file parses as a single column, so fields[1..3] are out of bounds -> segfault (issue #2449). - create_agtype_from_list()/_i() pair header[i] with fields[i] for all i < nfields, so a row with more fields than the header reads header[i] out of bounds. Add bounds validation that turns these into clear errors: - Edge header must have >= 4 columns; a smaller count almost always means the wrong delimiter, so the error carries a hint. - Each edge row must have >= 4 columns and no more than the header's. - Each label row must have no more than the header's column count. Rows with fewer trailing columns than the header remain allowed, matching existing behavior (exercised by the existing conversion tests). This closes the segfault and out-of-bounds reads. The silent mis-parsing of a non-comma file whose header and rows share the same (wrong) column count is not detectable here; adding a delimiter option to the load functions is a separate follow-up. Adds a regression test in age_load using a pipe-delimited edge file. Addresses #2449. * loader guards: clarify error wording and add per-row regression coverage Address review feedback on the nfields guards: - Error messages now say "the header's %d columns" (was "the header's %d"), making the count's unit explicit. - Add regression cases exercising the per-row guards, which previously only had coverage for the mis-delimited-header path: * an edge row with fewer than 4 columns * an edge row with more columns than the header * a label row with more columns than the header Each asserts a clean ERROR (these were the out-of-bounds reads the guards now catch). --- regress/age_load/data/bad_delim_edges.csv | 2 + regress/age_load/data/edges_long_row.csv | 2 + regress/age_load/data/edges_short_row.csv | 2 + regress/age_load/data/labels_long_row.csv | 2 + regress/expected/age_load.out | 57 +++++++++++++++++++++++ regress/sql/age_load.sql | 29 ++++++++++++ src/backend/utils/load/ag_load_edges.c | 39 ++++++++++++++++ src/backend/utils/load/ag_load_labels.c | 17 +++++++ 8 files changed, 150 insertions(+) create mode 100644 regress/age_load/data/bad_delim_edges.csv create mode 100644 regress/age_load/data/edges_long_row.csv create mode 100644 regress/age_load/data/edges_short_row.csv create mode 100644 regress/age_load/data/labels_long_row.csv diff --git a/regress/age_load/data/bad_delim_edges.csv b/regress/age_load/data/bad_delim_edges.csv new file mode 100644 index 000000000..c72170a6a --- /dev/null +++ b/regress/age_load/data/bad_delim_edges.csv @@ -0,0 +1,2 @@ +start_id|start_vertex_type|end_id|end_vertex_type +1|V|2|V diff --git a/regress/age_load/data/edges_long_row.csv b/regress/age_load/data/edges_long_row.csv new file mode 100644 index 000000000..2036f534a --- /dev/null +++ b/regress/age_load/data/edges_long_row.csv @@ -0,0 +1,2 @@ +start_id,start_vertex_type,end_id,end_vertex_type +1,V,2,V,extra diff --git a/regress/age_load/data/edges_short_row.csv b/regress/age_load/data/edges_short_row.csv new file mode 100644 index 000000000..e307927b3 --- /dev/null +++ b/regress/age_load/data/edges_short_row.csv @@ -0,0 +1,2 @@ +start_id,start_vertex_type,end_id,end_vertex_type +1,V diff --git a/regress/age_load/data/labels_long_row.csv b/regress/age_load/data/labels_long_row.csv new file mode 100644 index 000000000..72ec2a305 --- /dev/null +++ b/regress/age_load/data/labels_long_row.csv @@ -0,0 +1,2 @@ +id,name +1,Alice,extra diff --git a/regress/expected/age_load.out b/regress/expected/age_load.out index 1f76c31ce..17c5ecc27 100644 --- a/regress/expected/age_load.out +++ b/regress/expected/age_load.out @@ -454,6 +454,63 @@ NOTICE: graph "agload_conversion" has been dropped (1 row) +-- +-- Issue 2449: mis-delimited / malformed load files must fail with a clear +-- error instead of segfaulting or silently corrupting data. Edge files +-- require the 4 fixed columns; a file that is not comma-delimited parses as +-- a single column, so this must be rejected at the header. +-- +SELECT create_graph('agload_delim'); +NOTICE: graph "agload_delim" has been created + create_graph +-------------- + +(1 row) + +SELECT create_vlabel('agload_delim', 'V'); +NOTICE: VLabel "V" has been created + create_vlabel +--------------- + +(1 row) + +SELECT create_elabel('agload_delim', 'E'); +NOTICE: ELabel "E" has been created + create_elabel +--------------- + +(1 row) + +-- pipe-delimited edge file -> parses to 1 column -> clean error at the header +-- (was a segfault) +SELECT load_edges_from_file('agload_delim', 'E', 'age_load/bad_delim_edges.csv'); +ERROR: edge file must have at least 4 columns (start_id, start_vertex_type, end_id, end_vertex_type), but the header has 1 +HINT: load_edges_from_file expects a comma-delimited CSV; check the file's delimiter. +-- per-row guards (header is valid, but an individual data row is ragged): +-- an edge row with fewer than 4 columns -> clean error (was an OOB read of +-- the fixed fields[1..3]) +SELECT load_edges_from_file('agload_delim', 'E', 'age_load/edges_short_row.csv'); +ERROR: edge file row has 2 columns; expected at least 4 and no more than the header's 4 columns +-- an edge row with more columns than the header -> clean error (was an OOB +-- read of header[i] in create_agtype_from_list_i) +SELECT load_edges_from_file('agload_delim', 'E', 'age_load/edges_long_row.csv'); +ERROR: edge file row has 5 columns; expected at least 4 and no more than the header's 4 columns +-- a label row with more columns than the header -> clean error (was an OOB +-- read of header[i] in create_agtype_from_list) +SELECT load_labels_from_file('agload_delim', 'V', 'age_load/labels_long_row.csv'); +ERROR: label file row has 3 columns, more than the header's 2 columns +SELECT drop_graph('agload_delim', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table agload_delim._ag_label_vertex +drop cascades to table agload_delim._ag_label_edge +drop cascades to table agload_delim."V" +drop cascades to table agload_delim."E" +NOTICE: graph "agload_delim" has been dropped + drop_graph +------------ + +(1 row) + -- -- Test security and permissions -- diff --git a/regress/sql/age_load.sql b/regress/sql/age_load.sql index 976f050af..196b09806 100644 --- a/regress/sql/age_load.sql +++ b/regress/sql/age_load.sql @@ -194,6 +194,35 @@ SELECT load_edges_from_file('agload_conversion', 'Edges1', '../../etc/passwd', t -- SELECT drop_graph('agload_conversion', true); +-- +-- Issue 2449: mis-delimited / malformed load files must fail with a clear +-- error instead of segfaulting or silently corrupting data. Edge files +-- require the 4 fixed columns; a file that is not comma-delimited parses as +-- a single column, so this must be rejected at the header. +-- +SELECT create_graph('agload_delim'); +SELECT create_vlabel('agload_delim', 'V'); +SELECT create_elabel('agload_delim', 'E'); + +-- pipe-delimited edge file -> parses to 1 column -> clean error at the header +-- (was a segfault) +SELECT load_edges_from_file('agload_delim', 'E', 'age_load/bad_delim_edges.csv'); + +-- per-row guards (header is valid, but an individual data row is ragged): +-- an edge row with fewer than 4 columns -> clean error (was an OOB read of +-- the fixed fields[1..3]) +SELECT load_edges_from_file('agload_delim', 'E', 'age_load/edges_short_row.csv'); + +-- an edge row with more columns than the header -> clean error (was an OOB +-- read of header[i] in create_agtype_from_list_i) +SELECT load_edges_from_file('agload_delim', 'E', 'age_load/edges_long_row.csv'); + +-- a label row with more columns than the header -> clean error (was an OOB +-- read of header[i] in create_agtype_from_list) +SELECT load_labels_from_file('agload_delim', 'V', 'age_load/labels_long_row.csv'); + +SELECT drop_graph('agload_delim', true); + -- -- Test security and permissions -- diff --git a/src/backend/utils/load/ag_load_edges.c b/src/backend/utils/load/ag_load_edges.c index c05bf3352..01585bab0 100644 --- a/src/backend/utils/load/ag_load_edges.c +++ b/src/backend/utils/load/ag_load_edges.c @@ -56,6 +56,24 @@ static void process_edge_row(char **fields, int nfields, char *end_vertex_type; agtype *edge_properties; + /* + * Guard the fixed fields[0..3] accesses below and the header[i]/fields[i] + * pairing in create_agtype_from_list_i() against out-of-bounds reads on + * malformed or mis-delimited rows. A row must have at least the 4 fixed + * columns and no more columns than the header (rows with fewer trailing + * property columns than the header are allowed, matching existing + * behavior). A single-column row from a non-comma-delimited file is + * rejected here (previously it segfaulted). + */ + if (nfields < 4 || nfields > header_count) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("edge file row has %d columns; expected at least 4 " + "and no more than the header's %d columns", + nfields, header_count))); + } + /* Generate edge ID */ entry_id = nextval_internal(label_seq_relid, true); edge_id = make_graphid(label_id, entry_id); @@ -219,6 +237,27 @@ int create_edges_from_csv_file(char *file_path, header[i] = trim_whitespace(fields[i]); } + /* + * Edge files require the four fixed columns start_id, + * start_vertex_type, end_id and end_vertex_type. A smaller + * count almost always means the file is not comma-delimited + * (COPY defaults to comma). Fail clearly here instead of + * reading past the parsed fields in process_edge_row(), which + * previously caused a segfault. + */ + if (header_count < 4) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("edge file must have at least 4 columns " + "(start_id, start_vertex_type, end_id, " + "end_vertex_type), but the header has %d", + header_count), + errhint("load_edges_from_file expects a " + "comma-delimited CSV; check the file's " + "delimiter."))); + } + is_first_row = false; } else diff --git a/src/backend/utils/load/ag_load_labels.c b/src/backend/utils/load/ag_load_labels.c index 5b11f68b8..236d47a1d 100644 --- a/src/backend/utils/load/ag_load_labels.c +++ b/src/backend/utils/load/ag_load_labels.c @@ -46,6 +46,23 @@ static void process_vertex_row(char **fields, int nfields, TupleTableSlot *slot; agtype *vertex_properties; + /* + * Guard the header[i]/fields[i] pairing in create_agtype_from_list() + * against out-of-bounds reads on malformed rows that have more fields + * than the header. Rows with fewer fields than the header are allowed + * (matching existing behavior). Note: a file delimited by something + * other than comma is parsed as a single column throughout, so header + * and rows still match and the data lands in properties verbatim -- + * specifying the delimiter is the separate fix for that. + */ + if (nfields > header_count) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("label file row has %d columns, more than the " + "header's %d columns", nfields, header_count))); + } + /* Generate or use provided entry_id */ if (id_field_exists) { From c1617a2628eedbfe10c3332791bc7dde2f3cba44 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Thu, 2 Jul 2026 13:02:07 -0400 Subject: [PATCH 095/100] ci: pin Build/Regression runner to ubuntu-24.04 and guard Bison version for GLR grammar (#2445) * ci: pin runner to ubuntu-24.04 + guard Bison version for GLR grammar The Cypher GLR grammar pins exact shift/reduce and reduce/reduce conflict counts via %expect / %expect-rr in cypher_gram.y, and Bison treats %expect as exact-match: a different Bison version can report different counts and break the build. ubuntu-latest floats to new Ubuntu releases (and new Bison versions), which would silently shift those counts. Pin runs-on to ubuntu-24.04 to freeze Bison at 3.8.x, and add a guard step that fails loudly with a pointer to cypher_gram.y if Bison ever drifts off 3.8.x. Reproducibility comes from pinning the variable rather than widening the conflict-count tolerance, keeping the exact-match alarm for genuinely new grammar conflicts intact. * ci: make Bison version parse robust (awk + explicit empty guard) Address Copilot review on #2445: the previous grep-based parse could silently yield an empty version and fail with a confusing 'Bison != 3.8.x' message. Parse the version field with awk and error explicitly when it can't be determined. --- .github/workflows/installcheck.yaml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/installcheck.yaml b/.github/workflows/installcheck.yaml index 886dc08f1..aa30ef3d7 100644 --- a/.github/workflows/installcheck.yaml +++ b/.github/workflows/installcheck.yaml @@ -9,7 +9,12 @@ on: jobs: build: - runs-on: ubuntu-latest + # Pinned (not ubuntu-latest) so the Bison version stays fixed at 3.8.x. + # The Cypher GLR grammar pins exact conflict counts via %expect / %expect-rr + # in src/backend/parser/cypher_gram.y, and Bison treats %expect as exact-match: + # a different Bison version can report different counts and break the build. + # Freezing the runner image freezes Bison; bump both together, intentionally. + runs-on: ubuntu-24.04 steps: - name: Get latest commit id of PostgreSQL 18 @@ -28,6 +33,27 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential libreadline-dev zlib1g-dev flex bison + - name: Verify Bison version (grammar conflict counts are pinned) + run: | + ver=$(bison --version | awk 'NR==1 {print $NF}') + if [ -z "$ver" ]; then + echo "::error::Could not determine Bison version from 'bison --version'." + echo "::error::Expected the first line to end with a version (e.g. '... 3.8.2')." + exit 1 + fi + echo "bison $ver" + case "$ver" in + 3.8.*) ;; + *) + echo "::error::Bison $ver != 3.8.x. The Cypher GLR grammar pins exact" + echo "::error::%expect / %expect-rr conflict counts in src/backend/parser/cypher_gram.y." + echo "::error::A new Bison version may report different counts. Re-run bison locally," + echo "::error::update the %expect/%expect-rr numbers (and the comment block), then bump" + echo "::error::the pinned runner image and this guard together." + exit 1 + ;; + esac + - name: Install PostgreSQL 18 and some extensions if: steps.pg18cache.outputs.cache-hit != 'true' run: | From d84b4578fe7551211d8bf5fe8d6adf3bcc086e78 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Thu, 2 Jul 2026 10:23:39 -0700 Subject: [PATCH 096/100] Add automatic header-dependency tracking to the Makefile (#2454) AGE lists OBJS explicitly and relies on PGXS, whose built-in dependency tracking only runs when the server was built with --enable-depend (often disabled). Consequently, editing a header did not recompile the .c files that include it, leaving stale .o files. This is especially dangerous for node/struct headers: a stale ag_nodes.o keeps an outdated node_size, so _readExtensibleNode under-allocates and readNode corrupts the heap ("unrecognized node type"). Emit a .d file beside each object via -MMD -MP and -include them, deriving DEPFILES from OBJS. The mechanism is self-contained (independent of --enable-depend): -MMD skips system headers and -MP tolerates deleted headers. On servers built with --enable-depend, PGXS appends its own -MF after CFLAGS (last -MF wins), so this degrades cleanly to PGXS's tracking. Add DEPFILES to EXTRA_CLEAN and *.d to .gitignore. Co-authored-by: Copilot modified: .gitignore modified: Makefile --- .gitignore | 1 + Makefile | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1e2f8f674..98f4a7f9a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.o +*.d *.so build.sh .idea diff --git a/Makefile b/Makefile index 21c7f81f2..32f34d6e6 100644 --- a/Makefile +++ b/Makefile @@ -162,6 +162,10 @@ OBJS = src/backend/age.o \ src/backend/utils/name_validation.o \ src/backend/utils/ag_guc.o +# Per-object header-dependency files (see "Automatic header-dependency +# tracking" below the PGXS include). One .d is generated beside each .o. +DEPFILES = $(OBJS:.o=.d) + # ===== Extension SQL & data files ===== EXTENSION = age @@ -258,7 +262,8 @@ EXTRA_CLEAN = $(addprefix $(ag_regress_dir)/, $(ag_regress_out)) \ $(all_age_sql) \ $(age_init_sql) \ $(age_upgrade_test_sql) \ - $(ag_regress_dir)/age_upgrade_cleanup.sh + $(ag_regress_dir)/age_upgrade_cleanup.sh \ + $(DEPFILES) GEN_KEYWORDLIST = $(PERL) -I ./tools/ ./tools/gen_keywordlist.pl GEN_KEYWORDLIST_DEPS = ./tools/gen_keywordlist.pl tools/PerfectHash.pm @@ -271,6 +276,23 @@ PG_CONFIG ?= pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) +# ===== Automatic header-dependency tracking ===== +# +# AGE lists OBJS explicitly, and PGXS's built-in .deps tracking only runs when +# the *server* was built with --enable-depend (often off). Without the lines +# below, editing a header does NOT rebuild the .c files that include it, leaving +# STALE .o files. This is especially dangerous for node/struct headers: a stale +# ag_nodes.o keeps an old node_size, so _readExtensibleNode under-allocates and +# readNode corrupts the heap ("unrecognized node type: "). +# +# The compiler emits a .d file next to each object (-MMD = user headers only; +# -MP adds phony targets so deleting a header does not break the build). With +# "-o foo.o", -MMD writes "foo.d" automatically (no -MF, no basename clashes). +# On servers that DO set --enable-depend, PGXS appends its own "-MF .deps/*.Po" +# after $(CFLAGS) (last -MF wins), so this degrades cleanly to that mechanism. +override CFLAGS += -MMD -MP +-include $(DEPFILES) + # ===== Build rules ===== # 32-bit platform support: pass SIZEOF_DATUM=4 to enable (e.g., make SIZEOF_DATUM=4) From 39273ca8f6687d6cf0e723045ae1407c879f263f Mon Sep 17 00:00:00 2001 From: Muhammad Taha Naveed Date: Fri, 3 Jul 2026 16:20:59 +0500 Subject: [PATCH 097/100] Propagate null through unnest and single() three-valued logic (#2406) Fix age_unnest to emit SQL NULL for AGTV_NULL elements and rewrite single() to proper three-valued logic, fixing UNWIND [null] count, list-comprehension null filters, and single() semantics per openCypher. Closes #2383 Closes #2393 --- regress/expected/list_comprehension.out | 33 ++++ regress/expected/predicate_functions.out | 116 +++++++++++-- regress/sql/list_comprehension.sql | 8 + regress/sql/predicate_functions.sql | 75 +++++++-- src/backend/parser/cypher_clause.c | 199 ++++++++++++++--------- src/backend/parser/cypher_gram.y | 37 +---- src/backend/utils/adt/agtype.c | 20 ++- 7 files changed, 355 insertions(+), 133 deletions(-) diff --git a/regress/expected/list_comprehension.out b/regress/expected/list_comprehension.out index e47b2d569..63b246479 100644 --- a/regress/expected/list_comprehension.out +++ b/regress/expected/list_comprehension.out @@ -809,6 +809,39 @@ SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | -x] $$ [-1, null, -2] (1 row) +-- Issue 2393 - WHERE filter over null elements should use openCypher's +-- three-valued logic: IS NULL must keep nulls, IS NOT NULL must drop them. +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [null, 1] WHERE x IS NULL] $$) AS (result agtype); + result +-------- + [null] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [null, 1, null] WHERE x IS NULL] $$) AS (result agtype); + result +-------------- + [null, null] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [null, 1] WHERE x IS NOT NULL] $$) AS (result agtype); + result +-------- + [1] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, 2, 3] WHERE x IS NULL] $$) AS (result agtype); + result +-------- + [] +(1 row) + +SELECT * FROM cypher('list_comprehension', $$ UNWIND [null, 1] AS x RETURN x, x IS NULL, x IS NOT NULL $$) AS (x agtype, a agtype, b agtype); + x | a | b +---+-------+------- + | true | false + 1 | false | true +(2 rows) + -- Clean up SELECT * FROM drop_graph('list_comprehension', true); NOTICE: drop cascades to 4 other objects diff --git a/regress/expected/predicate_functions.out b/regress/expected/predicate_functions.out index 9ee673a4e..f6957c49f 100644 --- a/regress/expected/predicate_functions.out +++ b/regress/expected/predicate_functions.out @@ -194,20 +194,20 @@ $$) AS (result agtype); -- -- NULL predicate results: three-valued logic -- --- Note: In AGE's agtype, null is a first-class value. The comparison --- agtype_null > agtype_integer evaluates to true (not SQL NULL). --- Three-valued logic only applies when the predicate itself is a --- literal null constant, which becomes SQL NULL after coercion. --- agtype null in list: null > 0 = true in AGE, so any() = true +-- Null list elements arrive at the predicate as SQL NULL, so the usual +-- strict-operator short-circuit applies: `null > 0` yields NULL, and the +-- predicate functions combine NULLs with the openCypher three-valued +-- logic (true trumps null in any(), false trumps null in all(), etc.). +-- [null]: only null predicate, no true -> any() = NULL SELECT * FROM cypher('predicate_functions', $$ RETURN any(x IN [null] WHERE x > 0) $$) AS (result agtype); result -------- - true + (1 row) --- agtype null + real values: all comparisons are true +-- one true (1 > 0) is enough: any() = true SELECT * FROM cypher('predicate_functions', $$ RETURN any(x IN [null, 1, 2] WHERE x > 0) $$) AS (result agtype); @@ -226,16 +226,16 @@ $$) AS (result agtype); (1 row) --- agtype null in list: null > 0 = true in AGE, so all() = true +-- no false, but one null -> all() = NULL SELECT * FROM cypher('predicate_functions', $$ RETURN all(x IN [1, null, 2] WHERE x > 0) $$) AS (result agtype); result -------- - true + (1 row) --- -1 > 0 = false, so all() = false +-- -1 > 0 = false dominates the null -> all() = false SELECT * FROM cypher('predicate_functions', $$ RETURN all(x IN [1, null, -1] WHERE x > 0) $$) AS (result agtype); @@ -244,16 +244,16 @@ $$) AS (result agtype); false (1 row) --- agtype null > 0 = true in AGE, so none() = false +-- [null]: only null predicate, no true -> none() = NULL SELECT * FROM cypher('predicate_functions', $$ RETURN none(x IN [null] WHERE x > 0) $$) AS (result agtype); result -------- - false + (1 row) --- 5 > 0 = true, so none() = false +-- one true (5 > 0) dominates: none() = false SELECT * FROM cypher('predicate_functions', $$ RETURN none(x IN [null, 5] WHERE x > 0) $$) AS (result agtype); @@ -262,24 +262,108 @@ $$) AS (result agtype); false (1 row) --- agtype null > 0 = true AND 5 > 0 = true: 2 matches, single = false +-- one definite true (5 > 0) and one null predicate: the null could also +-- be a match, so we cannot conclude exactly-one -> single() = NULL SELECT * FROM cypher('predicate_functions', $$ RETURN single(x IN [null, 5] WHERE x > 0) $$) AS (result agtype); result -------- + +(1 row) + +-- two definite trues dominate any null -> single() = false +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [null, 5, 6] WHERE x > 0) +$$) AS (result agtype); + result +-------- false (1 row) --- single() with null list: NULL (same as other predicate functions) +-- only null predicates -> single() = NULL SELECT * FROM cypher('predicate_functions', $$ - RETURN single(x IN null WHERE x > 0) + RETURN single(x IN [null, null] WHERE x > 0) +$$) AS (result agtype); + result +-------- + +(1 row) + +-- one true + one false + one null: the null could be the second true +-- so we cannot conclude exactly-one -> single() = NULL +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [1, null, -1] WHERE x > 0) +$$) AS (result agtype); + result +-------- + +(1 row) + +-- +-- Additional null/three-valued coverage for any()/all()/none() +-- +-- any() with no true and one null -> NULL +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [null, -1] WHERE x > 0) +$$) AS (result agtype); + result +-------- + +(1 row) + +-- all() with one true and one null -> NULL (the null might be false) +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1, null] WHERE x > 0) +$$) AS (result agtype); + result +-------- + +(1 row) + +-- all() with one definite false dominates any null -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [null, -1] WHERE x > 0) +$$) AS (result agtype); + result +-------- + false +(1 row) + +-- none() with no true and one null -> NULL +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [null, -1] WHERE x > 0) $$) AS (result agtype); result -------- (1 row) +-- IS NULL predicate now sees unwound null elements (issue #2393) +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [1, null] WHERE x IS NULL) +$$) AS (result agtype); + result +-------- + true +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [1, null] WHERE x IS NULL) +$$) AS (result agtype); + result +-------- + false +(1 row) + +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [null, null] WHERE x IS NULL) +$$) AS (result agtype); + result +-------- + true +(1 row) + -- -- Integration with graph data -- diff --git a/regress/sql/list_comprehension.sql b/regress/sql/list_comprehension.sql index 57b43221b..e493da6df 100644 --- a/regress/sql/list_comprehension.sql +++ b/regress/sql/list_comprehension.sql @@ -194,5 +194,13 @@ SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x % 2] SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | x ^ 2] $$) AS (result agtype); SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, null, 2] | -x] $$) AS (result agtype); +-- Issue 2393 - WHERE filter over null elements should use openCypher's +-- three-valued logic: IS NULL must keep nulls, IS NOT NULL must drop them. +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [null, 1] WHERE x IS NULL] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [null, 1, null] WHERE x IS NULL] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [null, 1] WHERE x IS NOT NULL] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ RETURN [x IN [1, 2, 3] WHERE x IS NULL] $$) AS (result agtype); +SELECT * FROM cypher('list_comprehension', $$ UNWIND [null, 1] AS x RETURN x, x IS NULL, x IS NOT NULL $$) AS (x agtype, a agtype, b agtype); + -- Clean up SELECT * FROM drop_graph('list_comprehension', true); \ No newline at end of file diff --git a/regress/sql/predicate_functions.sql b/regress/sql/predicate_functions.sql index 82933f87a..ad99a5825 100644 --- a/regress/sql/predicate_functions.sql +++ b/regress/sql/predicate_functions.sql @@ -123,17 +123,17 @@ $$) AS (result agtype); -- -- NULL predicate results: three-valued logic -- --- Note: In AGE's agtype, null is a first-class value. The comparison --- agtype_null > agtype_integer evaluates to true (not SQL NULL). --- Three-valued logic only applies when the predicate itself is a --- literal null constant, which becomes SQL NULL after coercion. +-- Null list elements arrive at the predicate as SQL NULL, so the usual +-- strict-operator short-circuit applies: `null > 0` yields NULL, and the +-- predicate functions combine NULLs with the openCypher three-valued +-- logic (true trumps null in any(), false trumps null in all(), etc.). --- agtype null in list: null > 0 = true in AGE, so any() = true +-- [null]: only null predicate, no true -> any() = NULL SELECT * FROM cypher('predicate_functions', $$ RETURN any(x IN [null] WHERE x > 0) $$) AS (result agtype); --- agtype null + real values: all comparisons are true +-- one true (1 > 0) is enough: any() = true SELECT * FROM cypher('predicate_functions', $$ RETURN any(x IN [null, 1, 2] WHERE x > 0) $$) AS (result agtype); @@ -144,34 +144,83 @@ SELECT * FROM cypher('predicate_functions', $$ RETURN all(x IN [1] WHERE null) $$) AS (result agtype); --- agtype null in list: null > 0 = true in AGE, so all() = true +-- no false, but one null -> all() = NULL SELECT * FROM cypher('predicate_functions', $$ RETURN all(x IN [1, null, 2] WHERE x > 0) $$) AS (result agtype); --- -1 > 0 = false, so all() = false +-- -1 > 0 = false dominates the null -> all() = false SELECT * FROM cypher('predicate_functions', $$ RETURN all(x IN [1, null, -1] WHERE x > 0) $$) AS (result agtype); --- agtype null > 0 = true in AGE, so none() = false +-- [null]: only null predicate, no true -> none() = NULL SELECT * FROM cypher('predicate_functions', $$ RETURN none(x IN [null] WHERE x > 0) $$) AS (result agtype); --- 5 > 0 = true, so none() = false +-- one true (5 > 0) dominates: none() = false SELECT * FROM cypher('predicate_functions', $$ RETURN none(x IN [null, 5] WHERE x > 0) $$) AS (result agtype); --- agtype null > 0 = true AND 5 > 0 = true: 2 matches, single = false +-- one definite true (5 > 0) and one null predicate: the null could also +-- be a match, so we cannot conclude exactly-one -> single() = NULL SELECT * FROM cypher('predicate_functions', $$ RETURN single(x IN [null, 5] WHERE x > 0) $$) AS (result agtype); --- single() with null list: NULL (same as other predicate functions) +-- two definite trues dominate any null -> single() = false SELECT * FROM cypher('predicate_functions', $$ - RETURN single(x IN null WHERE x > 0) + RETURN single(x IN [null, 5, 6] WHERE x > 0) +$$) AS (result agtype); + +-- only null predicates -> single() = NULL +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [null, null] WHERE x > 0) +$$) AS (result agtype); + +-- one true + one false + one null: the null could be the second true +-- so we cannot conclude exactly-one -> single() = NULL +SELECT * FROM cypher('predicate_functions', $$ + RETURN single(x IN [1, null, -1] WHERE x > 0) +$$) AS (result agtype); + +-- +-- Additional null/three-valued coverage for any()/all()/none() +-- + +-- any() with no true and one null -> NULL +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [null, -1] WHERE x > 0) +$$) AS (result agtype); + +-- all() with one true and one null -> NULL (the null might be false) +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [1, null] WHERE x > 0) +$$) AS (result agtype); + +-- all() with one definite false dominates any null -> false +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [null, -1] WHERE x > 0) +$$) AS (result agtype); + +-- none() with no true and one null -> NULL +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [null, -1] WHERE x > 0) +$$) AS (result agtype); + +-- IS NULL predicate now sees unwound null elements (issue #2393) +SELECT * FROM cypher('predicate_functions', $$ + RETURN any(x IN [1, null] WHERE x IS NULL) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + RETURN none(x IN [1, null] WHERE x IS NULL) +$$) AS (result agtype); + +SELECT * FROM cypher('predicate_functions', $$ + RETURN all(x IN [null, null] WHERE x IS NULL) $$) AS (result agtype); -- diff --git a/src/backend/parser/cypher_clause.c b/src/backend/parser/cypher_clause.c index 72370ba06..147e3e74e 100644 --- a/src/backend/parser/cypher_clause.c +++ b/src/backend/parser/cypher_clause.c @@ -56,6 +56,10 @@ #include "utils/ag_func.h" #include "utils/ag_guc.h" +#ifndef INT8PASSBYVAL +#define INT8PASSBYVAL FLOAT8PASSBYVAL +#endif + /* * Variable string names for makeTargetEntry. As they are going to be variable * names that will be hidden from the user, we need to do our best to make sure @@ -1764,25 +1768,71 @@ static Node *make_bool_or_agg(ParseState *pstate, Node *arg) return (Node *) agg; } +/* + * Helper: build a fully-transformed `count(*) FILTER (WHERE filter)` Aggref. + * + * The filter must already be a transformed boolean expression. + */ +static Node *make_count_star_filter_agg(ParseState *pstate, Node *filter) +{ + Aggref *agg; + Oid count_oid; + + /* count() -- the zero-argument count-star form */ + count_oid = LookupFuncName(list_make1(makeString("count")), + 0, NULL, false); + + agg = makeNode(Aggref); + agg->aggfnoid = count_oid; + agg->aggtype = INT8OID; + agg->aggcollid = InvalidOid; + agg->inputcollid = InvalidOid; + agg->aggtranstype = InvalidOid; /* filled by planner */ + agg->aggargtypes = NIL; + agg->aggdirectargs = NIL; + agg->args = NIL; + agg->aggorder = NIL; + agg->aggdistinct = NIL; + agg->aggfilter = (Expr *) filter; + agg->aggstar = true; + agg->aggvariadic = false; + agg->aggkind = AGGKIND_NORMAL; + agg->aggpresorted = false; + agg->agglevelsup = 0; + agg->aggsplit = AGGSPLIT_SIMPLE; + agg->aggno = -1; + agg->aggtransno = -1; + agg->location = -1; + + pstate->p_hasAggs = true; + + return (Node *) agg; +} + /* * Helper: build a transformed CASE expression implementing three-valued - * predicate logic for all(), any(), and none(). + * predicate logic for all(), any(), none(), and single(). + * + * any(): CASE WHEN bool_or(pred IS TRUE) THEN true + * WHEN bool_or(pred IS NULL) THEN NULL + * ELSE false END * - * any(): CASE WHEN bool_or(pred IS TRUE) THEN true - * WHEN bool_or(pred IS NULL) THEN NULL - * ELSE false END + * all(): CASE WHEN bool_or(pred IS FALSE) THEN false + * WHEN bool_or(pred IS NULL) THEN NULL + * ELSE true END * - * all(): CASE WHEN bool_or(pred IS FALSE) THEN false - * WHEN bool_or(pred IS NULL) THEN NULL - * ELSE true END + * none(): CASE WHEN bool_or(pred IS TRUE) THEN false + * WHEN bool_or(pred IS NULL) THEN NULL + * ELSE true END * - * none(): CASE WHEN bool_or(pred IS TRUE) THEN false - * WHEN bool_or(pred IS NULL) THEN NULL - * ELSE true END + * single(): CASE WHEN count(*) FILTER (WHERE pred IS TRUE) >= 2 THEN false + * WHEN bool_or(pred IS NULL) THEN NULL + * WHEN count(*) FILTER (WHERE pred IS TRUE) = 1 THEN true + * ELSE false END * - * Empty list: both bool_or calls return NULL (no rows), so the CASE - * falls through to the default: false for any(), true for all()/none(). - * This matches Cypher's vacuous truth semantics. + * Empty list: bool_or returns NULL on zero rows and count(*) returns 0, so + * the CASE falls through to the default: false for any()/single(), true for + * all()/none(). This matches Cypher's vacuous truth semantics. */ static Node *make_predicate_case_expr(ParseState *pstate, Node *pred, cypher_predicate_function_kind kind) @@ -1840,7 +1890,7 @@ static Node *make_predicate_case_expr(ParseState *pstate, Node *pred, cexpr->defresult = (Expr *) false_const; cexpr->location = -1; } - else /* CPFK_NONE */ + else if (kind == CPFK_NONE) { /* bool_or(pred IS TRUE) -> false */ bool_or_first = make_bool_or_agg(pstate, @@ -1857,6 +1907,62 @@ static Node *make_predicate_case_expr(ParseState *pstate, Node *pred, cexpr->defresult = (Expr *) true_const; cexpr->location = -1; } + else /* CPFK_SINGLE */ + { + /* + * Three WHEN arms, in order: + * count(*) FILTER (pred IS TRUE) >= 2 -> false (definitely >1) + * bool_or(pred IS NULL) -> NULL (unknown arm) + * count(*) FILTER (pred IS TRUE) = 1 -> true (exactly one) + * else -> false (zero true) + * + * The >=2 arm is tested first so that 2-or-more definite trues + * win over any null predicates, matching Neo4j semantics. + */ + CaseWhen *when0, *when3; + Node *count_ge_2, *count_eq_1; + Node *count_true_a, *count_true_b; + Node *int_2 = (Node *) makeConst(INT8OID, -1, InvalidOid, sizeof(int64), + Int64GetDatum(2), false, INT8PASSBYVAL); + Node *int_1 = (Node *) makeConst(INT8OID, -1, InvalidOid, sizeof(int64), + Int64GetDatum(1), false, INT8PASSBYVAL); + + /* + * Two separate Aggref nodes for count(*) FILTER (pred IS TRUE); + * the planner will common-subexpression them back to a single + * aggregate evaluation. + */ + count_true_a = make_count_star_filter_agg(pstate, + make_boolean_test(pred, IS_TRUE)); + count_true_b = make_count_star_filter_agg(pstate, + make_boolean_test(pred, IS_TRUE)); + + count_ge_2 = (Node *) make_op(pstate, list_make1(makeString(">=")), + count_true_a, int_2, + pstate->p_last_srf, -1); + count_eq_1 = (Node *) make_op(pstate, list_make1(makeString("=")), + count_true_b, int_1, + pstate->p_last_srf, -1); + + /* first arm: count >= 2 -> false */ + when0 = makeNode(CaseWhen); + when0->expr = (Expr *) count_ge_2; + when0->result = (Expr *) false_const; + when0->location = -1; + + /* third arm: count = 1 -> true */ + when3 = makeNode(CaseWhen); + when3->expr = (Expr *) count_eq_1; + when3->result = (Expr *) true_const; + when3->location = -1; + + cexpr = makeNode(CaseExpr); + cexpr->casetype = BOOLOID; + cexpr->arg = NULL; + cexpr->args = list_make3(when0, when2, when3); + cexpr->defresult = (Expr *) false_const; + cexpr->location = -1; + } return (Node *) cexpr; } @@ -1933,67 +2039,12 @@ static Query *transform_cypher_predicate_function(cypher_parsestate *cpstate, pred = coerce_to_boolean(child_pstate, pred, "WHERE"); } - if (pred_func->kind == CPFK_SINGLE) - { - /* - * single(): SELECT count(*) FROM unnest(list) AS x - * WHERE pred IS TRUE - * - * Using IS TRUE ensures NULL predicates are not counted as - * matches, preserving correct semantics. The grammar layer - * compares the result = 1. - * - * Note: a LIMIT 2 optimization (to short-circuit after two - * matches) would require a nested subquery that breaks - * correlated variable references. Deferred to a future - * optimization pass. - */ - FuncCall *count_call; - Node *count_expr; - Node *is_true_qual; - - /* WHERE pred IS TRUE -- NULLs are not counted */ - is_true_qual = make_boolean_test(pred, IS_TRUE); - - count_call = makeFuncCall(list_make1(makeString("count")), - NIL, COERCE_SQL_SYNTAX, -1); - count_call->agg_star = true; - - count_expr = transformExpr(child_pstate, (Node *) count_call, - EXPR_KIND_SELECT_TARGET); - - te = makeTargetEntry((Expr *) count_expr, - (AttrNumber) child_pstate->p_next_resno++, - "count", false); - - query->targetList = lappend(query->targetList, te); - query->jointree = makeFromExpr(child_pstate->p_joinlist, - is_true_qual); - query->rtable = child_pstate->p_rtable; - query->rteperminfos = child_pstate->p_rteperminfos; - query->hasAggs = child_pstate->p_hasAggs; - query->hasSubLinks = child_pstate->p_hasSubLinks; - query->hasTargetSRFs = child_pstate->p_hasTargetSRFs; - - assign_query_collations(child_pstate, query); - - if (child_pstate->p_hasAggs || - query->groupClause || query->groupingSets || query->havingQual) - { - parse_check_aggregates(child_pstate, query); - } - - free_cypher_parsestate(child_cpstate); - - return query; - } - else + /* + * Build a CASE expression that preserves three-valued NULL semantics. + * No WHERE clause -- the logic is entirely in the SELECT list, so the + * CASE can see null predicates and react to them. + */ { - /* - * all()/any()/none(): Build a CASE expression with bool_or() - * aggregates that preserves three-valued NULL semantics. - * No WHERE clause -- the logic is entirely in the SELECT list. - */ Node *case_expr; case_expr = make_predicate_case_expr(child_pstate, pred, diff --git a/src/backend/parser/cypher_gram.y b/src/backend/parser/cypher_gram.y index ddd62d7ee..71b95bde3 100644 --- a/src/backend/parser/cypher_gram.y +++ b/src/backend/parser/cypher_gram.y @@ -3538,12 +3538,10 @@ static Node *build_list_comprehension_node(Node *var, Node *expr, * none(x IN list WHERE predicate) * single(x IN list WHERE predicate) * - * All four use EXPR_SUBLINK (scalar subquery). The transform layer - * generates aggregate-based queries (using bool_or + CASE) for - * all/any/none to preserve three-valued NULL semantics, and - * count(*) with IS TRUE filtering for single(). - * - * For single(), the subquery result is compared = 1. + * All four use EXPR_SUBLINK (scalar subquery). The transform layer + * generates aggregate-based queries that preserve three-valued NULL + * semantics: bool_or + CASE for all/any/none, and a CASE built on + * count(*) FILTER for single(). */ static Node *build_predicate_function_node(cypher_predicate_function_kind kind, Node *var, Node *expr, @@ -3583,28 +3581,11 @@ static Node *build_predicate_function_node(cypher_predicate_function_kind kind, sub->location = location; sub->subLinkType = EXPR_SUBLINK; - if (kind == CPFK_SINGLE) - { - /* - * single() -> (subquery) = 1 - * The subquery returns count(*) with IS TRUE filtering. - */ - Node *eq_expr; - - eq_expr = (Node *) makeSimpleA_Expr(AEXPR_OP, "=", - (Node *) sub, - make_int_const(1, location), - location); - result = (Node *) node_to_agtype(eq_expr, "boolean", location); - } - else - { - /* - * all()/any()/none(): the subquery returns a boolean directly - * from the CASE+bool_or() aggregate expression. - */ - result = (Node *) node_to_agtype((Node *) sub, "boolean", location); - } + /* + * The subquery returns a boolean directly from the CASE+bool_or() + * aggregate expression (count(*) FILTER for single()). + */ + result = (Node *) node_to_agtype((Node *) sub, "boolean", location); /* * NULL-list guard: CASE WHEN expr IS NULL THEN NULL ELSE result END diff --git a/src/backend/utils/adt/agtype.c b/src/backend/utils/adt/agtype.c index 10d95ee43..cc9cc7717 100644 --- a/src/backend/utils/adt/agtype.c +++ b/src/backend/utils/adt/agtype.c @@ -13103,12 +13103,28 @@ Datum age_unnest(PG_FUNCTION_ARGS) HeapTuple tuple; Datum values[1]; bool nulls[1] = {false}; - agtype *val = agtype_value_to_agtype(&v); /* use the tmp context so we can clean up after each tuple is done */ old_cxt = MemoryContextSwitchTo(tmp_cxt); - values[0] = PointerGetDatum(val); + /* + * Emit an agtype AGTV_NULL element as a SQL-NULL row so that + * `IS NULL` / `IS NOT NULL` on the unwound variable match + * openCypher's null semantics (issue #2393). Wrapping it as a + * non-SQL-NULL agtype container would leave SQL's IS NULL blind + * to it, dropping `WHERE x IS NULL` filters and passing through + * `WHERE x IS NOT NULL` filters. + */ + if (v.type == AGTV_NULL) + { + values[0] = (Datum) 0; + nulls[0] = true; + } + else + { + agtype *val = agtype_value_to_agtype(&v); + values[0] = PointerGetDatum(val); + } tuple = heap_form_tuple(ret_tdesc, values, nulls); From 53ecacf81c7a4d8620a3502e8153193520afcbc8 Mon Sep 17 00:00:00 2001 From: Greg Felice Date: Fri, 3 Jul 2026 08:27:23 -0400 Subject: [PATCH 098/100] Fix segfault on CREATE/MERGE/SET with generated or extra label columns (#2458) AGE builds the insert/update tuple slot from the full label-table descriptor (table_slot_create), but only populates the columns it manages: id/properties for vertices and id/start_id/end_id/properties for edges. A user-added column on the label table -- most notably a GENERATED ALWAYS ... STORED column, but also any plain ADD COLUMN -- therefore left an uninitialized slot entry. ExecStoreVirtualTuple marks all attributes valid, so heap_form_tuple() -> heap_compute_data_size() read the uninitialized tts_isnull[]/tts_values[] for that column and segfaulted dereferencing garbage as a varlena. This crashed the backend on CREATE, MERGE and SET. Fix in two parts: - clear_entity_slot() clears the slot and marks every attribute NULL before AGE fills in its own columns, so columns AGE does not manage default to NULL instead of stale slot memory. It is used at every slot build site: create_vertex, create_edge, merge_vertex, merge_edge, and the populate_vertex_tts/populate_edge_tts helpers used by SET. - insert_entity_tuple_cid (CREATE/MERGE) and update_entity_tuple (SET) call ExecComputeStoredGenerated() before materializing the heap tuple when the relation has stored generated columns, so those columns are computed on insert and recomputed on update per PostgreSQL semantics. Adds a generated_columns regression test covering stored generated columns on vertex and edge labels across CREATE, MERGE and SET, plus a plain user column that must not crash. --- Makefile | 1 + regress/expected/generated_columns.out | 222 +++++++++++++++++++++++++ regress/sql/generated_columns.sql | 129 ++++++++++++++ src/backend/executor/cypher_create.c | 4 +- src/backend/executor/cypher_merge.c | 22 ++- src/backend/executor/cypher_set.c | 23 +++ src/backend/executor/cypher_utils.c | 47 ++++++ src/include/executor/cypher_utils.h | 1 + 8 files changed, 445 insertions(+), 4 deletions(-) create mode 100644 regress/expected/generated_columns.out create mode 100644 regress/sql/generated_columns.sql diff --git a/Makefile b/Makefile index 32f34d6e6..2ea8f64ed 100644 --- a/Makefile +++ b/Makefile @@ -223,6 +223,7 @@ REGRESS = scan \ age_reduce \ map_projection \ direct_field_access \ + generated_columns \ security \ reserved_keyword_alias \ agtype_jsonb_cast \ diff --git a/regress/expected/generated_columns.out b/regress/expected/generated_columns.out new file mode 100644 index 000000000..4e929110c --- /dev/null +++ b/regress/expected/generated_columns.out @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Regression test for issue #2450. + * + * A GENERATED ALWAYS ... STORED column (or any user-added column) on a label + * table adds an attribute AGE's CREATE/MERGE/SET paths do not populate. The + * slot's tuple descriptor is the full relation descriptor, so heap_form_tuple() + * used to read the uninitialized slot entry and segfault. AGE now null-inits the + * entity slot and computes stored generated columns before materializing the + * tuple. Generated columns must be computed on insert and recomputed on update; + * plain user columns must default to NULL rather than crash. + */ +LOAD 'age'; +SET search_path TO ag_catalog; +SELECT create_graph('generated_columns'); +NOTICE: graph "generated_columns" has been created + create_graph +-------------- + +(1 row) + +-- +-- GENERATED ALWAYS ... STORED column on a vertex label +-- +SELECT create_vlabel('generated_columns', 'Product'); +NOTICE: VLabel "Product" has been created + create_vlabel +--------------- + +(1 row) + +ALTER TABLE generated_columns."Product" + ADD COLUMN category varchar(25) + GENERATED ALWAYS AS (agtype_access_operator(properties, '"category"')) STORED; +-- CREATE: the generated column is computed from the new properties +SELECT * FROM cypher('generated_columns', $$ + CREATE (p:Product {category: 'disk', type: 'M1234'}) RETURN p +$$) AS (p agtype); + p +---------------------------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "Product", "properties": {"type": "M1234", "category": "disk"}}::vertex +(1 row) + +-- CREATE where the generation expression yields NULL (no such key) +SELECT * FROM cypher('generated_columns', $$ + CREATE (p:Product {type: 'no-cat'}) RETURN p +$$) AS (p agtype); + p +--------------------------------------------------------------------------------------- + {"id": 844424930131970, "label": "Product", "properties": {"type": "no-cat"}}::vertex +(1 row) + +-- MERGE also computes the generated column +SELECT * FROM cypher('generated_columns', $$ + MERGE (p:Product {category: 'merged'}) RETURN p +$$) AS (p agtype); + p +------------------------------------------------------------------------------------------- + {"id": 844424930131971, "label": "Product", "properties": {"category": "merged"}}::vertex +(1 row) + +-- MERGE again on the same pattern must MATCH (not re-insert); this exercises the +-- matched path where the slot is intentionally left uncleared (issue #2450) +SELECT * FROM cypher('generated_columns', $$ + MERGE (p:Product {category: 'merged'}) RETURN p +$$) AS (p agtype); + p +------------------------------------------------------------------------------------------- + {"id": 844424930131971, "label": "Product", "properties": {"category": "merged"}}::vertex +(1 row) + +-- SET recomputes the generated column from the updated properties +SELECT * FROM cypher('generated_columns', $$ + MATCH (p:Product {category: 'disk'}) SET p.category = 'ssd', p.note = 'x' RETURN p +$$) AS (p agtype); + p +---------------------------------------------------------------------------------------------------------------------- + {"id": 844424930131969, "label": "Product", "properties": {"note": "x", "type": "M1234", "category": "ssd"}}::vertex +(1 row) + +-- The stored generated column always mirrors properties.category +SELECT category, properties FROM generated_columns."Product" ORDER BY category NULLS LAST; + category | properties +----------+--------------------------------------------------- + "merged" | {"category": "merged"} + "ssd" | {"note": "x", "type": "M1234", "category": "ssd"} + | {"type": "no-cat"} +(3 rows) + +-- +-- GENERATED ALWAYS ... STORED column on an edge label +-- +SELECT create_elabel('generated_columns', 'REL'); +NOTICE: ELabel "REL" has been created + create_elabel +--------------- + +(1 row) + +ALTER TABLE generated_columns."REL" + ADD COLUMN kind varchar(25) + GENERATED ALWAYS AS (agtype_access_operator(properties, '"kind"')) STORED; +SELECT * FROM cypher('generated_columns', $$ + CREATE (:Product {category: 'a'})-[r:REL {kind: 'link'}]->(:Product {category: 'b'}) + RETURN r +$$) AS (r agtype); + r +---------------------------------------------------------------------------------------------------------------------------------------- + {"id": 1125899906842625, "label": "REL", "end_id": 844424930131973, "start_id": 844424930131972, "properties": {"kind": "link"}}::edge +(1 row) + +SELECT kind FROM generated_columns."REL"; + kind +-------- + "link" +(1 row) + +-- +-- Plain (non-generated) user column: must not crash; defaults to NULL +-- +SELECT create_vlabel('generated_columns', 'Plain'); +NOTICE: VLabel "Plain" has been created + create_vlabel +--------------- + +(1 row) + +ALTER TABLE generated_columns."Plain" ADD COLUMN tag text; +SELECT * FROM cypher('generated_columns', $$ + CREATE (p:Plain {n: 1}) RETURN p +$$) AS (p agtype); + p +---------------------------------------------------------------------------- + {"id": 1407374883553281, "label": "Plain", "properties": {"n": 1}}::vertex +(1 row) + +SELECT * FROM cypher('generated_columns', $$ + MATCH (p:Plain) SET p.n = 2 RETURN p +$$) AS (p agtype); + p +---------------------------------------------------------------------------- + {"id": 1407374883553281, "label": "Plain", "properties": {"n": 2}}::vertex +(1 row) + +SELECT properties, tag FROM generated_columns."Plain"; + properties | tag +------------+----- + {"n": 2} | +(1 row) + +-- +-- Generated column referencing the tableoid system column: the entity slot must +-- carry tts_tableOid before the generated columns are computed, otherwise the +-- expression sees InvalidOid instead of the label table's OID (issue #2450). +-- +SELECT create_vlabel('generated_columns', 'T'); +NOTICE: VLabel "T" has been created + create_vlabel +--------------- + +(1 row) + +ALTER TABLE generated_columns."T" + ADD COLUMN tbl oid GENERATED ALWAYS AS (tableoid) STORED; +SELECT * FROM cypher('generated_columns', $$ + CREATE (n:T {k: 1}) RETURN n +$$) AS (n agtype); + n +------------------------------------------------------------------------ + {"id": 1688849860263937, "label": "T", "properties": {"k": 1}}::vertex +(1 row) + +SELECT * FROM cypher('generated_columns', $$ + MATCH (n:T) SET n.k = 2 RETURN n +$$) AS (n agtype); + n +------------------------------------------------------------------------ + {"id": 1688849860263937, "label": "T", "properties": {"k": 2}}::vertex +(1 row) + +-- tbl must equal the real label-table OID on both the CREATE and SET paths +SELECT tbl = 'generated_columns."T"'::regclass::oid AS tableoid_ok +FROM generated_columns."T"; + tableoid_ok +------------- + t +(1 row) + +-- +-- Cleanup +-- +SELECT drop_graph('generated_columns', true); +NOTICE: drop cascades to 6 other objects +DETAIL: drop cascades to table generated_columns._ag_label_vertex +drop cascades to table generated_columns._ag_label_edge +drop cascades to table generated_columns."Product" +drop cascades to table generated_columns."REL" +drop cascades to table generated_columns."Plain" +drop cascades to table generated_columns."T" +NOTICE: graph "generated_columns" has been dropped + drop_graph +------------ + +(1 row) + diff --git a/regress/sql/generated_columns.sql b/regress/sql/generated_columns.sql new file mode 100644 index 000000000..5229c28f1 --- /dev/null +++ b/regress/sql/generated_columns.sql @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Regression test for issue #2450. + * + * A GENERATED ALWAYS ... STORED column (or any user-added column) on a label + * table adds an attribute AGE's CREATE/MERGE/SET paths do not populate. The + * slot's tuple descriptor is the full relation descriptor, so heap_form_tuple() + * used to read the uninitialized slot entry and segfault. AGE now null-inits the + * entity slot and computes stored generated columns before materializing the + * tuple. Generated columns must be computed on insert and recomputed on update; + * plain user columns must default to NULL rather than crash. + */ + +LOAD 'age'; +SET search_path TO ag_catalog; + +SELECT create_graph('generated_columns'); + +-- +-- GENERATED ALWAYS ... STORED column on a vertex label +-- +SELECT create_vlabel('generated_columns', 'Product'); +ALTER TABLE generated_columns."Product" + ADD COLUMN category varchar(25) + GENERATED ALWAYS AS (agtype_access_operator(properties, '"category"')) STORED; + +-- CREATE: the generated column is computed from the new properties +SELECT * FROM cypher('generated_columns', $$ + CREATE (p:Product {category: 'disk', type: 'M1234'}) RETURN p +$$) AS (p agtype); + +-- CREATE where the generation expression yields NULL (no such key) +SELECT * FROM cypher('generated_columns', $$ + CREATE (p:Product {type: 'no-cat'}) RETURN p +$$) AS (p agtype); + +-- MERGE also computes the generated column +SELECT * FROM cypher('generated_columns', $$ + MERGE (p:Product {category: 'merged'}) RETURN p +$$) AS (p agtype); + +-- MERGE again on the same pattern must MATCH (not re-insert); this exercises the +-- matched path where the slot is intentionally left uncleared (issue #2450) +SELECT * FROM cypher('generated_columns', $$ + MERGE (p:Product {category: 'merged'}) RETURN p +$$) AS (p agtype); + +-- SET recomputes the generated column from the updated properties +SELECT * FROM cypher('generated_columns', $$ + MATCH (p:Product {category: 'disk'}) SET p.category = 'ssd', p.note = 'x' RETURN p +$$) AS (p agtype); + +-- The stored generated column always mirrors properties.category +SELECT category, properties FROM generated_columns."Product" ORDER BY category NULLS LAST; + +-- +-- GENERATED ALWAYS ... STORED column on an edge label +-- +SELECT create_elabel('generated_columns', 'REL'); +ALTER TABLE generated_columns."REL" + ADD COLUMN kind varchar(25) + GENERATED ALWAYS AS (agtype_access_operator(properties, '"kind"')) STORED; + +SELECT * FROM cypher('generated_columns', $$ + CREATE (:Product {category: 'a'})-[r:REL {kind: 'link'}]->(:Product {category: 'b'}) + RETURN r +$$) AS (r agtype); + +SELECT kind FROM generated_columns."REL"; + +-- +-- Plain (non-generated) user column: must not crash; defaults to NULL +-- +SELECT create_vlabel('generated_columns', 'Plain'); +ALTER TABLE generated_columns."Plain" ADD COLUMN tag text; + +SELECT * FROM cypher('generated_columns', $$ + CREATE (p:Plain {n: 1}) RETURN p +$$) AS (p agtype); + +SELECT * FROM cypher('generated_columns', $$ + MATCH (p:Plain) SET p.n = 2 RETURN p +$$) AS (p agtype); + +SELECT properties, tag FROM generated_columns."Plain"; + +-- +-- Generated column referencing the tableoid system column: the entity slot must +-- carry tts_tableOid before the generated columns are computed, otherwise the +-- expression sees InvalidOid instead of the label table's OID (issue #2450). +-- +SELECT create_vlabel('generated_columns', 'T'); +ALTER TABLE generated_columns."T" + ADD COLUMN tbl oid GENERATED ALWAYS AS (tableoid) STORED; + +SELECT * FROM cypher('generated_columns', $$ + CREATE (n:T {k: 1}) RETURN n +$$) AS (n agtype); + +SELECT * FROM cypher('generated_columns', $$ + MATCH (n:T) SET n.k = 2 RETURN n +$$) AS (n agtype); + +-- tbl must equal the real label-table OID on both the CREATE and SET paths +SELECT tbl = 'generated_columns."T"'::regclass::oid AS tableoid_ok +FROM generated_columns."T"; + +-- +-- Cleanup +-- +SELECT drop_graph('generated_columns', true); diff --git a/src/backend/executor/cypher_create.c b/src/backend/executor/cypher_create.c index 40f446bf6..36ef61b32 100644 --- a/src/backend/executor/cypher_create.c +++ b/src/backend/executor/cypher_create.c @@ -396,7 +396,7 @@ static void create_edge(cypher_create_custom_scan_state *css, estate->es_result_relations = &resultRelInfo; - ExecClearTuple(elemTupleSlot); + clear_entity_slot(elemTupleSlot); /* Graph Id for the edge */ id = ExecEvalExpr(node->id_expr_state, econtext, &isNull); @@ -491,7 +491,7 @@ static Datum create_vertex(cypher_create_custom_scan_state *css, estate->es_result_relations = &resultRelInfo; - ExecClearTuple(elemTupleSlot); + clear_entity_slot(elemTupleSlot); /* get the next graphid for this vertex. */ id = ExecEvalExpr(node->id_expr_state, econtext, &isNull); diff --git a/src/backend/executor/cypher_merge.c b/src/backend/executor/cypher_merge.c index 6f2e9376e..9c52073c2 100644 --- a/src/backend/executor/cypher_merge.c +++ b/src/backend/executor/cypher_merge.c @@ -1199,7 +1199,16 @@ static Datum merge_vertex(cypher_merge_custom_scan_state *css, estate->es_result_relations = &resultRelInfo; - ExecClearTuple(elemTupleSlot); + /* + * Only clear and null-init the slot when we will actually insert. On a + * matched MERGE the slot is never materialized (the output is built from + * id/prop directly), so the null-init memset would be wasted work that + * scales with any user-added columns (issue #2450 review follow-up). + */ + if (should_insert) + { + clear_entity_slot(elemTupleSlot); + } /* if we not are going to insert, we need our structure pointers */ if (should_insert == false && @@ -1526,7 +1535,16 @@ static void merge_edge(cypher_merge_custom_scan_state *css, estate->es_result_relations = &resultRelInfo; - ExecClearTuple(elemTupleSlot); + /* + * Only clear and null-init the slot when we will actually insert. On a + * matched MERGE the slot is never materialized (the output is built from + * id/prop directly), so the null-init memset would be wasted work that + * scales with any user-added columns (issue #2450 review follow-up). + */ + if (should_insert) + { + clear_entity_slot(elemTupleSlot); + } /* if we not are going to insert, we need our structure pointers */ if (should_insert == false && diff --git a/src/backend/executor/cypher_set.c b/src/backend/executor/cypher_set.c index fe3633f8d..7a0d48f0c 100644 --- a/src/backend/executor/cypher_set.c +++ b/src/backend/executor/cypher_set.c @@ -21,6 +21,7 @@ #include "common/hashfn.h" #include "executor/executor.h" +#include "executor/nodeModifyTable.h" #include "storage/bufmgr.h" #include "utils/rls.h" @@ -131,6 +132,28 @@ static HeapTuple update_entity_tuple(ResultRelInfo *resultRelInfo, close_indices = true; } ExecStoreVirtualTuple(elemTupleSlot); + + /* + * Recompute any stored generated columns before materializing the heap + * tuple. The slot's tuple descriptor is the full relation descriptor, + * which may include a GENERATED ALWAYS ... STORED column that the SET + * path does not populate; leaving those slot entries uninitialized makes + * heap_form_tuple() segfault on the garbage values (issue #2450). + */ + if (resultRelInfo->ri_RelationDesc->rd_att->constr != NULL && + resultRelInfo->ri_RelationDesc->rd_att->constr->has_generated_stored) + { + /* + * A generation expression may reference the tableoid system column, + * so the slot must carry the relation's OID before we recompute the + * stored generated columns (mirrors PostgreSQL's own ExecUpdate path). + */ + elemTupleSlot->tts_tableOid = + RelationGetRelid(resultRelInfo->ri_RelationDesc); + ExecComputeStoredGenerated(resultRelInfo, estate, elemTupleSlot, + CMD_UPDATE); + } + tuple = ExecFetchSlotHeapTuple(elemTupleSlot, true, NULL); tuple->t_self = old_tuple->t_self; diff --git a/src/backend/executor/cypher_utils.c b/src/backend/executor/cypher_utils.c index 8237cdcce..c697a560c 100644 --- a/src/backend/executor/cypher_utils.c +++ b/src/backend/executor/cypher_utils.c @@ -25,6 +25,7 @@ #include "postgres.h" #include "executor/executor.h" +#include "executor/nodeModifyTable.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" @@ -128,6 +129,24 @@ void destroy_entity_result_rel_info(ResultRelInfo *result_rel_info) table_close(result_rel_info->ri_RelationDesc, RowExclusiveLock); } +/* + * Clear an entity slot and mark every attribute NULL before AGE fills in the + * columns it manages (id/start_id/end_id/properties). + * + * The slot's tuple descriptor is the full label-table descriptor, which may + * contain columns AGE does not populate -- e.g. a user-added plain column, or a + * GENERATED ALWAYS ... STORED column. Without this, those attributes keep stale + * slot memory and heap_form_tuple() segfaults dereferencing the garbage + * (issue #2450). Plain columns then default to NULL; generated columns are + * recomputed via ExecComputeStoredGenerated() before the tuple is materialized. + */ +void clear_entity_slot(TupleTableSlot *elemTupleSlot) +{ + ExecClearTuple(elemTupleSlot); + memset(elemTupleSlot->tts_isnull, true, + elemTupleSlot->tts_tupleDescriptor->natts * sizeof(bool)); +} + TupleTableSlot *populate_vertex_tts( TupleTableSlot *elemTupleSlot, agtype_value *id, agtype_value *properties) { @@ -139,6 +158,8 @@ TupleTableSlot *populate_vertex_tts( errmsg("vertex id field cannot be NULL"))); } + clear_entity_slot(elemTupleSlot); + properties_isnull = properties == NULL; elemTupleSlot->tts_values[vertex_tuple_id] = GRAPHID_GET_DATUM(id->val.int_value); @@ -174,6 +195,8 @@ TupleTableSlot *populate_edge_tts( errmsg("edge end_id field cannot be NULL"))); } + clear_entity_slot(elemTupleSlot); + properties_isnull = properties == NULL; elemTupleSlot->tts_values[edge_tuple_id] = @@ -318,6 +341,30 @@ HeapTuple insert_entity_tuple_cid(ResultRelInfo *resultRelInfo, HeapTuple tuple = NULL; ExecStoreVirtualTuple(elemTupleSlot); + + /* + * The slot's tuple descriptor is the full relation descriptor, which may + * contain columns AGE does not populate itself -- most notably a + * GENERATED ALWAYS ... STORED column added to the label table. Those slot + * entries are left uninitialized by the create/merge/set paths, so we must + * compute the stored generated columns here before materializing the heap + * tuple. Otherwise heap_form_tuple() reads the uninitialized slot values + * and segfaults dereferencing garbage (issue #2450). + */ + if (resultRelInfo->ri_RelationDesc->rd_att->constr != NULL && + resultRelInfo->ri_RelationDesc->rd_att->constr->has_generated_stored) + { + /* + * A generation expression may reference the tableoid system column, so + * the slot must carry the relation's OID before we compute the stored + * generated columns (mirrors PostgreSQL's own ExecInsert path). + */ + elemTupleSlot->tts_tableOid = + RelationGetRelid(resultRelInfo->ri_RelationDesc); + ExecComputeStoredGenerated(resultRelInfo, estate, elemTupleSlot, + CMD_INSERT); + } + tuple = ExecFetchSlotHeapTuple(elemTupleSlot, true, NULL); /* Check the constraints of the tuple */ diff --git a/src/include/executor/cypher_utils.h b/src/include/executor/cypher_utils.h index ac4b5ea5a..8b65bc964 100644 --- a/src/include/executor/cypher_utils.h +++ b/src/include/executor/cypher_utils.h @@ -119,6 +119,7 @@ typedef struct cypher_merge_custom_scan_state void apply_update_list(CustomScanState *node, cypher_update_information *set_info); +void clear_entity_slot(TupleTableSlot *elemTupleSlot); TupleTableSlot *populate_vertex_tts(TupleTableSlot *elemTupleSlot, agtype_value *id, agtype_value *properties); TupleTableSlot *populate_edge_tts( From fef547000e84ce73d44bbff81aa67e74e316858c Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Fri, 3 Jul 2026 08:23:05 -0700 Subject: [PATCH 099/100] Fix installcheck fetch-depth parameter to 100 (#2461) Fix the installcheck.yaml fetch-depth parameter to 100. This release, being rather large, exceeded the original value of 75 (commits) and caused the upgrade test to fail. The new value will resolve this and make it unlikely to become a problem in the future. NOTE: We need to keep releases under this number of commits. modified: .github/workflows/installcheck.yaml --- .github/workflows/installcheck.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/installcheck.yaml b/.github/workflows/installcheck.yaml index aa30ef3d7..b1f78a985 100644 --- a/.github/workflows/installcheck.yaml +++ b/.github/workflows/installcheck.yaml @@ -69,7 +69,7 @@ jobs: - uses: actions/checkout@v3 with: - fetch-depth: 75 + fetch-depth: 100 - name: Build AGE id: build From 9fb7df8c668f0ffb5607f8ecf99953fc5cebfb48 Mon Sep 17 00:00:00 2001 From: John Gemignani Date: Fri, 3 Jul 2026 09:37:05 -0700 Subject: [PATCH 100/100] Advance master branch to Apache AGE version 1.8.0 (#2455) Updated the following files to advance the Apache AGE version to 1.8.0 modified: META.json modified: README.md modified: RELEASE renamed: age--1.7.0--y.y.y.sql -> age--1.7.0--1.8.0.sql modified: age.control modified: docker/Dockerfile new file: age--1.8.0--y.y.y.sql --- META.json | 8 +- README.md | 2 +- RELEASE | 110 +++++++++++++----- ...-1.7.0--y.y.y.sql => age--1.7.0--1.8.0.sql | 10 +- age--1.8.0--y.y.y.sql | 35 ++++++ age.control | 2 +- docker/Dockerfile | 2 +- 7 files changed, 121 insertions(+), 48 deletions(-) rename age--1.7.0--y.y.y.sql => age--1.7.0--1.8.0.sql (98%) create mode 100644 age--1.8.0--y.y.y.sql diff --git a/META.json b/META.json index bb63a0563..9ac550a87 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "ApacheAGE", "abstract": "Apache AGE is a PostgreSQL Extension that provides graph database functionality", "description": "Apache AGE is a PostgreSQL Extension that provides graph database functionality. AGE is an acronym for A Graph Extension, and is inspired by Bitnine's fork of PostgreSQL 10, AgensGraph, which is a multi-model database. The goal of the project is to create single storage that can handle both relational and graph model data so that users can use standard ANSI SQL along with openCypher, the Graph query language. A graph consists of a set of vertices (also called nodes) and edges, where each individual vertex and edge possesses a map of properties. A vertex is the basic object of a graph, that can exist independently of everything else in the graph. An edge creates a directed connection between two vertices. A graph database is simply composed of vertices and edges. This type of database is useful when the meaning is in the relationships between the data. Relational databases can easily handle direct relationships, but indirect relationships are more difficult to deal with in relational databases. A graph database stores relationship information as a first-class entity. Apache AGE gives you the best of both worlds, simultaneously.", - "version": "1.3.0", + "version": "1.8.0", "maintainer": [ "users@age.apache.org" ], @@ -10,15 +10,15 @@ "provides": { "ApacheAGE": { "abstract": "Apache AGE is a PostgreSQL Extension that provides graph database functionality", - "file": "age--1.3.0.sql", + "file": "age--1.8.0.sql", "docfile": "README.md", - "version": "1.3.0" + "version": "1.8.0" } }, "prereqs": { "runtime": { "requires": { - "PostgreSQL": "14.0.0" + "PostgreSQL": "18.0.0" } } }, diff --git a/README.md b/README.md index 412b55e8d..262f06c02 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@   - +   diff --git a/RELEASE b/RELEASE index 4a5eb45f3..7776b6d50 100644 --- a/RELEASE +++ b/RELEASE @@ -15,44 +15,90 @@ # specific language governing permissions and limitations # under the License. -Release Notes for Apache AGE release 1.7.0 for master branch (currently PG18) +Release Notes for Apache AGE release 1.8.0 for master branch (currently PG18) # # WARNING! # -# Please note the upgrade script (age--1.6.0--1.7.0.sql) may take a while to -# complete for large graphs, due to creation of indexes for existing labels. +# Always back up your database before executing any upgrade scripts # # WARNING! # -Apache AGE 1.7.0 - Release Notes - -Master to PostgreSQL version 18 (#2315) -Add RLS support and fix permission checks (#2309) -Replace libcsv with pg COPY for csv loading (#2310) -Fix Issue 1884: Ambiguous column reference (#2306) -Upgrade Jest to v29 for node: protocol compatibility (#2307) -Optimize vertex/edge field access with direct array indexing (#2302) -feat: Add 32-bit platform support for graphid type (#2286) -Fix and improve index.sql addendum (#2301) -Fix and improve index.sql regression test coverage (#2300) -Fix Issue 2289: handle empty list in IN expression (#2294) -Revise README for Python driver updates (#2298) -Makefile: fix race condition on cypher_gram_def.h (#2273) -Restrict age_load commands (#2274) -Migrate python driver configuration to pyproject.toml (#2272) -Convert string to raw string to remove invalid escape sequence warning (#2267) -Update grammar file for maintainability (#2270) -Fix ORDER BY alias resolution with AS in Cypher queries (#2269) -Fix possible memory and file descriptors leaks (#2258) -Adjust 'could not find rte for' ERROR message (#2266) -Fix Issue 2256: segmentation fault when calling coalesce function (#2259) -Add index on id columns (#2117) -Fix issue 2245 - Creating more than 41 vlabels causes crash in drop_graph (#2248) -Fix issue 2243 - Regression in string concatenation (#2244) -Add fast functions for checking edge uniqueness (#2227) -Bump gopkg.in/yaml.v3 from 3.0.0 to 3.0.1 in /drivers/golang (#2212) -Update branch security (#2219) -Fix issue with CALL/YIELD for user defined and qualified functions. (#2217) +Apache AGE 1.8.0 - Release Notes +Fix installcheck fetch-depth parameter to 100 (#2461) +Fix segfault on CREATE/MERGE/SET with generated or extra label columns (#2458) +Propagate null through unnest and single() three-valued logic (#2406) +Add automatic header-dependency tracking to the Makefile (#2454) +ci: pin Build/Regression runner to ubuntu-24.04 and guard Bison version for GLR grammar (#2445) +Fix segfault and out-of-bounds reads in file loaders on malformed rows (#2453) +Support outer references in reduce() fold bodies (#2448) +Fix stack overflow and precision loss in toFloatList() conversion (#2451) +Fix Node.js driver CI build broken by @types/node drift (#2452) +Preserve null-valued keys in map literals (#2391) (#2412) +Fix single-node labeled pattern expressions not filtering by label (#2443) (#2444) +Support relationship-type filters and a minimum hop count (#2442) +resolve subgraph staging sequences via regclass (#2446) +Add reduce() list folding function (#2435) +Support pattern expressions as boolean expressions (#2360) +Add shortest_path / all_shortest_paths SRFs (#2430) +cypher_with: add ORDER BY to non-deterministic RETURN queries (#2436) +Make ag_catalog ownership and built-in resolution explicit (#2440) +Makefile: add installcheck-existing target and improve readability (#2437) +age_global_graph: stabilize regression tests (#2431) +cypher_vle: add ORDER BY to non-deterministic RETURN queries (#2434) +feature: add create_subgraph() (#2441) +Make age extension usable from shared_preload_libraries (#2438) +Fix locale-dependent string comparison in direct_field_access test (#2439) +Fix VLE [*0..N] zero-hop self-binding when edge label is missing (#2382) (#2419) +fix: remove pthread_mutex that causes self-deadlock on VLE queries (#2433) +Restore contsel/contjoinsel for containment & key-existence operators (#2356) (#2417) +perf: VLE hash-adjacency overhaul — agehash + flat-array VertexEdgeArray (#2421) +python driver: preserve trailing quotes in agtype string values (#2425) +perf(agtype): arena passthrough + binary-direct id extraction (#2424) +Add vertex and edge composite types with direct field access optimization (#2303) +perf: VLE terminal-qual rewrite (#2420) +Add agtype <-> jsonb bidirectional casts (#350) (#2361) +feat(python-driver): add public API for connection pooling and model dict conversion (#2374) +Fix upgrade test: allow function removal (#2422) +Zero-initialize parent_cpstate in analyze_cypher (#2423) +Fix upgrade test: replace data-integrity checks with catalog comparison (#2403) +Allow safe Cypher reserved keywords in alias positions (#2355) (#2415) +Propagate null through agtype arithmetic operators (#2405) +Update README.md (#2414) +Add VLE semantics and cost model design note (#2349) (#2413) +Fix property access on list comprehension / predicate loop variables (#2402) +Add MERGE ON CREATE SET / ON MATCH SET support (#2347) +Propagate null in agtype_access_slice bounds (#2400) +VLE cache + performance improvements (#2376) +CI: fail build on compiler and bison warnings (#2398) +fix(python-driver): add null-guards in ANTLR parser and relax runtime version pin (#2372) +Return an empty list from tail() for empty and singleton lists (#2399) +Fix substring() crash when start-offset is NULL and length is supplied (#2401) +fix(python-driver): quote-escape column aliases in buildCypher() (#2373) +Implement predicate functions: all(), any(), none(), single() (#2359) +Fix OPTIONAL MATCH dropping null-preserving rows with subquery WHERE (#2380) +Python driver: Add `skip_load` parameter to skip `LOAD 'age'` statement (#2366) +Fix upgrade test: build default install SQL from HEAD, not initial commit (#2397) +Improve extension upgrade regression test (addendum to #2364) (#2377) +Add missing include for PR: Add index scan (#2351) (#2379) +Add index scan (#2351) +Fix MATCH on brand-new label after CREATE returning 0 rows (#2341) +Fix nondeterministic age_global_graph regression test (#2365) +Add extension upgrade template regression test (#2364) +Fix crash in PREPARE with property parameter when enable_containment is off (#2339) +Fix entity_exists() CID visibility for CREATE + WITH + MERGE (#2343) +Support doubled-quote escaping in Cypher string literals (issue #2222) (#2342) +Fix MATCH after CREATE returning 0 rows (issue #2308) (#2340) +Fix chained MERGE not seeing sibling MERGE's changes (#1446) (#2344) +Fix VLE queries failing on read-only replicas (#2160) (#2345) +Fix VLE NULL handling for chained OPTIONAL MATCH (#2337) +fix incorrect variable assignment (#2336) +Remove labeler github action (#2335) +Fix ISO C90 forbids mixed declarations and code warning (#2334) +Fix null pointer handling in array iteration (#2313) +Fix security vulnerabilities in Node.js driver (#2329) +Update python-driver security and formatting (#2330) +Fix JDBC driver CI test failures (#2333) +Add pg_upgrade support functions for PostgreSQL (#2326) diff --git a/age--1.7.0--y.y.y.sql b/age--1.7.0--1.8.0.sql similarity index 98% rename from age--1.7.0--y.y.y.sql rename to age--1.7.0--1.8.0.sql index fd1f28160..fe4c9a5f2 100644 --- a/age--1.7.0--y.y.y.sql +++ b/age--1.7.0--1.8.0.sql @@ -17,19 +17,11 @@ * under the License. */ ---* This is a TEMPLATE for upgrading from the previous version of Apache AGE ---* Please adjust the below ALTER EXTENSION to reflect the -- correct version it ---* is upgrading to. - -- This will only work within a major version of PostgreSQL, not across -- major versions. -- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "ALTER EXTENSION age UPDATE TO '1.X.0'" to load this file. \quit - ---* Please add all additions, deletions, and modifications to the end of this ---* file. We need to keep the order of these changes. ---* REMOVE ALL LINES ABOVE, and this one, that start with --* +\echo Use "ALTER EXTENSION age UPDATE TO '1.8.0'" to load this file. \quit -- -- pg_upgrade support functions diff --git a/age--1.8.0--y.y.y.sql b/age--1.8.0--y.y.y.sql new file mode 100644 index 000000000..4c85b28ca --- /dev/null +++ b/age--1.8.0--y.y.y.sql @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +-- This will only work within a major version of PostgreSQL, not across +-- major versions. + +--* This is a TEMPLATE for upgrading from the previous version of Apache AGE +--* Please adjust the below ALTER EXTENSION to reflect the -- correct version it +--* is upgrading to. + +-- This will only work within a major version of PostgreSQL, not across +-- major versions. + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION age UPDATE TO '1.X.0'" to load this file. \quit + +--* Please add all additions, deletions, and modifications to the end of this +--* file. We need to keep the order of these changes. +--* REMOVE ALL LINES ABOVE, and this one, that start with --* diff --git a/age.control b/age.control index 57f9fbc12..8b1d3aef2 100644 --- a/age.control +++ b/age.control @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -default_version = '1.7.0' +default_version = '1.8.0' comment = 'AGE database extension' module_pathname = '$libdir/age' diff --git a/docker/Dockerfile b/docker/Dockerfile index abdf0ccd0..22d912c46 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -49,7 +49,7 @@ ENV LC_COLLATE=en_US.UTF-8 ENV LC_CTYPE=en_US.UTF-8 COPY --from=build /usr/lib/postgresql/18/lib/age.so /usr/lib/postgresql/18/lib/ -COPY --from=build /usr/share/postgresql/18/extension/age--1.7.0.sql /usr/share/postgresql/18/extension/ +COPY --from=build /usr/share/postgresql/18/extension/age--1.8.0.sql /usr/share/postgresql/18/extension/ COPY --from=build /usr/share/postgresql/18/extension/age.control /usr/share/postgresql/18/extension/ COPY docker/docker-entrypoint-initdb.d/00-create-extension-age.sql /docker-entrypoint-initdb.d/00-create-extension-age.sql