diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/chunk_graph_builder.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/chunk_graph_builder.py index 6c4bae3f2..a89aefe99 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/chunk_graph_builder.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/chunk_graph_builder.py @@ -5,7 +5,7 @@ from typing import Any from graphrag_toolkit.lexical_graph.config import GraphRAGConfig -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.chunk_store_factory import ChunkStoreFactory from graphrag_toolkit.lexical_graph.indexing.build.graph_builder import GraphBuilder @@ -112,7 +112,13 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): query_c = '\n'.join(statements_c) - graph_client.execute_query_with_retry(query_c, self._to_params(properties_c), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry( + query_c, + self._to_params(properties_c), + max_attempts=5, + max_wait=7, + operation=GraphQueryOperation.UPSERT_CHUNK, + ) source_info = node.relationships.get(NodeRelationship.SOURCE, None) @@ -135,7 +141,7 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): query_s = '\n'.join(statements_s) - graph_client.execute_query_with_retry(query_s, self._to_params(properties_s), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query_s, self._to_params(properties_s), max_attempts=5, max_wait=7, operation=GraphQueryOperation.LINK_CHUNK_SOURCE) else: logger.warning(f'source_id missing from chunk node [node_id: {chunk_id}]') @@ -153,12 +159,13 @@ def insert_chunk_to_chunk_relationship(node_id:str, relationship_type:str): properties_c2c = { 'chunk_id': chunk_id, - 'target_id': node_id + 'target_id': node_id, + '_relationship_type': relationship_type, } query_c2c = '\n'.join(statements_c2c) - graph_client.execute_query_with_retry(query_c2c, self._to_params(properties_c2c), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query_c2c, self._to_params(properties_c2c), max_attempts=5, max_wait=7, operation=GraphQueryOperation.LINK_CHUNKS) for node_relationship,relationship_info in node.relationships.items(): diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/entity_graph_builder.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/entity_graph_builder.py index efdcc3cbf..08e27449b 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/entity_graph_builder.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/entity_graph_builder.py @@ -5,7 +5,7 @@ from typing import Any from graphrag_toolkit.lexical_graph.indexing.model import Fact, Entity -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.graph.graph_utils import search_string_from, label_from, new_query_var, escape_cypher_label from graphrag_toolkit.lexical_graph.indexing.build.graph_builder import GraphBuilder from graphrag_toolkit.lexical_graph.indexing.constants import DEFAULT_CLASSIFICATION, LOCAL_ENTITY_CLASSIFICATION @@ -98,7 +98,7 @@ def insert_for_entity(entity:Entity): query = '\n'.join(statements) - graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=5, max_wait=7, operation=GraphQueryOperation.UPSERT_ENTITY) insert_for_entity(fact.subject) @@ -125,7 +125,17 @@ def insert_domain_entity(entity:Entity): e_label = escape_cypher_label(label_from(entity.classification or DEFAULT_CLASSIFICATION)) e_comment = f'// awsqid:{e_id}-{e_label}'.replace('\r', ' ').replace('\n', ' ') query_e = f"UNWIND $params AS params MERGE ({e_var}:`__Entity__`{{{graph_client.node_id('entityId')}: params.entityId}}) SET {e_var} :`{e_label}` {e_comment}" - graph_client.execute_query_with_retry(query_e, self._to_params({'entityId': e_id}), max_attempts=5, max_wait=7) + properties = self._to_params({ + 'entityId': e_id, + '_classification': entity.classification or DEFAULT_CLASSIFICATION, + }) + graph_client.execute_query_with_retry( + query_e, + properties, + max_attempts=5, + max_wait=7, + operation=GraphQueryOperation.ADD_ENTITY_TYPE, + ) insert_domain_entity(fact.subject) @@ -136,4 +146,4 @@ def insert_domain_entity(entity:Entity): insert_domain_entity(fact.complement) else: - logger.warning(f'fact_id missing from fact node [node_id: {node.node_id}]') \ No newline at end of file + logger.warning(f'fact_id missing from fact node [node_id: {node.node_id}]') diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/entity_relation_graph_builder.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/entity_relation_graph_builder.py index ad196c6db..ce96b006e 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/entity_relation_graph_builder.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/entity_relation_graph_builder.py @@ -5,7 +5,7 @@ from typing import Any from graphrag_toolkit.lexical_graph.indexing.model import Fact -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.graph.graph_utils import relationship_name_from, new_query_var from graphrag_toolkit.lexical_graph.indexing.build.graph_builder import GraphBuilder from graphrag_toolkit.lexical_graph.indexing.utils.fact_utils import string_complement_to_entity @@ -90,7 +90,7 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): query = '\n'.join(statements) - graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=5, max_wait=7, operation=GraphQueryOperation.LINK_ENTITIES) # if include_domain_labels: @@ -137,7 +137,7 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): query = '\n'.join(statements) - graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=5, max_wait=7, operation=GraphQueryOperation.LINK_ENTITIES) # if include_domain_labels: diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/fact_graph_builder.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/fact_graph_builder.py index 637cb1f71..56943ca21 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/fact_graph_builder.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/fact_graph_builder.py @@ -5,7 +5,7 @@ from typing import Any, Optional from graphrag_toolkit.lexical_graph.indexing.model import Fact -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore, Query, QueryTree +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore, Query, QueryTree from graphrag_toolkit.lexical_graph.indexing.build.graph_builder import GraphBuilder from graphrag_toolkit.lexical_graph.indexing.constants import LOCAL_ENTITY_CLASSIFICATION from graphrag_toolkit.lexical_graph.indexing.utils.fact_utils import string_complement_to_entity @@ -82,15 +82,26 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): 'MERGE (fact)-[:`__SUPPORTS__`]->(statement)' ] + subject_literal = None + if fact.subject.classification == LOCAL_ENTITY_CLASSIFICATION and not include_local_entities: + subject_literal = fact.subject.value + + object_literal = None + if not fact.object and fact.complement and not include_local_entities: + object_literal = fact.complement.value + properties = { 'statement_id': fact.statementId, 'fact_id': fact.factId, - 'fact': node.text + 'fact': node.text, + '_predicate': fact.predicate.value, + '_subject_literal': subject_literal, + '_object_literal': object_literal } query = '\n'.join(statements) - graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=5, max_wait=7, operation=GraphQueryOperation.UPSERT_FACT) def insert_entity_fact_relationship(relationship_type:str, entity_id:Optional[str]=None): @@ -107,10 +118,11 @@ def insert_entity_fact_relationship(relationship_type:str, entity_id:Optional[st if entity_id: properties_e2f['fact_id'] = fact.factId properties_e2f['entity_id'] = entity_id + properties_e2f['_relationship_type'] = relationship_type query_e2f = '\n'.join(statements_e2f) - graph_client.execute_query_with_retry(query_e2f, self._to_params(properties_e2f), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query_e2f, self._to_params(properties_e2f), max_attempts=5, max_wait=7, operation=GraphQueryOperation.LINK_FACT_ENTITY) insert_entity_fact_relationship('subject') diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/graph_batch_client.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/graph_batch_client.py index 93903d741..eaa2caf25 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/graph_batch_client.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/graph_batch_client.py @@ -46,6 +46,7 @@ def __init__(self, graph_client:GraphStore, batch_writes_enabled:bool, batch_wri self.batch_writes_enabled = batch_writes_enabled self.batch_write_size = batch_write_size self.batches:Dict[str, List] = {} + self.batch_operations:Dict[str, Any] = {} self.query_trees:Dict[str, QueryTree] = {} self.all_nodes = [] self.parameterless_queries:Dict[str, str] = {} @@ -138,8 +139,9 @@ def execute_query_with_retry(self, query:QueryTree, properties:Dict[str, Any], * if properties: if query not in self.batches: self.batches[query] = [] + self.batch_operations[query] = kwargs.get('operation') self.batches[query].extend(properties['params']) - else: + elif kwargs.get('operation') is None: self._add_parameterless_query(query) elif isinstance(query, QueryTree): properties = properties or {'params':[]} @@ -224,7 +226,7 @@ def _apply_batch_query(self, query, parameters): 'params': p } try: - self.graph_client.execute_query_with_retry(query, params, max_attempts=BATCH_MAX_ATTEMPTS, max_wait=BATCH_MAX_WAIT) + self._execute_batch(query, params) except Exception as e: logger.debug(f'Batch failed - queuing for retry: [query: {query}, params: {params}]') retry_batches.append((query, params)) @@ -233,12 +235,21 @@ def _apply_batch_query(self, query, parameters): for (query, params) in retry_batches: try: - self.graph_client.execute_query_with_retry(query, params, max_attempts=BATCH_MAX_ATTEMPTS, max_wait=BATCH_MAX_WAIT) + self._execute_batch(query, params) except Exception as e: logger.debug(f'Retry batch failed - queuing for return: [query: {query}, params: {params}]') failed_batches.append((query, params)) return failed_batches + + def _execute_batch(self, query, parameters): + return self.graph_client.execute_query_with_retry( + query, + parameters, + max_attempts=BATCH_MAX_ATTEMPTS, + max_wait=BATCH_MAX_WAIT, + operation=self.batch_operations.get(query), + ) def _retry_failed_batches(self, failed_batches): @@ -246,7 +257,7 @@ def _retry_failed_batches(self, failed_batches): for (query, params) in failed_batches: try: - self.graph_client.execute_query_with_retry(query, params, max_attempts=BATCH_MAX_ATTEMPTS, max_wait=BATCH_MAX_WAIT) + self._execute_batch(query, params) except Exception as e: logger.debug(f'Retry failed batch failed - queuing for individual writes retry: [query: {query}, params: {params}]') last_chance_batches.append((query, params)) @@ -259,7 +270,7 @@ def _retry_failed_batches(self, failed_batches): 'params': [p] } try: - self.graph_client.execute_query_with_retry(query, single_params, max_attempts=BATCH_MAX_ATTEMPTS, max_wait=BATCH_MAX_WAIT) + self._execute_batch(query, single_params) except Exception as e: logger.error(f'Failed single write: [query: {query}, params: {params}, error: {str(e)}]') raise e @@ -269,7 +280,7 @@ def _apply_batch_query_tree(self, query_tree_id, parameters): query_tree = self.query_trees[query_tree_id] - def graph_store_op(q, p): + def graph_store_op(q, p, **kwargs): all_params = p['params'] @@ -284,7 +295,7 @@ def graph_store_op(q, p): 'params': chunk } - results = self.graph_client.execute_query_with_retry(q, params, max_attempts=BATCH_MAX_ATTEMPTS, max_wait=BATCH_MAX_WAIT) + results = self.graph_client.execute_query_with_retry(q, params, max_attempts=BATCH_MAX_ATTEMPTS, max_wait=BATCH_MAX_WAIT, **kwargs) for r in results: yield r diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/graph_summary_builder.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/graph_summary_builder.py index 18c65806c..b68b40441 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/graph_summary_builder.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/graph_summary_builder.py @@ -5,7 +5,7 @@ from typing import Any from graphrag_toolkit.lexical_graph.indexing.model import Fact -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.graph.graph_utils import label_from, relationship_name_from from graphrag_toolkit.lexical_graph.indexing.build.graph_builder import GraphBuilder from graphrag_toolkit.lexical_graph.indexing.constants import DEFAULT_CLASSIFICATION, LOCAL_ENTITY_CLASSIFICATION @@ -116,7 +116,7 @@ def build(self, node:BaseNode, graph_client:GraphStore, **kwargs:Any): query = '\n'.join(statements) - graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=10, max_wait=10) + graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=10, max_wait=10, operation=GraphQueryOperation.UPDATE_GRAPH_SUMMARY) else: logger.warning(f'fact_id missing from fact node [node_id: {node.node_id}]') \ No newline at end of file diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/local_entity_rewrites_graph_builder.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/local_entity_rewrites_graph_builder.py index b23253549..40074e295 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/local_entity_rewrites_graph_builder.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/local_entity_rewrites_graph_builder.py @@ -5,7 +5,7 @@ from typing import Any from graphrag_toolkit.lexical_graph.indexing.model import Fact -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore, Query, QueryTree +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore, Query, QueryTree from graphrag_toolkit.lexical_graph.indexing.build.graph_builder import GraphBuilder from graphrag_toolkit.lexical_graph.indexing.utils.fact_utils import string_complement_to_entity from graphrag_toolkit.lexical_graph.indexing.constants import LOCAL_ENTITY_CLASSIFICATION @@ -43,7 +43,8 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): WHERE {graph_client.node_id('n.entityId')} = params.n_id AND {graph_client.node_id('c.entityId')} = params.c_id MERGE (s)-[:`__RELATION__`{{value:r.value}}]->(n) MERGE (n)-[:`__OBJECT__`]->(f) - """ + """, + operation=GraphQueryOperation.COPY_COMPLEMENT_RELATIONSHIPS, ) delete_complement_relationships = Query( @@ -54,7 +55,8 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): DELETE r1 DELETE r2 DETACH DELETE c - """ + """, + operation=GraphQueryOperation.DELETE_COMPLEMENT, ) if fact.subject: @@ -72,7 +74,8 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): child_queries=[ copy_complement_relationships_to_subject, delete_complement_relationships - ] + ], + operation=GraphQueryOperation.FIND_COMPLEMENTS, ) params = { @@ -98,7 +101,8 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): child_queries=[ copy_complement_relationships_to_subject, delete_complement_relationships - ] + ], + operation=GraphQueryOperation.FIND_SUBJECTS, ) params = { diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/source_graph_builder.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/source_graph_builder.py index 51ff59a83..1eebfcf29 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/source_graph_builder.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/source_graph_builder.py @@ -4,7 +4,7 @@ import logging from typing import Any -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.indexing.build.graph_builder import GraphBuilder from graphrag_toolkit.lexical_graph.versioning import VALID_FROM, VALID_TO, VERSION_INDEPENDENT_ID_FIELDS from graphrag_toolkit.lexical_graph.versioning import EXTRACT_TIMESTAMP, BUILD_TIMESTAMP, PREV_VERSIONS @@ -111,7 +111,8 @@ def format_assigment(key): query = '\n'.join(statements) - graph_client.execute_query_with_retry(query, self._to_params(clean_metadata)) + clean_metadata['_source_id'] = source_id + graph_client.execute_query_with_retry(query, self._to_params(clean_metadata), operation=GraphQueryOperation.UPSERT_SOURCE) # prev_source_ids = source_metadata.get('prev_versions', []) diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/statement_graph_builder.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/statement_graph_builder.py index f075dfd03..4bf9b485a 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/statement_graph_builder.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/statement_graph_builder.py @@ -5,7 +5,7 @@ from typing import Any from graphrag_toolkit.lexical_graph.indexing.model import Statement -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.indexing.build.graph_builder import GraphBuilder from llama_index.core.schema import BaseNode @@ -83,7 +83,7 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): query = '\n'.join(statements) - graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query, self._to_params(properties), max_attempts=5, max_wait=7, operation=GraphQueryOperation.UPSERT_STATEMENT) if statement.chunkId: @@ -103,7 +103,7 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): query_c = '\n'.join(statements_c) - graph_client.execute_query_with_retry(query_c, self._to_params(properties_c), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query_c, self._to_params(properties_c), max_attempts=5, max_wait=7, operation=GraphQueryOperation.LINK_STATEMENT_CHUNK) if statement.topicId: @@ -122,7 +122,7 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): query_t = '\n'.join(statements_t) - graph_client.execute_query_with_retry(query_t, self._to_params(properties_t), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query_t, self._to_params(properties_t), max_attempts=5, max_wait=7, operation=GraphQueryOperation.LINK_STATEMENT_TOPIC) if prev_statement: @@ -141,7 +141,7 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): query_p = '\n'.join(statements_p) - graph_client.execute_query_with_retry(query_p, self._to_params(properties_p), max_attempts=5, max_wait=7) + graph_client.execute_query_with_retry(query_p, self._to_params(properties_p), max_attempts=5, max_wait=7, operation=GraphQueryOperation.LINK_STATEMENTS) else: logger.warning(f'statement_id missing from statement node [node_id: {node.node_id}]') \ No newline at end of file diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/topic_graph_builder.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/topic_graph_builder.py index bb588ab9e..0ae0491e6 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/topic_graph_builder.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/build/topic_graph_builder.py @@ -5,7 +5,7 @@ from typing import Any from graphrag_toolkit.lexical_graph.indexing.model import Topic -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.indexing.build.graph_builder import GraphBuilder from llama_index.core.schema import BaseNode @@ -88,7 +88,7 @@ def build(self, node:BaseNode, graph_client: GraphStore, **kwargs:Any): query = '\n'.join(statements) - graph_client.execute_query_with_retry(query, self._to_params(properties)) + graph_client.execute_query_with_retry(query, self._to_params(properties), operation=GraphQueryOperation.UPSERT_TOPIC) else: logger.warning(f'topic_id missing from topic node [node_id: {node.node_id}]') diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_context_provider.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_context_provider.py index 313dde94d..bc2da8867 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_context_provider.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_context_provider.py @@ -6,7 +6,7 @@ import json from typing import List, Dict -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.graph.graph_utils import node_result from graphrag_toolkit.lexical_graph.retrieval.model import ScoredEntity, EntityContexts, EntityContext from graphrag_toolkit.lexical_graph.retrieval.processors import ProcessorArgs @@ -69,7 +69,7 @@ def _get_entity_id_context_tree(self, entities:List[ScoredEntity]) -> Dict[str, 'numNeighbours': depth + 2 } - results = self.graph_store.execute_query(cypher, params) + results = self.graph_store.execute_query(cypher, params, operation=GraphQueryOperation.FIND_ENTITY_NEIGHBORS) new_entity_id_contexts = {} @@ -138,7 +138,7 @@ def walk_tree(d): 'entityIds': list(neighbour_entity_ids) } - results = self.graph_store.execute_query(cypher, params) + results = self.graph_store.execute_query(cypher, params, operation=GraphQueryOperation.SCORE_ENTITIES) neighbour_entities = [ ScoredEntity.model_validate(result['result']) for result in results diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_provider.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_provider.py index 88498b15a..bb189f796 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_provider.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_provider.py @@ -4,7 +4,7 @@ import logging from typing import List, Iterator, cast, Optional -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.metadata import FilterConfig from graphrag_toolkit.lexical_graph.storage.graph.graph_utils import node_result, search_string_from, filter_config_to_opencypher_filters from graphrag_toolkit.lexical_graph.retrieval.model import ScoredEntity @@ -57,7 +57,7 @@ def _get_entities_for_keyword(self, keyword:str) -> List[ScoredEntity]: 'keyword': search_string_from(parts[0]) } - results = self.graph_store.execute_query(cypher, params) + results = self.graph_store.execute_query(cypher, params, operation=GraphQueryOperation.FIND_ENTITIES_BY_KEYWORD) entities = [ ScoredEntity.model_validate(result['result']) @@ -80,7 +80,9 @@ def _get_entities_for_keyword(self, keyword:str) -> List[ScoredEntity]: params = { 'keyword': search_string_from(parts[0]), - 'classification': parts[1] + 'classification': parts[1], + '_starts_with': True, + '_classification_starts_with': True, } else: cypher = f""" @@ -95,10 +97,11 @@ def _get_entities_for_keyword(self, keyword:str) -> List[ScoredEntity]: }} AS result""" params = { - 'keyword': search_string_from(parts[0]) + 'keyword': search_string_from(parts[0]), + '_starts_with': True, } - results = self.graph_store.execute_query(cypher, params) + results = self.graph_store.execute_query(cypher, params, operation=GraphQueryOperation.FIND_ENTITIES_BY_KEYWORD) entities = [ ScoredEntity.model_validate(result['result']) diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_vss_provider.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_vss_provider.py index 9c7b7667c..a6b67acc2 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_vss_provider.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/entity_vss_provider.py @@ -5,7 +5,7 @@ from typing import List, Optional from graphrag_toolkit.lexical_graph.metadata import FilterConfig -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.vector import VectorStore from graphrag_toolkit.lexical_graph.storage.vector import DummyVectorIndex from graphrag_toolkit.lexical_graph.storage.graph.graph_utils import node_result @@ -43,6 +43,7 @@ def _get_node_ids(self, keywords:List[str]) -> List[str]: def _get_entities_for_nodes(self, node_ids:List[str]) -> List[ScoredEntity]: if self.index_name == 'topic': + operation = GraphQueryOperation.FIND_ENTITIES_BY_TOPICS cypher = f""" // get entities for topic ids MATCH (t:`__Topic__`)<-[:`__BELONGS_TO__`]-(:`__Statement__`) @@ -58,6 +59,7 @@ def _get_entities_for_nodes(self, node_ids:List[str]) -> List[ScoredEntity]: }} AS result """ else: + operation = GraphQueryOperation.FIND_ENTITIES_BY_CHUNKS cypher = f""" // get entities for chunk ids MATCH (c:`__Chunk__`)<-[:`__MENTIONED_IN__`]-(:`__Statement__`) @@ -78,7 +80,7 @@ def _get_entities_for_nodes(self, node_ids:List[str]) -> List[ScoredEntity]: 'limit': self.args.intermediate_limit } - results = self.graph_store.execute_query(cypher, parameters) + results = self.graph_store.execute_query(cypher, parameters, operation=operation) scored_entities = [ ScoredEntity.model_validate(result['result']) diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/keyword_vss_provider.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/keyword_vss_provider.py index 22867c54c..d219425bc 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/keyword_vss_provider.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/query_context/keyword_vss_provider.py @@ -7,7 +7,7 @@ from graphrag_toolkit.lexical_graph.config import GraphRAGConfig from graphrag_toolkit.lexical_graph.utils import LLMCache, LLMCacheType from graphrag_toolkit.lexical_graph.metadata import FilterConfig -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.chunk_store_factory import ChunkStoreFactory from graphrag_toolkit.lexical_graph.storage.vector import VectorStore from graphrag_toolkit.lexical_graph.storage.vector import DummyVectorIndex @@ -106,7 +106,7 @@ def get_statements_for_topic(topic_id): 'statementLimit': self.args.intermediate_limit } - results = self.graph_store.execute_query(cypher, parameters) + results = self.graph_store.execute_query(cypher, parameters, operation=GraphQueryOperation.GET_TOPIC) return '\n'.join(format_statement(r) for r in results) @@ -156,4 +156,4 @@ def _get_keywords(self, query_bundle:QueryBundle) -> List[str]: content = self._get_content(node_ids) keywords = self._get_keywords_from_content(query_bundle.query_str, content) - return keywords \ No newline at end of file + return keywords diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/chunk_based_search.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/chunk_based_search.py index c470d0f6e..03789cb9f 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/chunk_based_search.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/chunk_based_search.py @@ -8,7 +8,7 @@ from graphrag_toolkit.lexical_graph.metadata import FilterConfig from graphrag_toolkit.lexical_graph.retrieval.model import SearchResultCollection from graphrag_toolkit.lexical_graph.storage.vector.vector_store import VectorStore -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.retrieval.processors import ProcessorBase, ProcessorArgs from graphrag_toolkit.lexical_graph.retrieval.retrievers.traversal_based_base_retriever import TraversalBasedBaseRetriever from graphrag_toolkit.lexical_graph.retrieval.utils.vector_utils import get_diverse_vss_elements @@ -96,7 +96,7 @@ def chunk_based_graph_search(self, chunk_id): 'statementLimit': self.args.intermediate_limit } - results = self.graph_store.execute_query(cypher, properties) + results = self.graph_store.execute_query(cypher, properties, operation=GraphQueryOperation.SEARCH_BY_CHUNK) statement_ids = [r['l'] for r in results] return statement_ids diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/entity_based_search.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/entity_based_search.py index f71e97993..2ad5dc53a 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/entity_based_search.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/entity_based_search.py @@ -7,7 +7,7 @@ from graphrag_toolkit.lexical_graph.metadata import FilterConfig from graphrag_toolkit.lexical_graph.retrieval.model import SearchResultCollection -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.vector.vector_store import VectorStore from graphrag_toolkit.lexical_graph.retrieval.processors import ProcessorBase, ProcessorArgs from graphrag_toolkit.lexical_graph.retrieval.retrievers.traversal_based_base_retriever import TraversalBasedBaseRetriever @@ -164,7 +164,7 @@ def _multiple_entity_based_graph_search(self, start_id, end_ids, query:QueryBund 'statementLimit': self.args.intermediate_limit } - results = self.graph_store.execute_query(cypher, properties) + results = self.graph_store.execute_query(cypher, properties, operation=GraphQueryOperation.SEARCH_BY_ENTITIES) statement_ids = [r['l'] for r in results] return statement_ids @@ -200,7 +200,7 @@ def _single_entity_based_graph_search(self, entity_id, query:QueryBundle): 'statementLimit': self.args.intermediate_limit } - results = self.graph_store.execute_query(cypher, properties) + results = self.graph_store.execute_query(cypher, properties, operation=GraphQueryOperation.SEARCH_BY_ENTITY) statement_ids = [r['l'] for r in results] return statement_ids diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/entity_network_search.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/entity_network_search.py index 33d32a006..14b032e68 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/entity_network_search.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/entity_network_search.py @@ -8,7 +8,7 @@ from graphrag_toolkit.lexical_graph.metadata import FilterConfig from graphrag_toolkit.lexical_graph.retrieval.model import SearchResultCollection -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.vector.vector_store import VectorStore from graphrag_toolkit.lexical_graph.storage.vector.dummy_vector_index import DummyVectorIndex from graphrag_toolkit.lexical_graph.retrieval.processors import ProcessorBase, ProcessorArgs @@ -62,12 +62,14 @@ def __init__(self, def _graph_search(self, node_id): if self.index_name == 'topic': + operation = GraphQueryOperation.SEARCH_BY_TOPIC cypher = f'''// topic-based entity network search MATCH (l)-[:`__BELONGS_TO__`]->(t:`__Topic__`) WHERE {self.graph_store.node_id("t.topicId")} = $nodeId RETURN DISTINCT {self.graph_store.node_id("l.statementId")} AS l LIMIT $statementLimit ''' else: + operation = GraphQueryOperation.SEARCH_BY_CHUNK cypher = f'''// chunk-based entity network search MATCH (l)-[:`__BELONGS_TO__`]->()-[:`__MENTIONED_IN__`]->(c:`__Chunk__`) WHERE {self.graph_store.node_id("c.chunkId")} = $nodeId @@ -79,7 +81,7 @@ def _graph_search(self, node_id): 'statementLimit': self.args.intermediate_limit } - results = self.graph_store.execute_query(cypher, properties) + results = self.graph_store.execute_query(cypher, properties, operation=operation) statement_ids = [r['l'] for r in results] return statement_ids diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/traversal_based_base_retriever.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/traversal_based_base_retriever.py index b93b0b6f6..838d3c419 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/traversal_based_base_retriever.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/retrieval/retrievers/traversal_based_base_retriever.py @@ -10,7 +10,7 @@ from graphrag_toolkit.lexical_graph.config import GraphRAGConfig from graphrag_toolkit.lexical_graph.metadata import FilterConfig from graphrag_toolkit.lexical_graph.versioning import VALID_FROM, VALID_TO, EXTRACT_TIMESTAMP, BUILD_TIMESTAMP, VERSION_INDEPENDENT_ID_FIELDS, TIMESTAMP_LOWER_BOUND, TIMESTAMP_UPPER_BOUND -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.chunk_store_factory import ChunkStoreFactory from graphrag_toolkit.lexical_graph.storage.vector.vector_store import VectorStore from graphrag_toolkit.lexical_graph.retrieval.query_context import KeywordProvider, KeywordVSSProvider, KeywordNLPProvider, KeywordProviderMode, PassThruKeywordProvider @@ -148,7 +148,8 @@ def get_statements_by_topic_and_source(self, statement_ids): statements_params = { 'statementLimit': self.args.intermediate_limit, 'limit': self.args.query_limit, - 'statementIds': statement_ids + 'statementIds': statement_ids, + 'includeChunkMetadata': self.args.include_chunk_details, } chunk_metadata = 'properties(c)' if self.args.include_chunk_details else '{}' @@ -191,7 +192,7 @@ def get_statements_by_topic_and_source(self, statement_ids): topics: topics }} as result ORDER BY result.score DESC LIMIT $limit''' - statements_results = self.graph_store.execute_query(statements_cypher, statements_params) + statements_results = self.graph_store.execute_query(statements_cypher, statements_params, operation=GraphQueryOperation.GET_STATEMENTS) statement_facts_cypher = f'''// get facts for statements MATCH (f)-[:`__SUPPORTS__`]->(l:`__Statement__`) @@ -202,7 +203,7 @@ def get_statements_by_topic_and_source(self, statement_ids): 'statementIds': statement_ids } - statement_facts_results = self.graph_store.execute_query(statement_facts_cypher, statement_facts_params) + statement_facts_results = self.graph_store.execute_query(statement_facts_cypher, statement_facts_params, operation=GraphQueryOperation.GET_FACTS) statement_facts = { r['statementId']:r['facts'] for r in statement_facts_results @@ -428,4 +429,4 @@ def do_graph_search(self, query_bundle: QueryBundle, start_node_ids:List[str]) - NotImplementedError: This method must be implemented by any subclass and cannot be invoked directly from the abstract base class. """ - pass \ No newline at end of file + pass diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/chunk/in_graph_chunk_store.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/chunk/in_graph_chunk_store.py index c133e9717..01a14615f 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/chunk/in_graph_chunk_store.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/chunk/in_graph_chunk_store.py @@ -5,7 +5,7 @@ from typing import Dict, List from graphrag_toolkit.lexical_graph.storage.chunk.chunk_store import ChunkStore -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.graph.graph_utils import node_result, to_params logger = logging.getLogger(__name__) @@ -47,7 +47,11 @@ def get_batch(self, chunk_ids: List[str]) -> Dict[str, str]: ''' params = {'chunk_ids': chunk_ids} - rows = self.graph_client.execute_query(query, params) + rows = self.graph_client.execute_query( + query, + params, + operation=GraphQueryOperation.GET_CHUNKS, + ) return { row['result']['chunk']['chunkId']: row['result']['chunk']['value'] @@ -86,4 +90,10 @@ def put_batch(self, chunks: Dict[str, str]) -> None: ] } - self.graph_client.execute_query_with_retry(query, params, max_attempts=5, max_wait=7) + self.graph_client.execute_query_with_retry( + query, + params, + max_attempts=5, + max_wait=7, + operation=GraphQueryOperation.UPSERT_CHUNK, + ) diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/__init__.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/__init__.py index efd1711c4..8d24e0bee 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/__init__.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/__init__.py @@ -5,4 +5,5 @@ from .graph_store_factory_method import GraphStoreFactoryMethod from .multi_tenant_graph_store import MultiTenantGraphStore from .dummy_graph_store import DummyGraphStore +from .graph_query_operation import GraphQueryOperation from .query_tree import Query, QueryTree \ No newline at end of file diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/graph_query_operation.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/graph_query_operation.py new file mode 100644 index 000000000..9d963e65c --- /dev/null +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/graph_query_operation.py @@ -0,0 +1,45 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from enum import Enum + + +class GraphQueryOperation(str, Enum): + """Backend-neutral lexical-graph operations. + + Query-language-specific stores use these identifiers to select their native + implementation. + """ + + UPSERT_SOURCE = 'upsert_source' + UPSERT_CHUNK = 'upsert_chunk' + LINK_CHUNK_SOURCE = 'link_chunk_source' + LINK_CHUNKS = 'link_chunks' + UPSERT_TOPIC = 'upsert_topic' + UPSERT_STATEMENT = 'upsert_statement' + LINK_STATEMENT_CHUNK = 'link_statement_chunk' + LINK_STATEMENT_TOPIC = 'link_statement_topic' + LINK_STATEMENTS = 'link_statements' + UPSERT_FACT = 'upsert_fact' + LINK_FACT_ENTITY = 'link_fact_entity' + UPSERT_ENTITY = 'upsert_entity' + LINK_ENTITIES = 'link_entities' + ADD_ENTITY_TYPE = 'add_entity_type' + UPDATE_GRAPH_SUMMARY = 'update_graph_summary' + FIND_COMPLEMENTS = 'find_complements' + FIND_SUBJECTS = 'find_subjects' + COPY_COMPLEMENT_RELATIONSHIPS = 'copy_complement_relationships' + DELETE_COMPLEMENT = 'delete_complement' + GET_STATEMENTS = 'get_statements' + GET_FACTS = 'get_facts' + GET_CHUNKS = 'get_chunks' + GET_TOPIC = 'get_topic' + SEARCH_BY_CHUNK = 'search_by_chunk' + SEARCH_BY_TOPIC = 'search_by_topic' + FIND_ENTITIES_BY_KEYWORD = 'find_entities_by_keyword' + FIND_ENTITIES_BY_CHUNKS = 'find_entities_by_chunks' + FIND_ENTITIES_BY_TOPICS = 'find_entities_by_topics' + FIND_ENTITY_NEIGHBORS = 'find_entity_neighbors' + SCORE_ENTITIES = 'score_entities' + SEARCH_BY_ENTITY = 'search_by_entity' + SEARCH_BY_ENTITIES = 'search_by_entities' diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/graph_store.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/graph_store.py index 61bd85ef9..38ac8844e 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/graph_store.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/graph_store.py @@ -11,6 +11,7 @@ from graphrag_toolkit.lexical_graph import TenantId, GraphQueryError from graphrag_toolkit.lexical_graph.storage.graph.query_tree import QueryTree +from graphrag_toolkit.lexical_graph.storage.graph.graph_query_operation import GraphQueryOperation from llama_index.core.bridge.pydantic import BaseModel, Field @@ -67,7 +68,7 @@ class NodeId: def __str__(self): return self.value - + def format_id(id_name:str): """ Parses and formats the given ID string into a NodeId object. @@ -390,7 +391,7 @@ def __exit__(self, exception_type, exception_value, traceback): def unretriable_exception_types(self) -> Tuple: return () - def execute_query_with_retry(self, query:str, parameters:Dict[str, Any], max_attempts=3, max_wait=5, **kwargs) -> Dict[str, Any]: + def execute_query_with_retry(self, query:str, parameters:Dict[str, Any], max_attempts=3, max_wait=5, operation:Optional[GraphQueryOperation]=None, **kwargs) -> Dict[str, Any]: """ Executes a database query with a retry mechanism, allowing multiple attempts with delays between them. @@ -414,8 +415,9 @@ def execute_query_with_retry(self, query:str, parameters:Dict[str, Any], max_att """ correlation_id = uuid.uuid4().hex[:5] - if 'correlation_id' in kwargs: - correlation_id = f'{kwargs["correlation_id"]}/{correlation_id}' + supplied_correlation_id = kwargs.get('correlation_id') + if supplied_correlation_id: + correlation_id = f'{supplied_correlation_id}/{correlation_id}' kwargs['correlation_id'] = correlation_id log_entry_parameters = self.log_formatting.format_log_entry(f'{correlation_id}/*', query, parameters) @@ -435,9 +437,11 @@ def execute_query_with_retry(self, query:str, parameters:Dict[str, Any], max_att attempt_number += 1 attempt.retry_state.attempt_number if isinstance(query, str): + if operation is not None: + return self._execute_operation(operation, query, parameters, **kwargs) return self._execute_query(query, parameters, **kwargs) elif isinstance(query, QueryTree): - return query.run(parameters, self.execute_query_with_retry) + return query.run(parameters, self.execute_query_with_retry, **kwargs) else: raise ValueError(f'Invalid query type. Expected string or Query Tree but received {type(query).__name__}.') @@ -496,22 +500,19 @@ def property_assigment_fn(self, key:str, value:Any) -> Callable[[str], str]: """ return lambda x: x - def execute_query(self, query:QUERY_TYPE, parameters={}, correlation_id:Optional[str]=None) -> List[Any]: - if correlation_id: - return self.execute_query_with_retry( - query=query, - parameters=parameters, - max_attempts=1, - max_wait=0, - correlation_id=correlation_id - ) - else: - return self.execute_query_with_retry( - query=query, - parameters=parameters, - max_attempts=1, - max_wait=0 - ) + def execute_query(self, query:QUERY_TYPE, parameters={}, correlation_id:Optional[str]=None, operation:Optional[GraphQueryOperation]=None) -> List[Any]: + return self.execute_query_with_retry( + query=query, + parameters=parameters, + max_attempts=1, + max_wait=0, + correlation_id=correlation_id, + operation=operation, + ) + + def _execute_operation(self, operation:GraphQueryOperation, query:str, parameters:Dict[str, Any], correlation_id=None, **kwargs) -> List[Any]: + """Execute the existing native query for stores without an override.""" + return self._execute_query(query, parameters, correlation_id=correlation_id) @abc.abstractmethod @@ -528,10 +529,6 @@ def _execute_query(self, cypher, parameters={}, correlation_id=None) -> List[Any A dictionary containing the results of the executed Cypher query. """ raise NotImplementedError - + def init(self, graph_store=None): pass - - - - \ No newline at end of file diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/multi_tenant_graph_store.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/multi_tenant_graph_store.py index b2ef156a7..61932e131 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/multi_tenant_graph_store.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/multi_tenant_graph_store.py @@ -6,6 +6,7 @@ from graphrag_toolkit.lexical_graph import TenantId from graphrag_toolkit.lexical_graph.storage.constants import LEXICAL_GRAPH_LABELS from graphrag_toolkit.lexical_graph.storage.graph import GraphStore, NodeId +from graphrag_toolkit.lexical_graph.storage.graph.query_tree import QueryTree class MultiTenantGraphStore(GraphStore): """ @@ -64,7 +65,12 @@ def execute_query_with_retry(self, query:str, parameters:Dict[str, Any], max_att **kwargs: Additional optional keyword arguments to be passed to the `execute_query_with_retry` method of the `inner` object. """ - return self.inner.execute_query_with_retry(query=self._rewrite_query(query), parameters=parameters, max_attempts=max_attempts, max_wait=max_wait) + if isinstance(query, QueryTree): + kwargs.setdefault('tenant_id', self.tenant_id.value) + return super().execute_query_with_retry(query=query, parameters=parameters, max_attempts=max_attempts, max_wait=max_wait, **kwargs) + if kwargs.get('operation') is not None: + kwargs.setdefault('tenant_id', self.tenant_id.value) + return self.inner.execute_query_with_retry(query=self._rewrite_query(query), parameters=parameters, max_attempts=max_attempts, max_wait=max_wait, **kwargs) def _logging_prefix(self, query_id:str, correlation_id:Optional[str]=None): """ diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/query_tree.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/query_tree.py index 9f36f2db5..50ee2dad6 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/query_tree.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/storage/graph/query_tree.py @@ -3,6 +3,8 @@ from typing import Any, List, Dict, Optional, Callable, Union, Iterable, Generator +from .graph_query_operation import GraphQueryOperation + def _default_params_adapter(v): def _dedup(parameters:List): @@ -26,19 +28,25 @@ class Query(): def __init__(self, query:str, params_adapter:Optional[Callable[[Any], Dict]]=None, - child_queries:Optional[List]=None): + child_queries:Optional[List]=None, + operation:Optional[GraphQueryOperation]=None): self.query = query self.params_adapter = params_adapter or DEFAULT_PARAMS_ADAPTER self.child_queries = child_queries or [] + self.operation = operation class Job(): def __init__(self, query:Query, params:Any): self.query = query self.params = params - def run(self, graph_store_fn:Callable[[str, Dict], List[Any]]): + def run(self, graph_store_fn:Callable[[str, Dict], List[Any]], **kwargs): parameters = self.query.params_adapter(self.params) - return graph_store_fn(self.query.query, parameters) + if self.query.operation is not None: + kwargs['operation'] = self.query.operation + else: + kwargs.pop('tenant_id', None) + return graph_store_fn(self.query.query, parameters, **kwargs) class QueryTree(): @@ -46,7 +54,7 @@ def __init__(self, name:str, root_query:Query): self.id = f'query-tree-{name}' self.root_query = root_query - def run(self, params, graph_store_fn:Callable[[str, Dict], List[Any]]) -> Iterable[Any]: + def run(self, params, graph_store_fn:Callable[[str, Dict], List[Any]], **kwargs) -> Iterable[Any]: job_queue = [] @@ -54,7 +62,7 @@ def run(self, params, graph_store_fn:Callable[[str, Dict], List[Any]]) -> Iterab while job: - results = job.run(graph_store_fn) + results = job.run(graph_store_fn, **kwargs) if job.query.child_queries: for q in job.query.child_queries: diff --git a/lexical-graph/tests/unit/indexing/build/test_entity_graph_builder_domain_label_batch.py b/lexical-graph/tests/unit/indexing/build/test_entity_graph_builder_domain_label_batch.py index ef2ea1145..294214d15 100644 --- a/lexical-graph/tests/unit/indexing/build/test_entity_graph_builder_domain_label_batch.py +++ b/lexical-graph/tests/unit/indexing/build/test_entity_graph_builder_domain_label_batch.py @@ -82,4 +82,7 @@ def test_domain_entity_insert_uses_batch_params_shape(): assert 'UNWIND $params AS params' in query assert 'params.entityId' in query assert '$entityId' not in query - assert params == [{'entityId': ENTITY_ID}] + assert params == [{ + 'entityId': ENTITY_ID, + '_classification': 'Company', + }] diff --git a/lexical-graph/tests/unit/indexing/build/test_graph_batch_client.py b/lexical-graph/tests/unit/indexing/build/test_graph_batch_client.py index bc5c02977..11cc40e15 100644 --- a/lexical-graph/tests/unit/indexing/build/test_graph_batch_client.py +++ b/lexical-graph/tests/unit/indexing/build/test_graph_batch_client.py @@ -4,6 +4,7 @@ import pytest from unittest.mock import Mock from graphrag_toolkit.lexical_graph.indexing.build.graph_batch_client import GraphBatchClient +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, Query, QueryTree class TestGraphBatchClientInitialization: @@ -117,3 +118,43 @@ def test_execute_query_rejects_unsupported_kwargs(self, mock_neptune_store): ) with pytest.raises(TypeError): client.execute_query('MATCH (n) RETURN n', {}, max_attempts=3) + + +class TestGraphBatchClientOperations: + def test_batch_forwards_operation(self, mock_neptune_store): + client = GraphBatchClient(mock_neptune_store, True, 10) + + client.execute_query_with_retry( + 'UPSERT', + {'params': [{'chunk_id': 'c1'}]}, + operation=GraphQueryOperation.UPSERT_CHUNK, + ) + client.apply_batch_operations() + + kwargs = mock_neptune_store.execute_query_with_retry.call_args.kwargs + assert kwargs['operation'] is GraphQueryOperation.UPSERT_CHUNK + + def test_query_tree_batch_forwards_operation(self, mock_neptune_store): + client = GraphBatchClient(mock_neptune_store, True, 10) + tree = QueryTree( + 'lookup', + Query('SELECT', operation=GraphQueryOperation.FIND_COMPLEMENTS), + ) + + client.execute_query_with_retry(tree, {'params': [{'nId': 'e1'}]}) + client.apply_batch_operations() + + kwargs = mock_neptune_store.execute_query_with_retry.call_args.kwargs + assert kwargs['operation'] is GraphQueryOperation.FIND_COMPLEMENTS + + def test_empty_operation_is_a_noop(self, mock_neptune_store): + client = GraphBatchClient(mock_neptune_store, True, 10) + + client.execute_query_with_retry( + 'UNWIND $params AS params', + {'params': []}, + operation=GraphQueryOperation.LINK_FACT_ENTITY, + ) + client.apply_batch_operations() + + mock_neptune_store.execute_query_with_retry.assert_not_called() diff --git a/lexical-graph/tests/unit/storage/chunk/test_in_graph_chunk_store.py b/lexical-graph/tests/unit/storage/chunk/test_in_graph_chunk_store.py index 059866f8d..2f808d6c1 100644 --- a/lexical-graph/tests/unit/storage/chunk/test_in_graph_chunk_store.py +++ b/lexical-graph/tests/unit/storage/chunk/test_in_graph_chunk_store.py @@ -6,7 +6,7 @@ from unittest.mock import Mock from graphrag_toolkit.lexical_graph.storage.chunk import ChunkStore, InGraphChunkStore -from graphrag_toolkit.lexical_graph.storage.graph import GraphStore +from graphrag_toolkit.lexical_graph.storage.graph import GraphQueryOperation, GraphStore from graphrag_toolkit.lexical_graph.storage.graph.graph_store import format_id @@ -49,6 +49,10 @@ def test_get_batch_returns_values_keyed_by_chunk_id(self): 'chunk-1': 'text one', 'chunk-2': 'text two', } + assert ( + graph_client.execute_query.call_args.kwargs['operation'] + is GraphQueryOperation.GET_CHUNKS + ) def test_get_batch_omits_chunks_with_no_match(self): graph_client = Mock(spec=GraphStore) @@ -97,6 +101,10 @@ def test_put_writes_chunk_value_with_retry(self): query, params = graph_client.execute_query_with_retry.call_args.args[:2] assert 'chunk.value' in query assert params['params'] == [{'chunk_id': 'chunk-1', 'text': 'new text'}] + assert ( + graph_client.execute_query_with_retry.call_args.kwargs['operation'] + is GraphQueryOperation.UPSERT_CHUNK + ) class TestInGraphChunkStoreIsChunkStore: diff --git a/lexical-graph/tests/unit/storage/graph/test_dummy_graph_store.py b/lexical-graph/tests/unit/storage/graph/test_dummy_graph_store.py index ba302897f..463687c82 100644 --- a/lexical-graph/tests/unit/storage/graph/test_dummy_graph_store.py +++ b/lexical-graph/tests/unit/storage/graph/test_dummy_graph_store.py @@ -88,6 +88,43 @@ def test_execute_query_with_correlation_id(self): assert isinstance(result, list) assert len(result) == 0 + @pytest.mark.parametrize('correlation_id', [None, '']) + def test_empty_correlation_id_uses_only_generated_id(self, correlation_id): + store = DummyGraphStore() + generated = Mock(hex='abcde12345') + + with patch( + 'graphrag_toolkit.lexical_graph.storage.graph.graph_store.uuid.uuid4', + return_value=generated, + ), patch.object( + DummyGraphStore, + '_execute_query', + autospec=True, + return_value=[], + ) as execute_query: + store.execute_query('MATCH (n) RETURN n', correlation_id=correlation_id) + + assert execute_query.call_args.kwargs['correlation_id'] == 'abcde' + + def test_supplied_correlation_id_prefixes_generated_id(self): + store = DummyGraphStore() + generated = Mock(hex='abcde12345') + + with patch( + 'graphrag_toolkit.lexical_graph.storage.graph.graph_store.uuid.uuid4', + return_value=generated, + ), patch.object( + DummyGraphStore, + '_execute_query', + autospec=True, + return_value=[], + ) as execute_query: + store.execute_query( + 'MATCH (n) RETURN n', correlation_id='request-123' + ) + + assert execute_query.call_args.kwargs['correlation_id'] == 'request-123/abcde' + def test_context_manager_enter_exit(self): """Verify DummyGraphStore works as context manager.""" store = DummyGraphStore() diff --git a/lexical-graph/tests/unit/storage/graph/test_multi_tenant_graph_store.py b/lexical-graph/tests/unit/storage/graph/test_multi_tenant_graph_store.py index f74443b16..c7072ad55 100644 --- a/lexical-graph/tests/unit/storage/graph/test_multi_tenant_graph_store.py +++ b/lexical-graph/tests/unit/storage/graph/test_multi_tenant_graph_store.py @@ -10,6 +10,8 @@ from graphrag_toolkit.lexical_graph.storage.graph.multi_tenant_graph_store import ( MultiTenantGraphStore, ) +from graphrag_toolkit.lexical_graph.storage.graph.graph_query_operation import GraphQueryOperation +from graphrag_toolkit.lexical_graph.storage.graph.query_tree import Query, QueryTree def _wrap(tenant_value=None, labels=None): @@ -65,6 +67,34 @@ def test_execute_query_with_retry_rewrites_and_delegates(self): called_query = inner.execute_query_with_retry.call_args.kwargs['query'] assert '`Sourceacme__`' in called_query assert inner.execute_query_with_retry.call_args.kwargs['parameters'] == {'k': 1} + assert 'tenant_id' not in inner.execute_query_with_retry.call_args.kwargs + + def test_operation_receives_tenant_id(self): + store, inner = _wrap(tenant_value='acme', labels=['Source']) + + store.execute_query_with_retry( + 'MATCH (n:`Source`)', + {'k': 1}, + operation=GraphQueryOperation.GET_FACTS, + ) + + kwargs = inner.execute_query_with_retry.call_args.kwargs + assert kwargs['operation'] is GraphQueryOperation.GET_FACTS + assert kwargs['tenant_id'] == 'acme' + + def test_query_tree_operations_receive_tenant_id(self): + store, inner = _wrap(tenant_value='acme', labels=['Source']) + inner.execute_query_with_retry.return_value = [] + tree = QueryTree( + 'lookup', + Query('MATCH (n:`Source`)', operation=GraphQueryOperation.GET_FACTS), + ) + + list(store.execute_query_with_retry(tree, {'statementIds': ['s1']})) + + kwargs = inner.execute_query_with_retry.call_args.kwargs + assert kwargs['operation'] is GraphQueryOperation.GET_FACTS + assert kwargs['tenant_id'] == 'acme' def test_execute_query_rewrites_and_delegates(self): store, inner = _wrap(tenant_value='acme', labels=['Source']) diff --git a/lexical-graph/tests/unit/storage/graph/test_query_tree.py b/lexical-graph/tests/unit/storage/graph/test_query_tree.py index 0d378039d..3352f033c 100644 --- a/lexical-graph/tests/unit/storage/graph/test_query_tree.py +++ b/lexical-graph/tests/unit/storage/graph/test_query_tree.py @@ -14,6 +14,7 @@ QueryTree, _default_params_adapter, ) +from graphrag_toolkit.lexical_graph.storage.graph.graph_query_operation import GraphQueryOperation class TestDefaultParamsAdapter: @@ -88,6 +89,32 @@ def test_run_uses_custom_adapter(self): store.assert_called_once_with('MATCH (n)', {'wrapped': 'raw'}) + def test_run_drops_operation_context_for_native_query(self): + job = Job(Query('MATCH (n)'), params={}) + store = MagicMock(return_value=[]) + + job.run(store, tenant_id='acme', correlation_id='request-1') + + store.assert_called_once_with( + 'MATCH (n)', {}, correlation_id='request-1', + ) + + def test_run_forwards_semantic_operation_and_context(self): + job = Job( + Query('MATCH (n)', operation=GraphQueryOperation.GET_FACTS), + params={'statementIds': ['s1']}, + ) + store = MagicMock(return_value=[]) + + job.run(store, tenant_id='acme') + + store.assert_called_once_with( + 'MATCH (n)', + {'statementIds': ['s1']}, + operation=GraphQueryOperation.GET_FACTS, + tenant_id='acme', + ) + class TestQueryTree: def test_id_is_prefixed(self):