From cf9aba65d1ecae48b44beed088f2c3d7244be353 Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Mon, 17 Aug 2026 12:47:43 -0700 Subject: [PATCH 01/12] Add search tutorial --- docs/tutorials/python/search.md | 371 ++++++++++++ .../python/tutorial_scripts/search.py | 539 ++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 911 insertions(+) create mode 100644 docs/tutorials/python/search.md create mode 100644 docs/tutorials/python/tutorial_scripts/search.py diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md new file mode 100644 index 000000000..8b01d1071 --- /dev/null +++ b/docs/tutorials/python/search.md @@ -0,0 +1,371 @@ +# Search Indexes + +A [SearchIndex][synapseclient.models.SearchIndex] is a Synapse entity whose content is +defined by a Synapse SQL query. Synapse builds an OpenSearch index from the rows that +query returns, which gives you full-text search, relevance ranking, faceting, and +autocomplete over a table or view. + +This is a different way of asking questions than a +[Table](table.md) or a [Materialized View](materializedview.md). A table is queried with +Synapse SQL and answers "which rows match these exact conditions?". A search index is +queried with the +[OpenSearch Query DSL](https://docs.opensearch.org/latest/query-dsl/) and answers +"which rows are most relevant to this text?" — matching word stems, ignoring +punctuation and case, ranking the best matches first, and counting how many rows fall +into each category. It is what you would put behind a search box. + +This tutorial will walk you through creating a search index and querying it with the +Synapse Python client. + +## Tutorial Purpose +In this tutorial, you will: + +1. Log in, get your project, and create a table to index +2. Create a SearchIndex and wait for it to build +3. Run a full-text search +4. Highlight where the match happened +5. Combine scored clauses with unscored filters, and sort the results +6. Count facets with aggregations +7. Power a type-ahead box with autocomplete +8. Page through results +9. Tune matching with synonyms and analyzers + +## Prerequisites +* This tutorial assumes that you have a Synapse project. +* Pandas must also be installed as shown in the [installation documentation](../installation.md). +* Creating a SearchIndex may be restricted on some Synapse stacks. If `store()` fails + with a 403, your account is not permitted to create search indexes there. + +## 1. Log in, get your project, and create a table to index + +A search index is always defined over an existing table-like entity, so we first create +a small table of study summaries to search over. + +You will want to replace `"My uniquely named project about Alzheimer's Disease"` with +the name of your project. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:setup" +``` + +The steps below use two small helpers — one to print the rows a query matched, and one +to wait out the index build. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:helpers" +``` + +## 2. Create a SearchIndex and wait for it to build + +The `defining_sql` decides which rows and columns are indexed. Unlike a Materialized +View, it must reference exactly one table-like entity — JOIN and UNION across several +entities are not supported. If you need to search across several tables, build a +[Materialized View](materializedview.md) first and index that. + +Storing the entity returns as soon as Synapse has accepted it, but the OpenSearch index +behind it is built in the background. Until the build finishes, queries against the +index either raise an error or report zero hits, which is why we poll with +`wait_for_index`. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:create_index" +``` + +
+ Creating the index should look like: +``` +Created SearchIndex with ID: syn68123456 +Waiting for the search index to build... +Waiting for the search index to build... +Index syn68123456 is queryable with 6 rows +``` +
+ +**Note**: The index tracks its source. When rows in the underlying table change, the +index is updated in the background — you do not need to re-store the SearchIndex. + +## 3. Run a full-text search + +A [`match`](https://docs.opensearch.org/latest/query-dsl/full-text/match/) clause is the +workhorse of full-text search: the text you pass is analyzed the same way the column was +analyzed, so `"alzheimer"` matches `"Alzheimer's disease"`. Every clause kind Synapse +accepts is listed on [Query][synapseclient.models.search_dsl.Query]. + +By default a hit carries every indexed column. `source` narrows that down, and +`response_parts` asks for extras beyond the hits themselves — here the total hit count +and the columns each hit carries. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:full_text_search" +``` + +
+ The results of your searches should look like: +``` +Abstracts mentioning Alzheimer's: +columns: ['study_name', 'diagnosis'] +total_hits=3, returned=3 + ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} + +Anything mentioning tau: +total_hits=1, returned=1 + ROW_ID=5 {'study_name': 'MCI Plasma Biomarkers'} +``` +
+ +Hits come back ranked by relevance, and each one carries its score on +[`hit.score`][synapseclient.models.SearchHit] along with the `row_id` and `row_version` +of the source row. + +## 4. Highlight where the match happened + +A result list is much easier to read when it shows the matching text in context. +`highlight` returns short fragments of the matched columns with the matching terms +wrapped in `` tags. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:columns_and_highlights" +``` + +
+ The result of your highlighted search should look like: +``` +Studies that sequenced something: + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'assay': 'rnaSeq'} + abstract: ['Bulk RNA sequencing across four brain regions in a cohort'] + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'assay': 'wholeGenomeSeq'} + abstract: ['Whole genome sequencing of temporal cortex samples from'] + ROW_ID=4 {'study_name': 'Healthy Aging Single Cell Atlas', 'assay': 'snrnaSeq'} + abstract: ['Single nucleus RNA sequencing of hippocampus from'] +``` +
+ +**Note**: Highlighting, like relevance scoring, depends on the column being indexed as +analyzed text. Step 9 covers how to control that with a +[SearchConfiguration][synapseclient.models.SearchConfiguration]. + +## 5. Combine scored clauses with unscored filters, and sort the results + +A [`bool`](https://docs.opensearch.org/latest/query-dsl/compound/bool/) clause is how +you build a real search request out of several conditions: + +* `must` clauses have to match and **do** contribute to the relevance score +* `filter` and `must_not` clauses have to match (or not match) but **do not** affect + the score — use these for hard constraints like a numeric cutoff +* `should` clauses boost the rows that match them without excluding the rows that don't + +Passing `sort` replaces relevance ranking with an ordering of your choosing. Only column +and `_score` sorts are accepted. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:filters_and_sorting" +``` + +
+ The result of your filtered search should look like: +``` +Sequencing studies with at least 200 participants, largest first: +total_hits=2, returned=2 + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'participant_count': '350'} + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'participant_count': '300'} +``` +
+ +## 6. Count facets with aggregations + +Aggregations answer "how many rows are there of each kind?" — the counts you see next to +the checkboxes in a faceted search UI. A +[`terms`](https://docs.opensearch.org/latest/aggregations/bucket/terms/) aggregation +produces one bucket per distinct value of a column; metric aggregations like `avg` and +`stats` summarize a numeric column. Results come back on `aggregation_results` as the +raw OpenSearch response, with field references rewritten back to your column names. + +`post_filter` is what keeps a facet list usable: it narrows the hits *after* the +aggregations have been computed, so selecting one diagnosis does not make the other +diagnosis counts disappear. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:aggregations" +``` + +
+ The result of your faceted search should look like: +``` +Hits after the post filter: +total_hits=3, returned=3 + ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} + +Facet counts across all studies: +{ + "by_diagnosis": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "Alzheimer's Disease", + "doc_count": 3 + }, + { + "key": "Cognitively Normal", + "doc_count": 1 + }, + { + "key": "Mild Cognitive Impairment", + "doc_count": 1 + }, + { + "key": "Parkinson's Disease", + "doc_count": 1 + } + ] + }, + "mean_cohort_size": { + "value": 261.6666666666667 + } +} +``` +
+ +## 7. Power a type-ahead box with autocomplete + +[`autocomplete()`][synapseclient.models.SearchIndex.autocomplete] is a separate, +synchronous endpoint meant for search-as-you-type: it returns its hits directly instead +of going through the asynchronous job service, so it is fast enough to call on every +keystroke. In exchange, it only accepts prefix-style clauses — `prefix`, +`match_phrase_prefix`, or `match_bool_prefix` — and returns at most 8 hits. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:autocomplete" +``` + +
+ The result of your autocomplete request should look like: +``` +Suggestions for 'Mayo Cl': + ['Mayo Clinic Whole Genome'] +``` +
+ +## 8. Page through results + +A query returns at most 100 hits at a time (25 by default). `from_` and `size` walk +through the result set the way page numbers do. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:pagination" +``` + +
+ The result of paging through your index should look like: +``` +Page starting at offset 0: +total_hits=6, returned=2 + ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'participant_count': '400'} + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'participant_count': '350'} +Page starting at offset 2: +total_hits=6, returned=2 + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'participant_count': '300'} + ROW_ID=5 {'study_name': 'MCI Plasma Biomarkers', 'participant_count': '220'} +Page starting at offset 4: +total_hits=6, returned=2 + ROW_ID=6 {'study_name': 'Parkinson Comparative Cohort', 'participant_count': '180'} + ROW_ID=4 {'study_name': 'Healthy Aging Single Cell Atlas', 'participant_count': '120'} +``` +
+ +**Note**: Offset paging gets expensive deep into a large result set. For that case each +response carries a `next_search_after` cursor — pass it back unchanged as +`SearchQuery(search_after=...)` on the following request and leave `from_` unset. + +## 9. Tune matching with synonyms and analyzers + +Everything above relies on how each column was analyzed when the index was built: how +text is split into tokens, which tokens are dropped, and how they are normalized. Four +org-scoped resources let you control that: + +* [SynonymSet][synapseclient.models.SynonymSet] — terms that should be treated as + equivalent, so someone searching `AD` finds abstracts that say + "Alzheimer's disease" +* [TextAnalyzer][synapseclient.models.TextAnalyzer] — a named OpenSearch analyzer: a + tokenizer plus a chain of token filters, which may reference a SynonymSet +* [ColumnAnalyzerOverride][synapseclient.models.ColumnAnalyzerOverride] — a reusable + bundle assigning specific analyzers to specific columns +* [SearchConfiguration][synapseclient.models.SearchConfiguration] — bundles a default + analyzer with any column overrides; this is what a SearchIndex actually points at + +Each resource belongs to an [Organization][synapseclient.models.Organization] and is +referenced from another resource by its qualified name, +`{organization_name}-{name}`, written as `{"$ref": "my.org-my_analyzer"}`. + +!!! warning "Restricted and permanent" + Creating and updating these resources is restricted to Sage Bionetworks employees, + and the REST API has no delete endpoint for any of them. Once created, a + SynonymSet, TextAnalyzer, ColumnAnalyzerOverride, or SearchConfiguration cannot be + removed, and its owning Organization can no longer be deleted either. Choose names + deliberately. + +Note where the synonym filter goes below. The analyzer declares both a `default` chain, +used when rows are indexed, and a `default_search` chain, used when a query is analyzed. +Putting the synonyms only in `default_search` expands the incoming query instead of +storing every synonym for every row. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:search_configuration" +``` + +A SearchIndex resolves its configuration when the index is built, so set it up front. +Either point the index straight at a configuration with `search_configuration_id`, or +bind a configuration to the parent folder or project — an index with no +`search_configuration_id` of its own walks up the entity hierarchy and uses the first +[SearchConfigBinding][synapseclient.models.SearchConfigBinding] it finds, falling back +to the platform defaults. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:apply_search_configuration" +``` + +
+ Searching the abbreviation against the new index should look like: +``` +Created SearchIndex syn68123457 using config 4321 +Index syn68123457 is queryable with 6 rows +Bound configuration 4321 to syn12345678 +Abstracts matching the abbreviation 'AD': +total_hits=3, returned=3 + ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} +``` +
+ +## Source Code for this Tutorial + +
+ Click to show me + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py" +``` +
+ +## References +- [SearchIndex][synapseclient.models.SearchIndex] +- [SearchQuery][synapseclient.models.SearchQuery] +- [SearchQueryPart][synapseclient.models.SearchQueryPart] +- [SearchIndexQuery][synapseclient.models.SearchIndexQuery] +- [SearchHit][synapseclient.models.SearchHit] +- [Query][synapseclient.models.search_dsl.Query] +- [SearchConfiguration][synapseclient.models.SearchConfiguration] +- [TextAnalyzer][synapseclient.models.TextAnalyzer] +- [SynonymSet][synapseclient.models.SynonymSet] +- [ColumnAnalyzerOverride][synapseclient.models.ColumnAnalyzerOverride] +- [SearchConfigBinding][synapseclient.models.SearchConfigBinding] +- [Organization][synapseclient.models.Organization] +- [Table][synapseclient.models.Table] +- [syn.login][synapseclient.Synapse.login] +- [OpenSearch query DSL](https://docs.opensearch.org/latest/query-dsl/) +- [OpenSearch aggregations](https://docs.opensearch.org/latest/aggregations/) diff --git a/docs/tutorials/python/tutorial_scripts/search.py b/docs/tutorials/python/tutorial_scripts/search.py new file mode 100644 index 000000000..fc017ed21 --- /dev/null +++ b/docs/tutorials/python/tutorial_scripts/search.py @@ -0,0 +1,539 @@ +"""Here is where you'll find the code for the SearchIndex tutorial.""" + +# --8<-- [start:setup] +import json +import time + +import pandas as pd + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseError +from synapseclient.models import ( + Column, + ColumnType, + Project, + SearchIndex, + SearchIndexQuery, + SearchQuery, + SearchQueryPart, + Table, +) +from synapseclient.models.search_dsl import ( + Aggregation, + AvgAggregation, + BoolQuery, + Highlight, + HighlightField, + MatchBoolPrefixFieldOptions, + MatchFieldOptions, + MatchPhraseFieldOptions, + MultiMatchQuery, + Query, + RangeFieldOptions, + SourceFilter, + TermsAggregation, +) + +# Initialize Synapse client +syn = Synapse() +syn.login() + +# Get the project where we want to create the search index +project = Project(name="My uniquely named project about Alzheimer's Disease").get() +project_id = project.id +print(f"Got project with ID: {project_id}") + +# Create the table that will be indexed +table = Table( + name="Study Summaries", + parent_id=project_id, + columns=[ + Column(name="study_name", column_type=ColumnType.STRING), + Column(name="abstract", column_type=ColumnType.LARGETEXT), + Column(name="diagnosis", column_type=ColumnType.STRING), + Column(name="assay", column_type=ColumnType.STRING), + Column(name="participant_count", column_type=ColumnType.INTEGER), + ], +).store() +print(f"Created table with ID: {table.id}") + +# Add the rows we are going to search over +studies = pd.DataFrame( + [ + { + "study_name": "ROSMAP Cortex Proteomics", + "abstract": "Quantitative proteomics of dorsolateral prefrontal cortex " + "from donors with Alzheimer's disease and cognitively normal controls.", + "diagnosis": "Alzheimer's Disease", + "assay": "TMT quantitation", + "participant_count": 400, + }, + { + "study_name": "MSBB RNA Sequencing", + "abstract": "Bulk RNA sequencing across four brain regions in a cohort " + "spanning the full range of Alzheimer's disease neuropathology.", + "diagnosis": "Alzheimer's Disease", + "assay": "rnaSeq", + "participant_count": 300, + }, + { + "study_name": "Mayo Clinic Whole Genome", + "abstract": "Whole genome sequencing of temporal cortex samples from " + "donors with Alzheimer's disease, progressive supranuclear palsy, " + "and controls.", + "diagnosis": "Alzheimer's Disease", + "assay": "wholeGenomeSeq", + "participant_count": 350, + }, + { + "study_name": "Healthy Aging Single Cell Atlas", + "abstract": "Single nucleus RNA sequencing of hippocampus from " + "cognitively normal aged donors, establishing a baseline atlas.", + "diagnosis": "Cognitively Normal", + "assay": "snrnaSeq", + "participant_count": 120, + }, + { + "study_name": "MCI Plasma Biomarkers", + "abstract": "Plasma biomarker panel measuring phosphorylated tau and " + "neurofilament light chain in mild cognitive impairment.", + "diagnosis": "Mild Cognitive Impairment", + "assay": "immunoassay", + "participant_count": 220, + }, + { + "study_name": "Parkinson Comparative Cohort", + "abstract": "Comparative transcriptomic profiling of substantia nigra " + "in Parkinson disease versus age-matched controls.", + "diagnosis": "Parkinson's Disease", + "assay": "rnaSeq", + "participant_count": 180, + }, + ] +) +table.upsert_rows(values=studies, primary_keys=["study_name"]) +print(f"Stored {len(studies)} rows in {table.id}") +# --8<-- [end:setup] + + +# --8<-- [start:helpers] +def print_hits(results: SearchIndexQuery) -> None: + """Print the rows a query matched, one line per hit.""" + print(f"total_hits={results.total_hits}, returned={len(results.hits)}") + for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") + + +def wait_for_index(index: SearchIndex, timeout: int = 600) -> None: + """Wait until the search index has finished building. + + Building the OpenSearch index behind a SearchIndex happens in the background + after `store()` returns. Until that build completes, a query against the + index either raises an error or reports zero hits. + """ + deadline = time.time() + timeout + while time.time() < deadline: + try: + results = index.query( + search_query=SearchQuery(query=Query(match_all={}), size=1), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + if results.total_hits: + print(f"Index {index.id} is queryable with {results.total_hits} rows") + return + except SynapseError: + pass # The index has not been created yet + print("Waiting for the search index to build...") + time.sleep(10) + raise TimeoutError(f"{index.id} did not finish building within {timeout} seconds") + + +# --8<-- [end:helpers] + + +# --8<-- [start:create_index] +def create_search_index() -> SearchIndex: + """ + Example: Create a SearchIndex over a single table and wait for it to build. + """ + index = SearchIndex( + name="Study Summaries Search Index", + description="Full text search over the study summary table", + parent_id=project_id, + # The defining SQL must reference exactly one table-like entity + defining_sql=f"SELECT * FROM {table.id}", + ) + index = index.store() + print(f"Created SearchIndex with ID: {index.id}") + + wait_for_index(index) + return index + + +# --8<-- [end:create_index] + + +# --8<-- [start:full_text_search] +def search_free_text(index: SearchIndex) -> None: + """ + Example: Find every study whose abstract mentions Alzheimer's disease, then + search across several columns at once. + """ + results = index.query( + search_query=SearchQuery( + query=Query(match={"abstract": MatchFieldOptions(query="alzheimer")}), + # Every indexed column comes back on each hit unless a source filter + # narrows them down, and the abstracts are long + source=SourceFilter(includes=["study_name", "diagnosis"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS, SearchQueryPart.SELECT_COLUMNS], + ) + print("Abstracts mentioning Alzheimer's:") + print(f"columns: {[column.name for column in results.select_columns]}") + print_hits(results) + + # A multi_match clause runs the same text across several columns, so the + # person searching does not need to know which column holds the term. + results = index.query( + search_query=SearchQuery( + query=Query( + multi_match=MultiMatchQuery( + query="tau", + # ^2 boosts a match in the study name over one in the abstract + fields=["study_name^2", "abstract"], + ) + ), + source=SourceFilter(includes=["study_name"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print("\nAnything mentioning tau:") + print_hits(results) + + +# --8<-- [end:full_text_search] + + +# --8<-- [start:columns_and_highlights] +def search_with_highlighting(index: SearchIndex) -> None: + """ + Example: Ask for a snippet of the matching text alongside each hit, so a + result list can show why the row matched. + """ + results = index.query( + search_query=SearchQuery( + query=Query(match={"abstract": MatchFieldOptions(query="sequencing")}), + source=SourceFilter(includes=["study_name", "assay"]), + highlight=Highlight( + fields={"abstract": HighlightField(number_of_fragments=1)} + ), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print("Studies that sequenced something:") + for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") + for highlight in hit.highlights: + print(f" {highlight.name}: {highlight.snippets}") + + +# --8<-- [end:columns_and_highlights] + + +# --8<-- [start:filters_and_sorting] +def search_with_filters(index: SearchIndex) -> None: + """ + Example: Combine a scored clause with unscored filters using a bool query, + then order the results by a numeric column instead of by relevance. + """ + results = index.query( + search_query=SearchQuery( + query=Query( + bool=BoolQuery( + # Scored: how well the abstract matches drives relevance + must=[ + Query(match={"abstract": MatchFieldOptions(query="sequencing")}) + ], + # Unscored: a hard cutoff on cohort size + filter=[ + Query(range={"participant_count": RangeFieldOptions(gte=200)}) + ], + # Unscored: drop a diagnosis we are not interested in + must_not=[ + Query( + match_phrase={ + "diagnosis": MatchPhraseFieldOptions( + query="Parkinson's Disease" + ) + } + ) + ], + ) + ), + source=SourceFilter(includes=["study_name", "participant_count"]), + sort=[{"participant_count": "desc"}], + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print("Sequencing studies with at least 200 participants, largest first:") + print_hits(results) + + +# --8<-- [end:filters_and_sorting] + + +# --8<-- [start:aggregations] +def facet_the_results(index: SearchIndex) -> None: + """ + Example: Count how many studies fall under each diagnosis and average their + cohort sizes, while the hit list itself shows only one diagnosis. + """ + results = index.query( + search_query=SearchQuery( + query=Query(match_all={}), + aggregations={ + "by_diagnosis": Aggregation( + terms=TermsAggregation(field="diagnosis", size=10) + ), + "mean_cohort_size": Aggregation( + avg=AvgAggregation(field="participant_count") + ), + }, + # post_filter narrows the hits but not the aggregations, so the facet + # counts still show every option a person could pick next + post_filter=Query( + match_phrase={ + "diagnosis": MatchPhraseFieldOptions(query="Alzheimer's Disease") + } + ), + source=SourceFilter(includes=["study_name", "diagnosis"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print("Hits after the post filter:") + print_hits(results) + print("\nFacet counts across all studies:") + print(json.dumps(results.aggregation_results, indent=2)) + + +# --8<-- [end:aggregations] + + +# --8<-- [start:autocomplete] +def autocomplete_study_names(index: SearchIndex) -> None: + """ + Example: Back a type-ahead box with the autocomplete endpoint, which returns + its results directly instead of running as an asynchronous job. + """ + hits = index.autocomplete( + query=Query( + match_bool_prefix={ + "study_name": MatchBoolPrefixFieldOptions(query="Mayo Cl") + } + ), + source=SourceFilter(includes=["study_name"]), + ) + print("Suggestions for 'Mayo Cl':") + for hit in hits: + print(f" {[field.value for field in hit.fields]}") + + +# --8<-- [end:autocomplete] + + +# --8<-- [start:pagination] +def page_through_results(index: SearchIndex) -> None: + """ + Example: Walk every row in the index two hits at a time. + """ + page_size = 2 + offset = 0 + while True: + results = index.query( + search_query=SearchQuery( + query=Query(match_all={}), + source=SourceFilter(includes=["study_name", "participant_count"]), + sort=[{"participant_count": "desc"}], + from_=offset, + size=page_size, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print(f"Page starting at offset {offset}:") + print_hits(results) + + offset += page_size + if offset >= results.total_hits: + break + + +# --8<-- [end:pagination] + + +# --8<-- [start:search_configuration] +def create_search_configuration() -> str: + """ + Example: Teach the index that "AD" means "Alzheimer's disease" by building a + SynonymSet, wrapping it in a TextAnalyzer, and bundling that analyzer into a + SearchConfiguration. + + These resources belong to an Organization, and creating them is restricted + to Sage Bionetworks employees. None of them can be deleted once created. + """ + from synapseclient.models import ( + ColumnAnalyzerOverride, + ColumnAnalyzerOverrideEntry, + Organization, + SearchConfiguration, + SynonymSet, + TextAnalyzer, + ) + + organization_name = "my.uniquely.named.organization" + organization = Organization(name=organization_name).store() + print(f"Using organization: {organization.id} ({organization.name})") + + # Comma-separated entries are interchangeable in both directions; entries + # written with "=>" expand the left side to the right side only. + synonyms = SynonymSet( + organization_name=organization_name, + name="ad_synonyms", + description="Abbreviations used across Alzheimer's disease studies", + definition={ + "type": "synonym_graph", + "synonyms": [ + "rna sequencing, rna-seq, rnaseq", + "ad => alzheimer's disease, alzheimers disease", + "mci => mild cognitive impairment", + ], + }, + ).store() + print(f"Created SynonymSet: {synonyms.id} ({synonyms.qualified_name})") + + # The synonym filter is applied in `default_search` only, so synonyms expand + # the incoming query rather than bloating the stored index. + analyzer = TextAnalyzer( + organization_name=organization_name, + name="ad_synonym_analyzer", + description="English analyzer that expands AD abbreviations at search time", + settings={ + "filter": { + "english_stop": {"type": "stop", "stopwords": "_english_"}, + "english_stemmer": {"type": "stemmer", "language": "english"}, + # A $ref resolves to the SynonymSet by its qualified name + "ad_synonyms": {"$ref": synonyms.qualified_name}, + }, + "analyzer": { + "default": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase", "english_stop", "english_stemmer"], + }, + "default_search": { + "type": "custom", + "tokenizer": "standard", + "filter": [ + "lowercase", + "ad_synonyms", + "english_stop", + "english_stemmer", + ], + }, + }, + }, + ).store() + print(f"Created TextAnalyzer: {analyzer.id} ({analyzer.qualified_name})") + + # Columns not named here fall back to the configuration's default analyzer + overrides = ColumnAnalyzerOverride( + organization_name=organization_name, + name="study_column_overrides", + description="Treat the diagnosis column as a single exact value", + overrides=[ + ColumnAnalyzerOverrideEntry( + column_name="diagnosis", + analyzer={"analyzer": {"default": {"type": "keyword"}}}, + ), + ], + ).store() + print(f"Created ColumnAnalyzerOverride: {overrides.id}") + + configuration = SearchConfiguration( + organization_name=organization_name, + name="study_search_config", + description="Analyzer settings for the study summary search index", + default_analyzer={"$ref": analyzer.qualified_name}, + column_analyzer_overrides=[{"$ref": overrides.qualified_name}], + ).store() + print(f"Created SearchConfiguration: {configuration.id}") + return configuration.id + + +# --8<-- [end:search_configuration] + + +# --8<-- [start:apply_search_configuration] +def create_index_with_configuration(search_configuration_id: str) -> SearchIndex: + """ + Example: Build an index that uses a specific SearchConfiguration, and bind + the same configuration to the project so later indexes inherit it. + """ + from synapseclient.models import SearchConfigBinding + + index = SearchIndex( + name="Study Summaries Search Index With Synonyms", + parent_id=project_id, + defining_sql=f"SELECT * FROM {table.id}", + search_configuration_id=search_configuration_id, + ).store() + print(f"Created SearchIndex {index.id} using config {search_configuration_id}") + wait_for_index(index) + + # Any index created under this project without its own + # search_configuration_id now inherits this configuration + binding = SearchConfigBinding( + object_id=project_id, + search_configuration_id=search_configuration_id, + ).store() + print(f"Bound configuration {binding.search_configuration_id} to {project_id}") + + # "AD" now matches the abstracts that spell out "Alzheimer's disease" + results = index.query( + search_query=SearchQuery( + query=Query(match={"abstract": MatchFieldOptions(query="AD")}), + source=SourceFilter(includes=["study_name", "diagnosis"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print("Abstracts matching the abbreviation 'AD':") + print_hits(results) + return index + + +# --8<-- [end:apply_search_configuration] + + +def main(): + index = create_search_index() + search_free_text(index) + search_with_highlighting(index) + search_with_filters(index) + facet_the_results(index) + autocomplete_study_names(index) + page_through_results(index) + + # Requires an Organization you can write to + # search_configuration_id = create_search_configuration() + # create_index_with_configuration(search_configuration_id) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 60aab2d64..d759fa3b8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -39,6 +39,7 @@ nav: - Dataset: tutorials/python/dataset.md - Dataset Collection: tutorials/python/dataset_collection.md - Materialized View: tutorials/python/materializedview.md + - Search Index: tutorials/python/search.md - Submission View: tutorials/python/submissionview.md - Sharing Settings: tutorials/python/sharing_settings.md - Wiki: tutorials/python/wiki.md From eea9ab81b20db1407c4b3d2b273cc0ab4323f970 Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Thu, 20 Aug 2026 19:37:05 -0700 Subject: [PATCH 02/12] Simplify search index tutorial --- docs/tutorials/python/search.md | 34 +- .../python/tutorial_scripts/search.py | 375 +++++++----------- 2 files changed, 171 insertions(+), 238 deletions(-) diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md index 8b01d1071..4c99d19f0 100644 --- a/docs/tutorials/python/search.md +++ b/docs/tutorials/python/search.md @@ -21,7 +21,7 @@ Synapse Python client. In this tutorial, you will: 1. Log in, get your project, and create a table to index -2. Create a SearchIndex and wait for it to build +2. Create a SearchIndex 3. Run a full-text search 4. Highlight where the match happened 5. Combine scored clauses with unscored filters, and sort the results @@ -33,8 +33,6 @@ In this tutorial, you will: ## Prerequisites * This tutorial assumes that you have a Synapse project. * Pandas must also be installed as shown in the [installation documentation](../installation.md). -* Creating a SearchIndex may be restricted on some Synapse stacks. If `store()` fails - with a 403, your account is not permitted to create search indexes there. ## 1. Log in, get your project, and create a table to index @@ -55,17 +53,26 @@ to wait out the index build. --8<-- "docs/tutorials/python/tutorial_scripts/search.py:helpers" ``` -## 2. Create a SearchIndex and wait for it to build +## 2. Create a SearchIndex Entity -The `defining_sql` decides which rows and columns are indexed. Unlike a Materialized -View, it must reference exactly one table-like entity — JOIN and UNION across several +The `defining_sql` decides which rows and columns are indexed. It must reference exactly +one table-like entity — unlike a Materialized View, JOIN and UNION across several entities are not supported. If you need to search across several tables, build a [Materialized View](materializedview.md) first and index that. -Storing the entity returns as soon as Synapse has accepted it, but the OpenSearch index -behind it is built in the background. Until the build finishes, queries against the -index either raise an error or report zero hits, which is why we poll with -`wait_for_index`. +Any of these can be the source, whichever one the SQL selects from: + +* a [Table][synapseclient.models.Table] +* an [EntityView][synapseclient.models.EntityView] +* a [DatasetCollection][synapseclient.models.DatasetCollection] +* a [MaterializedView][synapseclient.models.MaterializedView] + +The SQL can pin a specific version of the source (`SELECT * FROM syn12345.7`), select a +subset of columns, and carry `WHERE`, `ORDER BY`, and `LIMIT` clauses — only those rows +and columns end up in the index. + +Storing the search index entity returns as soon as Synapse has accepted it, but the OpenSearch index +behind it is built in the background. ```python --8<-- "docs/tutorials/python/tutorial_scripts/search.py:create_index" @@ -73,6 +80,7 @@ index either raise an error or report zero hits, which is why we poll with
Creating the index should look like: + ``` Created SearchIndex with ID: syn68123456 Waiting for the search index to build... @@ -101,6 +109,7 @@ and the columns each hit carries.
The results of your searches should look like: + ``` Abstracts mentioning Alzheimer's: columns: ['study_name', 'diagnosis'] @@ -131,6 +140,7 @@ wrapped in `` tags.
The result of your highlighted search should look like: + ``` Studies that sequenced something: ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'assay': 'rnaSeq'} @@ -165,6 +175,7 @@ and `_score` sorts are accepted.
The result of your filtered search should look like: + ``` Sequencing studies with at least 200 participants, largest first: total_hits=2, returned=2 @@ -192,6 +203,7 @@ diagnosis counts disappear.
The result of your faceted search should look like: + ``` Hits after the post filter: total_hits=3, returned=3 @@ -250,7 +262,7 @@ Suggestions for 'Mayo Cl': ```
-## 8. Page through results +## 8. Pagination A query returns at most 100 hits at a time (25 by default). `from_` and `size` walk through the result set the way page numbers do. diff --git a/docs/tutorials/python/tutorial_scripts/search.py b/docs/tutorials/python/tutorial_scripts/search.py index fc017ed21..4126b83f5 100644 --- a/docs/tutorials/python/tutorial_scripts/search.py +++ b/docs/tutorials/python/tutorial_scripts/search.py @@ -2,7 +2,6 @@ # --8<-- [start:setup] import json -import time import pandas as pd @@ -125,254 +124,195 @@ def print_hits(results: SearchIndexQuery) -> None: print(f" ROW_ID={hit.row_id} {fields}") -def wait_for_index(index: SearchIndex, timeout: int = 600) -> None: - """Wait until the search index has finished building. - - Building the OpenSearch index behind a SearchIndex happens in the background - after `store()` returns. Until that build completes, a query against the - index either raises an error or reports zero hits. - """ - deadline = time.time() + timeout - while time.time() < deadline: - try: - results = index.query( - search_query=SearchQuery(query=Query(match_all={}), size=1), - response_parts=[SearchQueryPart.TOTAL_HITS], - ) - if results.total_hits: - print(f"Index {index.id} is queryable with {results.total_hits} rows") - return - except SynapseError: - pass # The index has not been created yet - print("Waiting for the search index to build...") - time.sleep(10) - raise TimeoutError(f"{index.id} did not finish building within {timeout} seconds") - - # --8<-- [end:helpers] # --8<-- [start:create_index] -def create_search_index() -> SearchIndex: - """ - Example: Create a SearchIndex over a single table and wait for it to build. - """ - index = SearchIndex( - name="Study Summaries Search Index", - description="Full text search over the study summary table", - parent_id=project_id, - # The defining SQL must reference exactly one table-like entity - defining_sql=f"SELECT * FROM {table.id}", - ) - index = index.store() - print(f"Created SearchIndex with ID: {index.id}") - - wait_for_index(index) - return index - +# Create a SearchIndex over a single table and wait for it to build. +index = SearchIndex( + name="Study Summaries Search Index", + description="Full text search over the study summary table", + parent_id=project_id, + # The defining SQL must reference exactly one table-like entity + defining_sql=f"SELECT * FROM {table.id}", +) +index = index.store() +print(f"Created SearchIndex with ID: {index.id}") # --8<-- [end:create_index] # --8<-- [start:full_text_search] -def search_free_text(index: SearchIndex) -> None: - """ - Example: Find every study whose abstract mentions Alzheimer's disease, then - search across several columns at once. - """ - results = index.query( - search_query=SearchQuery( - query=Query(match={"abstract": MatchFieldOptions(query="alzheimer")}), - # Every indexed column comes back on each hit unless a source filter - # narrows them down, and the abstracts are long - source=SourceFilter(includes=["study_name", "diagnosis"]), - size=10, - ), - response_parts=[SearchQueryPart.TOTAL_HITS, SearchQueryPart.SELECT_COLUMNS], - ) - print("Abstracts mentioning Alzheimer's:") - print(f"columns: {[column.name for column in results.select_columns]}") - print_hits(results) - - # A multi_match clause runs the same text across several columns, so the - # person searching does not need to know which column holds the term. - results = index.query( - search_query=SearchQuery( - query=Query( - multi_match=MultiMatchQuery( - query="tau", - # ^2 boosts a match in the study name over one in the abstract - fields=["study_name^2", "abstract"], - ) - ), - source=SourceFilter(includes=["study_name"]), - size=10, +# Find every study whose abstract mentions Alzheimer's disease, then +# search across several columns at once. +results = index.query( + search_query=SearchQuery( + query=Query(match={"abstract": MatchFieldOptions(query="alzheimer")}), + # Every indexed column comes back on each hit unless a source filter + # narrows them down, and the abstracts are long + source=SourceFilter(includes=["study_name", "diagnosis"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS, SearchQueryPart.SELECT_COLUMNS], +) +print("Abstracts mentioning Alzheimer's:") +print(f"columns: {[column.name for column in results.select_columns]}") +print_hits(results) + +# A multi_match clause runs the same text across several columns, so the +# person searching does not need to know which column holds the term. +results = index.query( + search_query=SearchQuery( + query=Query( + multi_match=MultiMatchQuery( + query="tau", + # ^2 boosts a match in the study name over one in the abstract + fields=["study_name^2", "abstract"], + ) ), - response_parts=[SearchQueryPart.TOTAL_HITS], - ) - print("\nAnything mentioning tau:") - print_hits(results) - + source=SourceFilter(includes=["study_name"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], +) +print("\nAnything mentioning tau:") +print_hits(results) # --8<-- [end:full_text_search] # --8<-- [start:columns_and_highlights] -def search_with_highlighting(index: SearchIndex) -> None: - """ - Example: Ask for a snippet of the matching text alongside each hit, so a - result list can show why the row matched. - """ - results = index.query( - search_query=SearchQuery( - query=Query(match={"abstract": MatchFieldOptions(query="sequencing")}), - source=SourceFilter(includes=["study_name", "assay"]), - highlight=Highlight( - fields={"abstract": HighlightField(number_of_fragments=1)} - ), - size=10, - ), - response_parts=[SearchQueryPart.TOTAL_HITS], - ) - print("Studies that sequenced something:") - for hit in results.hits: - fields = {field.name: field.value for field in hit.fields} - print(f" ROW_ID={hit.row_id} {fields}") - for highlight in hit.highlights: - print(f" {highlight.name}: {highlight.snippets}") +# Ask for a snippet of the matching text alongside each hit, so a +# result list can show why the row matched. +results = index.query( + search_query=SearchQuery( + query=Query(match={"abstract": MatchFieldOptions(query="sequencing")}), + source=SourceFilter(includes=["study_name", "assay"]), + highlight=Highlight(fields={"abstract": HighlightField(number_of_fragments=1)}), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], +) +print("Studies that sequenced something:") +for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") + for highlight in hit.highlights: + print(f" {highlight.name}: {highlight.snippets}") # --8<-- [end:columns_and_highlights] # --8<-- [start:filters_and_sorting] -def search_with_filters(index: SearchIndex) -> None: - """ - Example: Combine a scored clause with unscored filters using a bool query, - then order the results by a numeric column instead of by relevance. - """ - results = index.query( - search_query=SearchQuery( - query=Query( - bool=BoolQuery( - # Scored: how well the abstract matches drives relevance - must=[ - Query(match={"abstract": MatchFieldOptions(query="sequencing")}) - ], - # Unscored: a hard cutoff on cohort size - filter=[ - Query(range={"participant_count": RangeFieldOptions(gte=200)}) - ], - # Unscored: drop a diagnosis we are not interested in - must_not=[ - Query( - match_phrase={ - "diagnosis": MatchPhraseFieldOptions( - query="Parkinson's Disease" - ) - } - ) - ], - ) - ), - source=SourceFilter(includes=["study_name", "participant_count"]), - sort=[{"participant_count": "desc"}], - size=10, - ), - response_parts=[SearchQueryPart.TOTAL_HITS], - ) - print("Sequencing studies with at least 200 participants, largest first:") - print_hits(results) +# Combine a scored clause with unscored filters using a bool query, +# then order the results by a numeric column instead of by relevance. +results = index.query( + search_query=SearchQuery( + query=Query( + bool=BoolQuery( + # Scored: how well the abstract matches drives relevance + must=[Query(match={"abstract": MatchFieldOptions(query="sequencing")})], + # Unscored: a hard cutoff on cohort size + filter=[Query(range={"participant_count": RangeFieldOptions(gte=200)})], + # Unscored: drop a diagnosis we are not interested in + must_not=[ + Query( + match_phrase={ + "diagnosis": MatchPhraseFieldOptions( + query="Parkinson's Disease" + ) + } + ) + ], + ) + ), + source=SourceFilter(includes=["study_name", "participant_count"]), + sort=[{"participant_count": "desc"}], + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], +) +print("Sequencing studies with at least 200 participants, largest first:") +print_hits(results) # --8<-- [end:filters_and_sorting] # --8<-- [start:aggregations] -def facet_the_results(index: SearchIndex) -> None: - """ - Example: Count how many studies fall under each diagnosis and average their - cohort sizes, while the hit list itself shows only one diagnosis. - """ - results = index.query( - search_query=SearchQuery( - query=Query(match_all={}), - aggregations={ - "by_diagnosis": Aggregation( - terms=TermsAggregation(field="diagnosis", size=10) - ), - "mean_cohort_size": Aggregation( - avg=AvgAggregation(field="participant_count") - ), - }, - # post_filter narrows the hits but not the aggregations, so the facet - # counts still show every option a person could pick next - post_filter=Query( - match_phrase={ - "diagnosis": MatchPhraseFieldOptions(query="Alzheimer's Disease") - } +# Count how many studies fall under each diagnosis and average their +# cohort sizes, while the hit list itself shows only one diagnosis. + +results = index.query( + search_query=SearchQuery( + query=Query(match_all={}), + aggregations={ + "by_diagnosis": Aggregation( + terms=TermsAggregation(field="diagnosis", size=10) ), - source=SourceFilter(includes=["study_name", "diagnosis"]), - size=10, + "mean_cohort_size": Aggregation( + avg=AvgAggregation(field="participant_count") + ), + }, + # post_filter narrows the hits but not the aggregations, so the facet + # counts still show every option a person could pick next + post_filter=Query( + match_phrase={ + "diagnosis": MatchPhraseFieldOptions(query="Alzheimer's Disease") + } ), - response_parts=[SearchQueryPart.TOTAL_HITS], - ) - print("Hits after the post filter:") - print_hits(results) - print("\nFacet counts across all studies:") - print(json.dumps(results.aggregation_results, indent=2)) - + source=SourceFilter(includes=["study_name", "diagnosis"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], +) +print("Hits after the post filter:") +print_hits(results) +print("\nFacet counts across all studies:") +print(json.dumps(results.aggregation_results, indent=2)) # --8<-- [end:aggregations] # --8<-- [start:autocomplete] -def autocomplete_study_names(index: SearchIndex) -> None: - """ - Example: Back a type-ahead box with the autocomplete endpoint, which returns - its results directly instead of running as an asynchronous job. - """ - hits = index.autocomplete( - query=Query( - match_bool_prefix={ - "study_name": MatchBoolPrefixFieldOptions(query="Mayo Cl") - } - ), - source=SourceFilter(includes=["study_name"]), - ) - print("Suggestions for 'Mayo Cl':") - for hit in hits: - print(f" {[field.value for field in hit.fields]}") +# Back a type-ahead box with the autocomplete endpoint, which returns +# its results directly instead of running as an asynchronous job. +hits = index.autocomplete( + query=Query( + match_bool_prefix={"study_name": MatchBoolPrefixFieldOptions(query="Mayo Cl")} + ), + source=SourceFilter(includes=["study_name"]), +) +print("Suggestions for 'Mayo Cl':") +for hit in hits: + print(f" {[field.value for field in hit.fields]}") # --8<-- [end:autocomplete] # --8<-- [start:pagination] -def page_through_results(index: SearchIndex) -> None: - """ - Example: Walk every row in the index two hits at a time. - """ - page_size = 2 - offset = 0 - while True: - results = index.query( - search_query=SearchQuery( - query=Query(match_all={}), - source=SourceFilter(includes=["study_name", "participant_count"]), - sort=[{"participant_count": "desc"}], - from_=offset, - size=page_size, - ), - response_parts=[SearchQueryPart.TOTAL_HITS], - ) - print(f"Page starting at offset {offset}:") - print_hits(results) - - offset += page_size - if offset >= results.total_hits: - break +# Walk every row in the index two hits at a time. +page_size = 2 +offset = 0 +while True: + results = index.query( + search_query=SearchQuery( + query=Query(match_all={}), + source=SourceFilter(includes=["study_name", "participant_count"]), + sort=[{"participant_count": "desc"}], + from_=offset, + size=page_size, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print(f"Page starting at offset {offset}:") + print_hits(results) + offset += page_size + if offset >= results.total_hits: + break # --8<-- [end:pagination] @@ -494,7 +434,6 @@ def create_index_with_configuration(search_configuration_id: str) -> SearchIndex search_configuration_id=search_configuration_id, ).store() print(f"Created SearchIndex {index.id} using config {search_configuration_id}") - wait_for_index(index) # Any index created under this project without its own # search_configuration_id now inherits this configuration @@ -519,21 +458,3 @@ def create_index_with_configuration(search_configuration_id: str) -> SearchIndex # --8<-- [end:apply_search_configuration] - - -def main(): - index = create_search_index() - search_free_text(index) - search_with_highlighting(index) - search_with_filters(index) - facet_the_results(index) - autocomplete_study_names(index) - page_through_results(index) - - # Requires an Organization you can write to - # search_configuration_id = create_search_configuration() - # create_index_with_configuration(search_configuration_id) - - -if __name__ == "__main__": - main() From bee271340a66f961da21e5b1a93fed139e12daf4 Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Thu, 20 Aug 2026 20:00:26 -0700 Subject: [PATCH 03/12] Simplify tutorial --- docs/tutorials/python/search.md | 11 +--- .../python/tutorial_scripts/search.py | 56 ++++++++++--------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md index 4c99d19f0..891fb2840 100644 --- a/docs/tutorials/python/search.md +++ b/docs/tutorials/python/search.md @@ -46,18 +46,13 @@ the name of your project. --8<-- "docs/tutorials/python/tutorial_scripts/search.py:setup" ``` -The steps below use two small helpers — one to print the rows a query matched, and one -to wait out the index build. - -```python ---8<-- "docs/tutorials/python/tutorial_scripts/search.py:helpers" -``` - ## 2. Create a SearchIndex Entity The `defining_sql` decides which rows and columns are indexed. It must reference exactly one table-like entity — unlike a Materialized View, JOIN and UNION across several -entities are not supported. If you need to search across several tables, build a +entities are not supported. + +If you need to search across several tables, build a [Materialized View](materializedview.md) first and index that. Any of these can be the source, whichever one the SQL selects from: diff --git a/docs/tutorials/python/tutorial_scripts/search.py b/docs/tutorials/python/tutorial_scripts/search.py index 4126b83f5..d28565964 100644 --- a/docs/tutorials/python/tutorial_scripts/search.py +++ b/docs/tutorials/python/tutorial_scripts/search.py @@ -114,19 +114,6 @@ print(f"Stored {len(studies)} rows in {table.id}") # --8<-- [end:setup] - -# --8<-- [start:helpers] -def print_hits(results: SearchIndexQuery) -> None: - """Print the rows a query matched, one line per hit.""" - print(f"total_hits={results.total_hits}, returned={len(results.hits)}") - for hit in results.hits: - fields = {field.name: field.value for field in hit.fields} - print(f" ROW_ID={hit.row_id} {fields}") - - -# --8<-- [end:helpers] - - # --8<-- [start:create_index] # Create a SearchIndex over a single table and wait for it to build. index = SearchIndex( @@ -153,11 +140,18 @@ def print_hits(results: SearchIndexQuery) -> None: source=SourceFilter(includes=["study_name", "diagnosis"]), size=10, ), - response_parts=[SearchQueryPart.TOTAL_HITS, SearchQueryPart.SELECT_COLUMNS], + response_parts=[ + SearchQueryPart.HITS, + SearchQueryPart.TOTAL_HITS, + SearchQueryPart.SELECT_COLUMNS, + ], ) print("Abstracts mentioning Alzheimer's:") print(f"columns: {[column.name for column in results.select_columns]}") -print_hits(results) +print(f"total_hits={results.total_hits}, returned={len(results.hits)}") +for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") # A multi_match clause runs the same text across several columns, so the # person searching does not need to know which column holds the term. @@ -173,10 +167,13 @@ def print_hits(results: SearchIndexQuery) -> None: source=SourceFilter(includes=["study_name"]), size=10, ), - response_parts=[SearchQueryPart.TOTAL_HITS], + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], ) print("\nAnything mentioning tau:") -print_hits(results) +print(f"total_hits={results.total_hits}, returned={len(results.hits)}") +for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") # --8<-- [end:full_text_search] @@ -191,7 +188,7 @@ def print_hits(results: SearchIndexQuery) -> None: highlight=Highlight(fields={"abstract": HighlightField(number_of_fragments=1)}), size=10, ), - response_parts=[SearchQueryPart.TOTAL_HITS], + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], ) print("Studies that sequenced something:") for hit in results.hits: @@ -232,10 +229,13 @@ def print_hits(results: SearchIndexQuery) -> None: sort=[{"participant_count": "desc"}], size=10, ), - response_parts=[SearchQueryPart.TOTAL_HITS], + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], ) print("Sequencing studies with at least 200 participants, largest first:") -print_hits(results) +print(f"total_hits={results.total_hits}, returned={len(results.hits)}") +for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") # --8<-- [end:filters_and_sorting] @@ -265,10 +265,13 @@ def print_hits(results: SearchIndexQuery) -> None: source=SourceFilter(includes=["study_name", "diagnosis"]), size=10, ), - response_parts=[SearchQueryPart.TOTAL_HITS], + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], ) print("Hits after the post filter:") -print_hits(results) +print(f"total_hits={results.total_hits}, returned={len(results.hits)}") +for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") print("\nFacet counts across all studies:") print(json.dumps(results.aggregation_results, indent=2)) @@ -305,10 +308,13 @@ def print_hits(results: SearchIndexQuery) -> None: from_=offset, size=page_size, ), - response_parts=[SearchQueryPart.TOTAL_HITS], + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], ) print(f"Page starting at offset {offset}:") - print_hits(results) + print(f"total_hits={results.total_hits}, returned={len(results.hits)}") + for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") offset += page_size if offset >= results.total_hits: @@ -450,7 +456,7 @@ def create_index_with_configuration(search_configuration_id: str) -> SearchIndex source=SourceFilter(includes=["study_name", "diagnosis"]), size=10, ), - response_parts=[SearchQueryPart.TOTAL_HITS], + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], ) print("Abstracts matching the abbreviation 'AD':") print_hits(results) From e9cf4cf3194022a7fe567d5c9fdd2028818c15d2 Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Thu, 20 Aug 2026 20:17:36 -0700 Subject: [PATCH 04/12] Update pagination result --- docs/tutorials/python/search.md | 68 +++++++++++++++---- .../python/tutorial_scripts/search.py | 48 +++++++++++-- 2 files changed, 96 insertions(+), 20 deletions(-) diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md index 891fb2840..a135f0c07 100644 --- a/docs/tutorials/python/search.md +++ b/docs/tutorials/python/search.md @@ -27,7 +27,7 @@ In this tutorial, you will: 5. Combine scored clauses with unscored filters, and sort the results 6. Count facets with aggregations 7. Power a type-ahead box with autocomplete -8. Page through results +8. Paginated results 9. Tune matching with synonyms and analyzers ## Prerequisites @@ -257,13 +257,21 @@ Suggestions for 'Mayo Cl': ```
-## 8. Pagination +## 8. Paginated results -A query returns at most 100 hits at a time (25 by default). `from_` and `size` walk -through the result set the way page numbers do. +A query returns at most 100 hits at a time (25 by default), so anything larger requires special attention. There are two ways to do it. + +* Specifying the `from_` and `size` arguments on `SearchQuery` to an offset the way page numbers do. +* `search_after` picks up from where the last page ended. Each response has `next_search_after`; pass it back unchanged on the next request and leave `from_` unset. + +### Offset paging with `from_` and `size` + +Simple, and the right thing for the first few pages of a UI where someone clicks +"next". The cost grows with depth though — the server collects and discards every hit +before the offset — so it is the wrong tool for sweeping a large index. ```python ---8<-- "docs/tutorials/python/tutorial_scripts/search.py:pagination" +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:pagination_offset" ```
@@ -271,22 +279,54 @@ through the result set the way page numbers do. ``` Page starting at offset 0: total_hits=6, returned=2 - ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'participant_count': '400'} - ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'participant_count': '350'} + {'study_name': 'ROSMAP Cortex Proteomics', 'participant_count': '400'} + {'study_name': 'Mayo Clinic Whole Genome', 'participant_count': '350'} Page starting at offset 2: total_hits=6, returned=2 - ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'participant_count': '300'} - ROW_ID=5 {'study_name': 'MCI Plasma Biomarkers', 'participant_count': '220'} + {'study_name': 'MSBB RNA Sequencing', 'participant_count': '300'} + {'study_name': 'MCI Plasma Biomarkers', 'participant_count': '220'} Page starting at offset 4: total_hits=6, returned=2 - ROW_ID=6 {'study_name': 'Parkinson Comparative Cohort', 'participant_count': '180'} - ROW_ID=4 {'study_name': 'Healthy Aging Single Cell Atlas', 'participant_count': '120'} + {'study_name': 'Parkinson Comparative Cohort', 'participant_count': '180'} + {'study_name': 'Healthy Aging Single Cell Atlas', 'participant_count': '120'} +``` +
+ +### Cursor paging with `search_after` + +Cost per page stays flat no matter how far in you are, which makes this the one to +reach for when you need every row. + +The catch is that `search_after` is a position in a sort order, so the `sort` has to +place every row unambiguously. If two rows tie on every sort column, a page boundary +landing between them can skip or repeat rows. Sort on something unique, or append a +unique column as a final tie-breaker. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:pagination_cursor" +``` + +
+ The result of walking your index should look like: +``` +Page 0: +total_hits=6, returned=2 + {'study_name': 'ROSMAP Cortex Proteomics', 'participant_count': '400'} + {'study_name': 'Mayo Clinic Whole Genome', 'participant_count': '350'} +Page 1: +total_hits=6, returned=2 + {'study_name': 'MSBB RNA Sequencing', 'participant_count': '300'} + {'study_name': 'MCI Plasma Biomarkers', 'participant_count': '220'} +Page 2: +total_hits=6, returned=2 + {'study_name': 'Parkinson Comparative Cohort', 'participant_count': '180'} + {'study_name': 'Healthy Aging Single Cell Atlas', 'participant_count': '120'} ```
-**Note**: Offset paging gets expensive deep into a large result set. For that case each -response carries a `next_search_after` cursor — pass it back unchanged as -`SearchQuery(search_after=...)` on the following request and leave `from_` unset. +**Note**: Each page is a separate asynchronous job either way, not a cheap follow-up +GET, so ask for the largest `size` you can use rather than walking a big index in small +pages. ## 9. Tune matching with synonyms and analyzers diff --git a/docs/tutorials/python/tutorial_scripts/search.py b/docs/tutorials/python/tutorial_scripts/search.py index d28565964..6116a7598 100644 --- a/docs/tutorials/python/tutorial_scripts/search.py +++ b/docs/tutorials/python/tutorial_scripts/search.py @@ -6,13 +6,11 @@ import pandas as pd from synapseclient import Synapse -from synapseclient.core.exceptions import SynapseError from synapseclient.models import ( Column, ColumnType, Project, SearchIndex, - SearchIndexQuery, SearchQuery, SearchQueryPart, Table, @@ -295,8 +293,8 @@ # --8<-- [end:autocomplete] -# --8<-- [start:pagination] -# Walk every row in the index two hits at a time. +# --8<-- [start:pagination_offset] +# Walk every row in the index two hits at a time with a growing offset. page_size = 2 offset = 0 while True: @@ -314,13 +312,51 @@ print(f"total_hits={results.total_hits}, returned={len(results.hits)}") for hit in results.hits: fields = {field.name: field.value for field in hit.fields} - print(f" ROW_ID={hit.row_id} {fields}") + print(f" {fields}") offset += page_size if offset >= results.total_hits: break -# --8<-- [end:pagination] +# --8<-- [end:pagination_offset] + + +# --8<-- [start:pagination_cursor] +# The same walk, using the search_after cursor the server hands back instead of +# a growing offset. +page_size = 2 +search_after = None +page = 0 +while True: + results = index.query( + search_query=SearchQuery( + query=Query(match_all={}), + source=SourceFilter(includes=["study_name", "participant_count"]), + # search_after walks a sort order, so the sort has to put every row + # in a definite position. participant_count is unique in this table; + # on real data append a unique column to break ties, or a page + # boundary can skip or repeat rows. + sort=[{"participant_count": "desc"}], + # None on the first request, then the cursor from the previous one + search_after=search_after, + size=page_size, + ), + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], + ) + print(f"Page {page}:") + print(f"total_hits={results.total_hits}, returned={len(results.hits)}") + for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" {fields}") + + # The cursor is opaque -- pass it back unchanged. It goes None on the last + # page, which is what ends the walk. + search_after = results.next_search_after + if not search_after or not results.hits: + break + page += 1 + +# --8<-- [end:pagination_cursor] # --8<-- [start:search_configuration] From 7d2cfc242cce6e6846721e667ab0f40a818cb7c0 Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Thu, 20 Aug 2026 21:23:29 -0700 Subject: [PATCH 05/12] Update tutorial --- docs/tutorials/python/search.md | 39 ++++++++++++--------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md index a135f0c07..8deffc1a0 100644 --- a/docs/tutorials/python/search.md +++ b/docs/tutorials/python/search.md @@ -266,8 +266,7 @@ A query returns at most 100 hits at a time (25 by default), so anything larger r ### Offset paging with `from_` and `size` -Simple, and the right thing for the first few pages of a UI where someone clicks -"next". The cost grows with depth though — the server collects and discards every hit +Simple, and extracts the results in pages. The cost grows with depth and the server collects and discards every hit before the offset — so it is the wrong tool for sweeping a large index. ```python @@ -294,10 +293,7 @@ total_hits=6, returned=2 ### Cursor paging with `search_after` -Cost per page stays flat no matter how far in you are, which makes this the one to -reach for when you need every row. - -The catch is that `search_after` is a position in a sort order, so the `sort` has to +This is the solution if you need every row. The catch is that `search_after` is a position in a sort order, so the `sort` has to place every row unambiguously. If two rows tie on every sort column, a page boundary landing between them can skip or repeat rows. Sort on something unique, or append a unique column as a final tie-breaker. @@ -328,32 +324,25 @@ total_hits=6, returned=2 GET, so ask for the largest `size` you can use rather than walking a big index in small pages. -## 9. Tune matching with synonyms and analyzers +## Advanced: Tune matching with synonyms and analyzers + +!!! warning "Restricted and permanent" + Creating and updating the following resources is restricted to Sage Bionetworks employees, + and the REST API has no delete endpoint for any of them. Once created, a + SynonymSet, TextAnalyzer, ColumnAnalyzerOverride, or SearchConfiguration cannot be + removed, and its owning Organization can no longer be deleted either. Choose names deliberately. -Everything above relies on how each column was analyzed when the index was built: how -text is split into tokens, which tokens are dropped, and how they are normalized. Four -org-scoped resources let you control that: +Everything in this tutorial relies on how each column was analyzed when the index was built: how text is split into tokens, which tokens are dropped, and how they are normalized. There are four `Organization`-scoped resources that let you control that: -* [SynonymSet][synapseclient.models.SynonymSet] — terms that should be treated as - equivalent, so someone searching `AD` finds abstracts that say - "Alzheimer's disease" -* [TextAnalyzer][synapseclient.models.TextAnalyzer] — a named OpenSearch analyzer: a - tokenizer plus a chain of token filters, which may reference a SynonymSet -* [ColumnAnalyzerOverride][synapseclient.models.ColumnAnalyzerOverride] — a reusable - bundle assigning specific analyzers to specific columns -* [SearchConfiguration][synapseclient.models.SearchConfiguration] — bundles a default - analyzer with any column overrides; this is what a SearchIndex actually points at +* [SynonymSet][synapseclient.models.SynonymSet] — terms that should be treated as equivalent, so someone searching `AD` finds abstracts that say "Alzheimer's disease" +* [TextAnalyzer][synapseclient.models.TextAnalyzer] — a named OpenSearch analyzer: a tokenizer plus a chain of token filters, which may reference a SynonymSet +* [ColumnAnalyzerOverride][synapseclient.models.ColumnAnalyzerOverride] — a reusable bundle assigning specific analyzers to specific columns +* [SearchConfiguration][synapseclient.models.SearchConfiguration] — bundles a default analyzer with any column overrides; this is what a SearchIndex actually points at Each resource belongs to an [Organization][synapseclient.models.Organization] and is referenced from another resource by its qualified name, `{organization_name}-{name}`, written as `{"$ref": "my.org-my_analyzer"}`. -!!! warning "Restricted and permanent" - Creating and updating these resources is restricted to Sage Bionetworks employees, - and the REST API has no delete endpoint for any of them. Once created, a - SynonymSet, TextAnalyzer, ColumnAnalyzerOverride, or SearchConfiguration cannot be - removed, and its owning Organization can no longer be deleted either. Choose names - deliberately. Note where the synonym filter goes below. The analyzer declares both a `default` chain, used when rows are indexed, and a `default_search` chain, used when a query is analyzed. From 683f9c58680cc1fa1c4ba5b77ac9ff46fbf4fa9e Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Fri, 21 Aug 2026 09:27:50 -0700 Subject: [PATCH 06/12] Remove print hits --- docs/tutorials/python/tutorial_scripts/search.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/python/tutorial_scripts/search.py b/docs/tutorials/python/tutorial_scripts/search.py index 6116a7598..c320566cc 100644 --- a/docs/tutorials/python/tutorial_scripts/search.py +++ b/docs/tutorials/python/tutorial_scripts/search.py @@ -495,7 +495,10 @@ def create_index_with_configuration(search_configuration_id: str) -> SearchIndex response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], ) print("Abstracts matching the abbreviation 'AD':") - print_hits(results) + print(f"total_hits={results.total_hits}, returned={len(results.hits)}") + for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") return index From 86c975b1c90604931601e3ff78353b93bda60edc Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Fri, 21 Aug 2026 10:47:11 -0700 Subject: [PATCH 07/12] Remove irrelevant lines --- docs/tutorials/python/search.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md index 8deffc1a0..5aa73e138 100644 --- a/docs/tutorials/python/search.md +++ b/docs/tutorials/python/search.md @@ -78,8 +78,6 @@ behind it is built in the background. ``` Created SearchIndex with ID: syn68123456 -Waiting for the search index to build... -Waiting for the search index to build... Index syn68123456 is queryable with 6 rows ```
From e286eaed2501bf8fef578b49963c6ce815e4ff9a Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Fri, 21 Aug 2026 10:59:39 -0700 Subject: [PATCH 08/12] Add fuzzy matching and remove ROW_ID --- docs/tutorials/python/search.md | 42 ++++++++++++------- .../python/tutorial_scripts/search.py | 42 ++++++++++++++++--- 2 files changed, 63 insertions(+), 21 deletions(-) diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md index 5aa73e138..f967d583d 100644 --- a/docs/tutorials/python/search.md +++ b/docs/tutorials/python/search.md @@ -96,6 +96,12 @@ By default a hit carries every indexed column. `source` narrows that down, and `response_parts` asks for extras beyond the hits themselves — here the total hit count and the columns each hit carries. +Adding `fuzziness` to a `match` clause buys typo tolerance: the term someone typed will +still match a term in the index that is a few single-character edits away. `"AUTO"` +scales the allowance with term length, and `prefix_length` pins the first few characters +so unrelated short words don't start matching each other. Both options are available on +`match`, `match_bool_prefix`, and `multi_match`. + ```python --8<-- "docs/tutorials/python/tutorial_scripts/search.py:full_text_search" ``` @@ -107,13 +113,19 @@ and the columns each hit carries. Abstracts mentioning Alzheimer's: columns: ['study_name', 'diagnosis'] total_hits=3, returned=3 - ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} - ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} - ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} Anything mentioning tau: total_hits=1, returned=1 - ROW_ID=5 {'study_name': 'MCI Plasma Biomarkers'} + {'study_name': 'MCI Plasma Biomarkers'} + +Misspelling 'sequencng' still finds: +total_hits=3, returned=3 + {'study_name': 'MSBB RNA Sequencing'} + {'study_name': 'Mayo Clinic Whole Genome'} + {'study_name': 'Healthy Aging Single Cell Atlas'} ```
@@ -136,11 +148,11 @@ wrapped in `` tags. ``` Studies that sequenced something: - ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'assay': 'rnaSeq'} + {'study_name': 'MSBB RNA Sequencing', 'assay': 'rnaSeq'} abstract: ['Bulk RNA sequencing across four brain regions in a cohort'] - ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'assay': 'wholeGenomeSeq'} + {'study_name': 'Mayo Clinic Whole Genome', 'assay': 'wholeGenomeSeq'} abstract: ['Whole genome sequencing of temporal cortex samples from'] - ROW_ID=4 {'study_name': 'Healthy Aging Single Cell Atlas', 'assay': 'snrnaSeq'} + {'study_name': 'Healthy Aging Single Cell Atlas', 'assay': 'snrnaSeq'} abstract: ['Single nucleus RNA sequencing of hippocampus from'] ```
@@ -172,8 +184,8 @@ and `_score` sorts are accepted. ``` Sequencing studies with at least 200 participants, largest first: total_hits=2, returned=2 - ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'participant_count': '350'} - ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'participant_count': '300'} + {'study_name': 'Mayo Clinic Whole Genome', 'participant_count': '350'} + {'study_name': 'MSBB RNA Sequencing', 'participant_count': '300'} ``` @@ -200,9 +212,9 @@ diagnosis counts disappear. ``` Hits after the post filter: total_hits=3, returned=3 - ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} - ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} - ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} Facet counts across all studies: { @@ -370,9 +382,9 @@ Index syn68123457 is queryable with 6 rows Bound configuration 4321 to syn12345678 Abstracts matching the abbreviation 'AD': total_hits=3, returned=3 - ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} - ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} - ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} ``` diff --git a/docs/tutorials/python/tutorial_scripts/search.py b/docs/tutorials/python/tutorial_scripts/search.py index c320566cc..b9faa9387 100644 --- a/docs/tutorials/python/tutorial_scripts/search.py +++ b/docs/tutorials/python/tutorial_scripts/search.py @@ -149,7 +149,7 @@ print(f"total_hits={results.total_hits}, returned={len(results.hits)}") for hit in results.hits: fields = {field.name: field.value for field in hit.fields} - print(f" ROW_ID={hit.row_id} {fields}") + print(f" {fields}") # A multi_match clause runs the same text across several columns, so the # person searching does not need to know which column holds the term. @@ -171,7 +171,37 @@ print(f"total_hits={results.total_hits}, returned={len(results.hits)}") for hit in results.hits: fields = {field.name: field.value for field in hit.fields} - print(f" ROW_ID={hit.row_id} {fields}") + print(f" {fields}") + +# People misspell things. `fuzziness` tolerates a number of single-character +# edits -- insert, delete, substitute, or transpose -- between what was typed +# and what is in the index. +results = index.query( + search_query=SearchQuery( + query=Query( + match={ + "abstract": MatchFieldOptions( + # "sequencing" with a missing "i" + query="sequencng", + # "AUTO" scales the allowance with term length: 0 edits for + # very short terms, 1 for medium, 2 for long ones + fuzziness="AUTO", + # The first 3 characters still have to be exact. Without + # this, short unrelated words start matching each other + prefix_length=3, + ) + } + ), + source=SourceFilter(includes=["study_name"]), + size=10, + ), + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], +) +print("\nMisspelling 'sequencng' still finds:") +print(f"total_hits={results.total_hits}, returned={len(results.hits)}") +for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" {fields}") # --8<-- [end:full_text_search] @@ -191,7 +221,7 @@ print("Studies that sequenced something:") for hit in results.hits: fields = {field.name: field.value for field in hit.fields} - print(f" ROW_ID={hit.row_id} {fields}") + print(f" {fields}") for highlight in hit.highlights: print(f" {highlight.name}: {highlight.snippets}") @@ -233,7 +263,7 @@ print(f"total_hits={results.total_hits}, returned={len(results.hits)}") for hit in results.hits: fields = {field.name: field.value for field in hit.fields} - print(f" ROW_ID={hit.row_id} {fields}") + print(f" {fields}") # --8<-- [end:filters_and_sorting] @@ -269,7 +299,7 @@ print(f"total_hits={results.total_hits}, returned={len(results.hits)}") for hit in results.hits: fields = {field.name: field.value for field in hit.fields} - print(f" ROW_ID={hit.row_id} {fields}") + print(f" {fields}") print("\nFacet counts across all studies:") print(json.dumps(results.aggregation_results, indent=2)) @@ -498,7 +528,7 @@ def create_index_with_configuration(search_configuration_id: str) -> SearchIndex print(f"total_hits={results.total_hits}, returned={len(results.hits)}") for hit in results.hits: fields = {field.name: field.value for field in hit.fields} - print(f" ROW_ID={hit.row_id} {fields}") + print(f" {fields}") return index From 5680c672fccf14747b689ebc39c4b2d5f8ec3d2a Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Wed, 26 Aug 2026 10:27:43 -0700 Subject: [PATCH 09/12] Update tutorial to have setup and usage section --- docs/tutorials/python/search.md | 159 +++++----- .../python/tutorial_scripts/search.py | 292 +++++++++--------- 2 files changed, 233 insertions(+), 218 deletions(-) diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md index f967d583d..e1556e112 100644 --- a/docs/tutorials/python/search.md +++ b/docs/tutorials/python/search.md @@ -21,14 +21,15 @@ Synapse Python client. In this tutorial, you will: 1. Log in, get your project, and create a table to index -2. Create a SearchIndex -3. Run a full-text search -4. Highlight where the match happened -5. Combine scored clauses with unscored filters, and sort the results -6. Count facets with aggregations -7. Power a type-ahead box with autocomplete -8. Paginated results -9. Tune matching with synonyms and analyzers +2. Set up the index — create a SearchIndex, and optionally tune matching with synonyms + and analyzers +3. Use the search index + 1. Run a full-text search + 2. Highlight where the match happened + 3. Combine scored clauses with unscored filters, and sort the results + 4. Count facets with aggregations + 5. Power a type-ahead box with autocomplete + 6. Paginate through results ## Prerequisites * This tutorial assumes that you have a Synapse project. @@ -46,7 +47,15 @@ the name of your project. --8<-- "docs/tutorials/python/tutorial_scripts/search.py:setup" ``` -## 2. Create a SearchIndex Entity +## 2. Setup + +!!! warning "Restricted to Sage Bionetworks employees" + Everything in this section — creating a SearchIndex, and the synonym and analyzer + resources that configure one — is restricted to Sage Bionetworks employees. If that + is not you, read on for how indexes are built and configured, then pick up at + step 3 against an index someone has already created and shared with you. + +### 2.1 Create a SearchIndex entity The `defining_sql` decides which rows and columns are indexed. It must reference exactly one table-like entity — unlike a Materialized View, JOIN and UNION across several @@ -85,7 +94,67 @@ Index syn68123456 is queryable with 6 rows **Note**: The index tracks its source. When rows in the underlying table change, the index is updated in the background — you do not need to re-store the SearchIndex. -## 3. Run a full-text search +### 2.2 Advanced: Tune matching with synonyms and analyzers + +!!! warning "Permanent" + The REST API has no delete endpoint for any of the resources below. Once created, a + SynonymSet, TextAnalyzer, ColumnAnalyzerOverride, or SearchConfiguration cannot be + removed, and its owning Organization can no longer be deleted either. Choose names + deliberately. + +Everything in this tutorial relies on how each column was analyzed when the index was built: how text is split into tokens, which tokens are dropped, and how they are normalized. There are four `Organization`-scoped resources that let you control that: + +* [SynonymSet][synapseclient.models.SynonymSet] — terms that should be treated as equivalent, so someone searching `AD` finds abstracts that say "Alzheimer's disease" +* [TextAnalyzer][synapseclient.models.TextAnalyzer] — a named OpenSearch analyzer: a tokenizer plus a chain of token filters, which may reference a SynonymSet +* [ColumnAnalyzerOverride][synapseclient.models.ColumnAnalyzerOverride] — a reusable bundle assigning specific analyzers to specific columns +* [SearchConfiguration][synapseclient.models.SearchConfiguration] — bundles a default analyzer with any column overrides; this is what a SearchIndex actually points at + +Each resource belongs to an [Organization][synapseclient.models.Organization] and is +referenced from another resource by its qualified name, +`{organization_name}-{name}`, written as `{"$ref": "my.org-my_analyzer"}`. + + +Note where the synonym filter goes below. The analyzer declares both a `default` chain, +used when rows are indexed, and a `default_search` chain, used when a query is analyzed. +Putting the synonyms only in `default_search` expands the incoming query instead of +storing every synonym for every row. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:search_configuration" +``` + +A SearchIndex resolves its configuration when the index is built, so the configuration +has to exist before the index that uses it — that is why this comes before you run any +queries. Either point the index straight at a configuration with +`search_configuration_id`, or bind a configuration to the parent folder or project — an +index with no `search_configuration_id` of its own walks up the entity hierarchy and +uses the first [SearchConfigBinding][synapseclient.models.SearchConfigBinding] it finds, +falling back to the platform defaults. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:apply_search_configuration" +``` + +
+ Searching the abbreviation against the new index should look like: +``` +Created SearchIndex syn68123457 using config 4321 +Index syn68123457 is queryable with 6 rows +Bound configuration 4321 to syn12345678 +Abstracts matching the abbreviation 'AD': +total_hits=3, returned=3 + {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} + {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} +``` +
+ +## 3. Using a search index + +Everything from here on is querying an index that already exists, which does not require +any special permissions — read access to the SearchIndex entity is enough. + +### 3.1 Run a full-text search A [`match`](https://docs.opensearch.org/latest/query-dsl/full-text/match/) clause is the workhorse of full-text search: the text you pass is analyzed the same way the column was @@ -133,7 +202,7 @@ Hits come back ranked by relevance, and each one carries its score on [`hit.score`][synapseclient.models.SearchHit] along with the `row_id` and `row_version` of the source row. -## 4. Highlight where the match happened +### 3.2 Highlight where the match happened A result list is much easier to read when it shows the matching text in context. `highlight` returns short fragments of the matched columns with the matching terms @@ -158,10 +227,10 @@ Studies that sequenced something: **Note**: Highlighting, like relevance scoring, depends on the column being indexed as -analyzed text. Step 9 covers how to control that with a +analyzed text. Step 2.2 covers how to control that with a [SearchConfiguration][synapseclient.models.SearchConfiguration]. -## 5. Combine scored clauses with unscored filters, and sort the results +### 3.3 Combine scored clauses with unscored filters, and sort the results A [`bool`](https://docs.opensearch.org/latest/query-dsl/compound/bool/) clause is how you build a real search request out of several conditions: @@ -189,7 +258,7 @@ total_hits=2, returned=2 ``` -## 6. Count facets with aggregations +### 3.4 Count facets with aggregations Aggregations answer "how many rows are there of each kind?" — the counts you see next to the checkboxes in a faceted search UI. A @@ -247,7 +316,7 @@ Facet counts across all studies: ``` -## 7. Power a type-ahead box with autocomplete +### 3.5 Power a type-ahead box with autocomplete [`autocomplete()`][synapseclient.models.SearchIndex.autocomplete] is a separate, synchronous endpoint meant for search-as-you-type: it returns its hits directly instead @@ -267,14 +336,14 @@ Suggestions for 'Mayo Cl': ``` -## 8. Paginated results +### 3.6 Paginated results A query returns at most 100 hits at a time (25 by default), so anything larger requires special attention. There are two ways to do it. * Specifying the `from_` and `size` arguments on `SearchQuery` to an offset the way page numbers do. * `search_after` picks up from where the last page ended. Each response has `next_search_after`; pass it back unchanged on the next request and leave `from_` unset. -### Offset paging with `from_` and `size` +#### Offset paging with `from_` and `size` Simple, and extracts the results in pages. The cost grows with depth and the server collects and discards every hit before the offset — so it is the wrong tool for sweeping a large index. @@ -301,7 +370,7 @@ total_hits=6, returned=2 ``` -### Cursor paging with `search_after` +#### Cursor paging with `search_after` This is the solution if you need every row. The catch is that `search_after` is a position in a sort order, so the `sort` has to place every row unambiguously. If two rows tie on every sort column, a page boundary @@ -334,60 +403,6 @@ total_hits=6, returned=2 GET, so ask for the largest `size` you can use rather than walking a big index in small pages. -## Advanced: Tune matching with synonyms and analyzers - -!!! warning "Restricted and permanent" - Creating and updating the following resources is restricted to Sage Bionetworks employees, - and the REST API has no delete endpoint for any of them. Once created, a - SynonymSet, TextAnalyzer, ColumnAnalyzerOverride, or SearchConfiguration cannot be - removed, and its owning Organization can no longer be deleted either. Choose names deliberately. - -Everything in this tutorial relies on how each column was analyzed when the index was built: how text is split into tokens, which tokens are dropped, and how they are normalized. There are four `Organization`-scoped resources that let you control that: - -* [SynonymSet][synapseclient.models.SynonymSet] — terms that should be treated as equivalent, so someone searching `AD` finds abstracts that say "Alzheimer's disease" -* [TextAnalyzer][synapseclient.models.TextAnalyzer] — a named OpenSearch analyzer: a tokenizer plus a chain of token filters, which may reference a SynonymSet -* [ColumnAnalyzerOverride][synapseclient.models.ColumnAnalyzerOverride] — a reusable bundle assigning specific analyzers to specific columns -* [SearchConfiguration][synapseclient.models.SearchConfiguration] — bundles a default analyzer with any column overrides; this is what a SearchIndex actually points at - -Each resource belongs to an [Organization][synapseclient.models.Organization] and is -referenced from another resource by its qualified name, -`{organization_name}-{name}`, written as `{"$ref": "my.org-my_analyzer"}`. - - -Note where the synonym filter goes below. The analyzer declares both a `default` chain, -used when rows are indexed, and a `default_search` chain, used when a query is analyzed. -Putting the synonyms only in `default_search` expands the incoming query instead of -storing every synonym for every row. - -```python ---8<-- "docs/tutorials/python/tutorial_scripts/search.py:search_configuration" -``` - -A SearchIndex resolves its configuration when the index is built, so set it up front. -Either point the index straight at a configuration with `search_configuration_id`, or -bind a configuration to the parent folder or project — an index with no -`search_configuration_id` of its own walks up the entity hierarchy and uses the first -[SearchConfigBinding][synapseclient.models.SearchConfigBinding] it finds, falling back -to the platform defaults. - -```python ---8<-- "docs/tutorials/python/tutorial_scripts/search.py:apply_search_configuration" -``` - -
- Searching the abbreviation against the new index should look like: -``` -Created SearchIndex syn68123457 using config 4321 -Index syn68123457 is queryable with 6 rows -Bound configuration 4321 to syn12345678 -Abstracts matching the abbreviation 'AD': -total_hits=3, returned=3 - {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} - {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} - {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} -``` -
- ## Source Code for this Tutorial
diff --git a/docs/tutorials/python/tutorial_scripts/search.py b/docs/tutorials/python/tutorial_scripts/search.py index b9faa9387..e82c09632 100644 --- a/docs/tutorials/python/tutorial_scripts/search.py +++ b/docs/tutorials/python/tutorial_scripts/search.py @@ -127,6 +127,152 @@ # --8<-- [end:create_index] +# --8<-- [start:search_configuration] +def create_search_configuration() -> str: + """ + Example: Teach the index that "AD" means "Alzheimer's disease" by building a + SynonymSet, wrapping it in a TextAnalyzer, and bundling that analyzer into a + SearchConfiguration. + + These resources belong to an Organization, and creating them is restricted + to Sage Bionetworks employees. None of them can be deleted once created. + """ + from synapseclient.models import ( + ColumnAnalyzerOverride, + ColumnAnalyzerOverrideEntry, + Organization, + SearchConfiguration, + SynonymSet, + TextAnalyzer, + ) + + organization_name = "my.uniquely.named.organization" + organization = Organization(name=organization_name).store() + print(f"Using organization: {organization.id} ({organization.name})") + + # Comma-separated entries are interchangeable in both directions; entries + # written with "=>" expand the left side to the right side only. + synonyms = SynonymSet( + organization_name=organization_name, + name="ad_synonyms", + description="Abbreviations used across Alzheimer's disease studies", + definition={ + "type": "synonym_graph", + "synonyms": [ + "rna sequencing, rna-seq, rnaseq", + "ad => alzheimer's disease, alzheimers disease", + "mci => mild cognitive impairment", + ], + }, + ).store() + print(f"Created SynonymSet: {synonyms.id} ({synonyms.qualified_name})") + + # The synonym filter is applied in `default_search` only, so synonyms expand + # the incoming query rather than bloating the stored index. + analyzer = TextAnalyzer( + organization_name=organization_name, + name="ad_synonym_analyzer", + description="English analyzer that expands AD abbreviations at search time", + settings={ + "filter": { + "english_stop": {"type": "stop", "stopwords": "_english_"}, + "english_stemmer": {"type": "stemmer", "language": "english"}, + # A $ref resolves to the SynonymSet by its qualified name + "ad_synonyms": {"$ref": synonyms.qualified_name}, + }, + "analyzer": { + "default": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase", "english_stop", "english_stemmer"], + }, + "default_search": { + "type": "custom", + "tokenizer": "standard", + "filter": [ + "lowercase", + "ad_synonyms", + "english_stop", + "english_stemmer", + ], + }, + }, + }, + ).store() + print(f"Created TextAnalyzer: {analyzer.id} ({analyzer.qualified_name})") + + # Columns not named here fall back to the configuration's default analyzer + overrides = ColumnAnalyzerOverride( + organization_name=organization_name, + name="study_column_overrides", + description="Treat the diagnosis column as a single exact value", + overrides=[ + ColumnAnalyzerOverrideEntry( + column_name="diagnosis", + analyzer={"analyzer": {"default": {"type": "keyword"}}}, + ), + ], + ).store() + print(f"Created ColumnAnalyzerOverride: {overrides.id}") + + configuration = SearchConfiguration( + organization_name=organization_name, + name="study_search_config", + description="Analyzer settings for the study summary search index", + default_analyzer={"$ref": analyzer.qualified_name}, + column_analyzer_overrides=[{"$ref": overrides.qualified_name}], + ).store() + print(f"Created SearchConfiguration: {configuration.id}") + return configuration.id + + +# --8<-- [end:search_configuration] + + +# --8<-- [start:apply_search_configuration] +def create_index_with_configuration(search_configuration_id: str) -> SearchIndex: + """ + Example: Build an index that uses a specific SearchConfiguration, and bind + the same configuration to the project so later indexes inherit it. + """ + from synapseclient.models import SearchConfigBinding + + index = SearchIndex( + name="Study Summaries Search Index With Synonyms", + parent_id=project_id, + defining_sql=f"SELECT * FROM {table.id}", + search_configuration_id=search_configuration_id, + ).store() + print(f"Created SearchIndex {index.id} using config {search_configuration_id}") + + # Any index created under this project without its own + # search_configuration_id now inherits this configuration + binding = SearchConfigBinding( + object_id=project_id, + search_configuration_id=search_configuration_id, + ).store() + print(f"Bound configuration {binding.search_configuration_id} to {project_id}") + + # "AD" now matches the abstracts that spell out "Alzheimer's disease" + results = index.query( + search_query=SearchQuery( + query=Query(match={"abstract": MatchFieldOptions(query="AD")}), + source=SourceFilter(includes=["study_name", "diagnosis"]), + size=10, + ), + response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], + ) + print("Abstracts matching the abbreviation 'AD':") + print(f"total_hits={results.total_hits}, returned={len(results.hits)}") + for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" {fields}") + return index + + +# --8<-- [end:apply_search_configuration] + + # --8<-- [start:full_text_search] # Find every study whose abstract mentions Alzheimer's disease, then # search across several columns at once. @@ -387,149 +533,3 @@ page += 1 # --8<-- [end:pagination_cursor] - - -# --8<-- [start:search_configuration] -def create_search_configuration() -> str: - """ - Example: Teach the index that "AD" means "Alzheimer's disease" by building a - SynonymSet, wrapping it in a TextAnalyzer, and bundling that analyzer into a - SearchConfiguration. - - These resources belong to an Organization, and creating them is restricted - to Sage Bionetworks employees. None of them can be deleted once created. - """ - from synapseclient.models import ( - ColumnAnalyzerOverride, - ColumnAnalyzerOverrideEntry, - Organization, - SearchConfiguration, - SynonymSet, - TextAnalyzer, - ) - - organization_name = "my.uniquely.named.organization" - organization = Organization(name=organization_name).store() - print(f"Using organization: {organization.id} ({organization.name})") - - # Comma-separated entries are interchangeable in both directions; entries - # written with "=>" expand the left side to the right side only. - synonyms = SynonymSet( - organization_name=organization_name, - name="ad_synonyms", - description="Abbreviations used across Alzheimer's disease studies", - definition={ - "type": "synonym_graph", - "synonyms": [ - "rna sequencing, rna-seq, rnaseq", - "ad => alzheimer's disease, alzheimers disease", - "mci => mild cognitive impairment", - ], - }, - ).store() - print(f"Created SynonymSet: {synonyms.id} ({synonyms.qualified_name})") - - # The synonym filter is applied in `default_search` only, so synonyms expand - # the incoming query rather than bloating the stored index. - analyzer = TextAnalyzer( - organization_name=organization_name, - name="ad_synonym_analyzer", - description="English analyzer that expands AD abbreviations at search time", - settings={ - "filter": { - "english_stop": {"type": "stop", "stopwords": "_english_"}, - "english_stemmer": {"type": "stemmer", "language": "english"}, - # A $ref resolves to the SynonymSet by its qualified name - "ad_synonyms": {"$ref": synonyms.qualified_name}, - }, - "analyzer": { - "default": { - "type": "custom", - "tokenizer": "standard", - "filter": ["lowercase", "english_stop", "english_stemmer"], - }, - "default_search": { - "type": "custom", - "tokenizer": "standard", - "filter": [ - "lowercase", - "ad_synonyms", - "english_stop", - "english_stemmer", - ], - }, - }, - }, - ).store() - print(f"Created TextAnalyzer: {analyzer.id} ({analyzer.qualified_name})") - - # Columns not named here fall back to the configuration's default analyzer - overrides = ColumnAnalyzerOverride( - organization_name=organization_name, - name="study_column_overrides", - description="Treat the diagnosis column as a single exact value", - overrides=[ - ColumnAnalyzerOverrideEntry( - column_name="diagnosis", - analyzer={"analyzer": {"default": {"type": "keyword"}}}, - ), - ], - ).store() - print(f"Created ColumnAnalyzerOverride: {overrides.id}") - - configuration = SearchConfiguration( - organization_name=organization_name, - name="study_search_config", - description="Analyzer settings for the study summary search index", - default_analyzer={"$ref": analyzer.qualified_name}, - column_analyzer_overrides=[{"$ref": overrides.qualified_name}], - ).store() - print(f"Created SearchConfiguration: {configuration.id}") - return configuration.id - - -# --8<-- [end:search_configuration] - - -# --8<-- [start:apply_search_configuration] -def create_index_with_configuration(search_configuration_id: str) -> SearchIndex: - """ - Example: Build an index that uses a specific SearchConfiguration, and bind - the same configuration to the project so later indexes inherit it. - """ - from synapseclient.models import SearchConfigBinding - - index = SearchIndex( - name="Study Summaries Search Index With Synonyms", - parent_id=project_id, - defining_sql=f"SELECT * FROM {table.id}", - search_configuration_id=search_configuration_id, - ).store() - print(f"Created SearchIndex {index.id} using config {search_configuration_id}") - - # Any index created under this project without its own - # search_configuration_id now inherits this configuration - binding = SearchConfigBinding( - object_id=project_id, - search_configuration_id=search_configuration_id, - ).store() - print(f"Bound configuration {binding.search_configuration_id} to {project_id}") - - # "AD" now matches the abstracts that spell out "Alzheimer's disease" - results = index.query( - search_query=SearchQuery( - query=Query(match={"abstract": MatchFieldOptions(query="AD")}), - source=SourceFilter(includes=["study_name", "diagnosis"]), - size=10, - ), - response_parts=[SearchQueryPart.HITS, SearchQueryPart.TOTAL_HITS], - ) - print("Abstracts matching the abbreviation 'AD':") - print(f"total_hits={results.total_hits}, returned={len(results.hits)}") - for hit in results.hits: - fields = {field.name: field.value for field in hit.fields} - print(f" {fields}") - return index - - -# --8<-- [end:apply_search_configuration] From 9d2a02f01a9fbf1c00db34f5df0e7797e831f24e Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Wed, 26 Aug 2026 10:31:41 -0700 Subject: [PATCH 10/12] Update documentation --- docs/tutorials/python/search.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md index e1556e112..004b12a5b 100644 --- a/docs/tutorials/python/search.md +++ b/docs/tutorials/python/search.md @@ -198,9 +198,8 @@ total_hits=3, returned=3 ```
-Hits come back ranked by relevance, and each one carries its score on -[`hit.score`][synapseclient.models.SearchHit] along with the `row_id` and `row_version` -of the source row. +Hits come back ranked by relevance, and a score can be returned +[`hit.score`][synapseclient.models.SearchHit]. ### 3.2 Highlight where the match happened From f39a8795e1e102c90b5689398383ad17f4d5b46a Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Wed, 26 Aug 2026 10:36:32 -0700 Subject: [PATCH 11/12] Enhance pagination documentation --- docs/tutorials/python/search.md | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md index 004b12a5b..d91bf6365 100644 --- a/docs/tutorials/python/search.md +++ b/docs/tutorials/python/search.md @@ -337,15 +337,25 @@ Suggestions for 'Mayo Cl': ### 3.6 Paginated results -A query returns at most 100 hits at a time (25 by default), so anything larger requires special attention. There are two ways to do it. +A query returns at most 100 hits at a time (25 by default), so anything larger requires +paging. There are two mechanisms, and they answer different questions — pick by what you are trying to do: -* Specifying the `from_` and `size` arguments on `SearchQuery` to an offset the way page numbers do. -* `search_after` picks up from where the last page ended. Each response has `next_search_after`; pass it back unchanged on the next request and leave `from_` unset. +* **`from_` and `size`** jump to an arbitrary position, the way numbered pages in a UI + do. Reach for this when you want *the 504th result*. +* **`search_after`** picks up exactly where the previous page ended. Reach for this when + you want to *enumerate every result*. Each response carries `next_search_after`; pass + it back unchanged on the next request and leave `from_` unset. + +!!! warning "Offset paging stops at 10,000 hits" + OpenSearch caps `from_ + size` at its result window, 10,000 by default. Offset + paging therefore cannot reach past the 10,000th hit, and a sweep that might run + deeper than that has to use `search_after`. #### Offset paging with `from_` and `size` -Simple, and extracts the results in pages. The cost grows with depth and the server collects and discards every hit -before the offset — so it is the wrong tool for sweeping a large index. +Simple, and extracts the results in pages. The cost grows with depth — the server +collects and discards every hit before the offset — which is both why it is capped at +10,000 and why it is the wrong tool for sweeping a large index. ```python --8<-- "docs/tutorials/python/tutorial_scripts/search.py:pagination_offset" @@ -371,7 +381,11 @@ total_hits=6, returned=2 #### Cursor paging with `search_after` -This is the solution if you need every row. The catch is that `search_after` is a position in a sort order, so the `sort` has to +This is the solution if you need every row, and the only option once you are paging past +the 10,000-hit result window. It has no depth penalty: each request resumes from the +cursor instead of counting up from the start. + +The catch is that `search_after` is a position in a sort order, so the `sort` has to place every row unambiguously. If two rows tie on every sort column, a page boundary landing between them can skip or repeat rows. Sort on something unique, or append a unique column as a final tie-breaker. From 42e2f9fb56155d1fa49771b440f739f80932fdde Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Wed, 26 Aug 2026 10:37:38 -0700 Subject: [PATCH 12/12] Add note about lower case --- docs/tutorials/python/search.md | 8 ++++++++ docs/tutorials/python/tutorial_scripts/search.py | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md index d91bf6365..81197e8f6 100644 --- a/docs/tutorials/python/search.md +++ b/docs/tutorials/python/search.md @@ -119,6 +119,14 @@ used when rows are indexed, and a `default_search` chain, used when a query is a Putting the synonyms only in `default_search` expands the incoming query instead of storing every synonym for every row. +!!! tip "Write your synonyms in lowercase" + Token filters run in the order they are listed, and `lowercase` comes before the + synonym filter in the chain below. By the time a search term reaches the synonym + filter it has already been lowercased, so an entry written as `AD => Alzheimer's + disease` will never be matched and never expand. Lowercase every entry in the + SynonymSet — `ad => alzheimer's disease` — and the abbreviation still works no + matter how the person typed it. + ```python --8<-- "docs/tutorials/python/tutorial_scripts/search.py:search_configuration" ``` diff --git a/docs/tutorials/python/tutorial_scripts/search.py b/docs/tutorials/python/tutorial_scripts/search.py index e82c09632..4fbcf37d6 100644 --- a/docs/tutorials/python/tutorial_scripts/search.py +++ b/docs/tutorials/python/tutorial_scripts/search.py @@ -152,6 +152,11 @@ def create_search_configuration() -> str: # Comma-separated entries are interchangeable in both directions; entries # written with "=>" expand the left side to the right side only. + # + # Keep every entry lowercase. The `lowercase` filter runs before the synonym + # filter in the chain below, so a search term is already lowercased by the + # time the synonyms are applied -- an entry written "AD => ..." would never + # match and never expand. synonyms = SynonymSet( organization_name=organization_name, name="ad_synonyms",