From 3962160c12310450f9b66b637412bb0a85cb00e0 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:21:12 +0200 Subject: [PATCH 1/3] docs: add a Python generator example for server-side ingest The batch import page taught the iterable/generator form of the one-shot ingest call to TypeScript readers only, even though the Python client accepts any iterable too. Add the Python counterpart so both clients document the same capability. * New CI-executed snippet reads a JSONL file line by line and yields property dicts straight into the ingest call, so the file never has to fit in memory. Blank lines are skipped explicitly instead of relying on a null check that raises on them. * The snippet sits in the server-side batching tab, next to the streaming context, and links to the in-memory list section for the eager form. * "Stream data from large files" now names Python alongside TypeScript for the lazy-iterable option. Verified against Weaviate 1.38.0 with the CI-pinned client: 5 uuids returned, empty errors dict, all 5 titles persisted. --- _includes/code/howto/manage-data.import.py | 54 ++++++++++++++++++++++ docs/weaviate/manage-objects/import.mdx | 11 ++++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/_includes/code/howto/manage-data.import.py b/_includes/code/howto/manage-data.import.py index 4a7bf34f..dfe5eda7 100644 --- a/_includes/code/howto/manage-data.import.py +++ b/_includes/code/howto/manage-data.import.py @@ -616,6 +616,60 @@ def add_object(obj) -> None: client.collections.delete(collection.name) +# ================================================== +# ===== Server-side ingest from a generator ===== +# ================================================== + +# Re-create the collection +client.collections.delete("MyCollection") +client.collections.create( + "MyCollection", + vector_config=Configure.Vectors.self_provided() +) + +# Create the source file used by the example below +with open("my-data.jsonl", "w") as f: + for i in range(5): + f.write(json.dumps({"title": f"Object {i+1}"}) + "\n") + f.write("\n") # A blank line, to show that the generator skips it + +# START ServerSideIngestGeneratorExample +import json + +# Each line of the source file holds one JSON object +def read_objects(path): + with open(path) as f: + for line in f: + line = line.strip() + if not line: # Skip blank lines + continue + record = json.loads(line) + yield {"title": record["title"]} + +collection = client.collections.use("MyCollection") + +# highlight-start +# `ingest` accepts any iterable, so the generator supplies objects one at a +# time and the file is never loaded into memory in full +result = collection.data.ingest(read_objects("my-data.jsonl")) +# highlight-end + +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") +# END ServerSideIngestGeneratorExample + +# Tests +assert len(result.errors) == 0 +assert len(result.uuids) == 5 + +agg_result = collection.aggregate.over_all(total_count=True) +assert agg_result.total_count == 5 + +# Clean up +client.collections.delete(collection.name) +os.remove("my-data.jsonl") + + # ================================================== # ===== Server-side one-shot ingest ===== # ================================================== diff --git a/docs/weaviate/manage-objects/import.mdx b/docs/weaviate/manage-objects/import.mdx index b2642662..a053abbb 100644 --- a/docs/weaviate/manage-objects/import.mdx +++ b/docs/weaviate/manage-objects/import.mdx @@ -43,6 +43,15 @@ Open the `batch.stream()` context manager and add objects one at a time; the cli endMarker="# END ServerSideBatchImportExample" language="py" /> + +You can also stream from a data source with the one-shot import method. It accepts any iterable, so you can hand it a generator that reads a source file record by record. Objects go to the server as the generator produces them, so the source never has to fit in memory. To import objects that you already hold in a list, see [Ingest an in-memory list](#ingest-an-in-memory-list). + + @@ -398,7 +407,7 @@ You can batch create links from an object to another object through cross-refere If your dataset does not fit in memory, do not load it all at once. Instead, read the source file lazily and add objects to the import as you go: - With the server-side streaming context (`batch.stream()` in Python), add each object as you read it from the file. The client sends data at the pace the server requests, so memory usage stays flat. -- In TypeScript, `data.ingest()` accepts any `Iterable`, so you can pass a lazy source (such as a generator that reads the file record by record) instead of a fully loaded array. +- In Python and TypeScript, the [one-shot import method](#server-side-batching) accepts any iterable, so you can pass a lazy source, such as a generator that reads the file record by record, instead of a fully loaded list. - With manual batching, apply the same pattern: add objects to the batch as you read them. For JSON files, use a streaming parser that yields one object at a time (such as `ijson` in Python). For CSV files, read the file in chunks (such as `pandas` with the `chunksize` parameter) rather than loading it whole. From eea559ad3ba4d8a6fdba42d4328a406913d9a18b Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:24:08 +0200 Subject: [PATCH 2/3] docs: correct the ingest paragraph on the Python best practices page The page's only mention of the one-shot ingest call opened with "If your objects are already in an in-memory list", which is wrong by omission: the parameter is an iterable, so a generator works just as well. The page had no mention of generators anywhere. Rewrite that paragraph to cover both the eager and the lazy case, add a short generator snippet, and link to the batch import page for the version that reads a file line by line. Ingest is deliberately not added to the batch sizing table: it is a second entry point to server-side batching, not a fifth sizing mode. Snippet verified against Weaviate 1.38.0 with the CI-pinned client: 2 uuids returned, empty errors dict, both titles persisted. --- _includes/code/client-libraries/python_v4.py | 27 +++++++++++++++++++ .../python/notes-best-practices.mdx | 11 +++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/_includes/code/client-libraries/python_v4.py b/_includes/code/client-libraries/python_v4.py index 083c9d38..994931eb 100644 --- a/_includes/code/client-libraries/python_v4.py +++ b/_includes/code/client-libraries/python_v4.py @@ -360,6 +360,33 @@ # END BatchRateLimit +# START BatchIngestGenerator +import weaviate + +client = weaviate.connect_to_local() + +# A generator produces objects one at a time instead of building a list +def article_titles(): + for title in ["Multitenancy", "Database schema"]: + yield {"title": title} + +try: + articles = client.collections.use("WikiArticle") + # `ingest` accepts any iterable, so nothing is held in memory in full + result = articles.data.ingest(article_titles()) + + if result.errors: + print(f"Number of failed imports: {len(result.errors)}") + +finally: + client.close() +# END BatchIngestGenerator + +# Tests +assert len(result.errors) == 0 +assert len(result.uuids) == 2 + + import weaviate client = weaviate.connect_to_local() diff --git a/docs/weaviate/client-libraries/python/notes-best-practices.mdx b/docs/weaviate/client-libraries/python/notes-best-practices.mdx index 55d611e1..8a85488c 100644 --- a/docs/weaviate/client-libraries/python/notes-best-practices.mdx +++ b/docs/weaviate/client-libraries/python/notes-best-practices.mdx @@ -202,7 +202,16 @@ There are four methods to configure the batching behavior. They are `stream`, `d | `fixed_size` | The batch size and number of concurrent requests are fixed to sizes specified by the user. | When you want to specify fixed parameters. | | `rate_limit` | The number of objects sent to Weaviate is rate limited (specified as n_objects per minute). | When you want to avoid hitting third-party vectorization API rate limits. | -If your objects are already in an in-memory list, `collection.data.ingest(objs)` is a one-shot convenience that uses server-side batching under the hood (no batching context required). It accepts plain property dicts or `DataObject` instances and returns the same `BatchObjectReturn` object as `insert_many`, making it a drop-in replacement for `insert_many` for large lists. +`collection.data.ingest(objs)` is a one-shot convenience that uses server-side batching under the hood (no batching context required). It takes any iterable of plain property dicts or `DataObject` instances, and returns the same `BatchObjectReturn` object as `insert_many`. Pass a list of objects that you already hold in memory, which makes it a drop-in replacement for `insert_many` for large lists. Pass a generator, or any other lazy iterable, to import from a source that does not fit in memory: objects are sent to the server as the generator produces them. + + + +For a generator that reads a source file line by line, see [batch import](../../manage-objects/import.mdx#server-side-batching). #### Usage From c77b5e77225d50a6c433ab080830a4874da77c20 Mon Sep 17 00:00:00 2001 From: g-despot Date: Wed, 29 Jul 2026 14:05:07 +0200 Subject: [PATCH 3/3] docs: apply editor review to the Python ingest generator docs - import.mdx: name `data.ingest()` inside the Python tab, and drop the Python-only method name from the shared "Stream data from large files" prose, linking to the server-side batching section instead. - notes-best-practices.mdx: promote the one-shot ingest material out of "Batch sizing" into its own "One-shot ingest" section, so it no longer splits "#### Usage" from the batching-method table it refers to. - notes-best-practices.mdx: document the `BatchObjectReturn` that `data.ingest()` returns under "Error handling", which previously covered only the batching-context API. - Snippet cleanups: use `collections.use()`, and trim two code comments that overclaimed or duplicated the surrounding prose. --- _includes/code/client-libraries/python_v4.py | 2 +- _includes/code/howto/manage-data.import.py | 5 ++-- .../python/notes-best-practices.mdx | 26 +++++++++++-------- docs/weaviate/manage-objects/import.mdx | 4 +-- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/_includes/code/client-libraries/python_v4.py b/_includes/code/client-libraries/python_v4.py index 994931eb..b647c8ac 100644 --- a/_includes/code/client-libraries/python_v4.py +++ b/_includes/code/client-libraries/python_v4.py @@ -372,7 +372,7 @@ def article_titles(): try: articles = client.collections.use("WikiArticle") - # `ingest` accepts any iterable, so nothing is held in memory in full + # `ingest` accepts any iterable, including a generator result = articles.data.ingest(article_titles()) if result.errors: diff --git a/_includes/code/howto/manage-data.import.py b/_includes/code/howto/manage-data.import.py index dfe5eda7..ff064601 100644 --- a/_includes/code/howto/manage-data.import.py +++ b/_includes/code/howto/manage-data.import.py @@ -588,7 +588,7 @@ def add_object(obj) -> None: {"title": f"Object {i+1}"} for i in range(5) ] -collection = client.collections.get("MyCollection") +collection = client.collections.use("MyCollection") # highlight-start # Use `stream` for server-side batching. The client will send data @@ -649,8 +649,7 @@ def read_objects(path): collection = client.collections.use("MyCollection") # highlight-start -# `ingest` accepts any iterable, so the generator supplies objects one at a -# time and the file is never loaded into memory in full +# `ingest` pulls objects from the generator as it goes result = collection.data.ingest(read_objects("my-data.jsonl")) # highlight-end diff --git a/docs/weaviate/client-libraries/python/notes-best-practices.mdx b/docs/weaviate/client-libraries/python/notes-best-practices.mdx index 8a85488c..f3215cb4 100644 --- a/docs/weaviate/client-libraries/python/notes-best-practices.mdx +++ b/docs/weaviate/client-libraries/python/notes-best-practices.mdx @@ -202,17 +202,6 @@ There are four methods to configure the batching behavior. They are `stream`, `d | `fixed_size` | The batch size and number of concurrent requests are fixed to sizes specified by the user. | When you want to specify fixed parameters. | | `rate_limit` | The number of objects sent to Weaviate is rate limited (specified as n_objects per minute). | When you want to avoid hitting third-party vectorization API rate limits. | -`collection.data.ingest(objs)` is a one-shot convenience that uses server-side batching under the hood (no batching context required). It takes any iterable of plain property dicts or `DataObject` instances, and returns the same `BatchObjectReturn` object as `insert_many`. Pass a list of objects that you already hold in memory, which makes it a drop-in replacement for `insert_many` for large lists. Pass a generator, or any other lazy iterable, to import from a source that does not fit in memory: objects are sent to the server as the generator produces them. - - - -For a generator that reads a source file line by line, see [batch import](../../manage-objects/import.mdx#server-side-batching). - #### Usage We recommend using a context manager as shown below. @@ -256,6 +245,19 @@ These methods return a new context manager for each batch. Attributes that are r If the background thread that is responsible for sending the batches raises an exception during batch processing, the error is raised to the main thread. +### One-shot ingest + +`collection.data.ingest(objs)` is a one-shot convenience that uses server-side batching under the hood (no batching context required). It accepts any iterable of plain property dicts or `DataObject` instances, and returns the same `BatchObjectReturn` object as `insert_many`. Pass a list of objects that you already hold in memory to use `ingest` as a drop-in replacement for `insert_many` on large lists. Pass a generator, or any other lazy iterable, to import from a source that does not fit in memory: the client sends each object to the server as the generator produces it. + + + +For a generator that reads a source file line by line, see [Batch import](../../manage-objects/import.mdx#server-side-batching). + ### Error handling During a batch import, any failed objects or references will be stored for retrieval. Additionally, a running count of failed objects and references is maintained. @@ -273,6 +275,8 @@ Note that these lists are reset when a batching process is initialized. So make language="py" /> +`collection.data.ingest()` does not use a batching context, so it reports failures through its return value instead. The `BatchObjectReturn` object that it returns exposes `result.has_errors` as a summary flag, and `result.errors` as a dictionary keyed by the position of each failed object in the input. Check `result.errors` directly, because it holds an entry for every failed object. The [one-shot ingest](#one-shot-ingest) example above shows this pattern. + ### Batch vectorization import BatchVectorizationOverview from "/_includes/code/client-libraries/batch-import.mdx"; diff --git a/docs/weaviate/manage-objects/import.mdx b/docs/weaviate/manage-objects/import.mdx index a053abbb..b3656d0a 100644 --- a/docs/weaviate/manage-objects/import.mdx +++ b/docs/weaviate/manage-objects/import.mdx @@ -44,7 +44,7 @@ Open the `batch.stream()` context manager and add objects one at a time; the cli language="py" /> -You can also stream from a data source with the one-shot import method. It accepts any iterable, so you can hand it a generator that reads a source file record by record. Objects go to the server as the generator produces them, so the source never has to fit in memory. To import objects that you already hold in a list, see [Ingest an in-memory list](#ingest-an-in-memory-list). +You can also stream from a data source with `data.ingest()`. It accepts any iterable, so you can pass a generator that reads a source file record by record. Objects go to the server as the generator produces them, so the source never has to fit in memory. To import objects that you already hold in a list, see [Ingest an in-memory list](#ingest-an-in-memory-list).