diff --git a/_includes/code/csharp/QuickstartLocalTest.cs b/_includes/code/csharp/QuickstartLocalTest.cs index d6fc27dfc..6f9e71eb5 100644 --- a/_includes/code/csharp/QuickstartLocalTest.cs +++ b/_includes/code/csharp/QuickstartLocalTest.cs @@ -87,21 +87,25 @@ public async Task FullQuickstartWorkflowTest() ); } - // Call InsertMany with the list of objects converted to an array - var insertResponse = await questions.Data.InsertMany(questionsToInsert.ToArray()); + // `Batch.InsertMany` imports the list using server-side batching + var insertResponse = await questions.Batch.InsertMany(questionsToInsert); // highlight-end - // END Import // Check for errors if (insertResponse.HasErrors) { Console.WriteLine($"Number of failed imports: {insertResponse.Errors.Count()}"); - Console.WriteLine($"First failed object error: {insertResponse.Errors.First()}"); + // `Objects` holds one entry per object; `Index` is the position of the object in the input + foreach (var entry in insertResponse.Objects.Where(o => o.Error is not null)) + { + Console.WriteLine($"Failed object at index {entry.Index}: {entry.Error.Message}"); + } } else { - Console.WriteLine($"Successfully inserted {insertResponse.Objects.Count()} objects."); + Console.WriteLine($"Successfully inserted {insertResponse.Count} objects."); } + // END Import // START NearText // highlight-start diff --git a/_includes/code/csharp/QuickstartTest.cs b/_includes/code/csharp/QuickstartTest.cs index 894f41c68..07a144781 100644 --- a/_includes/code/csharp/QuickstartTest.cs +++ b/_includes/code/csharp/QuickstartTest.cs @@ -95,21 +95,25 @@ public static async Task FullQuickstartWorkflowTest() ); } - // Call InsertMany with the list of objects converted to an array - var insertResponse = await questions.Data.InsertMany(questionsToInsert.ToArray()); + // `Batch.InsertMany` imports the list using server-side batching + var insertResponse = await questions.Batch.InsertMany(questionsToInsert); // highlight-end - // END Import // Check for errors if (insertResponse.HasErrors) { Console.WriteLine($"Number of failed imports: {insertResponse.Errors.Count()}"); - Console.WriteLine($"First failed object error: {insertResponse.Errors.First()}"); + // `Objects` holds one entry per object; `Index` is the position of the object in the input + foreach (var entry in insertResponse.Objects.Where(o => o.Error is not null)) + { + Console.WriteLine($"Failed object at index {entry.Index}: {entry.Error.Message}"); + } } else { - Console.WriteLine($"Successfully inserted {insertResponse.Objects.Count()} objects."); + Console.WriteLine($"Successfully inserted {insertResponse.Count} objects."); } + // END Import // START NearText // highlight-start diff --git a/_includes/code/csharp/quickstart/QuickstartCreate.cs b/_includes/code/csharp/quickstart/QuickstartCreate.cs index e1a4f4c6f..256d8ff87 100644 --- a/_includes/code/csharp/quickstart/QuickstartCreate.cs +++ b/_includes/code/csharp/quickstart/QuickstartCreate.cs @@ -65,8 +65,8 @@ public static async Task Run() }, }; - // Insert objects using InsertMany - var insertResponse = await movies.Data.InsertMany(dataObjects.ToArray()); + // Insert the objects using server-side batching + var insertResponse = await movies.Batch.InsertMany(dataObjects); if (insertResponse.HasErrors) { diff --git a/_includes/code/csharp/quickstart/QuickstartCreateVectors.cs b/_includes/code/csharp/quickstart/QuickstartCreateVectors.cs index 9ebb7f3d1..330ccd720 100644 --- a/_includes/code/csharp/quickstart/QuickstartCreateVectors.cs +++ b/_includes/code/csharp/quickstart/QuickstartCreateVectors.cs @@ -92,8 +92,8 @@ public static async Task Run() ), }; - // Insert the objects with vectors - var insertResponse = await movies.Data.InsertMany(dataToInsert); + // Insert the objects with vectors using server-side batching + var insertResponse = await movies.Batch.InsertMany(dataToInsert); if (insertResponse.HasErrors) { diff --git a/_includes/code/csharp/quickstart/QuickstartLocalCreate.cs b/_includes/code/csharp/quickstart/QuickstartLocalCreate.cs index 6d00abfb2..cf0f4e75c 100644 --- a/_includes/code/csharp/quickstart/QuickstartLocalCreate.cs +++ b/_includes/code/csharp/quickstart/QuickstartLocalCreate.cs @@ -70,8 +70,8 @@ public static async Task Run() }, }; - // Insert objects using InsertMany - var insertResponse = await movies.Data.InsertMany(dataObjects.ToArray()); + // Insert the objects using server-side batching + var insertResponse = await movies.Batch.InsertMany(dataObjects); if (insertResponse.HasErrors) { diff --git a/_includes/code/csharp/quickstart/QuickstartLocalCreateVectors.cs b/_includes/code/csharp/quickstart/QuickstartLocalCreateVectors.cs index 1e08c3d1b..93cbf8b53 100644 --- a/_includes/code/csharp/quickstart/QuickstartLocalCreateVectors.cs +++ b/_includes/code/csharp/quickstart/QuickstartLocalCreateVectors.cs @@ -76,8 +76,8 @@ public static async Task Run() ), }; - // Insert the objects with vectors - var insertResponse = await movies.Data.InsertMany(dataToInsert); + // Insert the objects with vectors using server-side batching + var insertResponse = await movies.Batch.InsertMany(dataToInsert); if (insertResponse.HasErrors) { Console.WriteLine($"Errors during import: {insertResponse.Errors}"); diff --git a/_includes/code/java-v6/src/test/java/QuickstartLocalTest.java b/_includes/code/java-v6/src/test/java/QuickstartLocalTest.java index c067883b9..0a80c19ca 100644 --- a/_includes/code/java-v6/src/test/java/QuickstartLocalTest.java +++ b/_includes/code/java-v6/src/test/java/QuickstartLocalTest.java @@ -3,7 +3,8 @@ import io.weaviate.client6.v1.api.collections.Generative; import io.weaviate.client6.v1.api.collections.Property; import io.weaviate.client6.v1.api.collections.VectorConfig; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; import org.json.JSONArray; import org.json.JSONObject; @@ -94,16 +95,21 @@ void testImportDataWorkflow() throws Exception { questionsToInsert.add(properties); }); - // Call insertMany with the list of objects - InsertManyResponse insertResponse = questions.data.insertMany(questionsToInsert.toArray(new Map[0])); + // `batch.start()` opens a server-side batch + BatchContext> batch = questions.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (Map properties : questionsToInsert) { + batch.add(WeaviateObject.>of(o -> o.properties(properties))); + } + } // highlight-end // Check for errors - if (!insertResponse.errors().isEmpty()) { - System.err.printf("Number of failed imports: %d\n", insertResponse.errors().size()); - System.err.printf("First failed object error: %s\n", insertResponse.errors().get(0)); + if (batch.numberOfErrors() > 0) { + System.err.printf("Number of failed imports: %d\n", batch.numberOfErrors()); } else { - System.out.printf("Successfully inserted %d objects.\n", insertResponse.uuids().size()); + System.out.printf("Successfully inserted %d objects.\n", questionsToInsert.size()); } // END Import // client.collections.delete(collectionName); diff --git a/_includes/code/java-v6/src/test/java/QuickstartTest.java b/_includes/code/java-v6/src/test/java/QuickstartTest.java index 3de3fa0b5..2de260ead 100644 --- a/_includes/code/java-v6/src/test/java/QuickstartTest.java +++ b/_includes/code/java-v6/src/test/java/QuickstartTest.java @@ -3,7 +3,8 @@ import io.weaviate.client6.v1.api.collections.Generative; import io.weaviate.client6.v1.api.collections.Property; import io.weaviate.client6.v1.api.collections.VectorConfig; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; import org.json.JSONArray; import org.json.JSONObject; @@ -115,16 +116,21 @@ void testImportDataWorkflow() throws Exception { questionsToInsert.add(properties); }); - // Call insertMany with the list of objects - InsertManyResponse insertResponse = questions.data.insertMany(questionsToInsert.toArray(new Map[0])); + // `batch.start()` opens a server-side batch + BatchContext> batch = questions.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (Map properties : questionsToInsert) { + batch.add(WeaviateObject.>of(o -> o.properties(properties))); + } + } // highlight-end // Check for errors - if (!insertResponse.errors().isEmpty()) { - System.err.printf("Number of failed imports: %d\n", insertResponse.errors().size()); - System.err.printf("First failed object error: %s\n", insertResponse.errors().get(0)); + if (batch.numberOfErrors() > 0) { + System.err.printf("Number of failed imports: %d\n", batch.numberOfErrors()); } else { - System.out.printf("Successfully inserted %d objects.\n", insertResponse.uuids().size()); + System.out.printf("Successfully inserted %d objects.\n", questionsToInsert.size()); } // END Import // client.collections.delete(collectionName); diff --git a/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreate.java b/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreate.java index 745c1049f..d451389b1 100644 --- a/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreate.java +++ b/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreate.java @@ -5,7 +5,8 @@ import io.weaviate.client6.v1.api.collections.CollectionHandle; import io.weaviate.client6.v1.api.collections.Property; import io.weaviate.client6.v1.api.collections.VectorConfig; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; import java.util.List; import java.util.Map; @@ -54,18 +55,24 @@ public static void main(String[] args) throws Exception { "A meek Hobbit and his companions set out on a perilous journey to destroy a powerful ring and save Middle-earth.", "genre", "Fantasy")); - // Insert objects using insertMany + // Insert the objects using server-side batching CollectionHandle> movies = client.collections.use(collectionName); - InsertManyResponse insertResponse = - movies.data.insertMany(dataObjects.toArray(new Map[0])); + BatchContext> batch = movies.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (Map properties : dataObjects) { + batch.add( + WeaviateObject.>of(o -> o.properties(properties))); + } + } - if (!insertResponse.errors().isEmpty()) { - System.err.println("Errors during import: " + insertResponse.errors()); + if (batch.numberOfErrors() > 0) { + System.err + .println("Number of failed imports: " + batch.numberOfErrors()); } else { - System.out - .println("Imported & vectorized " + insertResponse.uuids().size() - + " objects into the Movie collection"); + System.out.println("Imported & vectorized " + dataObjects.size() + + " objects into the Movie collection"); } } finally { if (client != null) { diff --git a/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreateVectors.java b/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreateVectors.java index 6a6d930d8..e1f208d41 100644 --- a/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreateVectors.java +++ b/_includes/code/java-v6/src/test/java/quickstart/QuickstartCreateVectors.java @@ -7,7 +7,8 @@ import io.weaviate.client6.v1.api.collections.VectorConfig; import io.weaviate.client6.v1.api.collections.Vectors; import io.weaviate.client6.v1.api.collections.WeaviateObject; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; +import java.util.List; import java.util.Map; public class QuickstartCreateVectors { @@ -63,20 +64,30 @@ public static void main(String[] args) throws Exception { float[] vector3 = new float[] {0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.0f}; - // Insert the objects with vectors + // Insert the objects with vectors using server-side batching CollectionHandle> movies = client.collections.use(collectionName); - InsertManyResponse insertResponse = movies.data.insertMany( + List>> objectsToInsert = List.of( WeaviateObject.of(v -> v.properties(props1) .vectors(Vectors.of(vector1))), WeaviateObject.of(v -> v.properties(props2) .vectors(Vectors.of(vector2))), WeaviateObject.of(v -> v.properties(props3) .vectors(Vectors.of(vector3)))); - if (!insertResponse.errors().isEmpty()) { - System.err.println("Errors during import: " + insertResponse.errors()); + + BatchContext> batch = movies.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (WeaviateObject> object : objectsToInsert) { + batch.add(object); + } + } + + if (batch.numberOfErrors() > 0) { + System.err + .println("Number of failed imports: " + batch.numberOfErrors()); } else { - System.out.println("Imported " + insertResponse.uuids().size() + System.out.println("Imported " + objectsToInsert.size() + " objects with vectors into the Movie collection"); } } finally { diff --git a/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreate.java b/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreate.java index 16e7149be..6b6b45477 100644 --- a/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreate.java +++ b/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreate.java @@ -5,7 +5,8 @@ import io.weaviate.client6.v1.api.collections.CollectionHandle; import io.weaviate.client6.v1.api.collections.Property; import io.weaviate.client6.v1.api.collections.VectorConfig; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; import java.util.List; import java.util.Map; @@ -53,18 +54,24 @@ public static void main(String[] args) throws Exception { "A meek Hobbit and his companions set out on a perilous journey to destroy a powerful ring and save Middle-earth.", "genre", "Fantasy")); - // Insert objects using insertMany + // Insert the objects using server-side batching CollectionHandle> movies = client.collections.use(collectionName); - InsertManyResponse insertResponse = - movies.data.insertMany(dataObjects.toArray(new Map[0])); + BatchContext> batch = movies.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (Map properties : dataObjects) { + batch.add( + WeaviateObject.>of(o -> o.properties(properties))); + } + } - if (!insertResponse.errors().isEmpty()) { - System.err.println("Errors during import: " + insertResponse.errors()); + if (batch.numberOfErrors() > 0) { + System.err + .println("Number of failed imports: " + batch.numberOfErrors()); } else { - System.out - .println("Imported & vectorized " + insertResponse.uuids().size() - + " objects into the Movie collection"); + System.out.println("Imported & vectorized " + dataObjects.size() + + " objects into the Movie collection"); } } finally { if (client != null) { diff --git a/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreateVectors.java b/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreateVectors.java index f31eaad3d..66c88691d 100644 --- a/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreateVectors.java +++ b/_includes/code/java-v6/src/test/java/quickstart/QuickstartLocalCreateVectors.java @@ -7,7 +7,8 @@ import io.weaviate.client6.v1.api.collections.VectorConfig; import io.weaviate.client6.v1.api.collections.Vectors; import io.weaviate.client6.v1.api.collections.WeaviateObject; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; +import io.weaviate.client6.v1.api.collections.batch.BatchContext; +import java.util.List; import java.util.Map; public class QuickstartLocalCreateVectors { @@ -59,10 +60,10 @@ public static void main(String[] args) throws Exception { float[] vector3 = new float[] {0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.0f}; - // Insert the objects with vectors + // Insert the objects with vectors using server-side batching CollectionHandle> movies = client.collections.use(collectionName); - InsertManyResponse insertResponse = movies.data.insertMany( + List>> objectsToInsert = List.of( WeaviateObject.of(v -> v.properties(props1) .vectors(Vectors.of(vector1))), WeaviateObject.of(v -> v.properties(props2) @@ -70,10 +71,19 @@ public static void main(String[] args) throws Exception { WeaviateObject.of(v -> v.properties(props3) .vectors(Vectors.of(vector3)))); - if (!insertResponse.errors().isEmpty()) { - System.err.println("Errors during import: " + insertResponse.errors()); + BatchContext> batch = movies.batch.start(); + // Closing the batch sends the remaining objects and waits for the results + try (batch) { + for (WeaviateObject> object : objectsToInsert) { + batch.add(object); + } + } + + if (batch.numberOfErrors() > 0) { + System.err + .println("Number of failed imports: " + batch.numberOfErrors()); } else { - System.out.println("Imported " + insertResponse.uuids().size() + System.out.println("Imported " + objectsToInsert.size() + " objects with vectors into the Movie collection"); } } finally { diff --git a/_includes/code/llms-txt/python/quickstart.py b/_includes/code/llms-txt/python/quickstart.py index 30ad432ff..7f46cb2c1 100644 --- a/_includes/code/llms-txt/python/quickstart.py +++ b/_includes/code/llms-txt/python/quickstart.py @@ -46,9 +46,7 @@ movies = client.collections.use("Movie__QuickstartPy") # Import objects - with movies.batch.fixed_size(batch_size=200) as batch: - for obj in data_objects: - batch.add_object(properties=obj) + movies.data.ingest(data_objects) print(f"Imported & vectorized {len(data_objects)} objects into the Movie collection") diff --git a/_includes/code/llms-txt/typescript/quickstart.ts b/_includes/code/llms-txt/typescript/quickstart.ts index a83e17a1e..3270ee824 100644 --- a/_includes/code/llms-txt/typescript/quickstart.ts +++ b/_includes/code/llms-txt/typescript/quickstart.ts @@ -36,7 +36,7 @@ if (!(await client.collections.exists('Movie__QuickstartTs'))) { const movies = client.collections.use('Movie__QuickstartTs'); // Import objects -await movies.data.insertMany(dataObjects); +await movies.data.ingest(dataObjects.map((properties) => ({ properties }))); console.log(`Imported & vectorized ${dataObjects.length} objects into the Movie collection`); diff --git a/_includes/code/python/local.quickstart.import_objects.py b/_includes/code/python/local.quickstart.import_objects.py index 9f9c3d7b4..d46b4dd1c 100644 --- a/_includes/code/python/local.quickstart.import_objects.py +++ b/_includes/code/python/local.quickstart.import_objects.py @@ -9,27 +9,26 @@ ) data = json.loads(resp.text) -# highlight-start questions = client.collections.use("Question") -with questions.batch.fixed_size(batch_size=200) as batch: - for d in data: - batch.add_object( - { - "answer": d["Answer"], - "question": d["Question"], - "category": d["Category"], - } - ) - # highlight-end - if batch.number_errors > 10: - print("Batch import stopped due to excessive errors.") - break +# highlight-start +result = questions.data.ingest( + [ + { + "answer": d["Answer"], + "question": d["Question"], + "category": d["Category"], + } + for d in data + ] +) +# highlight-end -failed_objects = questions.batch.failed_objects -if failed_objects: - print(f"Number of failed imports: {len(failed_objects)}") - print(f"First failed object: {failed_objects[0]}") +# `errors` holds one entry per failed object, keyed by its position in the input +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") + for index, error in result.errors.items(): + print(f"Failed object at index {index}: {error.message}") client.close() # Free up resources # END Import diff --git a/_includes/code/python/quickstart.import_objects.py b/_includes/code/python/quickstart.import_objects.py index fa89fdf71..0f4041af9 100644 --- a/_includes/code/python/quickstart.import_objects.py +++ b/_includes/code/python/quickstart.import_objects.py @@ -17,27 +17,26 @@ ) data = json.loads(resp.text) -# highlight-start questions = client.collections.use("Question") -with questions.batch.fixed_size(batch_size=200) as batch: - for d in data: - batch.add_object( - { - "answer": d["Answer"], - "question": d["Question"], - "category": d["Category"], - } - ) - # highlight-end - if batch.number_errors > 10: - print("Batch import stopped due to excessive errors.") - break +# highlight-start +result = questions.data.ingest( + [ + { + "answer": d["Answer"], + "question": d["Question"], + "category": d["Category"], + } + for d in data + ] +) +# highlight-end -failed_objects = questions.batch.failed_objects -if failed_objects: - print(f"Number of failed imports: {len(failed_objects)}") - print(f"First failed object: {failed_objects[0]}") +# `errors` holds one entry per failed object, keyed by its position in the input +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") + for index, error in result.errors.items(): + print(f"Failed object at index {index}: {error.message}") client.close() # Free up resources # END Import diff --git a/_includes/code/python/quickstart.short.create_collection.py b/_includes/code/python/quickstart.short.create_collection.py index a4685294d..62becb3a9 100644 --- a/_includes/code/python/quickstart.short.create_collection.py +++ b/_includes/code/python/quickstart.short.create_collection.py @@ -42,9 +42,7 @@ # START CreateCollection movies = client.collections.use("Movie") - with movies.batch.fixed_size(batch_size=200) as batch: - for obj in data_objects: - batch.add_object(properties=obj) + movies.data.ingest(data_objects) print(f"Imported & vectorized {len(movies)} objects into the Movie collection") # END CreateCollection diff --git a/_includes/code/python/quickstart.short.import_vectors.create_collection.py b/_includes/code/python/quickstart.short.import_vectors.create_collection.py index 2211e39ae..76a15f1b8 100644 --- a/_includes/code/python/quickstart.short.import_vectors.create_collection.py +++ b/_includes/code/python/quickstart.short.import_vectors.create_collection.py @@ -1,6 +1,7 @@ # START CreateCollection import weaviate from weaviate.classes.config import Configure +from weaviate.classes.data import DataObject import os # Best practice: store your credentials in environment variables @@ -46,9 +47,10 @@ # Insert the objects with vectors movies = client.collections.get("Movie") - with movies.batch.fixed_size(batch_size=200) as batch: - for obj in data_objects: - batch.add_object(properties=obj["properties"], vector=obj["vector"]) + movies.data.ingest( + DataObject(properties=obj["properties"], vector=obj["vector"]) + for obj in data_objects + ) print( f"Imported {len(data_objects)} objects with vectors into the Movie collection" diff --git a/_includes/code/python/quickstart.short.local.create_collection.py b/_includes/code/python/quickstart.short.local.create_collection.py index 5d8cdfc56..ff8ce55a0 100644 --- a/_includes/code/python/quickstart.short.local.create_collection.py +++ b/_includes/code/python/quickstart.short.local.create_collection.py @@ -36,9 +36,7 @@ # START CreateCollection movies = client.collections.use("Movie") - with movies.batch.fixed_size(batch_size=200) as batch: - for obj in data_objects: - batch.add_object(properties=obj) + movies.data.ingest(data_objects) print(f"Imported & vectorized {len(movies)} objects into the Movie collection") # END CreateCollection diff --git a/_includes/code/python/quickstart.short.local.import_vectors.create_collection.py b/_includes/code/python/quickstart.short.local.import_vectors.create_collection.py index 208b7a31c..f5a874884 100644 --- a/_includes/code/python/quickstart.short.local.import_vectors.create_collection.py +++ b/_includes/code/python/quickstart.short.local.import_vectors.create_collection.py @@ -1,6 +1,7 @@ # START CreateCollection import weaviate from weaviate.classes.config import Configure +from weaviate.classes.data import DataObject # Step 1.1: Connect to your local Weaviate instance with weaviate.connect_to_local() as client: @@ -38,9 +39,10 @@ # Insert the objects with vectors movies = client.collections.get("Movie") - with movies.batch.fixed_size(batch_size=200) as batch: - for obj in data_objects: - batch.add_object(properties=obj["properties"], vector=obj["vector"]) + movies.data.ingest( + DataObject(properties=obj["properties"], vector=obj["vector"]) + for obj in data_objects + ) print( f"Imported {len(data_objects)} objects with vectors into the Movie collection" diff --git a/_includes/code/quickstart/local.quickstart.import_objects.mdx b/_includes/code/quickstart/local.quickstart.import_objects.mdx index 705df5b35..ad0c6222f 100644 --- a/_includes/code/quickstart/local.quickstart.import_objects.mdx +++ b/_includes/code/quickstart/local.quickstart.import_objects.mdx @@ -8,6 +8,8 @@ import JavaV6Code from "!!raw-loader!/_includes/code/java-v6/src/test/java/Quick import CSharpCode from "!!raw-loader!/_includes/code/csharp/QuickstartLocalTest.cs"; +Every import reports whether any objects failed. Check for failures in your own code to catch problems such as malformed data or a misconfigured model provider. + @@ -19,7 +21,7 @@ import CSharpCode from "!!raw-loader!/_includes/code/csharp/QuickstartLocalTest. title="quickstart_import.py" /> -During a batch import, any failed objects can be obtained through `batch.failed_objects`. Additionally, a running count of failed objects is maintained and can be accessed through `batch.number_errors` within the context manager. This counter can be used to stop the import process in order to investigate the failed objects or references. Find out more about error handling on the Python client [reference page](/weaviate/client-libraries/python/notes-best-practices#error-handling). +`data.ingest()` returns a `BatchObjectReturn`. Read `result.errors` to check for failures. It holds one entry per failed object, keyed by its position in the input. For more, see [error handling in the Python client reference](/weaviate/client-libraries/python/notes-best-practices#error-handling). @@ -33,6 +35,8 @@ During a batch import, any failed objects can be obtained through `batch.failed_ title="quickstart_import.ts" /> +`data.ingest()` returns a result object. Read `result.hasErrors` for a quick check, and `result.errors` for one entry per failed object, keyed by its position in the input. + @@ -52,6 +56,9 @@ During a batch import, any failed objects can be obtained through `batch.failed_ endMarker="// END Import" language="java" /> + +`batch.start()` opens a server-side batch. Closing the batch sends any remaining objects and waits for the server to report the results. Read `batch.numberOfErrors()` after the batch closes; before then, the tally is incomplete. + @@ -61,6 +68,9 @@ During a batch import, any failed objects can be obtained through `batch.failed_ endMarker="// END Import" language="csharp" /> + +`Batch.InsertMany()` returns a `BatchInsertResponse`. Read `HasErrors` for a quick check, `Errors` for the failures alone, and `Objects` for one entry per object. Each entry's `Index` is its position in the input, and failed entries carry an `Error`. + diff --git a/_includes/code/quickstart/quickstart.import_objects.mdx b/_includes/code/quickstart/quickstart.import_objects.mdx index 35ef8a581..5ca38bfca 100644 --- a/_includes/code/quickstart/quickstart.import_objects.mdx +++ b/_includes/code/quickstart/quickstart.import_objects.mdx @@ -8,6 +8,8 @@ import JavaV6Code from "!!raw-loader!/_includes/code/java-v6/src/test/java/Quick import CSharpCode from "!!raw-loader!/_includes/code/csharp/QuickstartTest.cs"; +Every import reports whether any objects failed. Check for failures in your own code to catch problems such as malformed data or a misconfigured model provider. + @@ -19,8 +21,7 @@ import CSharpCode from "!!raw-loader!/_includes/code/csharp/QuickstartTest.cs"; title="quickstart_import.py" /> - -During a batch import, any failed objects can be obtained through `batch.failed_objects`. Additionally, a running count of failed objects is maintained and can be accessed through `batch.number_errors` within the context manager. This counter can be used to stop the import process in order to investigate the failed objects or references. Find out more about error handling on the Python client [reference page](/weaviate/client-libraries/python/notes-best-practices#error-handling). +`data.ingest()` returns a `BatchObjectReturn`. Read `result.errors` to check for failures. It holds one entry per failed object, keyed by its position in the input. For more, see [error handling in the Python client reference](/weaviate/client-libraries/python/notes-best-practices#error-handling). @@ -34,6 +35,8 @@ During a batch import, any failed objects can be obtained through `batch.failed_ title="quickstart_import.ts" /> +`data.ingest()` returns a result object. Read `result.hasErrors` for a quick check, and `result.errors` for one entry per failed object, keyed by its position in the input. + @@ -53,6 +56,9 @@ During a batch import, any failed objects can be obtained through `batch.failed_ endMarker="// END Import" language="java" /> + +`batch.start()` opens a server-side batch. Closing the batch sends any remaining objects and waits for the server to report the results. Read `batch.numberOfErrors()` after the batch closes; before then, the tally is incomplete. + @@ -62,6 +68,9 @@ During a batch import, any failed objects can be obtained through `batch.failed_ endMarker="// END Import" language="csharp" /> + +`Batch.InsertMany()` returns a `BatchInsertResponse`. Read `HasErrors` for a quick check, `Errors` for the failures alone, and `Objects` for one entry per object. Each entry's `Index` is its position in the input, and failed entries carry an `Error`. + diff --git a/_includes/code/typescript/local.quickstart.import_objects.ts b/_includes/code/typescript/local.quickstart.import_objects.ts index ffe894279..dfa4a0861 100644 --- a/_includes/code/typescript/local.quickstart.import_objects.ts +++ b/_includes/code/typescript/local.quickstart.import_objects.ts @@ -11,18 +11,27 @@ async function getJsonData() { return file.json(); } -// highlight-start -// Note: The TS client does not have a `batch` method yet -// We use `insertMany` instead, which sends all of the data in one request async function importQuestions() { const questions = client.collections.use('Question'); const data = await getJsonData(); - const result = await questions.data.insertMany(data); - console.log('Insertion response: ', result); + + // highlight-start + // `ingest` imports the list using server-side batching + const result = await questions.data.ingest( + data.map((properties) => ({ properties })) + ); + // highlight-end + + if (result.hasErrors) { + console.log(`Number of failed imports: ${Object.keys(result.errors).length}`); + // `errors` is keyed by the position of the object in the input + for (const [index, error] of Object.entries(result.errors)) { + console.log(`Failed object at index ${index}: ${error.message}`); + } + } } await importQuestions(); -// highlight-end client.close(); // Close the client connection // END Import diff --git a/_includes/code/typescript/quickstart.import_objects.ts b/_includes/code/typescript/quickstart.import_objects.ts index b04257fd7..43aebb365 100644 --- a/_includes/code/typescript/quickstart.import_objects.ts +++ b/_includes/code/typescript/quickstart.import_objects.ts @@ -20,18 +20,27 @@ async function getJsonData() { return file.json(); } -// highlight-start -// Note: The TS client does not have a `batch` method yet -// We use `insertMany` instead, which sends all of the data in one request async function importQuestions() { const questions = client.collections.use('Question'); const data = await getJsonData(); - const result = await questions.data.insertMany(data); - console.log('Insertion response: ', result); + + // highlight-start + // `ingest` imports the list using server-side batching + const result = await questions.data.ingest( + data.map((properties) => ({ properties })) + ); + // highlight-end + + if (result.hasErrors) { + console.log(`Number of failed imports: ${Object.keys(result.errors).length}`); + // `errors` is keyed by the position of the object in the input + for (const [index, error] of Object.entries(result.errors)) { + console.log(`Failed object at index ${index}: ${error.message}`); + } + } } await importQuestions(); -// highlight-end client.close(); // Close the client connection // END Import diff --git a/_includes/code/typescript/quickstart.short.create_collection.ts b/_includes/code/typescript/quickstart.short.create_collection.ts index 3c7e5fb88..167ff7734 100644 --- a/_includes/code/typescript/quickstart.short.create_collection.ts +++ b/_includes/code/typescript/quickstart.short.create_collection.ts @@ -41,7 +41,9 @@ const dataObjects = [ // START CreateCollection const movieCollection = client.collections.get('Movie'); -const response = await movieCollection.data.insertMany(dataObjects); +await movieCollection.data.ingest( + dataObjects.map((properties) => ({ properties })) +); console.log(`Imported & vectorized ${dataObjects.length} objects into the Movie collection`); diff --git a/_includes/code/typescript/quickstart.short.import_vectors.create_collection.ts b/_includes/code/typescript/quickstart.short.import_vectors.create_collection.ts index a8f0cced0..82e4a8aae 100644 --- a/_includes/code/typescript/quickstart.short.import_vectors.create_collection.ts +++ b/_includes/code/typescript/quickstart.short.import_vectors.create_collection.ts @@ -51,7 +51,7 @@ const dataObjects = [ // START CreateCollection // Insert the objects with vectors const movieCollection = client.collections.get('Movie'); -const response = await movieCollection.data.insertMany(dataObjects); +await movieCollection.data.ingest(dataObjects); console.log(`Imported ${dataObjects.length} objects with vectors into the Movie collection`); diff --git a/_includes/code/typescript/quickstart.short.local.create_collection.ts b/_includes/code/typescript/quickstart.short.local.create_collection.ts index dabb209d9..239cae8fa 100644 --- a/_includes/code/typescript/quickstart.short.local.create_collection.ts +++ b/_includes/code/typescript/quickstart.short.local.create_collection.ts @@ -35,7 +35,9 @@ const dataObjects = [ // START CreateCollection const movieCollection = client.collections.get('Movie'); -const response = await movieCollection.data.insertMany(dataObjects); +await movieCollection.data.ingest( + dataObjects.map((properties) => ({ properties })) +); console.log(`Imported & vectorized ${dataObjects.length} objects into the Movie collection`); diff --git a/_includes/code/typescript/quickstart.short.local.import_vectors.create_collection.ts b/_includes/code/typescript/quickstart.short.local.import_vectors.create_collection.ts index 871ffe26c..6c667bcba 100644 --- a/_includes/code/typescript/quickstart.short.local.import_vectors.create_collection.ts +++ b/_includes/code/typescript/quickstart.short.local.import_vectors.create_collection.ts @@ -36,7 +36,7 @@ const dataObjects = [ // Insert the objects with vectors const movieCollection = client.collections.get('Movie'); -const response = await movieCollection.data.insertMany(dataObjects); +await movieCollection.data.ingest(dataObjects); console.log(`Imported ${dataObjects.length} objects with vectors into the Movie collection`); diff --git a/docs/cloud/quickstart.mdx b/docs/cloud/quickstart.mdx index 7039863d7..fd581c138 100644 --- a/docs/cloud/quickstart.mdx +++ b/docs/cloud/quickstart.mdx @@ -221,11 +221,11 @@ We can now add data to our collection. The following example: - Loads objects, and -- Adds objects to the target collection (`Question`) using a batch process. +- Adds objects to the target collection (`Question`) with a batch import. :::tip Batch imports -([Batch imports](/weaviate/manage-objects/import.mdx)) are the most efficient way to add large amounts of data, as it sends multiple objects in a single request. See the [How-to: Batch import](/weaviate/manage-objects/import.mdx) guide for more information. +Batch imports are the most efficient way to add large amounts of data, because they send objects in groups instead of one request per object. See the [How-to: Batch import](/weaviate/manage-objects/import.mdx) guide for the available methods, including [server-side batching](/weaviate/manage-objects/import.mdx#server-side-batching), where the server tells the client how much data to send next. ::: diff --git a/docs/weaviate/tutorials/quick-tour-of-weaviate.mdx b/docs/weaviate/tutorials/quick-tour-of-weaviate.mdx index 776f338cf..b55c4188f 100644 --- a/docs/weaviate/tutorials/quick-tour-of-weaviate.mdx +++ b/docs/weaviate/tutorials/quick-tour-of-weaviate.mdx @@ -212,11 +212,11 @@ We can now add data to our collection. The following example: - Loads objects, and -- Adds objects to the target collection (`Question`) using a batch process. +- Adds objects to the target collection (`Question`) with a batch import. :::tip Batch imports -([Batch imports](../manage-objects/import.mdx)) are the most efficient way to add large amounts of data, as it sends multiple objects in a single request. See the [How-to: Batch import](../manage-objects/import.mdx) guide for more information. +Batch imports are the most efficient way to add large amounts of data, because they send objects in groups instead of one request per object. See the [How-to: Batch import](../manage-objects/import.mdx) guide for the available methods, including [server-side batching](../manage-objects/import.mdx#server-side-batching), where the server tells the client how much data to send next. :::