Share paginated page merge logic for web managers

Co-authored-by: Shri Sukhani <shrisukhani@users.noreply.github.com>

Author: cursoragent

hyperbrowser/client/managers/async_manager/web/batch_fetch.py

@@ -10,7 +10,10 @@
 )
 from ...web_payload_utils import build_batch_fetch_start_payload
 from ...web_payload_utils import build_batch_fetch_get_params
-from ...web_pagination_utils import initialize_paginated_job_response
+from ...web_pagination_utils import (
+    initialize_paginated_job_response,
+    merge_paginated_page_response,
+)
 from ....polling import (
     build_fetch_operation_name,
     build_operation_name,
@@ -104,13 +107,7 @@ async def start_and_wait(
         )
 
         def merge_page_response(page_response: BatchFetchJobResponse) -> None:
-            if page_response.data:
-                job_response.data.extend(page_response.data)
-            job_response.current_page_batch = page_response.current_page_batch
-            job_response.total_pages = page_response.total_pages
-            job_response.total_page_batches = page_response.total_page_batches
-            job_response.batch_size = page_response.batch_size
-            job_response.error = page_response.error
+            merge_paginated_page_response(job_response, page_response)
 
         await collect_paginated_results_async(
             operation_name=operation_name,

hyperbrowser/client/managers/async_manager/web/crawl.py

@@ -10,7 +10,10 @@
 )
 from ...web_payload_utils import build_web_crawl_start_payload
 from ...web_payload_utils import build_web_crawl_get_params
-from ...web_pagination_utils import initialize_paginated_job_response
+from ...web_pagination_utils import (
+    initialize_paginated_job_response,
+    merge_paginated_page_response,
+)
 from ....polling import (
     build_fetch_operation_name,
     build_operation_name,
@@ -102,13 +105,7 @@ async def start_and_wait(
         )
 
         def merge_page_response(page_response: WebCrawlJobResponse) -> None:
-            if page_response.data:
-                job_response.data.extend(page_response.data)
-            job_response.current_page_batch = page_response.current_page_batch
-            job_response.total_pages = page_response.total_pages
-            job_response.total_page_batches = page_response.total_page_batches
-            job_response.batch_size = page_response.batch_size
-            job_response.error = page_response.error
+            merge_paginated_page_response(job_response, page_response)
 
         await collect_paginated_results_async(
             operation_name=operation_name,

hyperbrowser/client/managers/sync_manager/web/batch_fetch.py

@@ -10,7 +10,10 @@
 )
 from ...web_payload_utils import build_batch_fetch_start_payload
 from ...web_payload_utils import build_batch_fetch_get_params
-from ...web_pagination_utils import initialize_paginated_job_response
+from ...web_pagination_utils import (
+    initialize_paginated_job_response,
+    merge_paginated_page_response,
+)
 from ....polling import (
     build_fetch_operation_name,
     build_operation_name,
@@ -102,13 +105,7 @@ def start_and_wait(
         )
 
         def merge_page_response(page_response: BatchFetchJobResponse) -> None:
-            if page_response.data:
-                job_response.data.extend(page_response.data)
-            job_response.current_page_batch = page_response.current_page_batch
-            job_response.total_pages = page_response.total_pages
-            job_response.total_page_batches = page_response.total_page_batches
-            job_response.batch_size = page_response.batch_size
-            job_response.error = page_response.error
+            merge_paginated_page_response(job_response, page_response)
 
         collect_paginated_results(
             operation_name=operation_name,

hyperbrowser/client/managers/sync_manager/web/crawl.py

@@ -10,7 +10,10 @@
 )
 from ...web_payload_utils import build_web_crawl_start_payload
 from ...web_payload_utils import build_web_crawl_get_params
-from ...web_pagination_utils import initialize_paginated_job_response
+from ...web_pagination_utils import (
+    initialize_paginated_job_response,
+    merge_paginated_page_response,
+)
 from ....polling import (
     build_fetch_operation_name,
     build_operation_name,
@@ -102,13 +105,7 @@ def start_and_wait(
         )
 
         def merge_page_response(page_response: WebCrawlJobResponse) -> None:
-            if page_response.data:
-                job_response.data.extend(page_response.data)
-            job_response.current_page_batch = page_response.current_page_batch
-            job_response.total_pages = page_response.total_pages
-            job_response.total_page_batches = page_response.total_page_batches
-            job_response.batch_size = page_response.batch_size
-            job_response.error = page_response.error
+            merge_paginated_page_response(job_response, page_response)
 
         collect_paginated_results(
             operation_name=operation_name,

hyperbrowser/client/managers/web_pagination_utils.py

@@ -1,4 +1,4 @@
-from typing import Type, TypeVar
+from typing import Any, Type, TypeVar
 
 T = TypeVar("T")
 
@@ -19,3 +19,13 @@ def initialize_paginated_job_response(
         totalPages=0,
         batchSize=batch_size,
     )
+
+
+def merge_paginated_page_response(job_response: Any, page_response: Any) -> None:
+    if page_response.data:
+        job_response.data.extend(page_response.data)
+    job_response.current_page_batch = page_response.current_page_batch
+    job_response.total_pages = page_response.total_pages
+    job_response.total_page_batches = page_response.total_page_batches
+    job_response.batch_size = page_response.batch_size
+    job_response.error = page_response.error

tests/test_web_pagination_utils.py

@@ -1,5 +1,6 @@
 from hyperbrowser.client.managers.web_pagination_utils import (
     initialize_paginated_job_response,
+    merge_paginated_page_response,
 )
 from hyperbrowser.models import BatchFetchJobResponse, WebCrawlJobResponse
 
@@ -35,3 +36,29 @@ def test_initialize_paginated_job_response_for_web_crawl_with_custom_batch_size(
     assert response.total_page_batches == 0
     assert response.total_pages == 0
     assert response.batch_size == 25
+
+
+def test_merge_paginated_page_response_merges_page_data_and_metadata():
+    job_response = initialize_paginated_job_response(
+        model=BatchFetchJobResponse,
+        job_id="job-1",
+        status="running",
+    )
+    page_response = BatchFetchJobResponse(
+        jobId="job-1",
+        status="running",
+        data=[],
+        currentPageBatch=2,
+        totalPageBatches=4,
+        totalPages=10,
+        batchSize=100,
+        error="partial error",
+    )
+
+    merge_paginated_page_response(job_response, page_response)
+
+    assert job_response.current_page_batch == 2
+    assert job_response.total_page_batches == 4
+    assert job_response.total_pages == 10
+    assert job_response.batch_size == 100
+    assert job_response.error == "partial error"

tests/test_web_payload_helper_usage.py

@@ -36,9 +36,11 @@ def test_batch_fetch_managers_use_shared_start_payload_helper():
         assert "build_batch_fetch_start_payload(" in module_text
         assert "build_batch_fetch_get_params(" in module_text
         assert "initialize_paginated_job_response(" in module_text
+        assert "merge_paginated_page_response(" in module_text
         assert "inject_web_output_schemas(" not in module_text
         assert "serialize_model_dump_to_dict(" not in module_text
         assert "BatchFetchJobResponse(" not in module_text
+        assert "current_page_batch = page_response.current_page_batch" not in module_text
 
 
 def test_web_crawl_managers_use_shared_start_payload_helper():
@@ -47,6 +49,8 @@ def test_web_crawl_managers_use_shared_start_payload_helper():
         assert "build_web_crawl_start_payload(" in module_text
         assert "build_web_crawl_get_params(" in module_text
         assert "initialize_paginated_job_response(" in module_text
+        assert "merge_paginated_page_response(" in module_text
         assert "inject_web_output_schemas(" not in module_text
         assert "serialize_model_dump_to_dict(" not in module_text
         assert "WebCrawlJobResponse(" not in module_text
+        assert "current_page_batch = page_response.current_page_batch" not in module_text