Skip to content

Commit 00988eb

Browse files
cursoragentshrisukhani
cursoragent
and
shrisukhani
committed
Centralize shared job request execution helpers
Co-authored-by: Shri Sukhani <shrisukhani@users.noreply.github.com>
1 parent 10184ea commit 00988eb

12 files changed

Lines changed: 526 additions & 136 deletions

‎CONTRIBUTING.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ This runs lint, format checks, compile checks, tests, and package build.
110110
- `tests/test_job_poll_helper_boundary.py` (centralization boundary enforcement for terminal-status polling helper primitives),
111111
- `tests/test_job_poll_helper_usage.py` (shared terminal-status polling helper usage enforcement),
112112
- `tests/test_job_query_params_helper_usage.py` (shared scrape/crawl query-param helper usage enforcement),
113+
- `tests/test_job_request_helper_usage.py` (shared scrape/crawl/extract request-helper usage enforcement),
113114
- `tests/test_job_route_constants_usage.py` (shared scrape/crawl/extract route-constant usage enforcement),
114115
- `tests/test_job_start_payload_helper_usage.py` (shared scrape/crawl start-payload helper usage enforcement),
115116
- `tests/test_job_wait_helper_boundary.py` (centralization boundary enforcement for wait-for-job helper primitives),

‎hyperbrowser/client/managers/async_manager/crawl.py‎

Lines changed: 13 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,14 @@
1717
from ..job_status_utils import is_default_terminal_job_status
1818
from ..job_start_payload_utils import build_crawl_start_payload
1919
from ..job_operation_metadata import CRAWL_OPERATION_METADATA
20+
from ..job_request_utils import get_job_async, get_job_status_async, start_job_async
2021
from ..job_route_constants import CRAWL_JOB_ROUTE_PREFIX
2122
from ..job_query_params_utils import build_crawl_get_params
2223
from ..polling_defaults import (
2324
DEFAULT_MAX_WAIT_SECONDS,
2425
DEFAULT_POLLING_RETRY_ATTEMPTS,
2526
DEFAULT_POLL_INTERVAL_SECONDS,
2627
)
27-
from ..response_utils import parse_response_model
2828
from ..start_job_utils import build_started_job_context
2929
from ....models.crawl import (
3030
CrawlJobResponse,
@@ -44,22 +44,19 @@ def __init__(self, client):
4444

4545
async def start(self, params: StartCrawlJobParams) -> StartCrawlJobResponse:
4646
payload = build_crawl_start_payload(params)
47-
response = await self._client.transport.post(
48-
self._client._build_url(self._ROUTE_PREFIX),
49-
data=payload,
50-
)
51-
return parse_response_model(
52-
response.data,
47+
return await start_job_async(
48+
client=self._client,
49+
route_prefix=self._ROUTE_PREFIX,
50+
payload=payload,
5351
model=StartCrawlJobResponse,
5452
operation_name=self._OPERATION_METADATA.start_operation_name,
5553
)
5654

5755
async def get_status(self, job_id: str) -> CrawlJobStatusResponse:
58-
response = await self._client.transport.get(
59-
self._client._build_url(f"{self._ROUTE_PREFIX}/{job_id}/status")
60-
)
61-
return parse_response_model(
62-
response.data,
56+
return await get_job_status_async(
57+
client=self._client,
58+
route_prefix=self._ROUTE_PREFIX,
59+
job_id=job_id,
6360
model=CrawlJobStatusResponse,
6461
operation_name=self._OPERATION_METADATA.status_operation_name,
6562
)
@@ -68,12 +65,11 @@ async def get(
6865
self, job_id: str, params: Optional[GetCrawlJobParams] = None
6966
) -> CrawlJobResponse:
7067
query_params = build_crawl_get_params(params)
71-
response = await self._client.transport.get(
72-
self._client._build_url(f"{self._ROUTE_PREFIX}/{job_id}"),
68+
return await get_job_async(
69+
client=self._client,
70+
route_prefix=self._ROUTE_PREFIX,
71+
job_id=job_id,
7372
params=query_params,
74-
)
75-
return parse_response_model(
76-
response.data,
7773
model=CrawlJobResponse,
7874
operation_name=self._OPERATION_METADATA.job_operation_name,
7975
)

‎hyperbrowser/client/managers/async_manager/extract.py‎

Lines changed: 14 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
)
99
from ..extract_payload_utils import build_extract_start_payload
1010
from ..job_operation_metadata import EXTRACT_OPERATION_METADATA
11+
from ..job_request_utils import get_job_async, get_job_status_async, start_job_async
1112
from ..job_route_constants import EXTRACT_JOB_ROUTE_PREFIX
1213
from ..job_status_utils import is_default_terminal_job_status
1314
from ..job_wait_utils import wait_for_job_result_with_defaults_async
@@ -17,7 +18,6 @@
1718
DEFAULT_POLL_INTERVAL_SECONDS,
1819
)
1920
from ..start_job_utils import build_started_job_context
20-
from ..response_utils import parse_response_model
2121

2222

2323
class ExtractManager:
@@ -29,33 +29,29 @@ def __init__(self, client):
2929

3030
async def start(self, params: StartExtractJobParams) -> StartExtractJobResponse:
3131
payload = build_extract_start_payload(params)
32-
33-
response = await self._client.transport.post(
34-
self._client._build_url(self._ROUTE_PREFIX),
35-
data=payload,
36-
)
37-
return parse_response_model(
38-
response.data,
32+
return await start_job_async(
33+
client=self._client,
34+
route_prefix=self._ROUTE_PREFIX,
35+
payload=payload,
3936
model=StartExtractJobResponse,
4037
operation_name=self._OPERATION_METADATA.start_operation_name,
4138
)
4239

4340
async def get_status(self, job_id: str) -> ExtractJobStatusResponse:
44-
response = await self._client.transport.get(
45-
self._client._build_url(f"{self._ROUTE_PREFIX}/{job_id}/status")
46-
)
47-
return parse_response_model(
48-
response.data,
41+
return await get_job_status_async(
42+
client=self._client,
43+
route_prefix=self._ROUTE_PREFIX,
44+
job_id=job_id,
4945
model=ExtractJobStatusResponse,
5046
operation_name=self._OPERATION_METADATA.status_operation_name,
5147
)
5248

5349
async def get(self, job_id: str) -> ExtractJobResponse:
54-
response = await self._client.transport.get(
55-
self._client._build_url(f"{self._ROUTE_PREFIX}/{job_id}")
56-
)
57-
return parse_response_model(
58-
response.data,
50+
return await get_job_async(
51+
client=self._client,
52+
route_prefix=self._ROUTE_PREFIX,
53+
job_id=job_id,
54+
params=None,
5955
model=ExtractJobResponse,
6056
operation_name=self._OPERATION_METADATA.job_operation_name,
6157
)

‎hyperbrowser/client/managers/async_manager/scrape.py‎

Lines changed: 26 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
BATCH_SCRAPE_OPERATION_METADATA,
2121
SCRAPE_OPERATION_METADATA,
2222
)
23+
from ..job_request_utils import get_job_async, get_job_status_async, start_job_async
2324
from ..job_route_constants import (
2425
BATCH_SCRAPE_JOB_ROUTE_PREFIX,
2526
SCRAPE_JOB_ROUTE_PREFIX,
@@ -34,7 +35,6 @@
3435
build_batch_scrape_start_payload,
3536
build_scrape_start_payload,
3637
)
37-
from ..response_utils import parse_response_model
3838
from ..start_job_utils import build_started_job_context
3939
from ....models.scrape import (
4040
BatchScrapeJobResponse,
@@ -60,22 +60,19 @@ async def start(
6060
self, params: StartBatchScrapeJobParams
6161
) -> StartBatchScrapeJobResponse:
6262
payload = build_batch_scrape_start_payload(params)
63-
response = await self._client.transport.post(
64-
self._client._build_url(self._ROUTE_PREFIX),
65-
data=payload,
66-
)
67-
return parse_response_model(
68-
response.data,
63+
return await start_job_async(
64+
client=self._client,
65+
route_prefix=self._ROUTE_PREFIX,
66+
payload=payload,
6967
model=StartBatchScrapeJobResponse,
7068
operation_name=self._OPERATION_METADATA.start_operation_name,
7169
)
7270

7371
async def get_status(self, job_id: str) -> BatchScrapeJobStatusResponse:
74-
response = await self._client.transport.get(
75-
self._client._build_url(f"{self._ROUTE_PREFIX}/{job_id}/status")
76-
)
77-
return parse_response_model(
78-
response.data,
72+
return await get_job_status_async(
73+
client=self._client,
74+
route_prefix=self._ROUTE_PREFIX,
75+
job_id=job_id,
7976
model=BatchScrapeJobStatusResponse,
8077
operation_name=self._OPERATION_METADATA.status_operation_name,
8178
)
@@ -84,12 +81,11 @@ async def get(
8481
self, job_id: str, params: Optional[GetBatchScrapeJobParams] = None
8582
) -> BatchScrapeJobResponse:
8683
query_params = build_batch_scrape_get_params(params)
87-
response = await self._client.transport.get(
88-
self._client._build_url(f"{self._ROUTE_PREFIX}/{job_id}"),
84+
return await get_job_async(
85+
client=self._client,
86+
route_prefix=self._ROUTE_PREFIX,
87+
job_id=job_id,
8988
params=query_params,
90-
)
91-
return parse_response_model(
92-
response.data,
9389
model=BatchScrapeJobResponse,
9490
operation_name=self._OPERATION_METADATA.job_operation_name,
9591
)
@@ -162,32 +158,29 @@ def __init__(self, client):
162158

163159
async def start(self, params: StartScrapeJobParams) -> StartScrapeJobResponse:
164160
payload = build_scrape_start_payload(params)
165-
response = await self._client.transport.post(
166-
self._client._build_url(self._ROUTE_PREFIX),
167-
data=payload,
168-
)
169-
return parse_response_model(
170-
response.data,
161+
return await start_job_async(
162+
client=self._client,
163+
route_prefix=self._ROUTE_PREFIX,
164+
payload=payload,
171165
model=StartScrapeJobResponse,
172166
operation_name=self._OPERATION_METADATA.start_operation_name,
173167
)
174168

175169
async def get_status(self, job_id: str) -> ScrapeJobStatusResponse:
176-
response = await self._client.transport.get(
177-
self._client._build_url(f"{self._ROUTE_PREFIX}/{job_id}/status")
178-
)
179-
return parse_response_model(
180-
response.data,
170+
return await get_job_status_async(
171+
client=self._client,
172+
route_prefix=self._ROUTE_PREFIX,
173+
job_id=job_id,
181174
model=ScrapeJobStatusResponse,
182175
operation_name=self._OPERATION_METADATA.status_operation_name,
183176
)
184177

185178
async def get(self, job_id: str) -> ScrapeJobResponse:
186-
response = await self._client.transport.get(
187-
self._client._build_url(f"{self._ROUTE_PREFIX}/{job_id}")
188-
)
189-
return parse_response_model(
190-
response.data,
179+
return await get_job_async(
180+
client=self._client,
181+
route_prefix=self._ROUTE_PREFIX,
182+
job_id=job_id,
183+
params=None,
191184
model=ScrapeJobResponse,
192185
operation_name=self._OPERATION_METADATA.job_operation_name,
193186
)

‎hyperbrowser/client/managers/job_request_utils.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
from typing import Any, Dict, Optional, Type, TypeVar
2+
3+
from .response_utils import parse_response_model
4+
5+
T = TypeVar("T")
6+
7+
8+
def start_job(
9+
*,
10+
client: Any,
11+
route_prefix: str,
12+
payload: Dict[str, Any],
13+
model: Type[T],
14+
operation_name: str,
15+
) -> T:
16+
response = client.transport.post(
17+
client._build_url(route_prefix),
18+
data=payload,
19+
)
20+
return parse_response_model(
21+
response.data,
22+
model=model,
23+
operation_name=operation_name,
24+
)
25+
26+
27+
def get_job_status(
28+
*,
29+
client: Any,
30+
route_prefix: str,
31+
job_id: str,
32+
model: Type[T],
33+
operation_name: str,
34+
) -> T:
35+
response = client.transport.get(
36+
client._build_url(f"{route_prefix}/{job_id}/status"),
37+
)
38+
return parse_response_model(
39+
response.data,
40+
model=model,
41+
operation_name=operation_name,
42+
)
43+
44+
45+
def get_job(
46+
*,
47+
client: Any,
48+
route_prefix: str,
49+
job_id: str,
50+
params: Optional[Dict[str, Any]],
51+
model: Type[T],
52+
operation_name: str,
53+
) -> T:
54+
response = client.transport.get(
55+
client._build_url(f"{route_prefix}/{job_id}"),
56+
params=params,
57+
)
58+
return parse_response_model(
59+
response.data,
60+
model=model,
61+
operation_name=operation_name,
62+
)
63+
64+
65+
async def start_job_async(
66+
*,
67+
client: Any,
68+
route_prefix: str,
69+
payload: Dict[str, Any],
70+
model: Type[T],
71+
operation_name: str,
72+
) -> T:
73+
response = await client.transport.post(
74+
client._build_url(route_prefix),
75+
data=payload,
76+
)
77+
return parse_response_model(
78+
response.data,
79+
model=model,
80+
operation_name=operation_name,
81+
)
82+
83+
84+
async def get_job_status_async(
85+
*,
86+
client: Any,
87+
route_prefix: str,
88+
job_id: str,
89+
model: Type[T],
90+
operation_name: str,
91+
) -> T:
92+
response = await client.transport.get(
93+
client._build_url(f"{route_prefix}/{job_id}/status"),
94+
)
95+
return parse_response_model(
96+
response.data,
97+
model=model,
98+
operation_name=operation_name,
99+
)
100+
101+
102+
async def get_job_async(
103+
*,
104+
client: Any,
105+
route_prefix: str,
106+
job_id: str,
107+
params: Optional[Dict[str, Any]],
108+
model: Type[T],
109+
operation_name: str,
110+
) -> T:
111+
response = await client.transport.get(
112+
client._build_url(f"{route_prefix}/{job_id}"),
113+
params=params,
114+
)
115+
return parse_response_model(
116+
response.data,
117+
model=model,
118+
operation_name=operation_name,
119+
)

0 commit comments

Comments
 (0)