Skip to content

Commit 4b3dcf2

Browse files
authored
Fix GroupBy snippet tests for issue #30778 (#37672)
* Fix GroupBy snippet tests for issue #30778 - Add missing strawberry entry to GROCERY_LIST in all snippet files - Fix test() block scoping (moved inside with beam.Pipeline() context) - Remove beam.Map(print) from pipeline, add to else branch for standalone use - Fix check_simple_aggregate_result to compare Row objects directly - Set skip_due_to_30778 = False now that tests are passing - Restore [START]/[END] snippet markers and 2-space indentation * Fix yapf formatting issues in GroupBy snippet files * Fix skip flag, restore markers, remove to_grocery_row, fix check_simple_aggregate_result * Remove skip_due_to_30778 entirely and restore groupby_table markers * Fix pylint W0106 by assigning beam.Map(print) result to _ in else branches
1 parent 5bee123 commit 4b3dcf2

8 files changed

Lines changed: 34 additions & 44 deletions

File tree

sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_attr.py

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,22 +43,21 @@
4343
beam.Row(recipe='pie', fruit='blueberry', quantity=1, unit_price=2.00),
4444
beam.Row(recipe='muffin', fruit='blueberry', quantity=2, unit_price=2.00),
4545
beam.Row(recipe='muffin', fruit='banana', quantity=3, unit_price=1.00),
46+
beam.Row(recipe='pie', fruit='strawberry', quantity=3, unit_price=1.50),
4647
]
4748
# [END groupby_table]
4849

4950

5051
def groupby_attr(test=None):
5152
with beam.Pipeline() as p:
5253
# [START groupby_attr]
53-
grouped = (
54-
p
55-
| beam.Create(GROCERY_LIST)
56-
| beam.GroupBy('recipe')
57-
| beam.Map(print))
54+
grouped = (p | beam.Create(GROCERY_LIST) | beam.GroupBy('recipe'))
5855
# [END groupby_attr]
5956

60-
if test:
61-
test(grouped)
57+
if test:
58+
test(grouped)
59+
else:
60+
_ = grouped | beam.Map(print)
6261

6362

6463
if __name__ == '__main__':

sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_attr_expr.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
beam.Row(recipe='pie', fruit='blueberry', quantity=1, unit_price=2.00),
4444
beam.Row(recipe='muffin', fruit='blueberry', quantity=2, unit_price=2.00),
4545
beam.Row(recipe='muffin', fruit='banana', quantity=3, unit_price=1.00),
46+
beam.Row(recipe='pie', fruit='strawberry', quantity=3, unit_price=1.50),
4647
]
4748
# [END groupby_table]
4849

@@ -53,12 +54,13 @@ def groupby_attr_expr(test=None):
5354
grouped = (
5455
p
5556
| beam.Create(GROCERY_LIST)
56-
| beam.GroupBy('recipe', is_berry=lambda x: 'berry' in x.fruit)
57-
| beam.Map(print))
57+
| beam.GroupBy('recipe', is_berry=lambda x: 'berry' in x.fruit))
5858
# [END groupby_attr_expr]
5959

60-
if test:
61-
test(grouped)
60+
if test:
61+
test(grouped)
62+
else:
63+
_ = grouped | beam.Map(print)
6264

6365

6466
if __name__ == '__main__':

sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_expr.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,12 @@ def groupby_expr(test=None):
4444
p
4545
| beam.Create(
4646
['strawberry', 'raspberry', 'blueberry', 'blackberry', 'banana'])
47-
| beam.GroupBy(lambda s: s[0])
48-
| beam.Map(print))
47+
| beam.GroupBy(lambda s: s[0]))
4948
# [END groupby_expr]
5049
if test:
5150
test(grouped)
51+
else:
52+
_ = grouped | beam.Map(print)
5253

5354

5455
if __name__ == '__main__':

sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_expr_aggregate.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
beam.Row(recipe='pie', fruit='blueberry', quantity=1, unit_price=2.00),
4444
beam.Row(recipe='muffin', fruit='blueberry', quantity=2, unit_price=2.00),
4545
beam.Row(recipe='muffin', fruit='banana', quantity=3, unit_price=1.00),
46+
beam.Row(recipe='pie', fruit='strawberry', quantity=3, unit_price=1.50),
4647
]
4748
# [END groupby_table]
4849

@@ -55,12 +56,13 @@ def expr_aggregate(test=None):
5556
| beam.Create(GROCERY_LIST)
5657
| beam.GroupBy('recipe').aggregate_field(
5758
'quantity', sum, 'total_quantity').aggregate_field(
58-
lambda x: x.quantity * x.unit_price, sum, 'price')
59-
| beam.Map(print))
59+
lambda x: x.quantity * x.unit_price, sum, 'price'))
6060
# [END expr_aggregate]
6161

62-
if test:
63-
test(grouped)
62+
if test:
63+
test(grouped)
64+
else:
65+
_ = grouped | beam.Map(print)
6466

6567

6668
if __name__ == '__main__':

sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_global_aggregate.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
beam.Row(recipe='pie', fruit='blueberry', quantity=1, unit_price=2.00),
4545
beam.Row(recipe='muffin', fruit='blueberry', quantity=2, unit_price=2.00),
4646
beam.Row(recipe='muffin', fruit='banana', quantity=3, unit_price=1.00),
47+
beam.Row(recipe='pie', fruit='strawberry', quantity=3, unit_price=1.50),
4748
]
4849
# [END groupby_table]
4950

@@ -57,11 +58,12 @@ def global_aggregate(test=None):
5758
| beam.GroupBy().aggregate_field(
5859
'unit_price', min, 'min_price').aggregate_field(
5960
'unit_price', MeanCombineFn(), 'mean_price').aggregate_field(
60-
'unit_price', max, 'max_price')
61-
| beam.Map(print))
61+
'unit_price', max, 'max_price'))
6262
# [END global_aggregate]
6363
if test:
6464
test(grouped)
65+
else:
66+
_ = grouped | beam.Map(print)
6567

6668

6769
if __name__ == '__main__':

sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_simple_aggregate.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
beam.Row(recipe='pie', fruit='blueberry', quantity=1, unit_price=2.00),
4444
beam.Row(recipe='muffin', fruit='blueberry', quantity=2, unit_price=2.00),
4545
beam.Row(recipe='muffin', fruit='banana', quantity=3, unit_price=1.00),
46+
beam.Row(recipe='pie', fruit='strawberry', quantity=3, unit_price=1.50),
4647
]
4748
# [END groupby_table]
4849

@@ -54,9 +55,9 @@ def simple_aggregate(test=None):
5455
p
5556
| beam.Create(GROCERY_LIST)
5657
| beam.GroupBy('fruit').aggregate_field(
57-
'quantity', sum, 'total_quantity')
58-
| beam.Map(print))
58+
'quantity', sum, 'total_quantity'))
5959
# [END simple_aggregate]
60+
6061
if test:
6162
test(grouped)
6263

sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_test.py

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,6 @@
3838
from .groupby_simple_aggregate import simple_aggregate
3939
from .groupby_two_exprs import groupby_two_exprs
4040

41-
#
42-
# TODO: Remove early returns in check functions
43-
# https://github.com/apache/beam/issues/30778
44-
skip_due_to_30778 = True
4541

4642

4743
class UnorderedList(object):
@@ -80,8 +76,6 @@ def normalize_kv(k, v):
8076

8177

8278
def check_groupby_expr_result(grouped):
83-
if skip_due_to_30778:
84-
return
8579
assert_that(
8680
grouped | beam.MapTuple(normalize_kv),
8781
equal_to([
@@ -94,8 +88,6 @@ def check_groupby_expr_result(grouped):
9488

9589

9690
def check_groupby_two_exprs_result(grouped):
97-
if skip_due_to_30778:
98-
return
9991
assert_that(
10092
grouped | beam.MapTuple(normalize_kv),
10193
equal_to([
@@ -109,8 +101,6 @@ def check_groupby_two_exprs_result(grouped):
109101

110102

111103
def check_groupby_attr_result(grouped):
112-
if skip_due_to_30778:
113-
return
114104
assert_that(
115105
grouped | beam.MapTuple(normalize_kv),
116106
equal_to([
@@ -157,8 +147,6 @@ def check_groupby_attr_result(grouped):
157147

158148

159149
def check_groupby_attr_expr_result(grouped):
160-
if skip_due_to_30778:
161-
return
162150
assert_that(
163151
grouped | beam.MapTuple(normalize_kv),
164152
equal_to([
@@ -209,10 +197,8 @@ def check_groupby_attr_expr_result(grouped):
209197

210198

211199
def check_simple_aggregate_result(grouped):
212-
if skip_due_to_30778:
213-
return
214200
assert_that(
215-
grouped | beam.MapTuple(normalize_kv),
201+
grouped,
216202
equal_to([
217203
#[START simple_aggregate_result]
218204
NamedTuple(fruit='strawberry', total_quantity=3),
@@ -225,8 +211,6 @@ def check_simple_aggregate_result(grouped):
225211

226212

227213
def check_expr_aggregate_result(grouped):
228-
if skip_due_to_30778:
229-
return
230214
assert_that(
231215
grouped | beam.Map(normalize),
232216
equal_to([
@@ -238,8 +222,6 @@ def check_expr_aggregate_result(grouped):
238222

239223

240224
def check_global_aggregate_result(grouped):
241-
if skip_due_to_30778:
242-
return
243225
assert_that(
244226
grouped | beam.Map(normalize),
245227
equal_to([

sdks/python/apache_beam/examples/snippets/transforms/aggregation/groupby_two_exprs.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,13 @@ def groupby_two_exprs(test=None):
4444
p
4545
| beam.Create(
4646
['strawberry', 'raspberry', 'blueberry', 'blackberry', 'banana'])
47-
| beam.GroupBy(letter=lambda s: s[0], is_berry=lambda s: 'berry' in s)
48-
| beam.Map(print))
47+
| beam.GroupBy(letter=lambda s: s[0], is_berry=lambda s: 'berry' in s))
4948
# [END groupby_two_exprs]
5049

51-
if test:
52-
test(grouped)
50+
if test:
51+
test(grouped)
52+
else:
53+
_ = grouped | beam.Map(print)
5354

5455

5556
if __name__ == '__main__':

0 commit comments

Comments
 (0)