Skip to content

Commit 4250035

Browse files
yihong0618YvesDup
andcommitted
fix: profiling or tracing multiprocessing can cause error
Signed-off-by: yihong0618 <zouzou0208@gmail.com> Co-Authored-By: YvesDup <yduprat@gmail.com>
1 parent 92972ae commit 4250035

4 files changed

Lines changed: 77 additions & 4 deletions

File tree

Lib/profile.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -602,13 +602,16 @@ def main():
602602
code = compile(fp.read(), progname, 'exec')
603603
spec = importlib.machinery.ModuleSpec(name='__main__', loader=None,
604604
origin=progname)
605-
globs = {
606-
'__spec__': spec,
605+
module = importlib.util.module_from_spec(spec)
606+
sys.modules['__main__'] = module
607+
globs = module.__dict__
608+
globs.update({
609+
'__spec__': None,
607610
'__file__': spec.origin,
608611
'__name__': spec.name,
609612
'__package__': None,
610613
'__cached__': None,
611-
}
614+
})
612615
try:
613616
runctx(code, globs, None, options.outfile, options.sort)
614617
except BrokenPipeError as exc:

Lib/profiling/tracing/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ def main():
197197
# in the module's namespace.
198198
globs = module.__dict__
199199
globs.update({
200-
'__spec__': spec,
200+
'__spec__': None,
201201
'__file__': spec.origin,
202202
'__name__': spec.name,
203203
'__package__': None,

Lib/test/test_profile.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@
44
import pstats
55
import unittest
66
import os
7+
import subprocess
78
import warnings
89
from difflib import unified_diff
910
from io import StringIO
1011
from test.support.os_helper import TESTFN, unlink, temp_dir, change_cwd
12+
from test.support import script_helper, os_helper, SHORT_TIMEOUT
1113
from contextlib import contextmanager, redirect_stdout
1214

1315
# Suppress deprecation warning for profile module (PEP 799)
@@ -134,6 +136,39 @@ def test_output_file_when_changing_directory(self):
134136

135137
self.assertTrue(os.path.exists('out.pstats'))
136138

139+
def test_profile_multiprocessing(self):
140+
test_script = '''
141+
import multiprocessing
142+
def worker_proc(x):
143+
return x * 42
144+
145+
def main_proc():
146+
p = multiprocessing.Process(target=worker_proc, args=(10,))
147+
p.start()
148+
p.join()
149+
150+
if __name__ == "__main__":
151+
main_proc()
152+
'''
153+
with os_helper.temp_dir() as temp_dir:
154+
script = script_helper.make_script(
155+
temp_dir, 'test_profile_multiprocessing', test_script
156+
)
157+
with script_helper.spawn_python(
158+
"-m", self.profilermodule.__name__,
159+
script,
160+
stderr=subprocess.PIPE,
161+
text=True
162+
) as proc:
163+
proc.wait(timeout=SHORT_TIMEOUT)
164+
stdout = proc.stdout.read()
165+
stderr = proc.stderr.read()
166+
167+
self.assertIn("main_proc", stdout)
168+
self.assertNotIn("Can't pickle", stderr)
169+
self.assertNotIn("ModuleSpec", stderr)
170+
self.assertEqual(proc.returncode, 0)
171+
137172

138173
def regenerate_expected_output(filename, cls):
139174
filename = filename.rstrip('co')

Lib/test/test_profiling/test_tracing_profiler.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@
22

33
import sys
44
import unittest
5+
import subprocess
56

67
# rip off all interesting stuff from test_profile
78
import profiling.tracing as cProfile
89
import tempfile
910
import textwrap
1011
from test.test_profile import ProfileTest, regenerate_expected_output
12+
from test.support import script_helper, os_helper, SHORT_TIMEOUT
1113
from test.support.script_helper import assert_python_failure, assert_python_ok
1214
from test import support
1315

@@ -170,6 +172,39 @@ class Foo:
170172
f.close()
171173
assert_python_ok('-m', "cProfile", f.name)
172174

175+
def test_profile_multiprocessing(self):
176+
test_script = '''
177+
import multiprocessing
178+
179+
def worker_proc(x):
180+
return x * 42
181+
182+
def main_proc():
183+
p = multiprocessing.Process(target=worker_proc, args=(10,))
184+
p.start()
185+
p.join()
186+
print("SUCCESS")
187+
188+
if __name__ == "__main__":
189+
main_proc()
190+
'''
191+
with os_helper.temp_dir() as temp_dir:
192+
script = script_helper.make_script(
193+
temp_dir, 'test_cprofile_multiprocessing', test_script
194+
)
195+
with script_helper.spawn_python(
196+
"-m", "cProfile",
197+
script,
198+
stderr=subprocess.PIPE,
199+
text=True
200+
) as proc:
201+
proc.wait(timeout=SHORT_TIMEOUT)
202+
stdout = proc.stdout.read()
203+
stderr = proc.stderr.read()
204+
205+
self.assertIn("SUCCESS", stdout)
206+
self.assertNotIn("has no attribute \'worker_proc\'", stderr)
207+
173208

174209
def main():
175210
if '-r' not in sys.argv:

0 commit comments

Comments
 (0)