Merge pull request #646 from kroeschl/numprocesses-logical

Add explicit 'logical' choice to --numprocesses flag

Author: RonnyPfannschmidt

changelog/646.feature.rst

@@ -0,0 +1,3 @@
+Add ``--numprocesses=logical`` flag, which automatically uses the number of logical CPUs available, instead of physical CPUs with ``auto``.
+
+This is very useful for test suites which are not CPU-bound.

src/xdist/plugin.py

@@ -5,13 +5,14 @@
 import pytest
 
 
-def pytest_xdist_auto_num_workers():
+def pytest_xdist_auto_num_workers(config):
     try:
         import psutil
     except ImportError:
         pass
     else:
-        count = psutil.cpu_count(logical=False) or psutil.cpu_count()
+        use_logical = config.option.numprocesses == "logical"
+        count = psutil.cpu_count(logical=use_logical) or psutil.cpu_count()
         if count:
             return count
     try:
@@ -36,8 +37,8 @@ def cpu_count():
 
 
 def parse_numprocesses(s):
-    if s == "auto":
-        return "auto"
+    if s in ("auto", "logical"):
+        return s
     elif s is not None:
         return int(s)
 
@@ -51,9 +52,10 @@ def pytest_addoption(parser):
         metavar="numprocesses",
         action="store",
         type=parse_numprocesses,
-        help="shortcut for '--dist=load --tx=NUM*popen', "
-        "you can use 'auto' here for auto detection CPUs number on "
-        "host system and it will be 0 when used with --pdb",
+        help="Shortcut for '--dist=load --tx=NUM*popen'. With 'auto', attempt "
+        "to detect physical CPU count. With 'logical', detect logical CPU "
+        "count. If physical CPU count cannot be found, falls back to logical "
+        "count. This will be 0 when used with --pdb.",
     )
     group.addoption(
         "--maxprocesses",
@@ -190,7 +192,7 @@ def pytest_configure(config):
 @pytest.mark.tryfirst
 def pytest_cmdline_main(config):
     usepdb = config.getoption("usepdb", False)  # a core option
-    if config.option.numprocesses == "auto":
+    if config.option.numprocesses in ("auto", "logical"):
         if usepdb:
             config.option.numprocesses = 0
             config.option.dist = "no"

testing/test_plugin.py

@@ -69,6 +69,12 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     assert config.getoption("numprocesses") == 0
     assert config.getoption("dist") == "no"
 
+    config = testdir.parseconfigure("-nlogical", "--pdb")
+    check_options(config)
+    assert config.getoption("usepdb")
+    assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
+
     monkeypatch.delattr(os, "sched_getaffinity", raising=False)
     monkeypatch.setenv("TRAVIS", "true")
     config = testdir.parseconfigure("-nauto")
@@ -81,12 +87,16 @@ def test_auto_detect_cpus_psutil(testdir, monkeypatch):
 
     psutil = pytest.importorskip("psutil")
 
-    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 42)
+    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)
 
     config = testdir.parseconfigure("-nauto")
     check_options(config)
     assert config.getoption("numprocesses") == 42
 
+    config = testdir.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 84
+
 
 def test_hook_auto_num_workers(testdir, monkeypatch):
     from xdist.plugin import pytest_cmdline_main as check_options
@@ -101,6 +111,10 @@ def pytest_xdist_auto_num_workers():
     check_options(config)
     assert config.getoption("numprocesses") == 42
 
+    config = testdir.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
 
 def test_boxed_with_collect_only(testdir):
     from xdist.plugin import pytest_cmdline_main as check_options