Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve parallel default selection on macOS #5383

Merged
merged 12 commits into from
Mar 4, 2021
22 changes: 20 additions & 2 deletions qiskit/tools/parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,21 +50,39 @@

import os
from concurrent.futures import ProcessPoolExecutor
import sys

from qiskit.exceptions import QiskitError
from qiskit.utils.multiprocessing import local_hardware_info
from qiskit.tools.events.pubsub import Publisher
from qiskit import user_config

CONFIG = user_config.get_config()

if os.getenv('QISKIT_PARALLEL', None) is not None:
PARALLEL_DEFAULT = os.getenv('QISKIT_PARALLEL', None).lower() == 'true'
else:
# Default False on Windows
if sys.platform == 'win32':
PARALLEL_DEFAULT = False
# On macOS default false on Python >=3.8
elif sys.platform == 'darwin':
if sys.version_info[0] == 3 and sys.version_info[1] >= 8:
PARALLEL_DEFAULT = False
else:
PARALLEL_DEFAULT = True
# On linux (and other OSes) default to True
else:
PARALLEL_DEFAULT = True

# Set parallel flag
if os.getenv('QISKIT_IN_PARALLEL') is None:
os.environ['QISKIT_IN_PARALLEL'] = 'FALSE'

if os.getenv("QISKIT_NUM_PROCS") is not None:
CPU_COUNT = int(os.getenv('QISKIT_NUM_PROCS'))
else:
CPU_COUNT = CONFIG.get('num_processes') or local_hardware_info()['cpus']
CPU_COUNT = CONFIG.get('num_process', local_hardware_info()['cpus'])


def _task_wrapper(param):
Expand Down Expand Up @@ -118,7 +136,7 @@ def _callback(_):

# Run in parallel if not Win and not in parallel already
if num_processes > 1 and os.getenv('QISKIT_IN_PARALLEL') == 'FALSE' \
and CONFIG.get('parallel_enabled', user_config.PARALLEL_DEFAULT):
and CONFIG.get('parallel_enabled', PARALLEL_DEFAULT):
os.environ['QISKIT_IN_PARALLEL'] = 'TRUE'
try:
results = []
Expand Down
16 changes: 4 additions & 12 deletions qiskit/user_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,13 @@

import configparser
import os
import sys
from warnings import warn

from qiskit import exceptions

DEFAULT_FILENAME = os.path.join(os.path.expanduser("~"),
'.qiskit', 'settings.conf')

if os.getenv('QISKIT_PARALLEL', None) is not None:
PARALLEL_DEFAULT = os.getenv('QISKIT_PARALLEL', None).lower() == 'true'
else:
if sys.platform in {'darwin', 'win32'}:
PARALLEL_DEFAULT = False
else:
PARALLEL_DEFAULT = True


class UserConfig:
"""Class representing a user config file
Expand Down Expand Up @@ -132,8 +123,9 @@ def read_config_file(self):

# Parse parallel
parallel_enabled = self.config_parser.getboolean(
'default', 'parallel', fallback=PARALLEL_DEFAULT)
self.settings['parallel_enabled'] = parallel_enabled
'default', 'parallel', fallback=None)
if parallel_enabled is not None:
self.settings['parallel_enabled'] = parallel_enabled

# Parse num_processes
num_processes = self.config_parser.getint(
Expand All @@ -158,7 +150,7 @@ def get_config():
"""
filename = os.getenv('QISKIT_SETTINGS', DEFAULT_FILENAME)
if not os.path.isfile(filename):
return {'parallel_enabled': PARALLEL_DEFAULT}
return {}
user_config = UserConfig(filename)
user_config.read_config_file()
return user_config.settings
16 changes: 8 additions & 8 deletions releasenotes/notes/paralle-config-options-97c1ad44ab76e052.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ features:
config file.
upgrade:
- |
By default on macOS :func:`~qiskit.tools.parallel_map` will no longer run
in multiple processes. This is a change from previous releases where the
default behavior was that :func:`~qiskit.tools.parallel_map` would launch
multiple processes. This change was made because with newer versions of
macOS (especially with Python 3.8) multiprocessing is either unreliable
or adds significant overhead because of the change in Python 3.8 to launch
new processes with ``spawn`` instead of ``fork``. To re-enable parallel
execution on macOS you can use the user config file ``parallel` option or
By default on macOS with Python >=3.8 :func:`~qiskit.tools.parallel_map` will
no longer run in multiple processes. This is a change from previous releases
where the default behavior was that :func:`~qiskit.tools.parallel_map` would
launch multiple processes. This change was made because with newer versions of
macOS with Python 3.8 and 3.9 multiprocessing is either unreliable or adds
significant overhead because of the change in Python 3.8 to launch new
processes with ``spawn`` instead of ``fork``. To re-enable parallel execution on
macOS with Python >= 3.8 you can use the user config file ``parallel` option or
set the environment variable ``QISKIT_PARALLEL`` to ``True``.
9 changes: 3 additions & 6 deletions test/python/test_user_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@ def test_circuit_drawer_valid(self):
file.flush()
config = user_config.UserConfig(self.file_path)
config.read_config_file()
self.assertEqual({'circuit_drawer': 'latex',
'parallel_enabled': user_config.PARALLEL_DEFAULT},
self.assertEqual({'circuit_drawer': 'latex'},
config.settings)

def test_optimization_level_valid(self):
Expand All @@ -84,8 +83,7 @@ def test_optimization_level_valid(self):
config = user_config.UserConfig(self.file_path)
config.read_config_file()
self.assertEqual(
{'transpile_optimization_level': 1,
'parallel_enabled': user_config.PARALLEL_DEFAULT},
{'transpile_optimization_level': 1},
config.settings)

def test_invalid_num_processes(self):
Expand Down Expand Up @@ -113,8 +111,7 @@ def test_valid_num_processes(self):
config = user_config.UserConfig(self.file_path)
config.read_config_file()
self.assertEqual(
{'parallel_enabled': user_config.PARALLEL_DEFAULT,
'num_processes': 31},
{'num_processes': 31},
config.settings)

def test_valid_parallel(self):
Expand Down