diff --git a/awswrangler/__init__.py b/awswrangler/__init__.py index 17ca698d8..b105c07a9 100644 --- a/awswrangler/__init__.py +++ b/awswrangler/__init__.py @@ -31,6 +31,11 @@ ) from awswrangler.__metadata__ import __description__, __license__, __title__, __version__ # noqa from awswrangler._config import config # noqa +from awswrangler._distributed import initialize_ray + +if config.distributed: + initialize_ray() + __all__ = [ "athena", diff --git a/awswrangler/_config.py b/awswrangler/_config.py index bb9203fe9..eb1068485 100644 --- a/awswrangler/_config.py +++ b/awswrangler/_config.py @@ -1,5 +1,6 @@ """Configuration file for AWS Data Wrangler.""" +import importlib.util import inspect import logging import os @@ -49,6 +50,15 @@ class _ConfigArg(NamedTuple): # Botocore config "botocore_config": _ConfigArg(dtype=botocore.config.Config, nullable=True), "verify": _ConfigArg(dtype=str, nullable=True), + # Distributed + "distributed": _ConfigArg(dtype=bool, nullable=True), + "address": _ConfigArg(dtype=str, nullable=True), + "redis_password": _ConfigArg(dtype=str, nullable=True), + "ignore_reinit_error": _ConfigArg(dtype=bool, nullable=True), + "include_dashboard": _ConfigArg(dtype=bool, nullable=True), + "object_store_memory": _ConfigArg(dtype=int, nullable=True), + "cpu_count": _ConfigArg(dtype=int, nullable=True), + "gpu_count": _ConfigArg(dtype=int, nullable=True), } @@ -70,6 +80,7 @@ def __init__(self) -> None: self.secretsmanager_endpoint_url = None self.botocore_config = None self.verify = None + self.distributed = all(importlib.util.find_spec(pkg) for pkg in ("modin", "ray")) for name in _CONFIG_ARGS: self._load_config(name=name) @@ -155,7 +166,7 @@ def __getitem__(self, item: str) -> _ConfigValueType: def _reset_item(self, item: str) -> None: if item in self._loaded_values: - if item.endswith("_endpoint_url") or item == "verify": + if item.endswith("_endpoint_url") or item in ["verify", "distributed"]: self._loaded_values[item] = None else: del self._loaded_values[item] @@ -405,6 +416,78 @@ def verify(self) -> Optional[str]: def verify(self, value: Optional[str]) -> None: self._set_config_value(key="verify", value=value) + @property + def distributed(self) -> Optional[bool]: + """Property distributed.""" + return cast(Optional[bool], self["distributed"]) + + @distributed.setter + def distributed(self, value: Optional[bool]) -> None: + self._set_config_value(key="distributed", value=value) + + @property + def ignore_reinit_error(self) -> Optional[bool]: + """Property ignore_reinit_error.""" + return cast(Optional[bool], self["ignore_reinit_error"]) + + @ignore_reinit_error.setter + def ignore_reinit_error(self, value: Optional[bool]) -> None: + self._set_config_value(key="ignore_reinit_error", value=value) + + @property + def include_dashboard(self) -> Optional[bool]: + """Property include_dashboard.""" + return cast(Optional[bool], self["include_dashboard"]) + + @include_dashboard.setter + def include_dashboard(self, value: Optional[bool]) -> None: + self._set_config_value(key="include_dashboard", value=value) + + @property + def address(self) -> Optional[str]: + """Property address.""" + return cast(Optional[str], self["address"]) + + @address.setter + def address(self, value: Optional[str]) -> None: + self._set_config_value(key="address", value=value) + + @property + def redis_password(self) -> Optional[str]: + """Property redis_password.""" + return cast(Optional[str], self["redis_password"]) + + @redis_password.setter + def redis_password(self, value: Optional[str]) -> None: + self._set_config_value(key="redis_password", value=value) + + @property + def object_store_memory(self) -> int: + """Property object_store_memory.""" + return cast(int, self["object_store_memory"]) + + @object_store_memory.setter + def object_store_memory(self, value: int) -> None: + self._set_config_value(key="object_store_memory", value=value) + + @property + def cpu_count(self) -> int: + """Property cpu_count.""" + return cast(int, self["cpu_count"]) + + @cpu_count.setter + def cpu_count(self, value: int) -> None: + self._set_config_value(key="cpu_count", value=value) + + @property + def gpu_count(self) -> int: + """Property gpu_count.""" + return cast(int, self["gpu_count"]) + + @gpu_count.setter + def gpu_count(self, value: int) -> None: + self._set_config_value(key="gpu_count", value=value) + def _insert_str(text: str, token: str, insert: str) -> str: """Insert string into other.""" diff --git a/awswrangler/_distributed.py b/awswrangler/_distributed.py new file mode 100644 index 000000000..a751a1688 --- /dev/null +++ b/awswrangler/_distributed.py @@ -0,0 +1,126 @@ +"""Distributed Module (PRIVATE).""" + +import multiprocessing +import os +import sys +import warnings +from typing import TYPE_CHECKING, Any, Callable, Optional + +import psutil + +from awswrangler._config import apply_configs, config + +if config.distributed or TYPE_CHECKING: + import ray + + +def ray_remote(function: Callable[..., Any]) -> Callable[..., Any]: + """ + Decorate callable to wrap within ray.remote. + + Parameters + ---------- + function : Callable[..., Any] + Callable as input to ray.remote + + Returns + ------- + Callable[..., Any] + """ + if config.distributed: + + def wrapper(*args: Any, **kwargs: Any) -> Any: + return ray.remote(function).remote(*args, **kwargs) + + return wrapper + return function + + +@apply_configs +def initialize_ray( + address: Optional[str] = None, + redis_password: Optional[str] = None, + ignore_reinit_error: Optional[bool] = True, + include_dashboard: Optional[bool] = False, + object_store_memory: Optional[int] = None, + cpu_count: Optional[int] = None, + gpu_count: Optional[int] = 0, +) -> None: + """ + Connect to an existing Ray cluster or start one and connect to it. + + Parameters + ---------- + address : Optional[str] + Address of the Ray cluster to connect to, by default None + redis_password : Optional[str] + Password to the Redis cluster, by default None + ignore_reinit_error : Optional[bool] + If true, Ray suppress errors from calling ray.init() twice, by default True + include_dashboard : Optional[bool] + Boolean flag indicating whether or not to start the Ray dashboard, by default False + object_store_memory : Optional[int] + The amount of memory (in bytes) to start the object store with, by default None + cpu_count : Optional[int] + Number of CPUs to assign to each raylet, by default None + gpu_count : Optional[int] + Number of GPUs to assign to each raylet, by default 0 + """ + if address: + ray.init( + address=address, + include_dashboard=include_dashboard, + ignore_reinit_error=ignore_reinit_error, + ) + else: + if not object_store_memory: + object_store_memory = _get_ray_object_store_memory() + + mac_size_limit = getattr(ray.ray_constants, "MAC_DEGRADED_PERF_MMAP_SIZE_LIMIT", None) + if sys.platform == "darwin" and mac_size_limit is not None and object_store_memory > mac_size_limit: + warnings.warn( + "On Macs, Ray's performance is known to degrade with " + + "object store size greater than " + + f"{mac_size_limit / 2 ** 30:.4} GiB. Ray by default does " + + "not allow setting an object store size greater than " + + "that. This default is overridden to avoid " + + "spilling to disk more often. To override this " + + "behavior, you can initialize Ray yourself." + ) + os.environ["RAY_ENABLE_MAC_LARGE_OBJECT_STORE"] = "1" + + ray_init_kwargs = { + "num_cpus": cpu_count or multiprocessing.cpu_count(), + "num_gpus": gpu_count, + "include_dashboard": include_dashboard, + "ignore_reinit_error": ignore_reinit_error, + "object_store_memory": object_store_memory, + "_redis_password": redis_password, + "_memory": object_store_memory, + } + ray.init(**ray_init_kwargs) + + +def _get_ray_object_store_memory() -> Optional[int]: + virtual_memory = psutil.virtual_memory().total + if sys.platform.startswith("linux"): + shm_fd = os.open("/dev/shm", os.O_RDONLY) + try: + shm_stats = os.fstatvfs(shm_fd) + system_memory = shm_stats.f_bsize * shm_stats.f_bavail + if system_memory / (virtual_memory / 2) < 0.99: + warnings.warn( + f"The size of /dev/shm is too small ({system_memory} bytes). The required size " + + f"is at least half of RAM ({virtual_memory // 2} bytes). Please, delete files " + + "in /dev/shm or increase the size with --shm-size in Docker. Alternatively, set the " + + "memory size for each Ray worker in bytes with the RAY_OBJECT_STORE_MEMORY env var." + ) + finally: + os.close(shm_fd) + else: + system_memory = virtual_memory + object_store_memory: Optional[int] = int(0.6 * system_memory // 1e9 * 1e9) # type: ignore + # If the memory pool is smaller than 2GB, just use the default in ray. + if object_store_memory == 0: + object_store_memory = None + return object_store_memory diff --git a/poetry.lock b/poetry.lock index 8913d7f32..6fed96ea0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -130,7 +130,7 @@ python-versions = "*" [[package]] name = "astroid" -version = "2.11.5" +version = "2.11.6" description = "An abstract syntax tree for Python with inference support." category = "dev" optional = false @@ -173,7 +173,7 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (> [[package]] name = "babel" -version = "2.10.1" +version = "2.10.3" description = "Internationalization utilities" category = "dev" optional = false @@ -266,14 +266,14 @@ wcwidth = ">=0.1.4" [[package]] name = "boto3" -version = "1.24.7" +version = "1.24.10" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" [package.dependencies] -botocore = ">=1.27.7,<1.28.0" +botocore = ">=1.27.10,<1.28.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -282,7 +282,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.27.7" +version = "1.27.10" description = "Low-level, data-driven core of boto 3." category = "main" optional = false @@ -314,7 +314,7 @@ python-versions = "~=3.7" [[package]] name = "certifi" -version = "2022.5.18.1" +version = "2022.6.15" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -355,7 +355,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "main" optional = false @@ -574,7 +574,7 @@ tqdm = ["tqdm"] [[package]] name = "google-api-core" -version = "2.8.1" +version = "2.8.2" description = "Google API client core library" category = "main" optional = true @@ -583,17 +583,15 @@ python-versions = ">=3.6" [package.dependencies] google-auth = ">=1.25.0,<3.0dev" googleapis-common-protos = ">=1.56.2,<2.0dev" -protobuf = ">=3.15.0,<4.0.0dev" +protobuf = ">=3.15.0,<5.0.0dev" requests = ">=2.18.0,<3.0.0dev" [package.extras] grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio-status (>=1.33.2,<2.0dev)"] -grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] -grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] [[package]] name = "google-auth" -version = "2.7.0" +version = "2.8.0" description = "Google Authentication Library" category = "main" optional = true @@ -704,7 +702,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [[package]] name = "importlib-resources" -version = "5.7.1" +version = "5.8.0" description = "Read resources from Python packages" category = "main" optional = false @@ -727,7 +725,7 @@ python-versions = "*" [[package]] name = "ipykernel" -version = "6.13.1" +version = "6.15.0" description = "IPython Kernel for Jupyter" category = "dev" optional = false @@ -742,6 +740,7 @@ matplotlib-inline = ">=0.1" nest-asyncio = "*" packaging = "*" psutil = "*" +pyzmq = ">=17" tornado = ">=6.1" traitlets = ">=5.1.0" @@ -1713,14 +1712,14 @@ python-versions = ">=3.6" [[package]] name = "pylint" -version = "2.14.1" +version = "2.14.2" description = "python code static checker" category = "dev" optional = false python-versions = ">=3.7.2" [package.dependencies] -astroid = ">=2.11.5,<=2.12.0-dev0" +astroid = ">=2.11.6,<=2.12.0-dev0" colorama = {version = "*", markers = "sys_platform == \"win32\""} dill = ">=0.2" isort = ">=4.2.5,<6" @@ -2565,14 +2564,14 @@ docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [extras] -distributed = ["modin", "ray", "tqdm"] +distributed = ["modin", "ray", "psutil", "tqdm"] sparql = ["SPARQLWrapper"] sqlserver = ["pyodbc"] [metadata] lock-version = "1.1" python-versions = ">=3.8, <3.11" -content-hash = "164dec13d886f74c1ad583a1a16dd1191d1b8fb1a97f3203ab0735a0fd29257b" +content-hash = "9a35bdc2187eeea60ef3dac5bc722be970199536100edcf9d730c82c30d6c1b1" [metadata.files] aenum = [ @@ -2710,8 +2709,8 @@ asn1crypto = [ {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, ] astroid = [ - {file = "astroid-2.11.5-py3-none-any.whl", hash = "sha256:14ffbb4f6aa2cf474a0834014005487f7ecd8924996083ab411e7fa0b508ce0b"}, - {file = "astroid-2.11.5.tar.gz", hash = "sha256:f4e4ec5294c4b07ac38bab9ca5ddd3914d4bf46f9006eb5c0ae755755061044e"}, + {file = "astroid-2.11.6-py3-none-any.whl", hash = "sha256:ba33a82a9a9c06a5ceed98180c5aab16e29c285b828d94696bf32d6015ea82a9"}, + {file = "astroid-2.11.6.tar.gz", hash = "sha256:4f933d0bf5e408b03a6feb5d23793740c27e07340605f236496cd6ce552043d6"}, ] async-timeout = [ {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, @@ -2726,8 +2725,8 @@ attrs = [ {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] babel = [ - {file = "Babel-2.10.1-py3-none-any.whl", hash = "sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2"}, - {file = "Babel-2.10.1.tar.gz", hash = "sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13"}, + {file = "Babel-2.10.3-py3-none-any.whl", hash = "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb"}, + {file = "Babel-2.10.3.tar.gz", hash = "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51"}, ] backcall = [ {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, @@ -2775,12 +2774,12 @@ blessed = [ {file = "blessed-1.19.1.tar.gz", hash = "sha256:9a0d099695bf621d4680dd6c73f6ad547f6a3442fbdbe80c4b1daa1edbc492fc"}, ] boto3 = [ - {file = "boto3-1.24.7-py3-none-any.whl", hash = "sha256:925a34a55257219f4601e803951fd4d61ed6eac2208dc834a04fe150b03f265e"}, - {file = "boto3-1.24.7.tar.gz", hash = "sha256:6e243e28c804dccd2015935acfac0567e1861b20fdd96aa47f232b47aa214a69"}, + {file = "boto3-1.24.10-py3-none-any.whl", hash = "sha256:32ffc0fd50408acc710cf5ce40037aa3c14926d6e3f6fbf61ed5990fb63cd881"}, + {file = "boto3-1.24.10.tar.gz", hash = "sha256:88fd816274d4b64bcf90889441d4efa5f16a0048ed670bc33cbd0f5a678313a6"}, ] botocore = [ - {file = "botocore-1.27.7-py3-none-any.whl", hash = "sha256:3e0cbe26f08fe9a3f6df5de4dcc3bef686e01ba5f79ad03ffbe79d92f51ecea5"}, - {file = "botocore-1.27.7.tar.gz", hash = "sha256:dc83ef991c730ab0f06b51fcefda74f493b990903b882452aff78c123e3040e2"}, + {file = "botocore-1.27.10-py3-none-any.whl", hash = "sha256:24ec42b4f29a50f7ef78f9f863c3c25e00f65b5a48db669c8068457789a90803"}, + {file = "botocore-1.27.10.tar.gz", hash = "sha256:b39da97452c9e2c856e7778d8c908252394da81e2e5792f1d4cb0ece4ce1043a"}, ] bump2version = [ {file = "bump2version-1.0.1-py2.py3-none-any.whl", hash = "sha256:37f927ea17cde7ae2d7baf832f8e80ce3777624554a653006c9144f8017fe410"}, @@ -2791,8 +2790,8 @@ cachetools = [ {file = "cachetools-5.2.0.tar.gz", hash = "sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757"}, ] certifi = [ - {file = "certifi-2022.5.18.1-py3-none-any.whl", hash = "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"}, - {file = "certifi-2022.5.18.1.tar.gz", hash = "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7"}, + {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, + {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, ] cffi = [ {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, @@ -2855,8 +2854,8 @@ click = [ {file = "click-8.0.4.tar.gz", hash = "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb"}, ] colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] colorful = [ {file = "colorful-0.5.4-py2.py3-none-any.whl", hash = "sha256:8d264b52a39aae4c0ba3e2a46afbaec81b0559a99be0d2cfe2aba4cf94531348"}, @@ -3063,12 +3062,12 @@ fsspec = [ {file = "fsspec-2022.5.0.tar.gz", hash = "sha256:7a5459c75c44e760fbe6a3ccb1f37e81e023cde7da8ba20401258d877ec483b4"}, ] google-api-core = [ - {file = "google-api-core-2.8.1.tar.gz", hash = "sha256:958024c6aa3460b08f35741231076a4dd9a4c819a6a39d44da9627febe8b28f0"}, - {file = "google_api_core-2.8.1-py3-none-any.whl", hash = "sha256:ce1daa49644b50398093d2a9ad886501aa845e2602af70c3001b9f402a9d7359"}, + {file = "google-api-core-2.8.2.tar.gz", hash = "sha256:06f7244c640322b508b125903bb5701bebabce8832f85aba9335ec00b3d02edc"}, + {file = "google_api_core-2.8.2-py3-none-any.whl", hash = "sha256:93c6a91ccac79079ac6bbf8b74ee75db970cc899278b97d53bc012f35908cf50"}, ] google-auth = [ - {file = "google-auth-2.7.0.tar.gz", hash = "sha256:8a954960f852d5f19e6af14dd8e75c20159609e85d8db37e4013cc8c3824a7e1"}, - {file = "google_auth-2.7.0-py2.py3-none-any.whl", hash = "sha256:df549a1433108801b11bdcc0e312eaf0d5f0500db42f0523e4d65c78722e8475"}, + {file = "google-auth-2.8.0.tar.gz", hash = "sha256:819b70140d05501739e1387291d39f0de3b4dff3b00ae4aff8e7a05369957f89"}, + {file = "google_auth-2.8.0-py2.py3-none-any.whl", hash = "sha256:9b1da39ab8731c3061f36fefde9f8bb902dbee9eb28e3a67e8cfa7dc1be76227"}, ] googleapis-common-protos = [ {file = "googleapis-common-protos-1.56.2.tar.gz", hash = "sha256:b09b56f5463070c2153753ef123f07d2e49235e89148e9b2459ec8ed2f68d7d3"}, @@ -3140,16 +3139,16 @@ importlib-metadata = [ {file = "importlib_metadata-4.11.4.tar.gz", hash = "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700"}, ] importlib-resources = [ - {file = "importlib_resources-5.7.1-py3-none-any.whl", hash = "sha256:e447dc01619b1e951286f3929be820029d48c75eb25d265c28b92a16548212b8"}, - {file = "importlib_resources-5.7.1.tar.gz", hash = "sha256:b6062987dfc51f0fcb809187cffbd60f35df7acb4589091f154214af6d0d49d3"}, + {file = "importlib_resources-5.8.0-py3-none-any.whl", hash = "sha256:7952325ffd516c05a8ad0858c74dff2c3343f136fe66a6002b2623dd1d43f223"}, + {file = "importlib_resources-5.8.0.tar.gz", hash = "sha256:568c9f16cb204f9decc8d6d24a572eeea27dacbb4cee9e6b03a8025736769751"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] ipykernel = [ - {file = "ipykernel-6.13.1-py3-none-any.whl", hash = "sha256:fedc79bebd8a438162d056e0c7662d5ac8a47d1f6ef33a702e8460248dc4517f"}, - {file = "ipykernel-6.13.1.tar.gz", hash = "sha256:6f42070a5d87ecbf4a2fc27a7faae8d690fd3794825a090ddf6b00b9678a5b69"}, + {file = "ipykernel-6.15.0-py3-none-any.whl", hash = "sha256:b9ed519a29eb819eb82e87e0d3754088237b233e5c647b8bb0ff23c8c70ed16f"}, + {file = "ipykernel-6.15.0.tar.gz", hash = "sha256:b59f9d9672c3a483494bb75915a2b315e78b833a38b039b1ee36dc28683f0d89"}, ] ipython = [ {file = "ipython-7.34.0-py3-none-any.whl", hash = "sha256:c175d2440a1caff76116eb719d40538fbb316e214eda85c5515c303aacbfb23e"}, @@ -3858,8 +3857,8 @@ pygments = [ {file = "Pygments-2.12.0.tar.gz", hash = "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb"}, ] pylint = [ - {file = "pylint-2.14.1-py3-none-any.whl", hash = "sha256:bb71e6d169506de585edea997e48d9ff20c0dc0e2fbc1d166bad6b640120326b"}, - {file = "pylint-2.14.1.tar.gz", hash = "sha256:549261e0762c3466cc001024c4419c08252cb8c8d40f5c2c6966fea690e7fe2a"}, + {file = "pylint-2.14.2-py3-none-any.whl", hash = "sha256:592d0a4d2ffa8e33020209d255827c5a310499cdc023d156187bc677d86bd495"}, + {file = "pylint-2.14.2.tar.gz", hash = "sha256:482f1329d4b6b9e52599754a2e502c0ed91ebdfd0992a2299b7fa136a6c12349"}, ] pymysql = [ {file = "PyMySQL-1.0.2-py3-none-any.whl", hash = "sha256:41fc3a0c5013d5f039639442321185532e3e2c8924687abe6537de157d403641"}, diff --git a/pyproject.toml b/pyproject.toml index 7cfb2053f..4c3c6dfc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,12 +45,13 @@ SPARQLWrapper = { version = "^2.0.0", optional = true } pyodbc = { version = "~4.0.32", optional = true } modin = { version = "^0.15.0", optional = true } ray = { version = "^1.13.0", extras = ["default", "data"], optional = true} +psutil = { version = "^5.9.0", optional = true } tqdm = { version = "^4.64.0", optional = true } [tool.poetry.extras] sqlserver = ["pyodbc"] sparql = ["SPARQLWrapper"] -distributed = ["modin", "ray", "tqdm"] +distributed = ["modin", "ray", "psutil", "tqdm"] [tool.poetry.dev-dependencies] wheel = "^0.36.2"