Remove horovod (#16150)

Author: carmocca

.github/workflows/ci-tests-pytorch.yml

@@ -82,16 +82,6 @@ jobs:
         pip --version
         pip install -q -r .actions/requirements.txt
 
-    - name: Setup macOS
-      if: runner.os == 'macOS'
-      run: |
-        brew install openmpi libuv  # Horovod on macOS requires OpenMPI, Gloo not currently supported
-
-    - name: Setup Windows
-      if: runner.os == 'windows'
-      run: |
-        python .actions/assistant.py requirements_prune_pkgs "--packages=[horovod]"
-
     - name: Set min. dependencies
       if: ${{ matrix.requires == 'oldest' }}
       run: |
@@ -142,23 +132,6 @@ jobs:
         # pip install -r requirements/pytorch/strategies.txt --find-links ${TORCH_URL}
         pip list
 
-    - name: Reinstall Horovod if necessary
-      if: runner.os != 'windows'
-      env:
-        HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
-        HOROVOD_WITHOUT_MXNET: 1
-        HOROVOD_WITHOUT_TENSORFLOW: 1
-      run: |
-        HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')" || true)
-        if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
-          pip uninstall -y horovod
-          grep "horovod" requirements/pytorch/strategies.txt > requirements/pytorch/horovod.txt
-          pip install --no-cache-dir -r requirements/pytorch/horovod.txt
-        fi
-        horovodrun --check-build
-        python -c "import horovod.torch"
-        pip list
-
     - name: Cache datasets
       uses: actions/cache@v3
       with:

dockers/base-cuda/Dockerfile

@@ -28,7 +28,6 @@ ENV \
     TZ=Europe/Prague \
     PATH="$PATH:/root/.local/bin" \
     CUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda" \
-    TORCH_CUDA_ARCH_LIST="3.7;5.0;6.0;7.0;7.5;8.0" \
     MKL_THREADING_LAYER=GNU \
     # MAKEFLAGS="-j$(nproc)"
     MAKEFLAGS="-j2"
@@ -100,24 +99,6 @@ RUN \
     pip install -r requirements/pytorch/base.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \
     rm assistant.py
 
-ENV \
-    HOROVOD_CUDA_HOME=$CUDA_TOOLKIT_ROOT_DIR \
-    HOROVOD_GPU_OPERATIONS=NCCL \
-    HOROVOD_WITH_PYTORCH=1 \
-    HOROVOD_WITHOUT_TENSORFLOW=1 \
-    HOROVOD_WITHOUT_MXNET=1 \
-    HOROVOD_WITH_GLOO=1 \
-    HOROVOD_WITH_MPI=1
-
-RUN \
-    # CUDA 10.2 doesn't support ampere architecture (8.0).
-    if [[ "$CUDA_VERSION" < "11.0" ]]; then export TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST//";8.0"/}; echo $TORCH_CUDA_ARCH_LIST; fi && \
-    HOROVOD_BUILD_CUDA_CC_LIST=${TORCH_CUDA_ARCH_LIST//";"/","} && \
-    export HOROVOD_BUILD_CUDA_CC_LIST=${HOROVOD_BUILD_CUDA_CC_LIST//"."/""} && \
-    echo $HOROVOD_BUILD_CUDA_CC_LIST && \
-    cmake --version && \
-    pip install --no-cache-dir horovod && \
-    horovodrun --check-build
 
 RUN \
     CUDA_VERSION_MAJOR=$(python -c "import torch; print(torch.version.cuda.split('.')[0])") && \
@@ -156,7 +137,6 @@ RUN \
     # install rest of strategies
     # remove colossalai from requirements since they are installed separately
     python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'colossalai' not in line] ; open(fname, 'w').writelines(lines)" ; \
-    python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'horovod' not in line] ; open(fname, 'w').writelines(lines)" ; \
     cat requirements/pytorch/strategies.txt && \
     pip install -r requirements/pytorch/devel.txt -r requirements/pytorch/strategies.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
 

dockers/base-ipu/Dockerfile

@@ -60,7 +60,7 @@ COPY environment.yml environment.yml
 RUN conda init bash && \
     python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'python>=[\d\.]+', 'python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
     python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'- pytorch[>=]+[\d\.]+', '- pytorch=${PYTORCH_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
-    python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if not any(n in ln for n in ['pytorch>', 'horovod'])])" && \
+    python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if not any(n in ln for n in ['pytorch>'])])" && \
     cat environment.yml && \
     conda env create -f environment.yml && \
     conda clean -ya && \

dockers/base-xla/Dockerfile

@@ -61,7 +61,7 @@ RUN conda create -y --name $CONDA_ENV && \
     conda init bash && \
     python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'python>=[\d\.]+', 'python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
     # drop unnecessary packages
-    python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if not any(n in ln for n in ['pytorch>', 'horovod'])])" && \
+    python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if not any(n in ln for n in ['pytorch>'])])" && \
     cat environment.yml && \
     conda env update --file environment.yml && \
     conda clean -ya && \

requirements/pytorch/check-avail-strategies.py

@@ -2,7 +2,3 @@
     import bagua  # noqa: F401
     import deepspeed  # noqa: F401
     import fairscale  # noqa: F401
-    import horovod.torch
-
-    # returns an error code
-    assert horovod.torch.nccl_built()

requirements/pytorch/strategies.txt

@@ -4,6 +4,4 @@
 # colossalai>=0.1.10  # TODO: uncomment when there's a stable version released
 fairscale>=0.4.5, <0.4.13
 deepspeed>=0.6.0, <=0.7.0
-# no need to install with [pytorch] as pytorch is already installed
-horovod>=0.21.2, !=0.24.0, <=0.26.1
 hivemind==1.1.2; sys_platform == 'linux'

src/pytorch_lightning/core/optimizer.py

@@ -40,16 +40,7 @@ def __init__(self, optimizer: Optimizer):
         # copy most of the `Optimizer` methods into this instance. `__del__` is skipped in case the optimizer has
         # implemented custom logic which we would not want to call on destruction of the `LightningOptimizer`
         self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step", "__del__")}
-
-        # For Horovod
-        if hasattr(optimizer, "skip_synchronize"):
-            self.__class__ = type(
-                "Lightning" + optimizer.__class__.__name__, (self.__class__, optimizer.__class__.__bases__[0]), {}
-            )
-            self.skip_synchronize = optimizer.skip_synchronize
-            self.synchronize = optimizer.synchronize
-        else:
-            self.__class__ = type("Lightning" + optimizer.__class__.__name__, (self.__class__, optimizer.__class__), {})
+        self.__class__ = type("Lightning" + optimizer.__class__.__name__, (self.__class__, optimizer.__class__), {})
 
         self._optimizer = optimizer
         self._strategy: Optional[pl.strategies.Strategy] = None

src/pytorch_lightning/strategies/__init__.py

@@ -21,7 +21,6 @@
 from pytorch_lightning.strategies.fully_sharded import DDPFullyShardedStrategy  # noqa: F401
 from pytorch_lightning.strategies.fully_sharded_native import DDPFullyShardedNativeStrategy  # noqa: F401
 from pytorch_lightning.strategies.hivemind import HivemindStrategy  # noqa: F401
-from pytorch_lightning.strategies.horovod import HorovodStrategy  # noqa: F401
 from pytorch_lightning.strategies.hpu_parallel import HPUParallelStrategy  # noqa: F401
 from pytorch_lightning.strategies.ipu import IPUStrategy  # noqa: F401
 from pytorch_lightning.strategies.parallel import ParallelStrategy  # noqa: F401