-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathinstall.sh
More file actions
122 lines (107 loc) · 6.47 KB
/
Copy pathinstall.sh
File metadata and controls
122 lines (107 loc) · 6.47 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# theoretically this can work with any conda/mamba base install by setting variables appropriately below.
# if you install a fresh base (minconda/miniforge3) and are using bash,
# then this script should work without user interaction.
# if you like your existing clever environment, take this more as a requirement list than an install script.
# FIRST: install miniforge (mambaforge deprecated) or miniconda
# https://github.com/conda-forge/miniforge?tab=readme-ov-file#download
# https://docs.conda.io/en/latest/miniconda.html
# NOTE: it seems that miniforge3/mambaforge and conda do not play nice with each other,
# so if you want both installed, you will have to custom manage the init blocks in ~/.bashrc
# NOTE: installing mamba in a conda environment did also work when tested,
# but the mamba documentation claims that this is unsupported.
# NOTE: current best solution is using miniconda but with the libmamba solver:
# https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community
# python 3.11 does not work with aicspylibczi because the PyFrames object changed and pybind11 uses it.
# need <= 3.10 in order to build it, and the pip install for 2.8.0 is only available up to python 3.9.
# there are newer versions of aicspylibczi, but they made breaking changes starting with 3.0.0, and
# aicspylibczimsem was not updated to work with the breaking changes.
# the best strategy is probably to go back to our own minimalist version of pylibczi.
# aicspylibczi / aicspylibczimsem are only needed to support the legacy zen format.
#
# python 3.11 is also not compatible with the pinned version of pyfftw=0.13.0=py39h51d1ae8_0
# something about building with pthreads broke in this version and as of 1.1.2024 it is not fixed (see below).
# with the new version that does not have this fixed, this error occurs when running normxcorr with pyfftw:
# ValueError: threads > 1 requested, but pyFFTW was not built with multithreaded FFTW.
# pyfftw is only used as one option for normxcorrs in _template_match, that in all recent tests is slower
# than the mkl-enabled ffts anyways.
use_python_311=1 # comment to install the previous working env using python 3.9
if [[ -n "$use_python_311" ]]; then
pyver=3.11
use_pyfftw=pyfftw
use_aicspylibczi=
unset install_aicspylibczimsem
else
pyver=3.9
use_pyfftw="pyfftw=0.13.0=py39h51d1ae8_0"
use_aicspylibczi="aicspylibczi==2.8.0"
install_aicspylibczimsem=1
fi
#install_rcc_xcorr=1
if [[ -n "$install_rcc_xcorr" ]]; then
rcc_conda="nvtx"
rcc_pip="gputil tqdm"
else
rcc_conda=
rcc_pip=
fi
# name of the environment to use.
# WARNING: existing environment with this name is automatically deleted below.
env_name=msem
# allow this script to use either conda or mamba
#conda=conda
conda=mamba
# location of the root dir for the conda/mamba install.
#conda_dir=${HOME}/miniconda3
#conda_dir=${HOME}/mambaforge
conda_dir=${HOME}/miniforge3
# CAUTION: automatically deletes existing env with same name
${conda} env remove -y --name ${env_name}
# make an env specifically for the msem package.
# NOTE: suggest not mixing conda base channel with condaforge,
# likely to experience dependency problems.
# NOTE: needed to add intel channel for base install so that mkl
# and blas using mkl installs correctly and with more recent versions.
# NOTE: intel channel is foobar as of mid summer 2024:
# https://community.intel.com/t5/oneAPI-Registration-Download/Disappearance-of-intel-conda-chann
# Seems likely related to the anaconda license squeeze that started in 2024:
# https://www.theregister.com/2024/08/08/anaconda_puts_the_squeeze_on
#${conda} create -y --name ${env_name} -c intel -c conda-forge python=${pyver} matplotlib scikit-learn scikit-learn-intelex imageio scikit-image sympy hdf5plugin mkl blas mkl-service mkl_fft mkl_random
#${conda} create -y --name ${env_name} -c conda-forge python=${pyver} matplotlib scikit-learn scikit-learn-intelex imageio scikit-image sympy hdf5plugin mkl blas=*=*mkl* mkl-service mkl_fft mkl_random
# https://github.com/scipy/scipy/issues/20357
#${conda} create -y --name ${env_name} -c conda-forge python=${pyver} matplotlib scikit-learn scikit-learn-intelex imageio scikit-image sympy hdf5plugin mkl blas=*=*mkl* mkl-service mkl_fft mkl_random scipy=1.11.4
# faiss 1.7.4 requires numpy < 2.0, mkl_fft interface changes after 1.3.11, sklearn broken with sklearnex after 1.6.1
${conda} create -y --name ${env_name} -c conda-forge python=${pyver} matplotlib scikit-learn=1.6.1 scikit-learn-intelex imageio scikit-image sympy hdf5plugin mkl blas=*=*mkl* mkl-service mkl_fft=1.3.11 mkl_random numpy=1.26.4
# activate the new env, conda activate does not work within bash scripts:
#https://stackoverflow.com/questions/34534513/calling-conda-source-activate-from-bash-script
# CAUTION: this did not work in all user environments that were tested (why??).
source ${conda_dir}/bin/activate ${env_name}
python --version
# these are separated out of the initial conda create because
# they frequently have dependency compatibility issues.
# ongoing MP issue with pyfftw: https://github.com/conda-forge/pyfftw-feedstock/issues/51
# faiss-gpu > 1.7.4 does not install properly from conda-forge,
# AttributeError: module 'faiss' has no attribute 'StandardGpuResources'
#${conda} install -y -c conda-forge ${use_pyfftw} cudatoolkit=11.8 faiss-gpu cupy ${rcc_conda}
${conda} install -y -c conda-forge ${use_pyfftw} cudatoolkit=11.8 faiss-gpu=1.7.4 cupy ${rcc_conda}
# will get Qt errors without using opencv headless
#pip install opencv-contrib-python-headless tifffile dill ${use_aicspylibczi} scikit-fmm ${rcc_pip}
# opencv version for numpy < 2
pip install "opencv-contrib-python-headless==4.11.0.86" tifffile dill ${use_aicspylibczi} scikit-fmm ${rcc_pip}
# install sslock for gpfs file locking and job completion reporting
pip install git+ssh://git@github.com/mpinb/sslock.git
if [[ -n "$install_rcc_xcorr" ]]; then
# clone and install using local pip install with requirements commented out.
# reinstalls the mkl which causes problems, also forces unnecessary numpy/scipy updates.
#python -m pip install git+ssh://git@github.com/mpinb/rcc-xcorr.git
pip install ~/gits/rcc-xcorr
fi
# install msem and aicspylibczimsem as development
pip install --no-binary :all: -e ~/gits/emalign/python/msem
if [[ -n "$install_aicspylibczimsem" ]]; then
pip install --no-binary :all: -e ~/gits/emalign/python/aicspylibczimsem
fi
# for reference
# CAUTION: updates base conda but in some instances this can update more than just conda.
#conda update -y -n base conda
# CAUTION: clears conda caches / etc
#conda clean -y --all