From cac8a55028663fc1c303701325a2f9288cbffa1a Mon Sep 17 00:00:00 2001 From: Matthew K Defenderfer <mdefende@uab.edu> Date: Sat, 14 Dec 2024 13:10:11 -0600 Subject: [PATCH] Add installation instructions --- README.md | 28 +++++++ deps.yml | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 deps.yml diff --git a/README.md b/README.md index 69fb386..442196f 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,34 @@ The relavent [documentation is available from IBM](https://www.ibm.com/docs/en/s This project focuses on scheduled execution of lifecyle policies to gather and process data about file system objects and issue actions against those objects based on policy. +## Getting Started + +### Gitlab Registry Authentication + +To download from Gitlab's package repo, create a `~/.pypirc` file and add the following entry: + +```text +[gitlab] +repository = https://gitlab.rc.uab.edu/api/v4/projects/2550/packages/pypi +username = __token__ +password = <personal_access_token> +``` + +Go to your [Access Tokens](https://gitlab.rc.uab.edu/-/user_settings/personal_access_tokens) page and create a new token with `api` privileges. After the token is created, copy its value to the password field + +### Installation + +Package dependencies should be installed through `conda` due to some compilation errors on Cheaha when installed through PyPi. Use the following commands to create the environment. + +```bash +module load Anaconda3 +conda env create -n gpfs -f deps.yml +conda activate gpfs +pip install --index-url https://gitlab.rc.uab.edu/api/v4/projects/2550/packages/pypi/simple --no-deps rc-gpfs===<version> +``` + +`pip` by default will install the latest development version (based on the current version format) if a specific version isn't given. Be sure to include the version number if you would like a stable release as opposed to a development release. As well, be sure to `===` instead of `==` for simple version string matching. + ## Applying Policies Applying a policy to filesets is done through the `mmapplypolicy` command at a base level. This repo contains wrapper scripts to call that command with a specified policy file on a given fileset where each wrapper has different levels of functionality meant for different groups of users in RC. All scripts are stored in `src/run-policy` diff --git a/deps.yml b/deps.yml new file mode 100644 index 0000000..d27cda8 --- /dev/null +++ b/deps.yml @@ -0,0 +1,246 @@ +name: gpfs-dev +channels: + - conda-forge + - bioconda + - rapidsai + - nvidia +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - attr=2.5.1=h166bdaf_1 + - aws-c-auth=0.8.0=hb921021_15 + - aws-c-cal=0.8.1=h1a47875_3 + - aws-c-common=0.10.6=hb9d3cd8_0 + - aws-c-compression=0.3.0=h4e1184b_5 + - aws-c-event-stream=0.5.0=h7959bf6_11 + - aws-c-http=0.9.2=hefd7a92_4 + - aws-c-io=0.15.3=hbf5b6a4_4 + - aws-c-mqtt=0.11.0=h11f4f37_12 + - aws-c-s3=0.7.5=hf454442_5 + - aws-c-sdkutils=0.2.1=h4e1184b_4 + - aws-checksums=0.2.2=h4e1184b_4 + - aws-crt-cpp=0.29.7=hbe34e3b_6 + - aws-sdk-cpp=1.11.458=hc430e4a_4 + - azure-core-cpp=1.14.0=h5cfcd09_0 + - azure-identity-cpp=1.10.0=h113e628_0 + - azure-storage-blobs-cpp=12.13.0=h3cf044e_1 + - azure-storage-common-cpp=12.8.0=h736e048_1 + - azure-storage-files-datalake-cpp=12.12.0=ha633028_1 + - bokeh=3.6.2=pyhd8ed1ab_1 + - brotli-python=1.1.0=py311hfdbb021_2 + - bzip2=1.0.8=h4bc722e_7 + - c-ares=1.34.3=hb9d3cd8_1 + - ca-certificates=2024.8.30=hbcca054_0 + - cachetools=5.5.0=pyhd8ed1ab_1 + - cffi=1.17.1=py311hf29c0ef_0 + - click=8.1.7=unix_pyh707e725_1 + - cloudpickle=3.1.0=pyhd8ed1ab_1 + - contourpy=1.3.1=py311hd18a35c_0 + - cuda-cccl_linux-64=12.5.39=ha770c72_0 + - cuda-crt-dev_linux-64=12.5.82=ha770c72_0 + - cuda-crt-tools=12.5.82=ha770c72_0 + - cuda-cudart=12.5.82=he02047a_0 + - cuda-cudart-dev=12.5.82=he02047a_0 + - cuda-cudart-dev_linux-64=12.5.82=h85509e4_0 + - cuda-cudart-static=12.5.82=he02047a_0 + - cuda-cudart-static_linux-64=12.5.82=h85509e4_0 + - cuda-cudart_linux-64=12.5.82=h85509e4_0 + - cuda-nvcc-dev_linux-64=12.5.82=ha770c72_0 + - cuda-nvcc-impl=12.5.82=hd3aeb46_0 + - cuda-nvcc-tools=12.5.82=hd3aeb46_0 + - cuda-nvrtc=12.5.82=he02047a_0 + - cuda-nvvm-dev_linux-64=12.5.82=ha770c72_0 + - cuda-nvvm-impl=12.5.82=h59595ed_0 + - cuda-nvvm-tools=12.5.82=h59595ed_0 + - cuda-profiler-api=12.5.39=ha770c72_0 + - cuda-python=12.6.2=py311h817de4b_1 + - cuda-version=12.5=hd4f0392_3 + - cudf=24.10.01=cuda12_py311_241009_g7b0adfa253_0 + - cuml=24.10.00=cuda12_py311_241009_gba7e3ab9c_0 + - cupy=13.3.0=py311h1c6efab_2 + - cupy-core=13.3.0=py311ha4ffafd_2 + - cytoolz=1.0.1=py311h9ecbd09_0 + - dask=2024.9.0=pyhd8ed1ab_0 + - dask-core=2024.9.0=pyhd8ed1ab_0 + - dask-cuda=24.10.00=py311_241009_g4e45758_0 + - dask-cudf=24.10.01=cuda12_py311_241009_g7b0adfa253_0 + - dask-expr=1.1.14=pyhd8ed1ab_0 + - distributed=2024.9.0=pyhd8ed1ab_0 + - distributed-ucxx=0.40.00=py3.11_241009_g152901c_0 + - dlpack=0.8=h59595ed_3 + - fastrlock=0.8.2=py311hb755f60_2 + - fmt=11.0.2=h434a139_0 + - freetype=2.12.1=h267a509_2 + - fsspec=2024.10.0=pyhd8ed1ab_1 + - gflags=2.2.2=h5888daf_1005 + - glog=0.7.1=hbabe93e_0 + - h2=4.1.0=pyhd8ed1ab_1 + - hpack=4.0.0=pyhd8ed1ab_1 + - hyperframe=6.0.1=pyhd8ed1ab_1 + - importlib-metadata=8.5.0=pyha770c72_1 + - importlib_metadata=8.5.0=hd8ed1ab_1 + - jinja2=3.1.4=pyhd8ed1ab_1 + - joblib=1.4.2=pyhd8ed1ab_1 + - keyutils=1.6.1=h166bdaf_0 + - krb5=1.21.3=h659f571_0 + - lcms2=2.16=hb7c19ff_0 + - ld_impl_linux-64=2.43=h712a8e2_2 + - lerc=4.0.0=h27087fc_0 + - libabseil=20240722.0=cxx17_h5888daf_1 + - libarrow=17.0.0=h44a453e_38_cpu + - libarrow-acero=17.0.0=hcb10f89_38_cpu + - libarrow-dataset=17.0.0=hcb10f89_38_cpu + - libarrow-substrait=17.0.0=h3ee7192_38_cpu + - libblas=3.9.0=25_linux64_openblas + - libbrotlicommon=1.1.0=hb9d3cd8_2 + - libbrotlidec=1.1.0=hb9d3cd8_2 + - libbrotlienc=1.1.0=hb9d3cd8_2 + - libcap=2.71=h39aace5_0 + - libcblas=3.9.0=25_linux64_openblas + - libcrc32c=1.1.2=h9c3ff4c_0 + - libcublas=12.5.3.2=he02047a_0 + - libcublas-dev=12.5.3.2=he02047a_0 + - libcudf=24.10.01=cuda12_241009_g7b0adfa253_0 + - libcufft=11.2.3.61=he02047a_0 + - libcufile=1.10.1.7=he02047a_0 + - libcufile-dev=1.10.1.7=he02047a_0 + - libcuml=24.10.00=cuda12_241009_gba7e3ab9c_0 + - libcumlprims=24.10.00=cuda12_241009_g0848871_0 + - libcurand=10.3.6.82=he02047a_0 + - libcurand-dev=10.3.6.82=he02047a_0 + - libcurl=8.11.1=h332b0f4_0 + - libcusolver=11.6.3.83=he02047a_0 + - libcusolver-dev=11.6.3.83=he02047a_0 + - libcusparse=12.5.1.3=he02047a_0 + - libcusparse-dev=12.5.1.3=he02047a_0 + - libcuvs=24.10.00=cuda12_241009_g7de3a05_0 + - libdeflate=1.22=hb9d3cd8_0 + - libedit=3.1.20191231=he28a2e2_2 + - libev=4.33=hd590300_2 + - libevent=2.1.12=hf998b51_1 + - libexpat=2.6.4=h5888daf_0 + - libffi=3.4.2=h7f98852_5 + - libgcc=14.2.0=h77fa898_1 + - libgcc-ng=14.2.0=h69a702a_1 + - libgcrypt-lib=1.11.0=hb9d3cd8_2 + - libgfortran=14.2.0=h69a702a_1 + - libgfortran5=14.2.0=hd5240d6_1 + - libgomp=14.2.0=h77fa898_1 + - libgoogle-cloud=2.32.0=h804f50b_0 + - libgoogle-cloud-storage=2.32.0=h0121fbd_0 + - libgpg-error=1.51=hbd13f7d_1 + - libgrpc=1.67.1=hc2c308b_0 + - libiconv=1.17=hd590300_2 + - libjpeg-turbo=3.0.0=hd590300_1 + - libkvikio=24.10.00=cuda12_241009_g85a88a2_0 + - liblapack=3.9.0=25_linux64_openblas + - libllvm14=14.0.6=hcd5def8_4 + - liblzma=5.6.3=hb9d3cd8_1 + - libnghttp2=1.64.0=h161d5f1_0 + - libnl=3.11.0=hb9d3cd8_0 + - libnsl=2.0.1=hd590300_0 + - libnvjitlink=12.5.82=he02047a_0 + - libopenblas=0.3.28=pthreads_h94d23a6_1 + - libparquet=17.0.0=h081d1f1_38_cpu + - libpng=1.6.44=hadc24fc_0 + - libprotobuf=5.28.2=h5b01275_0 + - libraft=24.10.00=cuda12_241009_g397042a0_0 + - libraft-headers=24.10.00=cuda12_241009_g397042a0_0 + - libraft-headers-only=24.10.00=cuda12_241009_g397042a0_0 + - libre2-11=2024.07.02=hbbce691_1 + - librmm=24.10.00=cuda12_241009_g3223f841_0 + - libsqlite=3.47.2=hee588c1_0 + - libssh2=1.11.1=hf672d98_0 + - libstdcxx=14.2.0=hc0a3c3a_1 + - libstdcxx-ng=14.2.0=h4852527_1 + - libsystemd0=256.9=h0b6a36f_2 + - libthrift=0.21.0=h0e7cc3e_0 + - libtiff=4.7.0=hc4654cb_2 + - libucxx=0.40.00=cuda12_241009_g152901c_0 + - libudev1=256.9=h9a4d06a_2 + - libutf8proc=2.9.0=hb9d3cd8_1 + - libuuid=2.38.1=h0b41bf4_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.17.0=h8a09558_0 + - libxcrypt=4.4.36=hd590300_1 + - libxml2=2.13.5=h0d44e9d_1 + - libzlib=1.3.1=hb9d3cd8_2 + - llvmlite=0.43.0=py311h9c9ff8c_1 + - locket=1.0.0=pyhd8ed1ab_0 + - lz4=4.3.3=py311h8c6ae76_2 + - lz4-c=1.10.0=h5888daf_1 + - markdown-it-py=3.0.0=pyhd8ed1ab_1 + - markupsafe=3.0.2=py311h2dc5d0c_1 + - mdurl=0.1.2=pyhd8ed1ab_1 + - msgpack-python=1.1.0=py311hd18a35c_0 + - nccl=2.23.4.1=h2b5d15b_3 + - ncurses=6.5=he02047a_1 + - numba=0.60.0=py311h4bc866e_0 + - numpy=2.0.2=py311h71ddf71_1 + - nvcomp=4.0.1=hbc370b7_0 + - nvtx=0.2.10=py311h9ecbd09_2 + - openjpeg=2.5.3=h5fbd93e_0 + - openssl=3.4.0=hb9d3cd8_0 + - orc=2.0.3=h97ab989_1 + - packaging=24.2=pyhd8ed1ab_2 + - pandas=2.2.2=py311h14de704_1 + - partd=1.4.2=pyhd8ed1ab_0 + - pillow=11.0.0=py311h49e9ac3_0 + - pip=24.3.1=pyh8b19718_0 + - plotly=5.24.1=pyhd8ed1ab_1 + - psutil=6.1.0=py311h9ecbd09_0 + - pthread-stubs=0.4=hb9d3cd8_1002 + - pyarrow=17.0.0=py311hbd00459_2 + - pyarrow-core=17.0.0=py311h4854187_2_cpu + - pycparser=2.22=pyh29332c3_1 + - pygments=2.18.0=pyhd8ed1ab_1 + - pylibcudf=24.10.01=cuda12_py311_241009_g7b0adfa253_0 + - pylibraft=24.10.00=cuda12_py311_241009_g397042a0_0 + - pynvjitlink=0.4.0=py311hcc76453_0 + - pynvml=11.4.1=pyhd8ed1ab_0 + - pysocks=1.7.1=pyha55dd90_7 + - python=3.11.11=h9e4cc4f_1_cpython + - python-dateutil=2.9.0.post0=pyhff2d567_1 + - python-tzdata=2024.2=pyhd8ed1ab_1 + - python_abi=3.11=5_cp311 + - pytz=2024.2=pyhd8ed1ab_1 + - pyyaml=6.0.2=py311h9ecbd09_1 + - raft-dask=24.10.00=cuda12_py311_241009_g397042a0_0 + - rapids-dask-dependency=24.10.00=py_0 + - rdma-core=54.0=h5888daf_1 + - re2=2024.07.02=h77b4e00_1 + - readline=8.2=h8228510_1 + - rich=13.9.4=pyhd8ed1ab_1 + - rmm=24.10.00=cuda12_py311_241009_g3223f841_0 + - s2n=1.5.9=h0fd0ee4_0 + - scipy=1.14.1=py311he9a78e4_2 + - setuptools=75.6.0=pyhff2d567_1 + - six=1.17.0=pyhd8ed1ab_0 + - snappy=1.2.1=h8bd8927_1 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - spdlog=1.14.1=hed91bc2_1 + - tblib=3.0.0=pyhd8ed1ab_1 + - tenacity=9.0.0=pyhd8ed1ab_1 + - tk=8.6.13=noxft_h4845f30_101 + - toolz=1.0.0=pyhd8ed1ab_1 + - tornado=6.4.2=py311h9ecbd09_0 + - treelite=4.3.0=py311he8f9275_0 + - typing_extensions=4.12.2=pyha770c72_1 + - tzdata=2024b=hc8b5060_0 + - ucx=1.17.0=h05e919c_3 + - ucx-py=0.40.00=py311_241009_g773cd1e_0 + - ucxx=0.40.00=cuda12_py3.11_241009_g152901c_0 + - urllib3=2.2.3=pyhd8ed1ab_1 + - wheel=0.45.1=pyhd8ed1ab_1 + - xorg-libxau=1.0.11=hb9d3cd8_1 + - xorg-libxdmcp=1.1.5=hb9d3cd8_0 + - xyzservices=2024.9.0=pyhd8ed1ab_1 + - yaml=0.2.5=h7f98852_2 + - zict=3.0.0=pyhd8ed1ab_1 + - zipp=3.21.0=pyhd8ed1ab_1 + - zstandard=0.23.0=py311hbc35293_1 + - zstd=1.5.6=ha6fb4c9_0 + - pip: + - nvidia-ml-py==12.560.30 + - typeguard==4.4.1 +prefix: /home/mdefende/.conda/envs/gpfs-dev -- GitLab