diff --git a/environment-slurm-ds.yml b/environment-slurm-ds.yml new file mode 100644 index 0000000000000000000000000000000000000000..d2dabd098df70a4d4a15f4416b15068444e1db66 --- /dev/null +++ b/environment-slurm-ds.yml @@ -0,0 +1,112 @@ +name: slurm-ds +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main.conda + - backcall=0.1.0=py38_0.conda + - ca-certificates=2020.1.1=0.conda + - certifi=2019.11.28=py38_0.conda + - decorator=4.4.2=py_0.conda + - entrypoints=0.3=py38_0.conda + - ipykernel=5.1.4=py38h39e3cac_0.conda + - ipython=7.13.0=py38h5ca1d4c_0.conda + - ipython_genutils=0.2.0=py38_0.conda + - jedi=0.16.0=py38_0.conda + - jupyter_client=6.0.0=py_0.conda + - jupyter_core=4.6.1=py38_0.conda + - ld_impl_linux-64=2.33.1=h53a641e_7.conda + - libedit=3.1.20181209=hc058e9b_0.conda + - libffi=3.2.1=hd88cf55_4.conda + - libgcc-ng=9.1.0=hdf63c60_0.conda + - libsodium=1.0.16=h1bed415_0.conda + - libstdcxx-ng=9.1.0=hdf63c60_0.conda + - ncurses=6.2=he6710b0_0 + - openssl=1.1.1e=h7b6447c_0.conda + - parso=0.6.2=py_0.conda + - pexpect=4.8.0=py38_0.conda + - pickleshare=0.7.5=py38_1000.conda + - pip=20.0.2=py38_1.conda + - prompt_toolkit=3.0.3=py_0.conda + - ptyprocess=0.6.0=py38_0.conda + - pygments=2.6.1=py_0.conda + - python=3.8.1=h0371630_1.conda + - python-dateutil=2.8.1=py_0 + - pyzmq=18.1.1=py38he6710b0_0.conda + - readline=7.0=h7b6447c_5.conda + - setuptools=46.0.0=py38_0.conda + - six=1.14.0=py38_0.conda + - sqlite=3.31.1=h7b6447c_0.conda + - tk=8.6.8=hbc83047_0.conda + - tornado=6.0.4=py38h7b6447c_1.conda + - traitlets=4.3.3=py38_0.conda + - wcwidth=0.1.8=py_0.conda + - wheel=0.34.2=py38_0.conda + - xz=5.2.4=h14c3975_4.conda + - zeromq=4.3.1=he6710b0_3.conda + - zlib=1.2.11=h7b6447c_3.conda + - pip: + - astroid==2.3.3 + - astropy==4.0 + - attr==0.3.1 + - attrs==19.3.0 + - bleach==3.1.3 + - chardet==3.0.4 + - confuse==1.0.0 + - cycler==0.10.0 + - defusedxml==0.6.0 + - htmlmin==0.1.12 + - idna==2.8 + - ipywidgets==7.5.1 + - isort==4.3.21 + - jinja2==2.11.1 + - joblib==0.14.1 + - jsonschema==3.2.0 + - kaggle==1.5.6 + - kiwisolver==1.1.0 + - lazy-object-proxy==1.4.3 + - llvmlite==0.31.0 + - markupsafe==1.1.1 + - matplotlib==3.2.0 + - mccabe==0.6.1 + - missingno==0.4.2 + - mistune==0.8.4 + - more-itertools==8.2.0 + - nbconvert==5.6.1 + - nbformat==5.0.4 + - networkx==2.4 + - notebook==6.0.3 + - numba==0.48.0 + - numpy==1.18.2 + - packaging==20.3 + - pandas==0.25.3 + - pandas-profiling==2.5.0 + - pandocfilters==1.4.2 + - phik==0.9.9 + - pluggy==0.13.1 + - prometheus-client==0.7.1 + - py==1.8.1 + - pylint==2.4.4 + - pyparsing==2.4.6 + - pyrsistent==0.15.7 + - pytest==5.4.1 + - pytest-pylint==0.15.1 + - python-slugify==4.0.0 + - pytz==2019.3 + - pyyaml==5.3 + - requests==2.22.0 + - scipy==1.4.1 + - seaborn==0.10.0 + - send2trash==1.5.0 + - slurm2sql==0.9.0 + - tangled-up-in-unicode==0.0.3 + - terminado==0.8.3 + - testpath==0.4.4 + - text-unidecode==1.3 + - tqdm==4.42.0 + - urllib3==1.25.8 + - visions==0.2.2 + - webencodings==0.5.1 + - widgetsnbextension==3.5.1 + - wrapt==1.11.2 +prefix: /home/user/.conda/envs/slurm-ds + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3319e31e869876559896347734d7946c4cbb113b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,85 @@ +astroid==2.3.3 +astropy==4.0 +attr==0.3.1 +attrs==19.3.0 +backcall==0.1.0 +bleach==3.1.3 +certifi==2019.11.28 +chardet==3.0.4 +confuse==1.0.0 +cycler==0.10.0 +decorator==4.4.2 +defusedxml==0.6.0 +entrypoints==0.3 +htmlmin==0.1.12 +idna==2.8 +ipykernel==5.1.4 +ipython==7.13.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +isort==4.3.21 +jedi==0.16.0 +Jinja2==2.11.1 +joblib==0.14.1 +jsonschema==3.2.0 +jupyter-client==6.0.0 +jupyter-core==4.6.1 +kaggle==1.5.6 +kiwisolver==1.1.0 +lazy-object-proxy==1.4.3 +llvmlite==0.31.0 +MarkupSafe==1.1.1 +matplotlib==3.2.0 +mccabe==0.6.1 +missingno==0.4.2 +mistune==0.8.4 +more-itertools==8.2.0 +nbconvert==5.6.1 +nbformat==5.0.4 +networkx==2.4 +notebook==6.0.3 +numba==0.48.0 +numpy==1.18.2 +packaging==20.3 +pandas==0.25.3 +pandas-profiling==2.5.0 +pandocfilters==1.4.2 +parso==0.6.2 +pexpect==4.8.0 +phik==0.9.9 +pickleshare==0.7.5 +pluggy==0.13.1 +prometheus-client==0.7.1 +prompt-toolkit==3.0.3 +ptyprocess==0.6.0 +py==1.8.1 +Pygments==2.6.1 +pylint==2.4.4 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pytest==5.4.1 +pytest-pylint==0.15.1 +python-dateutil==2.8.1 +python-slugify==4.0.0 +pytz==2019.3 +PyYAML==5.3 +pyzmq==18.1.1 +requests==2.22.0 +scipy==1.4.1 +seaborn==0.10.0 +Send2Trash==1.5.0 +six==1.14.0 +slurm2sql==0.9.0 +tangled-up-in-unicode==0.0.3 +terminado==0.8.3 +testpath==0.4.4 +text-unidecode==1.3 +tornado==6.0.4 +tqdm==4.42.0 +traitlets==4.3.3 +urllib3==1.25.8 +visions==0.2.2 +wcwidth==0.1.8 +webencodings==0.5.1 +widgetsnbextension==3.5.1 +wrapt==1.11.2 diff --git a/slurm-2sql.ipynb b/slurm-2sql.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f2608ac87140174fb649f902e4175ef7b5bfc8eb --- /dev/null +++ b/slurm-2sql.ipynb @@ -0,0 +1,290 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import sqlite3\n", + "import slurm2sql\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db = sqlite3.connect('test.db')\n", + "slurm2sql.slurm2sql(db, ['-S', '2020-03-18', '-a'])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# For example, you can then convert to a dataframe:\n", + "df1 = pd.read_sql('SELECT * FROM slurm', db)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>JobID</th>\n", + " <th>ArrayJobID</th>\n", + " <th>ArrayTaskID</th>\n", + " <th>JobStep</th>\n", + " <th>JobIDSlurm</th>\n", + " <th>JobName</th>\n", + " <th>User</th>\n", + " <th>Group</th>\n", + " <th>Account</th>\n", + " <th>State</th>\n", + " <th>...</th>\n", + " <th>MaxDiskReadNode</th>\n", + " <th>MaxDiskReadTask</th>\n", + " <th>MaxDiskWrite</th>\n", + " <th>MaxDiskWriteNode</th>\n", + " <th>MaxDiskWriteTask</th>\n", + " <th>ReqGPUS</th>\n", + " <th>Comment</th>\n", + " <th>GPUMem</th>\n", + " <th>GPUEff</th>\n", + " <th>NGPU</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>3319116</td>\n", + " <td>3319116</td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>3319116_[43-45,47%5]</td>\n", + " <td>1mUD1MPa</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>PENDING</td>\n", + " <td>...</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>3927198</td>\n", + " <td>3887451</td>\n", + " <td>30.0</td>\n", + " <td>None</td>\n", + " <td>3887451_30</td>\n", + " <td>100kCrC20MPa</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>3927198</td>\n", + " <td>3887451</td>\n", + " <td>30.0</td>\n", + " <td>batch</td>\n", + " <td>3887451_30.batch</td>\n", + " <td>batch</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>1.222336e+10</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>3927198</td>\n", + " <td>3887451</td>\n", + " <td>30.0</td>\n", + " <td>extern</td>\n", + " <td>3887451_30.extern</td>\n", + " <td>extern</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>0.000000e+00</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>3927199</td>\n", + " <td>3887451</td>\n", + " <td>31.0</td>\n", + " <td>None</td>\n", + " <td>3887451_31</td>\n", + " <td>100kCrC20MPa</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>5 rows × 63 columns</p>\n", + "</div>" + ], + "text/plain": [ + " JobID ArrayJobID ArrayTaskID JobStep JobIDSlurm \\\n", + "0 3319116 3319116 NaN None 3319116_[43-45,47%5] \n", + "1 3927198 3887451 30.0 None 3887451_30 \n", + "2 3927198 3887451 30.0 batch 3887451_30.batch \n", + "3 3927198 3887451 30.0 extern 3887451_30.extern \n", + "4 3927199 3887451 31.0 None 3887451_31 \n", + "\n", + " JobName User Group Account State ... \\\n", + "0 1mUD1MPa user user user PENDING ... \n", + "1 100kCrC20MPa user user user COMPLETED ... \n", + "2 batch user COMPLETED ... \n", + "3 extern user COMPLETED ... \n", + "4 100kCrC20MPa user user user COMPLETED ... \n", + "\n", + " MaxDiskReadNode MaxDiskReadTask MaxDiskWrite MaxDiskWriteNode \\\n", + "0 NaN \n", + "1 NaN \n", + "2 c0088 0 1.222336e+10 c0088 \n", + "3 c0088 0 0.000000e+00 c0088 \n", + "4 NaN \n", + "\n", + " MaxDiskWriteTask ReqGPUS Comment GPUMem GPUEff NGPU \n", + "0 NaN None None None None \n", + "1 NaN None None None None \n", + "2 0 NaN None None None None \n", + "3 0 NaN None None None None \n", + "4 NaN None None None None \n", + "\n", + "[5 rows x 63 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1.head(5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:.conda-slurm-ds]", + "language": "python", + "name": "conda-env-.conda-slurm-ds-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}