CRI_XCBC/roles/gpu_build_vnfs/files/blacklist_modprobe

-blacklist nouveau

CRI_XCBC/roles/gpu_build_vnfs/files/create-cuda-devices.service

-[Unit]
-Description=Create_cuda_devices
-#After=
-Before=slurmd.service
-#Requires=
-
-[Service]
-Type=oneshot
-ExecStart=/bin/sh -c "/bin/nvidia-smi -c 1"
-
-[Install]
-WantedBy=multi-user.target

CRI_XCBC/roles/gpu_build_vnfs/files/gpu-nodes.tmpl

-#DESC: A clone of Red Hat Enterprise Linux 7
-
-# The general RHEL include has all of the necessary functions, but requires
-# some basic variables specific to each chroot type to be defined.
-# with additional procedure to add packages from variable EXTEND_COMPUTE
-. include-rhel-xcbc
-
-
-
-# Define the location of the YUM repository
-# YUM_MIRROR="http://mirror.centos.org/centos-7/7/os/\$basearch/"
-YUM_MIRROR="http://mirror.centos.org/centos-7/7/os/\$basearch/"
-
-# Include the basic packages
-. base_packages
-
-# Additional packages to get closer to the definition of compute node I had in rocks.
-. extend_compute_packages
-
-# include additional packages necessary for development environment
-. gpu_compute_packages
-
-ADDITIONALPACKAGES=( "$EXTEND_COMPUTE" "$GPU_COMPUTE" )
-
-
-# vim:filetype=sh:syntax=sh:expandtab:ts=4:sw=4:

CRI_XCBC/roles/gpu_build_vnfs/files/gpu_compute_packages

- GPU_COMPUTE="ElectricFence  GConf2-devel  PyQt4-devel  SDL-devel  alsa-lib-devel  babel  binutils-devel  boost-devel \
- boost-math  byacc  bzip2-devel  bzr  chrpath  cmake  compat-glibc  cscope \
- ctags  cups-devel  cvs-inetd  cyrus-sasl-devel  db4-devel  dbus-devel  dbus-glib-devel  dejagnu \
- diffstat  doxygen  e2fsprogs-devel  elfutils-devel  elfutils-libelf-devel  evolution-data-server-devel  expat-devel  freeglut-devel \
- freeipmi  fuse-devel  gcc-c++  gcc-gnat  gcc-objc  gcc-objc++  gdbm-devel  gdk-pixbuf2-devel \
- glib-networking  gmp-devel  gnutls-devel  gstreamer-devel  gstreamer-plugins-base-devel  gvfs-devel  hmaccalc  hunspell-devel \
- imake  indent  intltool  java-1.6.0-openjdk-devel  java-1.7.0-openjdk-devel  junit  kernel-abi-whitelists  ldapjdk \
- libarchive  libgdata  libgdata-devel  libXaw-devel  libXinerama-devel  libXmu-devel  libXpm-devel  libXrandr-devel \
- libacl-devel  libaio-devel  libattr-devel  libblkid-devel  libbonobo-devel  libcanberra-devel  libcap-devel  libcap-ng-devel \
- libcgroup-devel  libcurl-devel  libdrm-devel  libgcrypt-devel  libglade2-devel  libgnome-devel  libgnomeui-devel  libgudev1-devel \
- libhugetlbfs-devel  libnl-devel  libnotify-devel  librsvg2-devel  libselinux-devel  libsoup-devel  libstdc++-docs  libtiff-devel \
- libudev-devel  libusb-devel  libuuid-devel  libxml2-devel  libxslt-devel  mesa-private-llvm  mercurial  mod_dav_svn \
- nasm  ncurses-devel  net-snmp-devel  nspr-devel  nss-devel  nss-softokn-devel  nss-softokn-freebl-devel  nss-util-devel \
- numactl-devel  openmotif-devel  papi-devel  patchutils  pcre-devel  perl-Convert-ASN1  perl-Crypt-SSLeay  perl-XML-Dumper \
- perl-XML-Grove  perl-XML-NamespaceSupport  perl-XML-SAX  perl-XML-Twig  perl-libxml-perl  perltidy  polkit-devel  popt-devel \
- postgresql-devel  pulseaudio-libs-devel  pygtk2-devel  python-docs  python-ldap  readline-devel  sane-backends-devel  sqlite-devel \
- startup-notification-devel  subversion  systemtap  systemtap-sdt-devel  systemtap-server  tbb-devel  tcl-devel  tcp_wrappers-devel \
- texinfo  tk-devel  unique-devel  unixODBC-devel  xz-devel  ORBit2-devel  PyQt4  ant-antlr \
- ant-apache-bcel  ant-apache-bsf  ant-apache-log4j  ant-apache-oro  ant-apache-regexp  ant-apache-resolver  ant-commons-logging  ant-commons-net \
- ant-javamail  ant-jdepend  ant-jsch  ant-junit  ant-swing  antlr  apr-util  apr-util-ldap \
- atk-devel  avahi-glib  avalon-framework  batik  bcel  boost-date-time  boost-filesystem  boost-graph \
- boost-iostreams  boost-program-options  boost-python  boost-regex  boost-serialization  boost-signals  boost-test  boost-wave \
- bsf  check-devel  compat-glibc-headers  db4-cxx  docbook-dtds  docbook-style-dsssl  docbook-style-xsl  docbook-utils \
- ecj  enchant  evolution-data-server  evolution-data-server-doc  fakeroot  fakeroot-libs  file-devel  fop \
- fuse-libs  geronimo-specs  gettext-devel  gnome-icon-theme  gnome-keyring  gnome-keyring-devel  gnome-vfs2  gnome-vfs2-devel \
- gtk-doc  gtk2-devel  gtk2-engines  gvfs  hamcrest  hsqldb  httpd  httpd-tools \
- jakarta-commons-discovery  jakarta-commons-httpclient  jakarta-commons-io  jakarta-commons-lang  jakarta-commons-logging  jakarta-commons-net  jakarta-oro  java-1.6.0-openjdk-javadoc \
- java_cup  jdepend  jdom  jna  jsch  jss  junit4  jzlib \
- keyutils-libs-devel  krb5-devel  libICE-devel  libIDL-devel  libSM-devel  libXcomposite-devel  libXcursor-devel  libXdamage-devel \
- libXext-devel  libXfixes-devel  libXft-devel  libXi-devel  libXp-devel  libXres  libXt-devel  libXxf86vm-devel \
- libart_lgpl  libart_lgpl-devel  libatasmart  libbonobo  libbonoboui  libbonoboui-devel  libcanberra  libcanberra-gtk2 \
- libcdio  libcgroup  libcom_err-devel  libcroco-devel  liberation-fonts-common  liberation-sans-fonts  libexif  libexif-devel \
- libgnat  libgnat-devel  libgnome  libgnomecanvas  libgnomecanvas-devel  libgnomeui  libgpg-error-devel  libgphoto2 \
- libgphoto2-devel  libgsf  libgsf-devel  libgweather  libgweather-devel  libical  libical-devel  libidn-devel \
- libieee1284  libieee1284-devel  libnotify  librsvg2  libsepol-devel  libsoup  libstdc++-devel  libtasn1-devel \
- libtdb  libv4l  libwnck  lm_sensors-devel  lockdev  log4j  mesa-libGL-devel  mesa-libGLU-devel \
- mozilla-filesystem  neon  notification-daemon  objectweb-asm  openjade  opensp  openssl-devel  pakchois \
- pango-devel  perl-Compress-Zlib  perl-Error  perl-Git  perl-IO-Compress-Base  perl-IO-Compress-Zlib  perl-SGMLSpm  perl-XML-LibXML \
- perl-XML-Parser  polkit-docs  postgresql  postgresql-libs  pulseaudio-libs-glib2  pycairo-devel  pygobject2-codegen  pygobject2-devel \
- pygobject2-doc  pygtk2-codegen  pygtk2-doc  python-babel  python-enchant  python-magic  qdox  qt-devel \
- rarian  rarian-compat  regexp  rpm-devel  rpmdevtools  rpmlint  sac  sane-backends \
- sane-backends-libs  sip  sip-devel  slf4j  smp_utils  sound-theme-freedesktop  startup-notification  subversion-javahl \
- systemtap-client  systemtap-devel  systemtap-runtime  unique  unixODBC  w3m  ws-commons-util  ws-jaxme \
- wsdl4j  xalan-j2  xcb-util  xerces-j2  xml-commons-apis  xml-commons-resolver  xmlgraphics-commons  xmlrpc3-client \
- xmlrpc3-common  xulrunner  libvdpau libvdpau-devel dkms" 

CRI_XCBC/roles/gpu_build_vnfs/files/grub_default

-GRUB_CMDLINE_LINUX='quiet modprobe.blacklist=nouveau'

CRI_XCBC/roles/gpu_build_vnfs/files/rebuild-grub.service

-[Unit]
-Description=rebuild-grub
-#After=
-#Requires=
-
-[Service]
-Type=oneshot
-ExecStart=/bin/sh -c "if [[ -z $(grep nouveau /boot/grub2/grub.cfg) ]]; then /usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg; systemctl disable rebuild-grub;  fi"
-
-[Install]
-WantedBy=multi-user.target

CRI_XCBC/roles/gpu_build_vnfs/tasks/main.yml

----
-
-   - name: copy gpu_template.tmpl for wwmkchroot
-     copy: src=gpu-nodes.tmpl dest="{{ template_path }}{{ gpu_template }}.tmpl"
-
-   - name: copy gpu_compute_packages for wwmkchroot
-     copy: src=gpu_compute_packages dest="{{ template_path }}gpu_compute_packages"
-
-   - name: make chroot
-     command: wwmkchroot "{{ gpu_template }}" "{{ gpu_chroot_loc }}"
-
-   - name: copy resolve.conf into image
-     copy: src=/etc/resolv.conf dest="{{ gpu_chroot_loc }}/etc/resolv.conf" #"
-
-   - name: yum install into the image chroot
-     command: yum -y --installroot={{ gpu_chroot_loc }} install chrony kernel lmod-ohpc grub2 glibc-devel glibc-headers kernel-devel kernel-headers dkms gcc
-#" for vim
-
-   - name: yum install slurm client into image
-     command: yum -y --installroot={{ gpu_chroot_loc }} groupinstall ohpc-slurm-client
-
-   - name: create export dir
-     file: path="{{ gpu_chroot_loc }}/export" state=directory
-#" for vim
-#
-#     mount: src={{ item.src }} path={{ item.path }} opts=bind state=mounted fstype=none
-# commented ^ out because ansible adds these entries to fstab!!!
-   - name: mount necessary bits
-     command: mount -o rw,bind "{{ item.src }}" "{{ item.path }}"
-     with_items:
-      - { src: /proc/, path: "{{ gpu_chroot_loc }}/proc/" }
-      - { src: /dev/,  path: "{{ gpu_chroot_loc }}/dev/"  }
-      - { src: /export/,  path: "{{ gpu_chroot_loc }}/export/"  }
-      - { src: /var/run/utmp, path: "{{ gpu_chroot_loc }}/var/run/utmp" }
-
-   - name: get kernel version
-     shell: chroot "{{ gpu_chroot_loc }}" rpm -q --qf '%{version}-%{release}.%{arch}\n' kernel | tail -1
-     register: kernel_ver
-
-   - name: create build dir
-     file: path="{{ gpu_chroot_loc }}/root/nvidia_build/" state=directory
-
-   - name: get nvidia installation files
-     copy: src={{ item }} dest={{ gpu_chroot_loc }}/root/nvidia_build mode=0700
-     with_items:
-      - "{{ nvidia_driver_installer }}"
-      - "{{ cuda_toolkit_installer }}"
-      - "{{ cuda_samples_installer }}"
- 
-   - name: build nvidia kernel modules in the vnfs
-     shell: chroot {{ gpu_chroot_loc }} {{ nvidia_kernel_build_cmd }}
-     vars:
-       nvidia_kernel_build_cmd: "/root/nvidia_build/{{ nvidia_driver_installer }} -k {{ kernel_ver.stdout }} --kernel-install-path=/lib/modules/{{ kernel_ver.stdout }}/kernel/drivers/video -z -s -X"
-
-   - name: build cuda toolkit
-     shell: chroot {{ gpu_chroot_loc }} {{ cuda_toolkit_build_cmd }}
-     vars:
-       cuda_toolkit_build_cmd: "/root/nvidia_build/{{ cuda_toolkit_installer }} --noprompt --prefix=/export/cuda"
-
-   - name: build cuda samples
-     shell: chroot {{ gpu_chroot_loc }} {{ cuda_samples_build_cmd }}
-     vars:
-       cuda_samples_build_cmd: "/root/nvidia_build/{{ cuda_samples_installer }} --noprompt --prefix=/export/cuda/samples --cudaprefix=/export/cuda"
-
-#     mount: path={{ item.path }} state=unmounted
-   - name: unmount necessary bits
-     command: umount -l {{ item.path }}
-     with_items:
-      - { src: /proc/, path: "{{ gpu_chroot_loc }}/proc/" }
-      - { src: /dev/,  path: "{{ gpu_chroot_loc }}/dev/"  }
-      - { src: /export/,  path: "{{ gpu_chroot_loc }}/export/"  }
-      - { src: /var/run/utmp, path: "{{ gpu_chroot_loc }}/var/run/utmp" }
-
-   - name: blacklist nouveau in modprobe
-     copy: src=blacklist_modprobe dest={{ gpu_chroot_loc }}/etc/modprobe.d/blacklist.conf
-
-   - name: blacklist nouveau in grub
-     copy: src=grub_default dest={{ gpu_chroot_loc }}/etc/default/grub
-
-   - name: create rebuild-grub service
-     copy: src=rebuild-grub.service dest={{ gpu_chroot_loc }}/etc/systemd/system/rebuild-grub.service
- 
-   - name: enable rebuild-grub.service
-     command: chroot {{ gpu_chroot_loc }} systemctl enable rebuild-grub.service
-
-   - name: create service create-cuda-devices
-     copy: src=create-cuda-devices.service dest={{ gpu_chroot_loc }}/etc/systemd/system/create-cuda-devices.service
-
-   - name: enable create-cuda-devices
-     command: chroot {{ gpu_chroot_loc }} systemctl enable create-cuda-devices
-
-   - name: copy ssh keys over
-     copy: src=cluster_root.pub dest={{ gpu_chroot_loc }}/root/.ssh/authorized_keys
-
-   - name: put NFS home mount info in image
-     lineinfile: line="{{ headnode_private_ip }}:/home /home nfs nfsvers=3,rsize=1024,wsize=1024,cto 0 0" dest={{ gpu_chroot_loc }}/etc/fstab state=present
-
-   - name: put NFS opt mount info in image
-     lineinfile: line="{{ headnode_private_ip }}:/opt/ohpc/pub /opt/ohpc/pub-master nfs nfsvers=3 0 0" dest={{ gpu_chroot_loc }}/etc/fstab state=present
-
-   - name: put NFS opt mount info in image
-     lineinfile: line="{{ headnode_private_ip }}:/export /share nfs nfsvers=3 0 0" dest={{ gpu_chroot_loc }}/etc/fstab state=present
-
-   - name: chronyd on compute image enabled 
-     command: chroot '{{ gpu_chroot_loc }}' systemctl enable chronyd
-
-   - name: add headnode to compute chrony.conf
-     lineinfile: line="server {{ headnode_private_ip }}" dest={{ gpu_chroot_loc }}/etc/chrony.conf state=present
-
-   - name: wwimport file (gres.conf)
-     command: wwsh file import /etc/slurm/gres.conf
-
-   - name: wwimport file (passwd)
-     command: wwsh file import /etc/passwd
-
-   - name: wwimport file (group)
-     command: wwsh file import /etc/group
-
-   - name: wwimport file (shadow)
-     command: wwsh file import /etc/shadow
-
-   - name: wwimport file (slurm)
-     command: wwsh file import /etc/slurm/slurm.conf
-
-   - name: wwimport file (munge)
-     command: wwsh file import /etc/munge/munge.key
-
-   - name: build bootstrap image
-     shell: wwbootstrap {{ build_kernel_ver }}
-
-   - name: build the vnfs 
-     command: wwvnfs -y --chroot "{{ gpu_chroot_loc }}/"
-
-   - name: fix export of usr/include
-     lineinfile: line="hybridize=''" path=/etc/warewulf/vnfs/{{ gpu_chroot }}.conf state=present insertafter=EOF
-
-   - name: set up provisioning interface
-     lineinfile: line="GATEWAYDEV={{ private_interface }}" dest=/tmp/network.ww create=yes
-#" for vim
-#
-   - name: add network file to import
-     command: wwsh -y file import /tmp/network.ww --name network
-
-   - name: set network file path
-     command: wwsh -y file set network --path /etc/sysconfig/network --mode=0644 --uid=0
-
-   - name: add slurm.conf file to import
-     command: wwsh -y file import /etc/slurm/slurm.conf --name slurm.conf
-

CRI_XCBC/roles/login_build_nodes/tasks/main.yml

----
-
-#   - name: print single node info
-#     debug:
-#       var: item.mac
-#     with_items: "{{ login_nodes }}"
-
-   - name: add node to ww db
-     command: wwsh -y node new {{ item.name }} --ipaddr={{ item.ip }} --hwaddr={{ item.mac }} -D {{ private_interface }}
-     with_items: "{{ login_nodes }}"
-
-   - name: set nodes bootloader
-     command: wwsh -y object modify -s bootloader=sda -t node {{ item.name }}
-     with_items: "{{ login_nodes }}"
-
-   - name: set nodes partitions
-     command: wwsh -y object modify -s diskpartition=sda -t node {{ item.name }}
-     with_items: "{{ login_nodes }}"
-
-   - name: format partitions
-     command: wwsh -y object modify -s diskformat=sda1,sda2,sda3 -t node {{ item.name }}
-     with_items: "{{ login_nodes }}"
-
-   - name: define filesystems
-     command: wwsh -y object modify -s filesystems="{{ sda1 }},{{ sda2 }},{{ sda3 }}" -t node {{ item.name }}
-     with_items: "{{ login_nodes }}"
-#" for vim
-  
-   - name: remove node from slurm.conf if it exists already # to avoid duplication!
-     lineinfile:
-       dest: /etc/slurm/slurm.conf
-       regexp: "^NodeName={{ item.name }}"
-       state: absent
-     with_items: "{{ login_nodes }}"
-
-   - name: add node to slurm.conf
-     lineinfile:
-       dest: /etc/slurm/slurm.conf
-       line: "NodeName={{ item.name }} Sockets={{ item.sockets }} CoresPerSocket={{ item.corespersocket }} State=UNKNOWN"
-       insertbefore: "^# PARTITIONS"
-       state: present
-     with_items: "{{ login_nodes }}"
-
-   - name: set files to provision
-     command: wwsh -y provision set {{ item.name }} --vnfs={{ item.vnfs }} --bootstrap={{ build_kernel_ver }} --files=passwd,group,shadow,munge.key,slurm.conf,dynamic_hosts,network,gres.conf
-     with_items: "{{ login_nodes }}"
-
-   - name: restart dhcp
-     service: name=dhcpd state=restarted
- 
-   - name: update pxeconfig to let node boot from pxe
-     command: wwsh -y object modify -D bootlocal -t node {{ item.name }}
-     with_items: "{{ login_nodes }}"
- 
-   - name: wwsh pxe update
-     command: wwsh -v pxe update
-     register: command_result
-     failed_when: "'Building Pxelinux' not in command_result.stdout"

CRI_XCBC/roles/login_build_vnfs/files/login-nodes.tmpl

-#DESC: A clone of Red Hat Enterprise Linux 7
-
-# The general RHEL include has all of the necessary functions, but requires
-# some basic variables specific to each chroot type to be defined.
-# with additional procedure to add packages from variable EXTEND_COMPUTE
-. include-rhel-xcbc
-
-
-
-# Define the location of the YUM repository
-# YUM_MIRROR="http://mirror.centos.org/centos-7/7/os/\$basearch/"
-YUM_MIRROR="http://mirror.centos.org/centos-7/7/os/\$basearch/"
-
-# Include the basic packages
-. base_packages
-
-# Additional packages to get closer to the definition of compute node I had in rocks.
-. extend_compute_packages
-
-# include additional packages necessary for development environment
-. login_packages
-
-ADDITIONALPACKAGES=( "$EXTEND_COMPUTE" "$LOGIN_PACKAGES" )
-
-
-# vim:filetype=sh:syntax=sh:expandtab:ts=4:sw=4:

CRI_XCBC/roles/login_build_vnfs/files/login_packages

- LOGIN_PACKAGES="ElectricFence  GConf2-devel  PyQt4-devel  SDL-devel  alsa-lib-devel  babel  binutils-devel  boost-devel \
- boost-math  byacc  bzip2-devel  bzr  chrpath  cmake  compat-glibc  cscope \
- ctags  cups-devel  cvs-inetd  cyrus-sasl-devel  db4-devel  dbus-devel  dbus-glib-devel  dejagnu \
- diffstat  doxygen  e2fsprogs-devel  elfutils-devel  elfutils-libelf-devel  evolution-data-server-devel  expat-devel  freeglut-devel \
- freeipmi  fuse-devel  gcc-c++  gcc-gnat  gcc-objc  gcc-objc++  gdbm-devel  gdk-pixbuf2-devel \
- glib-networking  gmp-devel  gnutls-devel  gstreamer-devel  gstreamer-plugins-base-devel  gvfs-devel  hmaccalc  hunspell-devel \
- imake  indent  intltool  java-1.6.0-openjdk-devel  java-1.7.0-openjdk-devel  junit  kernel-abi-whitelists  ldapjdk \
- libarchive  libgdata  libgdata-devel  libXaw-devel  libXinerama-devel  libXmu-devel  libXpm-devel  libXrandr-devel \
- libacl-devel  libaio-devel  libattr-devel  libblkid-devel  libbonobo-devel  libcanberra-devel  libcap-devel  libcap-ng-devel \
- libcgroup-devel  libcurl-devel  libdrm-devel  libgcrypt-devel  libglade2-devel  libgnome-devel  libgnomeui-devel  libgudev1-devel \
- libhugetlbfs-devel  libnl-devel  libnotify-devel  librsvg2-devel  libselinux-devel  libsoup-devel  libstdc++-docs  libtiff-devel \
- libudev-devel  libusb-devel  libuuid-devel  libxml2-devel  libxslt-devel  mesa-private-llvm  mercurial  mod_dav_svn \
- nasm  ncurses-devel  net-snmp-devel  nspr-devel  nss-devel  nss-softokn-devel  nss-softokn-freebl-devel  nss-util-devel \
- numactl-devel  openmotif-devel  papi-devel  patchutils  pcre-devel  perl-Convert-ASN1  perl-Crypt-SSLeay  perl-XML-Dumper \
- perl-XML-Grove  perl-XML-NamespaceSupport  perl-XML-SAX  perl-XML-Twig  perl-libxml-perl  perltidy  polkit-devel  popt-devel \
- postgresql-devel  pulseaudio-libs-devel  pygtk2-devel  python-docs  python-ldap  readline-devel  sane-backends-devel  sqlite-devel \
- startup-notification-devel  subversion  systemtap  systemtap-sdt-devel  systemtap-server  tbb-devel  tcl-devel  tcp_wrappers-devel \
- texinfo  tk-devel  unique-devel  unixODBC-devel  xz-devel  ORBit2-devel  PyQt4  ant-antlr \
- ant-apache-bcel  ant-apache-bsf  ant-apache-log4j  ant-apache-oro  ant-apache-regexp  ant-apache-resolver  ant-commons-logging  ant-commons-net \
- ant-javamail  ant-jdepend  ant-jsch  ant-junit  ant-swing  antlr  apr-util  apr-util-ldap \
- atk-devel  avahi-glib  avalon-framework  batik  bcel  boost-date-time  boost-filesystem  boost-graph \
- boost-iostreams  boost-program-options  boost-python  boost-regex  boost-serialization  boost-signals  boost-test  boost-wave \
- bsf  check-devel  compat-glibc-headers  db4-cxx  docbook-dtds  docbook-style-dsssl  docbook-style-xsl  docbook-utils \
- ecj  enchant  evolution-data-server  evolution-data-server-doc  fakeroot  fakeroot-libs  file-devel  fop \
- fuse-libs  geronimo-specs  gettext-devel  gnome-icon-theme  gnome-keyring  gnome-keyring-devel  gnome-vfs2  gnome-vfs2-devel \
- gtk-doc  gtk2-devel  gtk2-engines  gvfs  hamcrest  hsqldb  httpd  httpd-tools \
- jakarta-commons-discovery  jakarta-commons-httpclient  jakarta-commons-io  jakarta-commons-lang  jakarta-commons-logging  jakarta-commons-net  jakarta-oro  java-1.6.0-openjdk-javadoc \
- java_cup  jdepend  jdom  jna  jsch  jss  junit4  jzlib \
- keyutils-libs-devel  krb5-devel  libICE-devel  libIDL-devel  libSM-devel  libXcomposite-devel  libXcursor-devel  libXdamage-devel \
- libXext-devel  libXfixes-devel  libXft-devel  libXi-devel  libXp-devel  libXres  libXt-devel  libXxf86vm-devel \
- libart_lgpl  libart_lgpl-devel  libatasmart  libbonobo  libbonoboui  libbonoboui-devel  libcanberra  libcanberra-gtk2 \
- libcdio  libcgroup  libcom_err-devel  libcroco-devel  liberation-fonts-common  liberation-sans-fonts  libexif  libexif-devel \
- libgnat  libgnat-devel  libgnome  libgnomecanvas  libgnomecanvas-devel  libgnomeui  libgpg-error-devel  libgphoto2 \
- libgphoto2-devel  libgsf  libgsf-devel  libgweather  libgweather-devel  libical  libical-devel  libidn-devel \
- libieee1284  libieee1284-devel  libnotify  librsvg2  libsepol-devel  libsoup  libstdc++-devel  libtasn1-devel \
- libtdb  libv4l  libwnck  lm_sensors-devel  lockdev  log4j  mesa-libGL-devel  mesa-libGLU-devel \
- mozilla-filesystem  neon  notification-daemon  objectweb-asm  openjade  opensp  openssl-devel  pakchois \
- pango-devel  perl-Compress-Zlib  perl-Error  perl-Git  perl-IO-Compress-Base  perl-IO-Compress-Zlib  perl-SGMLSpm  perl-XML-LibXML \
- perl-XML-Parser  polkit-docs  postgresql  postgresql-libs  pulseaudio-libs-glib2  pycairo-devel  pygobject2-codegen  pygobject2-devel \
- pygobject2-doc  pygtk2-codegen  pygtk2-doc  python-babel  python-enchant  python-magic  qdox  qt-devel \
- rarian  rarian-compat  regexp  rpm-devel  rpmdevtools  rpmlint  sac  sane-backends \
- sane-backends-libs  sip  sip-devel  slf4j  smp_utils  sound-theme-freedesktop  startup-notification  subversion-javahl \
- systemtap-client  systemtap-devel  systemtap-runtime  unique  unixODBC  w3m  ws-commons-util  ws-jaxme \
- wsdl4j  xalan-j2  xcb-util  xerces-j2  xml-commons-apis  xml-commons-resolver  xmlgraphics-commons  xmlrpc3-client \
- xmlrpc3-common  xulrunner  libvdpau libvdpau-devel dkms" 

CRI_XCBC/roles/login_build_vnfs/tasks/main.yml

----
-
-   - name: copy login_template.tmpl for wwmkchroot
-     copy: src=login-nodes.tmpl dest="{{ template_path }}{{ login_template }}.tmpl"
-
-   - name: copy login_packages for wwmkchroot
-     copy: src=login_packages dest="{{ template_path }}login_packages"
-
-   - name: make chroot
-     command: wwmkchroot "{{ login_template }}" "{{ login_chroot_loc }}"
-
-   - name: copy resolve.conf into image
-     copy: src=/etc/resolv.conf dest="{{ login_chroot_loc }}/etc/resolv.conf" #"
-
-   - name: yum install into the image chroot
-     command: yum -y --installroot={{ login_chroot_loc }} install chrony kernel lmod-ohpc grub2 glibc-devel glibc-headers kernel-devel kernel-headers dkms
-#" for vim
-
-   - name: yum install slurm client into image
-     command: yum -y --installroot={{ login_chroot_loc }} groupinstall ohpc-slurm-client
-#" for vim
-#
-##FIREWALL STUFF - DON'T LEAVE COMMENTED OUT
-#   - template: src=firewalld.conf dest={{ login_chroot_loc }}/etc/firewalld/firewalld.conf
-#
-#   - template: src=zones/firewall_public.xml.j2 dest={{ login_chroot_loc }}/etc/firewalld/zones/public.xml
-#
-#   - template: src=services/slurm.xml dest={{ login_chroot_loc }}/etc/firewalld/services/slurm.xml
-#
-#   - template: src=zones/firewall_internal.xml.j2 dest={{ login_chroot_loc }}/etc/firewalld/zones/internal.xml
- 
-   - name: copy ssh keys over
-     copy: src=cluster_root.pub dest={{ login_chroot_loc }}/root/.ssh/authorized_keys
-
-   - name: put NFS home mount info in image
-     lineinfile: line="{{ headnode_private_ip }}:/home /home nfs nfsvers=3,rsize=1024,wsize=1024,cto 0 0" dest={{ login_chroot_loc }}/etc/fstab state=present
-
-   - name: put NFS opt mount info in image
-     lineinfile: line="{{ headnode_private_ip }}:/opt/ohpc/pub /opt/ohpc/pub-master nfs nfsvers=3 0 0" dest={{ login_chroot_loc }}/etc/fstab state=present
-
-   - name: put NFS opt mount info in image
-     lineinfile: line="{{ headnode_private_ip }}:/export /share nfs nfsvers=3 0 0" dest={{ login_chroot_loc }}/etc/fstab state=present
-
-   - name: chronyd on login image enabled 
-     command: chroot '{{ login_chroot_loc }}' systemctl enable chronyd
-
-   - name: add headnode to compute chrony.conf
-     lineinfile: line="server {{ headnode_private_ip }}" dest={{ login_chroot_loc }}/etc/chrony.conf state=present
-
-   - name: wwimport file (gres.conf)
-     command: wwsh file import /etc/slurm/gres.conf
-
-   - name: wwimport file (passwd)
-     command: wwsh file import /etc/passwd
-
-   - name: wwimport file (group)
-     command: wwsh file import /etc/group
-
-   - name: wwimport file (shadow)
-     command: wwsh file import /etc/shadow
-
-   - name: wwimport file (slurm)
-     command: wwsh file import /etc/slurm/slurm.conf
-
-   - name: wwimport file (munge)
-     command: wwsh file import /etc/munge/munge.key
-
-   - name: build bootstrap image
-     shell: wwbootstrap {{ build_kernel_ver }}
- 
-   - name: fix export of usr/include
-     lineinfile: line="hybridize=''" dest=/etc/warewulf/vnfs/{{ login_chroot }}.conf state=present insertafter=EOF create=yes
-
-   - name: build the vnfs 
-     command: wwvnfs -y --chroot "{{ login_chroot_loc }}/"
-
-   - name: set up provisioning interface
-     lineinfile: line="GATEWAYDEV={{ private_interface }}" dest=/tmp/network.ww create=yes
-#" for vim
-#
-   - name: add network file to import
-     command: wwsh -y file import /tmp/network.ww --name network
-
-   - name: set network file path
-     command: wwsh -y file set network --path /etc/sysconfig/network --mode=0644 --uid=0
-
-   - name: add slurm.conf file to import
-     command: wwsh -y file import /etc/slurm/slurm.conf --name slurm.conf
-

CRI_XCBC/roles/login_build_vnfs/templates/firewalld.conf

-# firewalld config file
-
-# default zone
-# The default zone used if an empty zone string is used.
-# Default: public
-DefaultZone=public
-
-# Minimal mark
-# Marks up to this minimum are free for use for example in the direct 
-# interface. If more free marks are needed, increase the minimum
-# Default: 100
-MinimalMark=100
-
-# Clean up on exit
-# If set to no or false the firewall configuration will not get cleaned up
-# on exit or stop of firewalld
-# Default: yes
-CleanupOnExit=yes
-
-# Lockdown
-# If set to enabled, firewall changes with the D-Bus interface will be limited
-# to applications that are listed in the lockdown whitelist.
-# The lockdown whitelist file is lockdown-whitelist.xml
-# Default: no
-Lockdown=no
-
-# IPv6_rpfilter
-# Performs a reverse path filter test on a packet for IPv6. If a reply to the
-# packet would be sent via the same interface that the packet arrived on, the 
-# packet will match and be accepted, otherwise dropped.
-# The rp_filter for IPv4 is controlled using sysctl.
-# Default: yes
-IPv6_rpfilter=yes

CRI_XCBC/roles/login_build_vnfs/templates/services/slurm.xml

-<?xml version="1.0" encoding="utf-8"?>
-<service>
-  <short>Slurmctld</short>
-  <description> Slurmctld is the controller daemon for the Slurm 
-   scheduler, which communicates with slurmd on controlled nodes.
-  </description>
-  <port protocol="tcp" port="6817"/>
-  <port protocol="udp" port="6817"/>
-  <port protocol="tcp" port="6818"/>
-  <port protocol="udp" port="6818"/>
-  <port protocol="tcp" port="7321"/>
-  <port protocol="udp" port="7321"/>
-</service>

CRI_XCBC/roles/login_build_vnfs/templates/zones/external.xml

-<?xml version="1.0" encoding="utf-8"?>
-<zone>
-  <short>External</short>
-  <description>For use on external networks. You do not trust the other computers on networks to not harm your computer. Only selected incoming connections are accepted.</description>
-  <service name="ssh"/>
-  <masquerade/>
-</zone>

CRI_XCBC/roles/login_build_vnfs/templates/zones/internal.xml

-<?xml version="1.0" encoding="utf-8"?>
-<zone>
-  <short>Internal</short>
-  <description>For use on internal networks. You mostly trust the other computers on the networks to not harm your computer. Only selected incoming connections are accepted.</description>
-  <interface name="{{ login_private_interface }}"/>
-  <service name="mdns"/>
-  <service name="ipp-client"/>
-  <service name="dhcpv6-client"/>
-  <service name="ssh"/>
-  <service name="nfs"/>
-  <service name="mountd"/>
-  <service name="rpc-bind"/>
-  <service name="http"/>
-  <service name="https"/>
-  <service name="tftp"/>
-  <service name="samba-client"/>
-</zone>

CRI_XCBC/roles/login_build_vnfs/templates/zones/public.xml

-<?xml version="1.0" encoding="utf-8"?>
-<zone>
-  <short>Public</short>
-  <description>For use in public areas. You do not trust the other computers on networks to not harm your computer. Only selected incoming connections are accepted.</description>
-  <interface name="{{ login_public_interface }}"/>
-  <service name="ssh"/>
-</zone>

CRI_XCBC/roles/nodes_vivify/tasks/main.yml

----
-
-   - name: Waiting for the compute node to bootup
-     pause:
-       seconds: 150
-
-   - name: tell nodes to get files
-     command: pdsh -w $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') /warewulf/bin/wwgetfiles
-
-   - name: start munge
-     service: name=munge state=restarted enabled=yes
-
-   - name: start slurmctld
-     service: name=slurmctld state=restarted enabled=yes
-
-   - name: start munge on compute nodes
-     shell: pdsh -w $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') systemctl start munge
-
-   - name: enable munge on compute nodes
-     shell: pdsh -w $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') systemctl enable munge
-
-   - name: start slurmd on compute nodes
-     shell: pdsh -w $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') systemctl start slurmd
-
-   - name: enable slurmd on compute nodes
-     shell: pdsh -w $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') systemctl enable slurmd
-
-   - name: Waiting for slurmd to enable on the compute nodes
-     pause:
-       seconds: 30
-
-   - name: update slurm status on nodes
-     shell: scontrol update nodename=$(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ',') state=IDLE
-
-   - name: update pxeconfig to let node boot locally
-     shell: wwsh -y object modify -s bootlocal=EXIT -t node $(wwsh node list | tail --lines=+3 | cut -f 1 -d' '| tr '\n' ' ') 
-     when: stateful_nodes == true

CRI_XCBC/roles/ohpc_add_easybuild/tasks/main.yaml

----
-- name: Install pip for system python and build tools
-  yum:
-    state: present
-    name: 
-      - '@Development Tools'
-      - python-pip
-
-- name: Create user build
-  user:
-    name: build
-
-- name: Change owner and permission of easybuild installation directory
-  file:
-    path: "{{ easybuild_prefix }}"
-    owner: root
-    group: build
-    mode: "g+w"
-    recurse: yes
-
-# EasyBuild bootstrap will fail with outdated setuptools on CentOS
-# This is a work-around that updates the sytem installed setuptools
-# Ideal solution is to provide a python specific to EasyBuild
-# likely via software collections
-- name: Fix EasyBuild setuptools dependency on version > 17.1
-  shell: pip install -U setuptools
-
-- name: Download EasyBuild bootstrap script
-  get_url:
-    url: https://raw.githubusercontent.com/easybuilders/easybuild-framework/develop/easybuild/scripts/bootstrap_eb.py
-    dest: /tmp/bootstrap_eb.py
-
-- name: Boostrap EasyBuild
-  shell: |
-      source /opt/ohpc/admin/lmod/lmod/init/bash
-      python bootstrap_eb.py "{{ easybuild_prefix }}"
-  become_user: build
-  args:
-    executable: /bin/bash
-    chdir: /tmp
-
-- name: Check if EasyBuild installation path is in default MODULEPATH
-  shell: grep "{{ easybuild_prefix }}/modules/all" /etc/profile.d/lmod.sh
-  ignore_errors: yes
-  register: default_modulepath_sh
-
-- shell: grep "{{ easybuild_prefix }}/modules/all" /etc/profile.d/lmod.csh
-  ignore_errors: yes
-  register: default_modulepath_csh
-
-- name: Add EasyBuild installation path into default MODULEPATH
-  replace:
-    path: /etc/profile.d/lmod.sh
-    regexp: '(MODULEPATH=.*)'
-    replace: '\1:{{ easybuild_prefix }}/modules/all'
-    backup: yes
-  when: default_modulepath_sh.stdout == ""
-
-- name: Add EasyBuild installation path into default MODULEPATH
-  replace:
-    path: /etc/profile.d/lmod.csh
-    regexp: '(MODULEPATH "[^"]*)'
-    replace: '\1:{{ easybuild_prefix }}/modules/all'
-    backup: yes
-  when: default_modulepath_csh.stdout == ""
-
-- name: Update file in warewulf file database
-  command: wwsh file sync
-
-- name: Verify EasyBuild installation
-  shell: |
-      source /opt/ohpc/admin/lmod/lmod/init/bash
-      module use "{{ easybuild_prefix }}/modules/all"
-      module load EasyBuild
-      eb --version
-

CRI_XCBC/roles/ohpc_add_rstudio/tasks/main.yaml

----
-- name: Install Singularity
-  shell: |
-    source /etc/profile.d/lmod.sh
-    export EASYBUILD_PREFIX={{ easybuild_prefix }}
-    module load EasyBuild
-    eb Singularity-{{ singularity_ver }}-GCC-5.4.0-2.26.eb --try-toolchain-name=dummy -r --force
-  become_user: build
-  args:
-    executable: /bin/bash
-    chdir: /tmp
-
-- name: Singularity post install steps(1/3) - chown
-  file:
-    path: "{{ easybuild_prefix }}/software/Singularity/{{ item }}"
-    owner: root
-    group: root
-  with_items:
-    - "{{ singularity_ver }}/etc/singularity/singularity.conf"
-    - "{{ singularity_ver }}/libexec/singularity/bin/action-suid"
-    - "{{ singularity_ver }}/libexec/singularity/bin/mount-suid"
-    - "{{ singularity_ver }}/libexec/singularity/bin/start-suid"
-    - "{{ singularity_ver }}/var/singularity/mnt/container"
-
-- name: Singularity post install steps(2/3) - chmod
-  file:
-    path: "{{ easybuild_prefix }}/software/Singularity/{{ item }}"
-    mode: '4755'
-  with_items:
-    - "{{ singularity_ver }}/libexec/singularity/bin/action-suid"
-    - "{{ singularity_ver }}/libexec/singularity/bin/mount-suid"
-    - "{{ singularity_ver }}/libexec/singularity/bin/start-suid"
-
-- name: Singularity post install steps(3/3) - chmod
-  file:
-    path: "{{ easybuild_prefix }}/software/Singularity/{{ item }}"
-    mode: '+s'
-  with_items:
-    - "{{ singularity_ver }}/libexec/singularity/bin/action-suid"
-    - "{{ singularity_ver }}/libexec/singularity/bin/mount-suid"
-    - "{{ singularity_ver }}/libexec/singularity/bin/start-suid"
-
-- name: Clone template modulefiles from upstream
-  git:
-    repo: 'https://github.com/nickjer/singularity-rstudio.git'
-    dest: '/tmp/singularity-rstudio'
-    force: yes
-  become_user: build
-
-- name: Delete folder and modulefile of old version
-  file:
-    path: "{{ item }}"
-    state: absent
-  with_items:
-    - "/tmp/singularity-rstudio/example_module/3.4.3"
-    - "/tmp/singularity-rstudio/example_module/modulefiles/rstudio_singularity/3.4.3.lua"
-
-- name: Create folder for different R versions
-  file:
-    path: /tmp/singularity-rstudio/example_module/{{ item.full }}
-    state: directory
-  with_items:
-    "{{ r_versions }}"
-  become_user: build
-
-- name: Create library folder for different R versions
-  file:
-    path: /tmp/singularity-rstudio/example_module/library-{{ item.short }}
-    state: directory
-  with_items:
-    "{{ r_versions }}"
-  become_user: build
-
-- name: Download Rstudio singularity image
-  shell: |
-    source /etc/profile.d/lmod.sh
-    module load Singularity
-    SINGULARITY_PULLFOLDER={{ item.full }} singularity pull --name singularity-rstudio.simg shub://nickjer/singularity-rstudio:{{ item.full }}
-  with_items:
-    "{{ r_versions }}"
-  become_user: build
-  args:
-    chdir: /tmp/singularity-rstudio/example_module
-
-- name: Generate modulefiles for each version
-  template:
-    src: modulefiles/rstudio.j2
-    dest: /tmp/singularity-rstudio/example_module/modulefiles/rstudio_singularity/{{ item.full }}.lua
-  vars:
-    ver: "{{ item }}"
-  with_items:
-    "{{ r_versions }}"
-
-- name: Move modulefiles into shared folder
-  command: mv {{ src }} {{ dest }}
-  args:
-    creates: "{{ dest }}"
-    removes: "{{ src }}"
-  vars:
-    src: /tmp/singularity-rstudio/example_module/modulefiles/rstudio_singularity
-    dest: "{{ easybuild_prefix }}/modules/all/rstudio_singularity"
-
-- file:
-    path: /tmp/singularity-rstudio/example_module/modulefiles
-    state: absent
-
-- name: Move Singularity image and executable into shared folder
-  command: mv {{ src }} {{ dest }}
-  args:
-    creates: "{{ dest }}"
-    removes: "{{ src }}"
-  vars:
-    src: /tmp/singularity-rstudio/example_module
-    dest: "{{ easybuild_prefix }}/software/singularity-rstudio"
-
-- name: Clean up
-  file:
-    path: /tmp/singularity-rstudio
-    state: absent

CRI_XCBC/roles/ohpc_add_rstudio/templates/modulefiles/rstudio.j2

-help([[
-This module loads the RStudio Server environment which utilizes a Singularity
-image for portability.
-]])
-
-whatis([[Description: RStudio Server environment using Singularity]])
-
-local root = "{{ easybuild_prefix }}/software/singularity-rstudio"
-local bin = pathJoin(root, "/bin")
-local img = pathJoin(root, "/{{ ver.full }}/singularity-rstudio.simg")
-local library = pathJoin(root, "/library-{{ ver.short }}")
-local host_mnt = "/mnt"
-
-local user_library = os.getenv("HOME") .. "/R/library-{{ ver.short }}"
-
-load("Singularity")
-prepend_path("PATH", bin)
-prepend_path("RSTUDIO_SINGULARITY_BINDPATH", "/:" .. host_mnt, ",")
-prepend_path("RSTUDIO_SINGULARITY_BINDPATH", library .. ":/library", ",")
-setenv("RSTUDIO_SINGULARITY_IMAGE", img)
-setenv("RSTUDIO_SINGULARITY_HOST_MNT", host_mnt)
-setenv("RSTUDIO_SINGULARITY_CONTAIN", "1")
-setenv("RSTUDIO_SINGULARITY_HOME", os.getenv("HOME") .. ":/home/" .. os.getenv("USER"))
-setenv("R_LIBS_USER", pathJoin(host_mnt, user_library))
-
--- Note: Singularity on CentOS 6 fails to bind a directory to `/tmp` for some
--- reason. This is necessary for RStudio Server to work in a multi-user
--- environment. So to get around this we use a combination of:
---
---   - SINGULARITY_CONTAIN=1 (containerize /home, /tmp, and /var/tmp)
---   - SINGULARITY_HOME=$HOME (set back the home directory)
---   - SINGUARLITY_WORKDIR=$(mktemp -d) (bind a temp directory for /tmp and /var/tmp)
---
--- The last one is called from within the executable scripts found under `bin/`
--- as it makes the temp directory at runtime.
---
--- If your system does successfully bind a directory over `/tmp`, then you can
--- probably get away with just:
---
---   - SINGULARITY_BINDPATH=$(mktemp -d):/tmp,$SINGULARITY_BINDPATH