From 2ef47c76d2ee8b4aa550cfd985997e640000049b Mon Sep 17 00:00:00 2001
From: Cloud User <centos@ohpc.novalocal>
Date: Fri, 20 Mar 2020 18:34:12 +0000
Subject: [PATCH 001/188] Add agent to record the user reg dict object
---
user_reg_logger.py | 39 +++++++++++++++++++++++++++++++++++++++
1 file changed, 39 insertions(+)
create mode 100755 user_reg_logger.py
diff --git a/user_reg_logger.py b/user_reg_logger.py
new file mode 100755
index 0000000..6d80bd4
--- /dev/null
+++ b/user_reg_logger.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+import json
+import sys
+import dataset
+import logging
+from rc_rmq import RCRMQ
+
+# Define queue name
+task = 'reg_logger'
+
+# Instantiate logging object
+logging.basicConfig(format='%(asctime)s[%(levelname)s] - %(message)s', level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Instantiate rabbitmq object
+reg_logger = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+
+# Open registry table in DB
+db = dataset.connect('sqlite:///reg_logger.db')
+table = db['registry']
+
+# Define registration logger callback
+def log_registration(ch, method, properties, body):
+
+ account_req = json.loads(body)
+ table.insert(account_req)
+ logger.info("logged account request for %s", account_req['username'])
+
+ ch.basic_ack(delivery_tag=method.delivery_tag)
+
+logger.info("Start listening to queue: {}".format(task))
+
+# Start consuming messages from queue with callback function
+reg_logger.start_consume({
+ 'queue': task,
+ 'routing_key': "create.*",
+ 'cb': log_registration
+})
+
--
GitLab
From 02f5b2bb9bd6adbbb91c4351652944d974583d20 Mon Sep 17 00:00:00 2001
From: atlurie <atlurie@uab.edu>
Date: Sat, 21 Mar 2020 06:16:43 +0000
Subject: [PATCH 002/188] RabbitMQ agent to get next available UID,GID
This will inititate user creation in BrightCM
---
get-next-uid-gid.py | 80 +++++++++++++++++++++++++++++++++++++++++++++
requirements.txt | 1 +
2 files changed, 81 insertions(+)
create mode 100644 get-next-uid-gid.py
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
new file mode 100644
index 0000000..4afa8ad
--- /dev/null
+++ b/get-next-uid-gid.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+import sys
+import json
+import ldap
+from os import popen
+from rc_rmq import RCRMQ
+
+task = 'get_next_uid_gid'
+
+# Instantiate rabbitmq object
+rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+
+#Check if the username already exists via LDAP
+def user_exists(username):
+ try:
+ con = ldap.initialize('ldap://ldapserver')
+ ldap_base = "dc=cm,dc=cluster"
+ query = "(uid={})".format(username)
+ result = con.search_s(ldap_base, ldap.SCOPE_SUBTREE, query)
+ return result
+ except:
+ e = sys.exc_info()[0]
+ print("[{}]: Error: {}".format(task, e))
+
+# Define your callback function
+def get_next_uid_gid(ch, method, properties, body):
+
+ # Retrieve message
+ msg = json.loads(body)
+ print("Message received {}".format(msg))
+ username = msg['username']
+ success = False
+
+ # Determine next available UID
+ try:
+ if user_exists(username):
+ print("The user, {} already exists".format(username))
+ sys.exit(1)
+
+ cmd_uid = "/usr/bin/getent passwd | \
+ awk -F: '($3>10000) && ($3<20000) && ($3>maxuid) { maxuid=$3; } END { print maxuid+1; }'"
+
+ msg['uid'] = popen(cmd_uid).read().rstrip()
+
+ cmd_gid = "/usr/bin/getent group | \
+ awk -F: '($3>10000) && ($3<20000) && ($3>maxgid) { maxgid=$3; } END { print maxgid+1; }'"
+
+ msg['gid'] = popen(cmd_gid).read().rstrip()
+ success = True
+ except:
+ e = sys.exc_info()[0]
+ print("[{}]: Error: {}".format(task, e))
+
+ # Acknowledge message
+ ch.basic_ack(delivery_tag=method.delivery_tag)
+
+ # Send confirm message
+ rc_rmq.publish_msg({
+ 'routing_key': 'confirm.' + username,
+ 'msg': {
+ 'task': task,
+ 'success': success
+ }
+ })
+
+ if success:
+ # Send create message to BrightCM agent
+ rc_rmq.publish_msg({
+ 'routing_key': 'create.' + username,
+ 'msg': msg
+ })
+
+print("Start listening to queue: {}".format(task))
+rc_rmq.start_consume({
+ 'queue': task,
+ 'routing_key': "request.*",
+ 'cb': get_next_uid_gid
+})
+
+rc_rmq.disconnect()
diff --git a/requirements.txt b/requirements.txt
index becc2ce..4fd3ec6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
pika==1.1.0
+pyldap==3.0.0.post1
--
GitLab
From 54ba2a33860ac4fa0274cf3a173112cbd7179556 Mon Sep 17 00:00:00 2001
From: atlurie <atlurie@uab.edu>
Date: Fri, 20 Mar 2020 20:50:01 +0000
Subject: [PATCH 003/188] RabbitMQ agent to create a user in BrightCM.
---
brightcm_account_create.py | 61 ++++++++++++++++++++++++++++++++++++++
1 file changed, 61 insertions(+)
create mode 100644 brightcm_account_create.py
diff --git a/brightcm_account_create.py b/brightcm_account_create.py
new file mode 100644
index 0000000..3d96949
--- /dev/null
+++ b/brightcm_account_create.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+import sys
+import json
+from os import popen
+from rc_rmq import RCRMQ
+
+task = 'bright_account'
+
+# Instantiate rabbitmq object
+rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+
+# Define your callback function
+def bright_account_create(ch, method, properties, body):
+ # Retrieve message
+ msg = json.loads(body)
+ print("Received msg{}".format(msg))
+ username = msg['username']
+ uid = msg['uid']
+ email = msg['email']
+ fullname = msg['fullname']
+ success = False
+ try:
+ # Bright command to create user
+ cmd = '/cm/local/apps/cmd/bin/cmsh -c '
+ cmd += f'"user; add {username}; set userid {uid}; set email {email}; set commonname \\"{fullname}\\"; '
+ cmd += 'commit;"'
+
+ #popen(cmd)
+ print(cmd)
+ success = true
+ except:
+ e = sys.exc_info()[0]
+ print("[{}]: Error: {}".format(task, e))
+
+ # Acknowledge message
+ ch.basic_ack(delivery_tag=method.delivery_tag)
+
+ # send confirm message
+ rc_rmq.publish_msg({
+ 'routing_key': 'confirm.' + username,
+ 'msg': {
+ 'task': task,
+ 'success': success
+ }
+ })
+
+ if success:
+ # send create message to verify dir permissions agent
+ rc_rmq.publish_msg({
+ 'routing_key': 'verify.' + username,
+ 'msg': msg
+ })
+
+print("Start listening to queue: {}".format(task))
+rc_rmq.start_consume({
+ 'queue': task,
+ 'routing_key': "create.*",
+ 'cb': bright_account_create
+})
+
+rc_rmq.disconnect()
--
GitLab
From ed4d951077b0ebefcfabf400f7e3298e55c2baad Mon Sep 17 00:00:00 2001
From: atlurie <atlurie@uab.edu>
Date: Fri, 20 Mar 2020 21:07:58 +0000
Subject: [PATCH 004/188] RabbitMQ agent to subscribe a user to mailing lists.
---
subscribe_mail_lists.py | 71 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 71 insertions(+)
create mode 100644 subscribe_mail_lists.py
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
new file mode 100644
index 0000000..400608a
--- /dev/null
+++ b/subscribe_mail_lists.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+import sys
+import json
+import smtplib
+from email.message import EmailMessage
+from rc_rmq import RCRMQ
+
+task = 'subscribe_mail_list'
+
+# Instantiate rabbitmq object
+rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+
+# Define your callback function
+def mail_list_subscription(ch, method, properties, body):
+
+ # Retrieve message
+ msg = json.loads(body)
+ print("Received msg {}".format(msg))
+ username = msg['username']
+ fullname = msg['fullname']
+ user_email = msg['email']
+
+ mail_list_admin = 'admin@uab.edu' #change this during deploy
+ mail_list = 'LISTSERV@LISTSERV.UAB.EDU'
+
+ listserv_cmd = {}
+ listserv_cmd['hpc_announce'] = f'QUIET ADD hpc-announce {email} {fullname}'
+ listserv_cmd['hpc_users'] = f'QUIET ADD hpc-users {email} {fullname}'
+
+ print("Adding user{} to mail list".format(username))
+ try:
+ # Create a text/plain message
+ email_msg = EmailMessage()
+
+ email_msg['From'] = mail_list_admin
+ email_msg['To'] = mail_list
+ email_msg['Subject'] = ''
+
+ # Create an smtp object and send email
+ s = smtplib.SMTP('localhost')
+
+ for each_cmd in listserv_cmd:
+ email_msg.set_content(listserv_cmd[each_cmd])
+ #s.send_message(email_msg)
+ print(email_msg)
+
+ s.quit()
+ except:
+ e = sys.exc_info()[0]
+ print("[{}]: Error: {}".format(task, e))
+
+ # Acknowledge message
+ ch.basic_ack(delivery_tag=method.delivery_tag)
+
+ # send confirm message
+ rc_rmq.publish_msg({
+ 'routing_key': 'confirm.' + username,
+ 'msg': {
+ 'task': task,
+ 'success': success
+ }
+ })
+
+print("Start listening to queue: {}".format(task))
+rc_rmq.start_consume({
+ 'queue': task, # Define your Queue name
+ 'routing_key': "create.*", # Define your routing key
+ 'cb': mail_list_subscription # Pass in callback function you just define
+})
+
+rc_rmq.disconnect()
--
GitLab
From 52398bd52522792cb500d4ec3fb3dae7d93b79b6 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 8 Apr 2020 20:09:08 +0000
Subject: [PATCH 005/188] RabbitMQ agent for event logging user self reg app
---
user_reg_event_logger.py | 32 ++++++++++++++++++++++++++++++++
1 file changed, 32 insertions(+)
create mode 100644 user_reg_event_logger.py
diff --git a/user_reg_event_logger.py b/user_reg_event_logger.py
new file mode 100644
index 0000000..0676095
--- /dev/null
+++ b/user_reg_event_logger.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import sys
+import json
+from rc_rmq import RCRMQ
+
+task = 'user_reg_event_log'
+
+# Instantiate rabbitmq object
+rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+
+# Define your callback function
+def log_user_reg_events(ch, method, properties, body):
+
+ # Retrieve routing key
+ routing_key = method.routing_key
+ print(routing_key)
+ # Retrieve message
+ msg = json.loads(body)
+ print(msg)
+ # Do Something
+ print('[{}]: Callback called.'.format(task))
+
+ # Acknowledge message
+ ch.basic_ack(delivery_tag=method.delivery_tag)
+
+
+print("Start listening to queue: {}".format(task))
+rc_rmq.start_consume({
+ 'queue': task, # Define your Queue name
+ 'routing_key': "#", # Define your routing key
+ 'cb': log_user_reg_events # Pass in callback function you just define
+})
--
GitLab
From 1802791818078d086d115e8cb4d44c7df4d25afd Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Sun, 12 Apr 2020 02:34:02 +0000
Subject: [PATCH 006/188] Add logging functionality and dry run to
reolve-uid-gid agent
---
get-next-uid-gid.py | 54 +++++++++++++++++++++++++++++++++++----------
1 file changed, 42 insertions(+), 12 deletions(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index 4afa8ad..d18f56a 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -2,6 +2,8 @@
import sys
import json
import ldap
+import logging
+import argparse
from os import popen
from rc_rmq import RCRMQ
@@ -10,51 +12,76 @@ task = 'get_next_uid_gid'
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+# Parse arguments
+parser = argparse.ArgumentParser()
+parser.add_argument('-v', '--verbose', action='store_true', help='verbose output')
+parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
+args = parser.parse_args()
+
+#Default Log level
+log_lvl = logging.WARNING
+
+if args.verbose:
+ log_lvl = logging.DEBUG
+if args.dry_run:
+ log_lvl = logging.INFO
+
+# Logger
+logging.basicConfig(format='%(asctime)s %(levelname)s [%(module)s] - %(message)s', level=log_lvl)
+logger = logging.getLogger(__name__)
+
+
#Check if the username already exists via LDAP
def user_exists(username):
try:
+ logger.info(f"Searching LDAP for the user: {username}")
con = ldap.initialize('ldap://ldapserver')
ldap_base = "dc=cm,dc=cluster"
query = "(uid={})".format(username)
result = con.search_s(ldap_base, ldap.SCOPE_SUBTREE, query)
+ logging.debug(f"The search result is: {result}")
return result
- except:
- e = sys.exc_info()[0]
- print("[{}]: Error: {}".format(task, e))
+ except ldap.LDAPError:
+ logger.exception("Fatal LDAP error:")
# Define your callback function
def get_next_uid_gid(ch, method, properties, body):
# Retrieve message
msg = json.loads(body)
- print("Message received {}".format(msg))
+ logger.info("Received {}".format(msg))
username = msg['username']
success = False
# Determine next available UID
try:
- if user_exists(username):
- print("The user, {} already exists".format(username))
+ #if user_exists(username):
+ if False:
+ logger.info("The user, {} already exists".format(username))
sys.exit(1)
cmd_uid = "/usr/bin/getent passwd | \
awk -F: '($3>10000) && ($3<20000) && ($3>maxuid) { maxuid=$3; } END { print maxuid+1; }'"
+ if not args.dry_run:
+ msg['uid'] = popen(cmd_uid).read().rstrip()
- msg['uid'] = popen(cmd_uid).read().rstrip()
+ logger.info(f"UID query: {cmd_uid}")
cmd_gid = "/usr/bin/getent group | \
awk -F: '($3>10000) && ($3<20000) && ($3>maxgid) { maxgid=$3; } END { print maxgid+1; }'"
+ if not args.dry_run:
+ msg['gid'] = popen(cmd_gid).read().rstrip()
- msg['gid'] = popen(cmd_gid).read().rstrip()
+ logger.info(f"GID query: {cmd_gid}")
success = True
- except:
- e = sys.exc_info()[0]
- print("[{}]: Error: {}".format(task, e))
+ except Exception:
+ logger.exception("Fatal error:")
# Acknowledge message
ch.basic_ack(delivery_tag=method.delivery_tag)
# Send confirm message
+ logger.debug('rc_rmq.publish_msg()')
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
'msg': {
@@ -62,19 +89,22 @@ def get_next_uid_gid(ch, method, properties, body):
'success': success
}
})
+ logger.info('confirmation sent')
if success:
# Send create message to BrightCM agent
+ logger.info(f'The task {task} finished, sending create msg to next queue')
rc_rmq.publish_msg({
'routing_key': 'create.' + username,
'msg': msg
})
-print("Start listening to queue: {}".format(task))
+logger.info("Start listening to queue: {}".format(task))
rc_rmq.start_consume({
'queue': task,
'routing_key': "request.*",
'cb': get_next_uid_gid
})
+logger.info("Disconnected")
rc_rmq.disconnect()
--
GitLab
From 89df5eb50e543d7e27ba1e55a83199457f72fc92 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Mon, 13 Apr 2020 23:45:38 +0000
Subject: [PATCH 007/188] Use rc_util class methods for logging and arg parsing
in resolve-uid-gid
Conflicts:
get-next-uid-gid.py
---
get-next-uid-gid.py | 19 +++----------------
1 file changed, 3 insertions(+), 16 deletions(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index d18f56a..b4b5d5e 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -4,6 +4,7 @@ import json
import ldap
import logging
import argparse
+import rc_util
from os import popen
from rc_rmq import RCRMQ
@@ -12,24 +13,10 @@ task = 'get_next_uid_gid'
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
-# Parse arguments
-parser = argparse.ArgumentParser()
-parser.add_argument('-v', '--verbose', action='store_true', help='verbose output')
-parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
-args = parser.parse_args()
-
-#Default Log level
-log_lvl = logging.WARNING
-
-if args.verbose:
- log_lvl = logging.DEBUG
-if args.dry_run:
- log_lvl = logging.INFO
+args = rc_util.get_args()
# Logger
-logging.basicConfig(format='%(asctime)s %(levelname)s [%(module)s] - %(message)s', level=log_lvl)
-logger = logging.getLogger(__name__)
-
+logger = rc_util.get_logger()
#Check if the username already exists via LDAP
def user_exists(username):
--
GitLab
From bbc37013d46825a3b7e44ff52e5774768f65d641 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 10 Apr 2020 00:36:23 +0000
Subject: [PATCH 008/188] Fix syntax for boolean val of success
---
brightcm_account_create.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/brightcm_account_create.py b/brightcm_account_create.py
index 3d96949..a209aa3 100644
--- a/brightcm_account_create.py
+++ b/brightcm_account_create.py
@@ -27,7 +27,7 @@ def bright_account_create(ch, method, properties, body):
#popen(cmd)
print(cmd)
- success = true
+ success = True
except:
e = sys.exc_info()[0]
print("[{}]: Error: {}".format(task, e))
--
GitLab
From ced317741d34bb91c59565063e5c81005219aa2f Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Sun, 12 Apr 2020 07:01:36 +0000
Subject: [PATCH 009/188] Add logging and dry-run functionality to
bright-account-create agent
---
brightcm_account_create.py | 39 +++++++++++++++++++++++++++++++-------
1 file changed, 32 insertions(+), 7 deletions(-)
diff --git a/brightcm_account_create.py b/brightcm_account_create.py
index a209aa3..2e68459 100644
--- a/brightcm_account_create.py
+++ b/brightcm_account_create.py
@@ -1,6 +1,8 @@
#!/usr/bin/env python
import sys
import json
+import logging
+import argparse
from os import popen
from rc_rmq import RCRMQ
@@ -9,11 +11,29 @@ task = 'bright_account'
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+# Parse arguments
+parser = argparse.ArgumentParser()
+parser.add_argument('-v', '--verbose', action='store_true', help='verbose output')
+parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
+args = parser.parse_args()
+
+#Default Log level
+log_lvl = logging.WARNING
+
+if args.verbose:
+ log_lvl = logging.DEBUG
+if args.dry_run:
+ log_lvl = logging.INFO
+
+# Logger
+logging.basicConfig(format='%(asctime)s %(levelname)s [%(module)s] - %(message)s', level=log_lvl)
+logger = logging.getLogger(__name__)
+
# Define your callback function
def bright_account_create(ch, method, properties, body):
# Retrieve message
msg = json.loads(body)
- print("Received msg{}".format(msg))
+ logger.info("Received {}".format(msg))
username = msg['username']
uid = msg['uid']
email = msg['email']
@@ -25,17 +45,18 @@ def bright_account_create(ch, method, properties, body):
cmd += f'"user; add {username}; set userid {uid}; set email {email}; set commonname \\"{fullname}\\"; '
cmd += 'commit;"'
- #popen(cmd)
- print(cmd)
+ if not args.dry_run:
+ popen(cmd)
+ logger.info(f'Bright command to create user:{cmd}')
success = True
- except:
- e = sys.exc_info()[0]
- print("[{}]: Error: {}".format(task, e))
+ except Exception:
+ logger.exception("Fatal error:")
# Acknowledge message
ch.basic_ack(delivery_tag=method.delivery_tag)
# send confirm message
+ logger.debug('rc_rmq.publish_msg()')
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
'msg': {
@@ -43,19 +64,23 @@ def bright_account_create(ch, method, properties, body):
'success': success
}
})
+ logger.info('confirmation sent')
if success:
# send create message to verify dir permissions agent
+ logger.debug(f'The task {task} finished successfully')
rc_rmq.publish_msg({
'routing_key': 'verify.' + username,
'msg': msg
})
+ logger.info('verify msg sent to next agent')
-print("Start listening to queue: {}".format(task))
+logger.info("Start listening to queue: {}".format(task))
rc_rmq.start_consume({
'queue': task,
'routing_key': "create.*",
'cb': bright_account_create
})
+logger.info("Disconnected")
rc_rmq.disconnect()
--
GitLab
From 286b320d8bc1542ca0560455a7decde8a5e70643 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Mon, 13 Apr 2020 23:48:26 +0000
Subject: [PATCH 010/188] Use rc_util class methods for logging and arg parsing
in bright account create agent
---
brightcm_account_create.py | 17 +++--------------
1 file changed, 3 insertions(+), 14 deletions(-)
diff --git a/brightcm_account_create.py b/brightcm_account_create.py
index 2e68459..ed6c71b 100644
--- a/brightcm_account_create.py
+++ b/brightcm_account_create.py
@@ -3,6 +3,7 @@ import sys
import json
import logging
import argparse
+import rc_util
from os import popen
from rc_rmq import RCRMQ
@@ -12,22 +13,10 @@ task = 'bright_account'
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
# Parse arguments
-parser = argparse.ArgumentParser()
-parser.add_argument('-v', '--verbose', action='store_true', help='verbose output')
-parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
-args = parser.parse_args()
-
-#Default Log level
-log_lvl = logging.WARNING
-
-if args.verbose:
- log_lvl = logging.DEBUG
-if args.dry_run:
- log_lvl = logging.INFO
+args = rc_util.get_args()
# Logger
-logging.basicConfig(format='%(asctime)s %(levelname)s [%(module)s] - %(message)s', level=log_lvl)
-logger = logging.getLogger(__name__)
+logger = rc_util.get_logger()
# Define your callback function
def bright_account_create(ch, method, properties, body):
--
GitLab
From 7f4ff5809c2f8e0723f06a22be6a8ff02a9d63ff Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 10 Apr 2020 00:44:23 +0000
Subject: [PATCH 011/188] Change the var name to match the one used in cmds
---
subscribe_mail_lists.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index 400608a..7fae67d 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -18,7 +18,7 @@ def mail_list_subscription(ch, method, properties, body):
print("Received msg {}".format(msg))
username = msg['username']
fullname = msg['fullname']
- user_email = msg['email']
+ email = msg['email']
mail_list_admin = 'admin@uab.edu' #change this during deploy
mail_list = 'LISTSERV@LISTSERV.UAB.EDU'
--
GitLab
From 8ba5c7d2634b3b23e21960422acde61592c88dd1 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 10 Apr 2020 00:46:50 +0000
Subject: [PATCH 012/188] Add flags to denote task success
---
subscribe_mail_lists.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index 7fae67d..f731a9a 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -28,6 +28,7 @@ def mail_list_subscription(ch, method, properties, body):
listserv_cmd['hpc_users'] = f'QUIET ADD hpc-users {email} {fullname}'
print("Adding user{} to mail list".format(username))
+ success = False
try:
# Create a text/plain message
email_msg = EmailMessage()
@@ -45,6 +46,7 @@ def mail_list_subscription(ch, method, properties, body):
print(email_msg)
s.quit()
+ success = True
except:
e = sys.exc_info()[0]
print("[{}]: Error: {}".format(task, e))
--
GitLab
From e375efb72444ab1d216f2efdbc7bd557d434b6cd Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Sun, 12 Apr 2020 07:03:18 +0000
Subject: [PATCH 013/188] Add logging and dry-run functionality to
subscribe-mail-lists agent
---
subscribe_mail_lists.py | 43 +++++++++++++++++++++++++++++++----------
1 file changed, 33 insertions(+), 10 deletions(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index f731a9a..33d2a5c 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -2,6 +2,8 @@
import sys
import json
import smtplib
+import logging
+import argparse
from email.message import EmailMessage
from rc_rmq import RCRMQ
@@ -10,12 +12,30 @@ task = 'subscribe_mail_list'
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+# Parse arguments
+parser = argparse.ArgumentParser()
+parser.add_argument('-v', '--verbose', action='store_true', help='verbose output')
+parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
+args = parser.parse_args()
+
+#Default Log level
+log_lvl = logging.WARNING
+
+if args.verbose:
+ log_lvl = logging.DEBUG
+if args.dry_run:
+ log_lvl = logging.INFO
+
+# Logger
+logging.basicConfig(format='%(asctime)s %(levelname)s [%(module)s] - %(message)s', level=log_lvl)
+logger = logging.getLogger(__name__)
+
# Define your callback function
def mail_list_subscription(ch, method, properties, body):
# Retrieve message
msg = json.loads(body)
- print("Received msg {}".format(msg))
+ logger.info("Received msg {}".format(msg))
username = msg['username']
fullname = msg['fullname']
email = msg['email']
@@ -27,7 +47,7 @@ def mail_list_subscription(ch, method, properties, body):
listserv_cmd['hpc_announce'] = f'QUIET ADD hpc-announce {email} {fullname}'
listserv_cmd['hpc_users'] = f'QUIET ADD hpc-users {email} {fullname}'
- print("Adding user{} to mail list".format(username))
+ logger.info("Adding user{} to mail list".format(username))
success = False
try:
# Create a text/plain message
@@ -40,21 +60,22 @@ def mail_list_subscription(ch, method, properties, body):
# Create an smtp object and send email
s = smtplib.SMTP('localhost')
- for each_cmd in listserv_cmd:
- email_msg.set_content(listserv_cmd[each_cmd])
- #s.send_message(email_msg)
- print(email_msg)
+ for key,value in listserv_cmd.items():
+ email_msg.set_content(value)
+ if not args.dry_run:
+ s.send_message(email_msg)
+ logging.info(f'This email will add user {username} to {key}\n{email_msg}')
s.quit()
success = True
- except:
- e = sys.exc_info()[0]
- print("[{}]: Error: {}".format(task, e))
+ except Exception:
+ logger.exception("Fatal error:")
# Acknowledge message
ch.basic_ack(delivery_tag=method.delivery_tag)
# send confirm message
+ logger.debug('rc_rmq.publish_msg()')
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
'msg': {
@@ -62,12 +83,14 @@ def mail_list_subscription(ch, method, properties, body):
'success': success
}
})
+ logger.info('confirmation sent')
-print("Start listening to queue: {}".format(task))
+logger.info("Start listening to queue: {}".format(task))
rc_rmq.start_consume({
'queue': task, # Define your Queue name
'routing_key': "create.*", # Define your routing key
'cb': mail_list_subscription # Pass in callback function you just define
})
+logger.info("Disconnected")
rc_rmq.disconnect()
--
GitLab
From 1bb3b2b562b79ee9e422917b806c430c5b20a49d Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Mon, 13 Apr 2020 23:50:24 +0000
Subject: [PATCH 014/188] Use rc_util class methods for logging and arg parsing
in subscribe mail lists agent
---
subscribe_mail_lists.py | 18 +++---------------
1 file changed, 3 insertions(+), 15 deletions(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index 33d2a5c..98d6097 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -4,6 +4,7 @@ import json
import smtplib
import logging
import argparse
+import rc_util
from email.message import EmailMessage
from rc_rmq import RCRMQ
@@ -13,24 +14,11 @@ task = 'subscribe_mail_list'
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
# Parse arguments
-parser = argparse.ArgumentParser()
-parser.add_argument('-v', '--verbose', action='store_true', help='verbose output')
-parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
-args = parser.parse_args()
-
-#Default Log level
-log_lvl = logging.WARNING
-
-if args.verbose:
- log_lvl = logging.DEBUG
-if args.dry_run:
- log_lvl = logging.INFO
+args = rc_util.get_args()
# Logger
-logging.basicConfig(format='%(asctime)s %(levelname)s [%(module)s] - %(message)s', level=log_lvl)
-logger = logging.getLogger(__name__)
+logger = rc_util.get_logger()# Define your callback function
-# Define your callback function
def mail_list_subscription(ch, method, properties, body):
# Retrieve message
--
GitLab
From 47ec299f98919a392eb8455bc66b813f02d15660 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 14 Apr 2020 01:46:38 +0000
Subject: [PATCH 015/188] Add time of request to the account request object.
---
user_reg_logger.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/user_reg_logger.py b/user_reg_logger.py
index 6d80bd4..9b5e295 100755
--- a/user_reg_logger.py
+++ b/user_reg_logger.py
@@ -4,6 +4,7 @@ import sys
import dataset
import logging
from rc_rmq import RCRMQ
+from datetime import datetime
# Define queue name
task = 'reg_logger'
@@ -24,6 +25,7 @@ def log_registration(ch, method, properties, body):
account_req = json.loads(body)
table.insert(account_req)
+ account_req['req_time'] = datetime.now(),
logger.info("logged account request for %s", account_req['username'])
ch.basic_ack(delivery_tag=method.delivery_tag)
--
GitLab
From be566e137389ffe0cd2e338e44e3eba2e4bd1a04 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 14 Apr 2020 01:47:55 +0000
Subject: [PATCH 016/188] Add logging and argparsing functionality.
---
user_reg_logger.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/user_reg_logger.py b/user_reg_logger.py
index 9b5e295..cb05446 100755
--- a/user_reg_logger.py
+++ b/user_reg_logger.py
@@ -9,13 +9,15 @@ from datetime import datetime
# Define queue name
task = 'reg_logger'
-# Instantiate logging object
-logging.basicConfig(format='%(asctime)s[%(levelname)s] - %(message)s', level=logging.INFO)
-logger = logging.getLogger(__name__)
-
# Instantiate rabbitmq object
reg_logger = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+# Parse arguments
+args = rc_util.get_args()
+
+# Logger
+logger = rc_util.get_logger()
+
# Open registry table in DB
db = dataset.connect('sqlite:///reg_logger.db')
table = db['registry']
--
GitLab
From 07e06bc818f6c527c2fe89635c3a2a4cca2391c2 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 14 Apr 2020 01:49:21 +0000
Subject: [PATCH 017/188] Change the table name to represent account req
---
user_reg_logger.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/user_reg_logger.py b/user_reg_logger.py
index cb05446..dec5c18 100755
--- a/user_reg_logger.py
+++ b/user_reg_logger.py
@@ -20,14 +20,14 @@ logger = rc_util.get_logger()
# Open registry table in DB
db = dataset.connect('sqlite:///reg_logger.db')
-table = db['registry']
+account_req_table = db['registry']
# Define registration logger callback
def log_registration(ch, method, properties, body):
account_req = json.loads(body)
- table.insert(account_req)
account_req['req_time'] = datetime.now(),
+ account_req_table.insert(account_req)
logger.info("logged account request for %s", account_req['username'])
ch.basic_ack(delivery_tag=method.delivery_tag)
--
GitLab
From a0f2334de0d5a62a817a39be17383af6a1d784b6 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 10 Apr 2020 00:54:09 +0000
Subject: [PATCH 018/188] Add functionality for event logging
---
user_reg_event_logger.py | 18 ++++++++++++------
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/user_reg_event_logger.py b/user_reg_event_logger.py
index 0676095..d762e06 100644
--- a/user_reg_event_logger.py
+++ b/user_reg_event_logger.py
@@ -11,14 +11,20 @@ rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
# Define your callback function
def log_user_reg_events(ch, method, properties, body):
- # Retrieve routing key
- routing_key = method.routing_key
- print(routing_key)
# Retrieve message
msg = json.loads(body)
- print(msg)
- # Do Something
- print('[{}]: Callback called.'.format(task))
+ #print(msg)
+
+ # Retrieve routing key
+ routing_key = method.routing_key
+ action = routing_key.split(".")[0]
+ user = routing_key.split(".")[1]
+ if action != 'confirm':
+ print(f'Got a message for {user}: {routing_key}')
+ else:
+ task = msg['task']
+ status = msg['success']
+ print(f'Task {task} completed?: {status}')
# Acknowledge message
ch.basic_ack(delivery_tag=method.delivery_tag)
--
GitLab
From ff962a25e76041c04539efa3bfc408b24d61e243 Mon Sep 17 00:00:00 2001
From: "Bo-Chun Louis Chen(VM)" <louistw@uab.edu>
Date: Tue, 24 Mar 2020 22:14:45 +0000
Subject: [PATCH 019/188] Add directory verify agent
---
dir_verify.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 dir_verify.py
diff --git a/dir_verify.py b/dir_verify.py
new file mode 100644
index 0000000..9491bce
--- /dev/null
+++ b/dir_verify.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+import sys
+import json
+import shutil
+from pathlib import Path
+from rc_rmq import RCRMQ
+
+task = 'dir_verify'
+dirs = ['/home', '/data/user', '/data/scratch']
+
+# Instantiate rabbitmq object
+rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+
+def dir_verify(ch, method, properties, body):
+ msg = json.loads(body)
+ username = msg['username']
+ success = False
+
+ try:
+ for d in dirs:
+ path = Path(d) / msg['username']
+
+ if not path.exists():
+ # Make sure folder exists and with right permission
+ path.mkdir(mode=0o700)
+
+ # Make sure ownership is correct
+ shutil.chown(path, msg['uid'], msg['gid'])
+ success = True
+ except:
+ e = sys.exc_info()[0]
+ print("[{}]: Error: {}".format(task, e))
+
+ # send confirm message
+ rc_rmq.publish_msg({
+ 'routing_key': 'confirm.' + username,
+ 'msg': {
+ 'task': task,
+ 'success': success
+ }
+ })
+
+
+print("Start listening to queue: {}".format(task))
+rc_rmq.start_consume({
+ 'queue': task,
+ 'routing_key': "verify.*",
+ 'cb': dir_verify
+})
+
+rc_rmq.disconnect()
--
GitLab
From 8d7e98ae59de14b261bccaa06160987292ff8b77 Mon Sep 17 00:00:00 2001
From: "Bo-Chun Louis Chen(VM)" <louistw@uab.edu>
Date: Mon, 30 Mar 2020 21:37:59 +0000
Subject: [PATCH 020/188] Add notify user agent
---
notify_user.py | 155 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 155 insertions(+)
create mode 100644 notify_user.py
diff --git a/notify_user.py b/notify_user.py
new file mode 100644
index 0000000..c453424
--- /dev/null
+++ b/notify_user.py
@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+import sys
+import json
+import smtplib
+from rc_rmq import RCRMQ
+
+task = 'notify_user'
+
+# Some variable for email
+DEBUG = False
+passwd_str = '<Your BlazerID/XIAS Password>'
+hpcsrv = 'cheaha.rc.uab.edu'
+# idmap_srv = 'cheaha.rc.uab.edu'
+# idmap = '/home/useradmin/.useridmap'
+signature = 'UAB IT Research Computing'
+my_email = 'mhanby@uab.edu'
+info_url = 'http://docs.uabgrid.uab.edu/wiki/Cheaha'
+quickstart_url = 'https://docs.uabgrid.uab.edu/wiki/Cheaha_Quick_Start'
+# matlab_url = 'http://docs.uabgrid.uab.edu/wiki/MatLab_DCS'
+helpdesk_url = 'https://uabprod.service-now.com/ess_portal/' # Once we get a real helpdesk, change to URL instead of email
+support_email = 'askit@uab.edu'
+loginwiki_url = 'http://docs.uabgrid.uab.edu/wiki/Cheaha_GettingStarted#Login'
+queuewiki_url = 'https://docs.uabgrid.uab.edu/wiki/Slurm'
+# Old Mailman HPC mailing list
+# hpclist_url = 'http://lists.it.uab.edu/mailman/listinfo/hpcusers'
+# hpclist_email = 'hpcusers-subscribe@lists.it.uab.edu'
+# New Sympa based mailing lists
+mailinglist_admin = 'mhanby@uab.edu'
+mailinglist_email = 'LISTSERV@LISTSERV.UAB.EDU'
+#mailinglist_email = 'sympa@vo.uabgrid.uab.edu'
+#hpcannounce_url = 'http://vo.uabgrid.uab.edu/sympa/info/hpc-announce'
+hpcannounce_email = 'HPC-ANNOUNCE@LISTSERV.UAB.EDU'
+#hpcusers_url = 'http://vo.uabgrid.uab.edu/sympa/info/hpc-users'
+hpcusers_email = 'HPC-USERS@LISTSERV.UAB.EDU'
+ticket_url = 'https://gitlab.rc.uab.edu/mhanby/rc-users/issues'
+
+
+# Instantiate rabbitmq object
+rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+
+def send_email(to, opts=None):
+ opts['server'] = opts.get('server', 'localhost')
+ opts['from'] = opts.get('from', 'SUPPORT@LISTSERV.UAB.EDU')
+ opts['from_alias'] = opts.get('from_alias', 'Research Computing Services')
+ opts['subject'] = opts.get('subject', 'HPC New User Account')
+
+ msg = f"""From: {opts['from_alias']} <{opts['from']}>
+To: <{to}>
+Subject: {opts['subject']}
+{opts['body']}
+"""
+ if DEBUG:
+ print(msg)
+ return
+
+ smtp = smtplib.SMTP(opts['server'])
+ if 'bcc' in opts:
+ smtp.sendmail(opts['from'], [to, opts['bcc']], messagge)
+ else:
+ smtp.sendmail(opts['from'], [to], messagge)
+
+
+# Email instruction to user
+def notify_user(ch, method, properties, body):
+ msg = json.loads(body)
+ username = msg['username']
+ user_email = msg['email']
+ success = False
+
+ email_body = f"""
+Your account has been set up on {hpcsrv}
+
+====================================
+User ID: {username}
+Password: {passwd_str}
+====================================
+
+Important: You are responsible for backing up your files on Cheaha! The following storage locations are available:
+* $HOME - /home/$USER - 20G quota per user
+* $USER_DATA - /data/user/$USER - 5TB quota per user
+* $USER_SCRATCH - /data/scratch/$USER - 500TB shared amongst ALL users
+
+DO NOT compute out of $HOME, all computation must be done on the fast storage that is $USER_DATA and $USER_SCRATCH
+
+Please read the information available on our Wiki pages, especially the Cheaha Quick Start:
+
+Cheaha Quick Start
+ {quickstart_url}
+Additional Cluster Login Instructions
+ {loginwiki_url}
+Job Queuing information and examples
+ {queuewiki_url}
+General information about Cheaha
+ {info_url}
+
+If you encounter a problem or have any questions, please send a detailed email to: {support_email}
+
+You have been subscribed to two important email lists:
+ * hpc-announce : Important HPC related announcements are sent via this list, please read
+ all emails from {hpcannounce_email}
+ * hpc-users : This email list can be used by all HPC users to discuss anything
+ related to our HPC environment by sending email to {hpcusers_email}
+
+ Regards,
+
+{signature}
+{helpdesk_url}
+"""
+
+ try:
+ #Send email to user
+ if DEBUG:
+ send_email('louistw@uab.edu', {
+ 'from': support_email,
+ 'from_alias': 'Research Computing Services',
+ 'subject': f'[TEST] New User Account on {hpcsrv}',
+ 'body': email_body
+ })
+
+ else:
+ send_email(user_email, {
+ 'from': support_email,
+ 'from_alias': 'Research Computing Services',
+ 'subject': f'New User Account on {hpcsrv}',
+ 'body': email_body
+ })
+
+ success = True
+ except:
+ e = sys.exc_info()[0]
+ print("[{}]: Error: {}".format(task, e))
+
+ if not DEBUG:
+ # acknowledge the message
+ ch.basic_ack(delivery_tag=method.delivery_tag)
+
+ # send confirm message
+ rc_rmq.publish_msg({
+ 'routing_key': 'confirm.' + username,
+ 'msg': {
+ 'task': task,
+ 'success': success
+ }
+ })
+
+
+if __name__ == "__main__":
+ print("Start listening to queue: {}".format(task))
+ rc_rmq.start_consume({
+ 'queue': task,
+ 'routing_key': "notify.*",
+ 'cb': notify_user
+ })
+
+ rc_rmq.disconnect()
--
GitLab
From bbc071eea43162185a317191faf1683a57d9c350 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 9 Apr 2020 00:06:14 -0500
Subject: [PATCH 021/188] First draft task manager
---
task_manager.py | 94 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 94 insertions(+)
create mode 100644 task_manager.py
diff --git a/task_manager.py b/task_manager.py
new file mode 100644
index 0000000..acc9cdf
--- /dev/null
+++ b/task_manager.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+import sys
+import json
+from rc_rmq import RCRMQ
+from datetime import datetime
+
+task = 'task_manager'
+
+record = {
+ 'uid': -1,
+ 'gid': -1,
+ 'email': '',
+ 'last_update': datetime.now(),
+ 'request': {
+ 'uid_resolve': None
+ },
+ 'create': {
+ 'join_list': None,
+ 'create_account': None
+ },
+ 'verify': {
+ 'git_commit': None,
+ 'dir_verify': None
+ }
+}
+
+# Currently tracking users
+tracking = {}
+
+# Instantiate rabbitmq object
+rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+
+def task_manager(ch, method, properties, body):
+ msg = json.loads(body)
+ username = method.routing_key.split('.')[1]
+ task_name = msg['task']
+ success = msg['success']
+
+ if username not in tracking:
+ current = tracking[username] = record.copy()
+ current['uid'] = msg.get('uid', -1)
+ current['gid'] = msg.get('gid', -1)
+ current['email'] = msg.get('email', '')
+
+ try:
+ # Check each level
+ # task timeout
+ # failure agent(?
+ if task_name in current['request']:
+ current['request'][task_name] = success
+ routing_key = 'create.' + username
+ done = success
+
+ elif task_name in current['create']:
+ current['create'][task_name] = success
+ routing_key = 'verify.' + username
+ done = True
+ for status in current['create'].values():
+ if status is not True:
+ done = False
+
+ elif task_name in current['verify']:
+ current['verify'][task_name] = success
+ routing_key = 'notify.' + username
+ done = True
+ for status in current['verify'].values():
+ if status is not True:
+ done = False
+
+ except:
+ e = sys.exc_info()[0]
+ print("[{}]: Error: {}".format(task, e))
+
+ if done:
+ # send trigger message
+ rc_rmq.publish_msg({
+ 'routing_key': routing_key
+ 'msg': {
+ 'username': username,
+ 'email': current['email'],
+ 'uid': current['uid'],
+ 'gid': current['gid']
+ }
+ })
+
+print("Start listening to queue: {}".format(task))
+rc_rmq.start_consume({
+ 'queue': task,
+ 'routing_key': "confirm.*",
+ 'cb': task_manager
+})
+
+print("Disconnected")
+rc_rmq.disconnect()
--
GitLab
From 4d4c43402a8c9bc14fb512d23ee0a166ea3cfae7 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 10 Apr 2020 14:33:08 -0500
Subject: [PATCH 022/188] Set current to the record of current user
---
task_manager.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index acc9cdf..b36ce30 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -41,6 +41,8 @@ def task_manager(ch, method, properties, body):
current['uid'] = msg.get('uid', -1)
current['gid'] = msg.get('gid', -1)
current['email'] = msg.get('email', '')
+ else:
+ current = tracking[username]
try:
# Check each level
@@ -83,6 +85,7 @@ def task_manager(ch, method, properties, body):
}
})
+
print("Start listening to queue: {}".format(task))
rc_rmq.start_consume({
'queue': task,
--
GitLab
From 219fabfbcc5c6f4896991bc3c5f6ed400373175d Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 20 Apr 2020 15:10:55 -0500
Subject: [PATCH 023/188] Add last step once receive confirm fron verify
---
task_manager.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index b36ce30..31db06e 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -21,6 +21,9 @@ record = {
'verify': {
'git_commit': None,
'dir_verify': None
+ },
+ 'notify': {
+ 'notify_user': None
}
}
@@ -69,6 +72,11 @@ def task_manager(ch, method, properties, body):
if status is not True:
done = False
+ elif task_name in current['notify']:
+ current['verify'][task_name] = success
+ routing_key = 'complete.' + username
+ done = success
+
except:
e = sys.exc_info()[0]
print("[{}]: Error: {}".format(task, e))
--
GitLab
From 78001500443f3a73e4e202211d282b725368eca7 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 20 Apr 2020 17:48:44 -0500
Subject: [PATCH 024/188] Fix missing comma
---
task_manager.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index 31db06e..9e52616 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -84,7 +84,7 @@ def task_manager(ch, method, properties, body):
if done:
# send trigger message
rc_rmq.publish_msg({
- 'routing_key': routing_key
+ 'routing_key': routing_key,
'msg': {
'username': username,
'email': current['email'],
--
GitLab
From 8608ad85911679ea57fd61ea8328d21a9fa694c3 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 20 Apr 2020 21:13:08 -0500
Subject: [PATCH 025/188] Add delivery_tags field to track message received
---
task_manager.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index 9e52616..cc94af2 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -24,7 +24,8 @@ record = {
},
'notify': {
'notify_user': None
- }
+ },
+ 'delivery_tags': []
}
# Currently tracking users
@@ -47,6 +48,9 @@ def task_manager(ch, method, properties, body):
else:
current = tracking[username]
+ # save the delivery tags for future use
+ current['delivery_tags'].append(method.delivery_tag)
+
try:
# Check each level
# task timeout
@@ -82,6 +86,11 @@ def task_manager(ch, method, properties, body):
print("[{}]: Error: {}".format(task, e))
if done:
+ # acknowledge all message from last level
+ for tag in current['delivery_tags']:
+ ch.basic_ack(tag)
+ current['delivery_tags'] = []
+
# send trigger message
rc_rmq.publish_msg({
'routing_key': routing_key,
--
GitLab
From f87d41877f50cd4080f5047fe98f2d49bff7663f Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 20 Apr 2020 23:55:53 -0500
Subject: [PATCH 026/188] Use jinja2 template engine for email body
mail.py:
define variable related to email
`body` is the template we will use in agent
for variables that are pre-defined in the same file,
use single curly bracket for it.
e.g. {info_url}
for `username` or other variables that are undefined until msg is received,
use four curly brackets for both open and close
e.g. {{{{ username }}}}
---
mail.py | 19 ++++++++++++++
notify_user.py | 69 +++-----------------------------------------------
2 files changed, 22 insertions(+), 66 deletions(-)
create mode 100644 mail.py
diff --git a/mail.py b/mail.py
new file mode 100644
index 0000000..dd2df02
--- /dev/null
+++ b/mail.py
@@ -0,0 +1,19 @@
+# Some variable for email
+my_email = 'admin@localhost'
+info_url = 'https://www.google.com'
+
+body = f"""
+Hi {{{{ username }}}}
+Your account has been set up with:
+
+============================
+User ID: {{{{ username }}}}
+============================
+
+If you have any questions, please visit:
+{info_url}
+
+or email at {my_email}
+
+Cheers,
+"""
diff --git a/notify_user.py b/notify_user.py
index c453424..6c0c119 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -3,37 +3,12 @@ import sys
import json
import smtplib
from rc_rmq import RCRMQ
+from jinja2 import Template
+import mail
task = 'notify_user'
-# Some variable for email
DEBUG = False
-passwd_str = '<Your BlazerID/XIAS Password>'
-hpcsrv = 'cheaha.rc.uab.edu'
-# idmap_srv = 'cheaha.rc.uab.edu'
-# idmap = '/home/useradmin/.useridmap'
-signature = 'UAB IT Research Computing'
-my_email = 'mhanby@uab.edu'
-info_url = 'http://docs.uabgrid.uab.edu/wiki/Cheaha'
-quickstart_url = 'https://docs.uabgrid.uab.edu/wiki/Cheaha_Quick_Start'
-# matlab_url = 'http://docs.uabgrid.uab.edu/wiki/MatLab_DCS'
-helpdesk_url = 'https://uabprod.service-now.com/ess_portal/' # Once we get a real helpdesk, change to URL instead of email
-support_email = 'askit@uab.edu'
-loginwiki_url = 'http://docs.uabgrid.uab.edu/wiki/Cheaha_GettingStarted#Login'
-queuewiki_url = 'https://docs.uabgrid.uab.edu/wiki/Slurm'
-# Old Mailman HPC mailing list
-# hpclist_url = 'http://lists.it.uab.edu/mailman/listinfo/hpcusers'
-# hpclist_email = 'hpcusers-subscribe@lists.it.uab.edu'
-# New Sympa based mailing lists
-mailinglist_admin = 'mhanby@uab.edu'
-mailinglist_email = 'LISTSERV@LISTSERV.UAB.EDU'
-#mailinglist_email = 'sympa@vo.uabgrid.uab.edu'
-#hpcannounce_url = 'http://vo.uabgrid.uab.edu/sympa/info/hpc-announce'
-hpcannounce_email = 'HPC-ANNOUNCE@LISTSERV.UAB.EDU'
-#hpcusers_url = 'http://vo.uabgrid.uab.edu/sympa/info/hpc-users'
-hpcusers_email = 'HPC-USERS@LISTSERV.UAB.EDU'
-ticket_url = 'https://gitlab.rc.uab.edu/mhanby/rc-users/issues'
-
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
@@ -67,45 +42,7 @@ def notify_user(ch, method, properties, body):
user_email = msg['email']
success = False
- email_body = f"""
-Your account has been set up on {hpcsrv}
-
-====================================
-User ID: {username}
-Password: {passwd_str}
-====================================
-
-Important: You are responsible for backing up your files on Cheaha! The following storage locations are available:
-* $HOME - /home/$USER - 20G quota per user
-* $USER_DATA - /data/user/$USER - 5TB quota per user
-* $USER_SCRATCH - /data/scratch/$USER - 500TB shared amongst ALL users
-
-DO NOT compute out of $HOME, all computation must be done on the fast storage that is $USER_DATA and $USER_SCRATCH
-
-Please read the information available on our Wiki pages, especially the Cheaha Quick Start:
-
-Cheaha Quick Start
- {quickstart_url}
-Additional Cluster Login Instructions
- {loginwiki_url}
-Job Queuing information and examples
- {queuewiki_url}
-General information about Cheaha
- {info_url}
-
-If you encounter a problem or have any questions, please send a detailed email to: {support_email}
-
-You have been subscribed to two important email lists:
- * hpc-announce : Important HPC related announcements are sent via this list, please read
- all emails from {hpcannounce_email}
- * hpc-users : This email list can be used by all HPC users to discuss anything
- related to our HPC environment by sending email to {hpcusers_email}
-
- Regards,
-
-{signature}
-{helpdesk_url}
-"""
+ email_body = Template(mail.body).render(username=username)
try:
#Send email to user
--
GitLab
From 4a853f160409891216f518cf8037349a55940aaf Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 21 Apr 2020 22:05:21 -0500
Subject: [PATCH 027/188] Avoid list reference when copying dict
---
task_manager.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index cc94af2..b7542de 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -25,7 +25,7 @@ record = {
'notify': {
'notify_user': None
},
- 'delivery_tags': []
+ 'delivery_tags': None
}
# Currently tracking users
@@ -42,6 +42,7 @@ def task_manager(ch, method, properties, body):
if username not in tracking:
current = tracking[username] = record.copy()
+ current['delivery_tags'] = []
current['uid'] = msg.get('uid', -1)
current['gid'] = msg.get('gid', -1)
current['email'] = msg.get('email', '')
--
GitLab
From dca91e022c0ef1f9f65ec417fba941073e58e9c1 Mon Sep 17 00:00:00 2001
From: "Bo-Chun Louis Chen(VM)" <louistw@uab.edu>
Date: Tue, 24 Mar 2020 22:04:58 +0000
Subject: [PATCH 028/188] Add requirements.txt
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index becc2ce..6d35ddf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
pika==1.1.0
+sh==1.12.14
--
GitLab
From fda4d5c41cb3be7bac85f79bb229dd28a65afa9b Mon Sep 17 00:00:00 2001
From: "Bo-Chun Louis Chen(VM)" <louistw@uab.edu>
Date: Tue, 24 Mar 2020 22:08:12 +0000
Subject: [PATCH 029/188] Add git agent
---
git_commit.py | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 69 insertions(+)
create mode 100644 git_commit.py
diff --git a/git_commit.py b/git_commit.py
new file mode 100644
index 0000000..2cfcd35
--- /dev/null
+++ b/git_commit.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+import sh
+import sys
+import json
+from rc_rmq import RCRMQ
+
+task = 'git_commit'
+
+# Instantiate rabbitmq object
+rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+
+repo_location = '~/git/rc-users'
+cheaha_ldif = repo_location + '/users/cheaha-openldap.ldif'
+cheaha_ldapsearch_ldif = repo_location + '/users/cheaha-openldap-ldapsearch.ldif'
+
+git = sh.git.bake(repo_location)
+slapcat = sh.Command('/cm/local/apps/openldap/sbin/slapcat')
+ldapsearch = sh.Command('ldapsearch')
+
+def git_commit(ch, method, properties, body):
+ msg = json.loads(body)
+ username = msg['username']
+ ticketnum = msg['ticketnum']
+ success = False
+
+ try:
+
+ branch_name = 'issue-' + ticketnum
+ git.checkout('master')
+ git.pull()
+ git.checkout('-b', branch_name)
+
+ with open(cheaha_ldif, 'w') as ldif_f,\
+ open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
+ slapcat('-f', '/cm/local/apps/openldap/etc/slapd.conf', '-b', "'dc=cm,dc=cluster'", _out=ldif_f)
+ ldapsearch('-LLL', '-x', '-h', 'cheaha-master02', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
+
+ git.diff()
+ git.add(cheaha_ldif)
+ git.add(cheaha_ldapsearch_ldif)
+ git.commit(m="Added new cheaha user: " + username)
+ git.push('origin', branch_name)
+ git.checkout('master')
+ time.sleep(60)
+ git.pull()
+
+ success = True
+ except:
+ e = sys.exc_info()[0]
+ print("[{}]: Error: {}".format(task, e))
+
+ # send confirm message
+ rc_rmq.publish_msg({
+ 'routing_key': 'confirm.' + username,
+ 'msg': {
+ 'task': task,
+ 'success': success
+ }
+ })
+
+
+print("Start listening to queue: {}".format(task))
+rc_rmq.start_consume({
+ 'queue': task,
+ 'routing_key': "git.*",
+ 'cb': git_commit
+})
+
+rc_rmq.disconnect()
--
GitLab
From 70d3a4506b2831774552cadd3445f049532822d3 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 3 Apr 2020 09:59:08 -0500
Subject: [PATCH 030/188] Set default ticketnum
According to UserAdd-Template.md,
`ticketnum="add-users-`printf "%.40s\n" "$users" | tr ' ' '-'`"`
---
git_commit.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index 2cfcd35..d4f60ca 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -20,7 +20,7 @@ ldapsearch = sh.Command('ldapsearch')
def git_commit(ch, method, properties, body):
msg = json.loads(body)
username = msg['username']
- ticketnum = msg['ticketnum']
+ ticketnum = msg.get('ticketnum', 'add-users-' + username.lower())
success = False
try:
--
GitLab
From 0913a4245af91dd16608637139b232e6fc78c593 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 3 Apr 2020 10:10:28 -0500
Subject: [PATCH 031/188] Add debug variable
---
git_commit.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/git_commit.py b/git_commit.py
index d4f60ca..e18b541 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -21,6 +21,7 @@ def git_commit(ch, method, properties, body):
msg = json.loads(body)
username = msg['username']
ticketnum = msg.get('ticketnum', 'add-users-' + username.lower())
+ DEBUG = msg.get('debug', False)
success = False
try:
@@ -53,6 +54,7 @@ def git_commit(ch, method, properties, body):
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
'msg': {
+ 'debug': DEBUG,
'task': task,
'success': success
}
--
GitLab
From 123ce45c171d453a05c8d0de71f88bc76d909c62 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 3 Apr 2020 10:12:37 -0500
Subject: [PATCH 032/188] Acknowledge message after job's done
---
git_commit.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/git_commit.py b/git_commit.py
index e18b541..0a58864 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -60,6 +60,9 @@ def git_commit(ch, method, properties, body):
}
})
+ # Acknowledge message
+ ch.basic_ack(delivery_tag=method.delivery_tag)
+
print("Start listening to queue: {}".format(task))
rc_rmq.start_consume({
--
GitLab
From 108d8cdcce94019078dbc671d4fd573b814df2b5 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 3 Apr 2020 10:15:22 -0500
Subject: [PATCH 033/188] Relocate branch_name variable
---
git_commit.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index 0a58864..9f74da9 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -23,10 +23,10 @@ def git_commit(ch, method, properties, body):
ticketnum = msg.get('ticketnum', 'add-users-' + username.lower())
DEBUG = msg.get('debug', False)
success = False
+ branch_name = 'issue-' + ticketnum
try:
- branch_name = 'issue-' + ticketnum
git.checkout('master')
git.pull()
git.checkout('-b', branch_name)
--
GitLab
From 86631358149d49185c5af53daac0264cef10ee13 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 3 Apr 2020 10:17:09 -0500
Subject: [PATCH 034/188] Add debug message
---
git_commit.py | 50 +++++++++++++++++++++++++++++++++-----------------
1 file changed, 33 insertions(+), 17 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 9f74da9..13d5586 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -27,23 +27,39 @@ def git_commit(ch, method, properties, body):
try:
- git.checkout('master')
- git.pull()
- git.checkout('-b', branch_name)
-
- with open(cheaha_ldif, 'w') as ldif_f,\
- open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
- slapcat('-f', '/cm/local/apps/openldap/etc/slapd.conf', '-b', "'dc=cm,dc=cluster'", _out=ldif_f)
- ldapsearch('-LLL', '-x', '-h', 'cheaha-master02', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
-
- git.diff()
- git.add(cheaha_ldif)
- git.add(cheaha_ldapsearch_ldif)
- git.commit(m="Added new cheaha user: " + username)
- git.push('origin', branch_name)
- git.checkout('master')
- time.sleep(60)
- git.pull()
+ if DEBUG:
+ print('[{}]: git checkout master'.format(task))
+ print('[{}]: git pull'.format(task))
+ print('[{}]: git checkout -b {}'.format(task, branch_name))
+ print('[{}]: /cm/local/apps/openldap/sbin/slapcat\\'.format(task))
+ print("\t-f /cm/local/apps/openldap/etc/slapd.conf -b 'dc=cm,dc=cluster' > {}".format(cheaha_ldif))
+ print('[{}]: ldapsearch\\'.format(task))
+ print("\t-LLL -x -h cheaha-master02 -b 'dc=cm,dc=cluster'".format(cheaha_ldapsearch_ldif))
+ print('[{}]: git add {}'.format(task, cheaha_ldif))
+ print('[{}]: git add {}'.format(task, cheaha_ldapsearch_ldif))
+ print('[{}]: git commit -m "{}"'.format(task, "Added new cheaha user: " + username))
+ print('[{}]: git push origin'.format(task))
+ print('[{}]: git checkout master'.format(task))
+ print('[{}]: git pull'.format(task))
+
+ else:
+ git.checkout('master')
+ git.pull()
+ git.checkout('-b', branch_name)
+
+ with open(cheaha_ldif, 'w') as ldif_f,\
+ open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
+ slapcat('-f', '/cm/local/apps/openldap/etc/slapd.conf', '-b', "'dc=cm,dc=cluster'", _out=ldif_f)
+ ldapsearch('-LLL', '-x', '-h', 'cheaha-master02', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
+
+ git.diff()
+ git.add(cheaha_ldif)
+ git.add(cheaha_ldapsearch_ldif)
+ git.commit(m="Added new cheaha user: " + username)
+ git.push('origin', branch_name)
+ git.checkout('master')
+ time.sleep(60)
+ git.pull()
success = True
except:
--
GitLab
From 904c1bef6529a1b1ade42a6ba80975fd67c75d55 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 3 Apr 2020 10:22:39 -0500
Subject: [PATCH 035/188] Update comment
---
git_commit.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index 13d5586..139b556 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -66,7 +66,7 @@ def git_commit(ch, method, properties, body):
e = sys.exc_info()[0]
print("[{}]: Error: {}".format(task, e))
- # send confirm message
+ # Send confirm message
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
'msg': {
--
GitLab
From 0524864c1af2616edd0e9bb25d1620c7e3e9d9c7 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 3 Apr 2020 10:44:39 -0500
Subject: [PATCH 036/188] Update ldap hostname
---
git_commit.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 139b556..a60629c 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -34,7 +34,7 @@ def git_commit(ch, method, properties, body):
print('[{}]: /cm/local/apps/openldap/sbin/slapcat\\'.format(task))
print("\t-f /cm/local/apps/openldap/etc/slapd.conf -b 'dc=cm,dc=cluster' > {}".format(cheaha_ldif))
print('[{}]: ldapsearch\\'.format(task))
- print("\t-LLL -x -h cheaha-master02 -b 'dc=cm,dc=cluster'".format(cheaha_ldapsearch_ldif))
+ print("\t-LLL -x -h ldapserver -b 'dc=cm,dc=cluster'".format(cheaha_ldapsearch_ldif))
print('[{}]: git add {}'.format(task, cheaha_ldif))
print('[{}]: git add {}'.format(task, cheaha_ldapsearch_ldif))
print('[{}]: git commit -m "{}"'.format(task, "Added new cheaha user: " + username))
@@ -50,7 +50,7 @@ def git_commit(ch, method, properties, body):
with open(cheaha_ldif, 'w') as ldif_f,\
open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
slapcat('-f', '/cm/local/apps/openldap/etc/slapd.conf', '-b', "'dc=cm,dc=cluster'", _out=ldif_f)
- ldapsearch('-LLL', '-x', '-h', 'cheaha-master02', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
+ ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
git.diff()
git.add(cheaha_ldif)
--
GitLab
From 1a4db32ace39850678d6394074aa3441b8695683 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 8 Apr 2020 21:55:55 -0500
Subject: [PATCH 037/188] Add argparse
---
git_commit.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/git_commit.py b/git_commit.py
index a60629c..32b7597 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -2,6 +2,7 @@
import sh
import sys
import json
+import argparse
from rc_rmq import RCRMQ
task = 'git_commit'
@@ -13,6 +14,12 @@ repo_location = '~/git/rc-users'
cheaha_ldif = repo_location + '/users/cheaha-openldap.ldif'
cheaha_ldapsearch_ldif = repo_location + '/users/cheaha-openldap-ldapsearch.ldif'
+# Parse arguments
+parser = argparse.ArgumentParser()
+parser.add_argument('-v', '--verbose', action='store_true', help='verbose output')
+parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
+args = parser.parse_args()
+
git = sh.git.bake(repo_location)
slapcat = sh.Command('/cm/local/apps/openldap/sbin/slapcat')
ldapsearch = sh.Command('ldapsearch')
--
GitLab
From 8aba13925b8d1bc0c1e6d421d6eeb1e1c39251c0 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 8 Apr 2020 22:00:31 -0500
Subject: [PATCH 038/188] Remove DEBUG variable and related code
---
git_commit.py | 52 +++++++++++++++++----------------------------------
1 file changed, 17 insertions(+), 35 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 32b7597..4f29fe2 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -28,45 +28,28 @@ def git_commit(ch, method, properties, body):
msg = json.loads(body)
username = msg['username']
ticketnum = msg.get('ticketnum', 'add-users-' + username.lower())
- DEBUG = msg.get('debug', False)
success = False
branch_name = 'issue-' + ticketnum
try:
- if DEBUG:
- print('[{}]: git checkout master'.format(task))
- print('[{}]: git pull'.format(task))
- print('[{}]: git checkout -b {}'.format(task, branch_name))
- print('[{}]: /cm/local/apps/openldap/sbin/slapcat\\'.format(task))
- print("\t-f /cm/local/apps/openldap/etc/slapd.conf -b 'dc=cm,dc=cluster' > {}".format(cheaha_ldif))
- print('[{}]: ldapsearch\\'.format(task))
- print("\t-LLL -x -h ldapserver -b 'dc=cm,dc=cluster'".format(cheaha_ldapsearch_ldif))
- print('[{}]: git add {}'.format(task, cheaha_ldif))
- print('[{}]: git add {}'.format(task, cheaha_ldapsearch_ldif))
- print('[{}]: git commit -m "{}"'.format(task, "Added new cheaha user: " + username))
- print('[{}]: git push origin'.format(task))
- print('[{}]: git checkout master'.format(task))
- print('[{}]: git pull'.format(task))
-
- else:
- git.checkout('master')
- git.pull()
- git.checkout('-b', branch_name)
-
- with open(cheaha_ldif, 'w') as ldif_f,\
- open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
- slapcat('-f', '/cm/local/apps/openldap/etc/slapd.conf', '-b', "'dc=cm,dc=cluster'", _out=ldif_f)
- ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
-
- git.diff()
- git.add(cheaha_ldif)
- git.add(cheaha_ldapsearch_ldif)
- git.commit(m="Added new cheaha user: " + username)
- git.push('origin', branch_name)
- git.checkout('master')
- time.sleep(60)
- git.pull()
+ git.checkout('master')
+ git.pull()
+ git.checkout('-b', branch_name)
+
+ with open(cheaha_ldif, 'w') as ldif_f,\
+ open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
+ slapcat('-f', '/cm/local/apps/openldap/etc/slapd.conf', '-b', "'dc=cm,dc=cluster'", _out=ldif_f)
+ ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
+
+ git.diff()
+ git.add(cheaha_ldif)
+ git.add(cheaha_ldapsearch_ldif)
+ git.commit(m="Added new cheaha user: " + username)
+ git.push('origin', branch_name)
+ git.checkout('master')
+ time.sleep(60)
+ git.pull()
success = True
except:
@@ -77,7 +60,6 @@ def git_commit(ch, method, properties, body):
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
'msg': {
- 'debug': DEBUG,
'task': task,
'success': success
}
--
GitLab
From f5509fa1bd2369862b46bbfb90ccefbb86a967fe Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 8 Apr 2020 22:03:47 -0500
Subject: [PATCH 039/188] Fix git current working directory syntax
Also, add os module to expand ~ to user home
---
git_commit.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 4f29fe2..0bdd004 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+import os
import sh
import sys
import json
@@ -10,7 +11,7 @@ task = 'git_commit'
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
-repo_location = '~/git/rc-users'
+repo_location = os.path.expanduser('~/git/rc-users')
cheaha_ldif = repo_location + '/users/cheaha-openldap.ldif'
cheaha_ldapsearch_ldif = repo_location + '/users/cheaha-openldap-ldapsearch.ldif'
@@ -20,7 +21,7 @@ parser.add_argument('-v', '--verbose', action='store_true', help='verbose output
parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
args = parser.parse_args()
-git = sh.git.bake(repo_location)
+git = sh.git.bake('-C', repo_location)
slapcat = sh.Command('/cm/local/apps/openldap/sbin/slapcat')
ldapsearch = sh.Command('ldapsearch')
--
GitLab
From 6882eb65f7b570652778a4d58d4ce8559fa9f4a4 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 8 Apr 2020 22:09:50 -0500
Subject: [PATCH 040/188] Define some path as variable
---
git_commit.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 0bdd004..ee348a7 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -11,9 +11,12 @@ task = 'git_commit'
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+# Define some location
repo_location = os.path.expanduser('~/git/rc-users')
cheaha_ldif = repo_location + '/users/cheaha-openldap.ldif'
cheaha_ldapsearch_ldif = repo_location + '/users/cheaha-openldap-ldapsearch.ldif'
+slapcat_bin = '/cm/local/apps/openldap/sbin/slapcat'
+slapd_conf = '/cm/local/apps/openldap/etc/slapd.conf'
# Parse arguments
parser = argparse.ArgumentParser()
@@ -22,7 +25,7 @@ parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run
args = parser.parse_args()
git = sh.git.bake('-C', repo_location)
-slapcat = sh.Command('/cm/local/apps/openldap/sbin/slapcat')
+slapcat = sh.Command(slapcat_bin)
ldapsearch = sh.Command('ldapsearch')
def git_commit(ch, method, properties, body):
@@ -40,7 +43,7 @@ def git_commit(ch, method, properties, body):
with open(cheaha_ldif, 'w') as ldif_f,\
open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
- slapcat('-f', '/cm/local/apps/openldap/etc/slapd.conf', '-b', "'dc=cm,dc=cluster'", _out=ldif_f)
+ slapcat('-f', slapd_conf, '-b', "'dc=cm,dc=cluster'", _out=ldif_f)
ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
git.diff()
--
GitLab
From 4924e7f87b5ab55b4e08106a4824520dc358d679 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 8 Apr 2020 22:15:50 -0500
Subject: [PATCH 041/188] Remove two unnecessary steps
---
git_commit.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index ee348a7..4739cb7 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -52,8 +52,6 @@ def git_commit(ch, method, properties, body):
git.commit(m="Added new cheaha user: " + username)
git.push('origin', branch_name)
git.checkout('master')
- time.sleep(60)
- git.pull()
success = True
except:
--
GitLab
From 38b86f2ed3e59db24222b2f21c5f397b64e7f947 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 8 Apr 2020 22:17:57 -0500
Subject: [PATCH 042/188] Modify steps
Instead of pushing issue branch, and merge on web page
merging by agent than push master branch to remote
---
git_commit.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index 4739cb7..53a31ec 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -50,9 +50,11 @@ def git_commit(ch, method, properties, body):
git.add(cheaha_ldif)
git.add(cheaha_ldapsearch_ldif)
git.commit(m="Added new cheaha user: " + username)
- git.push('origin', branch_name)
git.checkout('master')
+ git.merge(branch_name, '--no-ff', '--no-edit')
+ git.push('origin', 'master')
+
success = True
except:
e = sys.exc_info()[0]
--
GitLab
From b2df7ebd38c86fa21bf817a0b55c692914b0ff8f Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 8 Apr 2020 22:25:27 -0500
Subject: [PATCH 043/188] Add logging module
replace print statement with logging
---
git_commit.py | 56 ++++++++++++++++++++++++++++++++++++++++++++-------
1 file changed, 49 insertions(+), 7 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 53a31ec..bde549c 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -4,6 +4,7 @@ import sh
import sys
import json
import argparse
+import logging
from rc_rmq import RCRMQ
task = 'git_commit'
@@ -18,15 +19,32 @@ cheaha_ldapsearch_ldif = repo_location + '/users/cheaha-openldap-ldapsearch.ldif
slapcat_bin = '/cm/local/apps/openldap/sbin/slapcat'
slapd_conf = '/cm/local/apps/openldap/etc/slapd.conf'
+# Default logger level
+logger_lvl = logging.WARNING
+
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true', help='verbose output')
parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
args = parser.parse_args()
-git = sh.git.bake('-C', repo_location)
-slapcat = sh.Command(slapcat_bin)
-ldapsearch = sh.Command('ldapsearch')
+if args.verbose:
+ logger_lvl = logging.DEBUG
+
+if not args.dry_run:
+ git = sh.git.bake('-C', repo_location)
+ slapcat = sh.Command(slapcat_bin)
+ ldapsearch = sh.Command('ldapsearch')
+else:
+ logger_lvl = logging.INFO
+ git = sh.echo.bake('git', '-C', repo_location)
+ slapcat = sh.echo.bake(slapcat_bin)
+ ldapsearch = sh.echo.bake('ldapsearch')
+
+# Logger
+logging.basicConfig(format='%(asctime)s [%(module)s] - %(message)s', level=logger_lvl)
+logger = logging.getLogger(__name__)
+
def git_commit(ch, method, properties, body):
msg = json.loads(body)
@@ -35,32 +53,53 @@ def git_commit(ch, method, properties, body):
success = False
branch_name = 'issue-' + ticketnum
+ logger.info("Received: %s", msg)
+ logger.debug("ticketnum: %s", ticketnum)
+ logger.debug("branch_name: %s", branch_name)
+
try:
+ logger.debug('git checkout master')
git.checkout('master')
+ logger.debug('git pull')
git.pull()
+ logger.debug('git checkout -b %s', branch_name)
git.checkout('-b', branch_name)
+ logger.debug("open(%s, 'w'), open(%s, 'w')", cheaha_ldif, cheaha_ldapsearch_ldif)
with open(cheaha_ldif, 'w') as ldif_f,\
open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
+ logger.debug("%s -f %s -b 'dc=cm,dc=cluster' > %s", slapcat_bin, slapcat_config, cheaha_ldif)
slapcat('-f', slapd_conf, '-b', "'dc=cm,dc=cluster'", _out=ldif_f)
+ logger.debug("ldapsearch -LLL -x -h ldapserver -b 'dc=cm,dc=cluster' > %s", ldapsearch_ldif)
ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
+ logger.info('ldif files generated.')
+ logger.debug('git diff')
git.diff()
+ logger.debug('git add %s', cheaha_ldif)
git.add(cheaha_ldif)
+ logger.debug('git add %s', cheaha_ldapsearch_ldif)
git.add(cheaha_ldapsearch_ldif)
+ logger.debug("git commit -m 'Added new cheaha user: %s'", branch_name)
git.commit(m="Added new cheaha user: " + username)
+ logger.debug('git checkout master')
git.checkout('master')
+ logger.debug('git merge %s --no-ff --no-edit', branch_name)
git.merge(branch_name, '--no-ff', '--no-edit')
+ logger.debug('git push origin master')
git.push('origin', 'master')
+ # merge with gitlab api
+
+ logger.info('updated ldif files and committed to git repo')
success = True
- except:
- e = sys.exc_info()[0]
- print("[{}]: Error: {}".format(task, e))
+ except Exception as exception:
+ logger.error('', exc_info=True)
# Send confirm message
+ logger.debug('rc_rmq.publish_msge()')
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
'msg': {
@@ -68,16 +107,19 @@ def git_commit(ch, method, properties, body):
'success': success
}
})
+ logger.info('confirmation sent')
# Acknowledge message
+ logger.debug('ch.basic_ack()')
ch.basic_ack(delivery_tag=method.delivery_tag)
-print("Start listening to queue: {}".format(task))
+logger.info("Start listening to queue: %s", task)
rc_rmq.start_consume({
'queue': task,
'routing_key': "git.*",
'cb': git_commit
})
+logger.info("Disconnected")
rc_rmq.disconnect()
--
GitLab
From 04856761370aa7a29fbc44ce2501563bf5083813 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 8 Apr 2020 22:26:21 -0500
Subject: [PATCH 044/188] Update routing key
replace with verify.*
---
git_commit.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index bde549c..0c6484c 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -117,7 +117,7 @@ def git_commit(ch, method, properties, body):
logger.info("Start listening to queue: %s", task)
rc_rmq.start_consume({
'queue': task,
- 'routing_key': "git.*",
+ 'routing_key': "verify.*",
'cb': git_commit
})
--
GitLab
From 12e99e55ca14e6a9874b5a2ce2aeb397abe00d09 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 20 Apr 2020 21:55:11 -0500
Subject: [PATCH 045/188] Remove slapcat and related
---
git_commit.py | 14 ++------------
1 file changed, 2 insertions(+), 12 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 0c6484c..122d3da 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -14,10 +14,7 @@ rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
# Define some location
repo_location = os.path.expanduser('~/git/rc-users')
-cheaha_ldif = repo_location + '/users/cheaha-openldap.ldif'
cheaha_ldapsearch_ldif = repo_location + '/users/cheaha-openldap-ldapsearch.ldif'
-slapcat_bin = '/cm/local/apps/openldap/sbin/slapcat'
-slapd_conf = '/cm/local/apps/openldap/etc/slapd.conf'
# Default logger level
logger_lvl = logging.WARNING
@@ -33,12 +30,10 @@ if args.verbose:
if not args.dry_run:
git = sh.git.bake('-C', repo_location)
- slapcat = sh.Command(slapcat_bin)
ldapsearch = sh.Command('ldapsearch')
else:
logger_lvl = logging.INFO
git = sh.echo.bake('git', '-C', repo_location)
- slapcat = sh.echo.bake(slapcat_bin)
ldapsearch = sh.echo.bake('ldapsearch')
# Logger
@@ -66,19 +61,14 @@ def git_commit(ch, method, properties, body):
logger.debug('git checkout -b %s', branch_name)
git.checkout('-b', branch_name)
- logger.debug("open(%s, 'w'), open(%s, 'w')", cheaha_ldif, cheaha_ldapsearch_ldif)
- with open(cheaha_ldif, 'w') as ldif_f,\
- open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
- logger.debug("%s -f %s -b 'dc=cm,dc=cluster' > %s", slapcat_bin, slapcat_config, cheaha_ldif)
- slapcat('-f', slapd_conf, '-b', "'dc=cm,dc=cluster'", _out=ldif_f)
+ logger.debug("open(%s, 'w')", cheaha_ldapsearch_ldif)
+ with open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
logger.debug("ldapsearch -LLL -x -h ldapserver -b 'dc=cm,dc=cluster' > %s", ldapsearch_ldif)
ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
logger.info('ldif files generated.')
logger.debug('git diff')
git.diff()
- logger.debug('git add %s', cheaha_ldif)
- git.add(cheaha_ldif)
logger.debug('git add %s', cheaha_ldapsearch_ldif)
git.add(cheaha_ldapsearch_ldif)
logger.debug("git commit -m 'Added new cheaha user: %s'", branch_name)
--
GitLab
From db39a688fca841e49d4f17be92b66103759705fd Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 20 Apr 2020 22:45:11 -0500
Subject: [PATCH 046/188] Separate ldif output file
~/git/rc-users/users/<username>.ldif
user info
~/git/rc-users/groups/<username>.ldif
group info
---
git_commit.py | 26 +++++++++++++++++---------
1 file changed, 17 insertions(+), 9 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 122d3da..3413117 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -14,7 +14,8 @@ rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
# Define some location
repo_location = os.path.expanduser('~/git/rc-users')
-cheaha_ldapsearch_ldif = repo_location + '/users/cheaha-openldap-ldapsearch.ldif'
+users_dir = repo_location + '/users'
+groups_dir = repo_location + '/groups'
# Default logger level
logger_lvl = logging.WARNING
@@ -47,6 +48,8 @@ def git_commit(ch, method, properties, body):
ticketnum = msg.get('ticketnum', 'add-users-' + username.lower())
success = False
branch_name = 'issue-' + ticketnum
+ user_ldif = users_dir + f'/{username}.ldif'
+ group_ldif = groups_dir + f'/{username}.ldif'
logger.info("Received: %s", msg)
logger.debug("ticketnum: %s", ticketnum)
@@ -61,16 +64,21 @@ def git_commit(ch, method, properties, body):
logger.debug('git checkout -b %s', branch_name)
git.checkout('-b', branch_name)
- logger.debug("open(%s, 'w')", cheaha_ldapsearch_ldif)
- with open(cheaha_ldapsearch_ldif, 'w') as ldapsearch_ldif_f:
- logger.debug("ldapsearch -LLL -x -h ldapserver -b 'dc=cm,dc=cluster' > %s", ldapsearch_ldif)
- ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'dc=cm,dc=cluster'", _out=ldapsearch_ldif_f)
- logger.info('ldif files generated.')
+ logger.debug("open(%s, 'w'), open(%s, 'w')", user_ldif, group_ldif)
+ with open(user_ldif, 'w') as ldif_u,\
+ open(group_ldif, 'w') as ldif_g:
+ logger.debug(f"ldapsearch -LLL -x -h ldapserver -b 'dc=cm,dc=cluster' uid={username} > {user_ldif}")
+ ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'dc=cm,dc=cluster'", f"uid={username}", _out=ldif_u)
+ logger.debug(f"ldapsearch -LLL -x -h ldapserver -b 'ou=Group,dc=cm,dc=cluster' cn={username} > {group_ldif}")
+ ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'ou=Group,dc=cm,dc=cluster'", f"cn={username}", _out=ldif_g)
+ logger.info('user ldif files generated.')
logger.debug('git diff')
git.diff()
- logger.debug('git add %s', cheaha_ldapsearch_ldif)
- git.add(cheaha_ldapsearch_ldif)
+ logger.debug('git add %s', user_ldif)
+ git.add(user_ldif)
+ logger.debug('git add %s', group_ldif)
+ git.add(group_ldif)
logger.debug("git commit -m 'Added new cheaha user: %s'", branch_name)
git.commit(m="Added new cheaha user: " + username)
logger.debug('git checkout master')
@@ -82,7 +90,7 @@ def git_commit(ch, method, properties, body):
git.push('origin', 'master')
# merge with gitlab api
- logger.info('updated ldif files and committed to git repo')
+ logger.info('Added ldif files and committed to git repo')
success = True
except Exception as exception:
--
GitLab
From 660def700c04fcfb5dad3c934dcc0715064a306d Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 20 Apr 2020 22:47:26 -0500
Subject: [PATCH 047/188] Remove git diff step
---
git_commit.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 3413117..d2a6cdd 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -73,8 +73,6 @@ def git_commit(ch, method, properties, body):
ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'ou=Group,dc=cm,dc=cluster'", f"cn={username}", _out=ldif_g)
logger.info('user ldif files generated.')
- logger.debug('git diff')
- git.diff()
logger.debug('git add %s', user_ldif)
git.add(user_ldif)
logger.debug('git add %s', group_ldif)
--
GitLab
From 859e1b749e46242cffd2d3c9edeb335fb6210206 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 20 Apr 2020 22:48:10 -0500
Subject: [PATCH 048/188] Fix debug message
---
git_commit.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index d2a6cdd..97c65f3 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -77,7 +77,7 @@ def git_commit(ch, method, properties, body):
git.add(user_ldif)
logger.debug('git add %s', group_ldif)
git.add(group_ldif)
- logger.debug("git commit -m 'Added new cheaha user: %s'", branch_name)
+ logger.debug("git commit -m 'Added new cheaha user: %s'", username)
git.commit(m="Added new cheaha user: " + username)
logger.debug('git checkout master')
git.checkout('master')
--
GitLab
From 9d7d69289ce38bbb611f7ccecfae1d22a5c07311 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 22 Apr 2020 14:53:21 -0500
Subject: [PATCH 049/188] Utilize logger from rc_util
---
git_commit.py | 21 +++------------------
1 file changed, 3 insertions(+), 18 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 97c65f3..c7391e9 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -3,8 +3,7 @@ import os
import sh
import sys
import json
-import argparse
-import logging
+import rc_util
from rc_rmq import RCRMQ
task = 'git_commit'
@@ -17,17 +16,8 @@ repo_location = os.path.expanduser('~/git/rc-users')
users_dir = repo_location + '/users'
groups_dir = repo_location + '/groups'
-# Default logger level
-logger_lvl = logging.WARNING
-
-# Parse arguments
-parser = argparse.ArgumentParser()
-parser.add_argument('-v', '--verbose', action='store_true', help='verbose output')
-parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
-args = parser.parse_args()
-
-if args.verbose:
- logger_lvl = logging.DEBUG
+args = rc_util.get_args()
+logger = rc_util.get_logger(args)
if not args.dry_run:
git = sh.git.bake('-C', repo_location)
@@ -37,11 +27,6 @@ else:
git = sh.echo.bake('git', '-C', repo_location)
ldapsearch = sh.echo.bake('ldapsearch')
-# Logger
-logging.basicConfig(format='%(asctime)s [%(module)s] - %(message)s', level=logger_lvl)
-logger = logging.getLogger(__name__)
-
-
def git_commit(ch, method, properties, body):
msg = json.loads(body)
username = msg['username']
--
GitLab
From 659708535c5cc7183f7bfaad46d6dbd6afaf5996 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 22 Apr 2020 23:49:56 -0500
Subject: [PATCH 050/188] Fix incorrect variable name
---
notify_user.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index 6c0c119..e0a6bed 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -30,9 +30,9 @@ Subject: {opts['subject']}
smtp = smtplib.SMTP(opts['server'])
if 'bcc' in opts:
- smtp.sendmail(opts['from'], [to, opts['bcc']], messagge)
+ smtp.sendmail(opts['from'], [to, opts['bcc']], msg)
else:
- smtp.sendmail(opts['from'], [to], messagge)
+ smtp.sendmail(opts['from'], [to], msg)
# Email instruction to user
--
GitLab
From 3436edbfcc79befe8523f19bd530e0e7293bbe5f Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 13:44:32 -0500
Subject: [PATCH 051/188] Update mail.py
Added more variables
Capitalize variable name
Added `Whole_mail`
---
mail.py | 21 ++++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
diff --git a/mail.py b/mail.py
index dd2df02..e34e862 100644
--- a/mail.py
+++ b/mail.py
@@ -1,8 +1,17 @@
# Some variable for email
-my_email = 'admin@localhost'
-info_url = 'https://www.google.com'
+Server = 'localhost'
+My_email = 'admin@localhost'
+Sender = 'ADMIN@LISTSERV.LOCALHOST'
+Sender_alias = 'Services'
+Subject = 'New User Account'
+Info_url = 'https://www.google.com'
-body = f"""
+Head = f"""From: {Sender_alias} <{Sender}>
+To: <{{{{ to }}}}>
+Subject: {Subject}
+"""
+
+Body = f"""
Hi {{{{ username }}}}
Your account has been set up with:
@@ -11,9 +20,11 @@ User ID: {{{{ username }}}}
============================
If you have any questions, please visit:
-{info_url}
+{Info_url}
-or email at {my_email}
+or email at {My_email}
Cheers,
"""
+
+Whole_mail = Head + Body
--
GitLab
From 906bac7233dfe456912e3f71f2bc9ccd9bbc9b64 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 13:50:06 -0500
Subject: [PATCH 052/188] Update confirmation message
---
notify_user.py | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index e0a6bed..d04866c 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -40,7 +40,8 @@ def notify_user(ch, method, properties, body):
msg = json.loads(body)
username = msg['username']
user_email = msg['email']
- success = False
+ msg['task'] = task
+ msg['success'] = False
email_body = Template(mail.body).render(username=username)
@@ -62,7 +63,7 @@ def notify_user(ch, method, properties, body):
'body': email_body
})
- success = True
+ msg['success'] = True
except:
e = sys.exc_info()[0]
print("[{}]: Error: {}".format(task, e))
@@ -74,10 +75,7 @@ def notify_user(ch, method, properties, body):
# send confirm message
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
- 'msg': {
- 'task': task,
- 'success': success
- }
+ 'msg': msg
})
--
GitLab
From 25d7518f7bdfaeec2c007ad1280709085facfcbf Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 13:54:45 -0500
Subject: [PATCH 053/188] Utilize rc_util
replace DEBUG with args.dry_run
replace print statement with logger
---
notify_user.py | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index d04866c..e6c45b2 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
import sys
import json
+import rc_util
import smtplib
from rc_rmq import RCRMQ
from jinja2 import Template
@@ -8,7 +9,8 @@ import mail
task = 'notify_user'
-DEBUG = False
+args = rc_util.get_args()
+logger = rc_util.get_logger(args)
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
@@ -24,7 +26,7 @@ To: <{to}>
Subject: {opts['subject']}
{opts['body']}
"""
- if DEBUG:
+ if args.dry_run:
print(msg)
return
@@ -47,7 +49,7 @@ def notify_user(ch, method, properties, body):
try:
#Send email to user
- if DEBUG:
+ if args.dry_run:
send_email('louistw@uab.edu', {
'from': support_email,
'from_alias': 'Research Computing Services',
@@ -64,11 +66,10 @@ def notify_user(ch, method, properties, body):
})
msg['success'] = True
- except:
- e = sys.exc_info()[0]
- print("[{}]: Error: {}".format(task, e))
+ except Exception as exception:
+ logger.error('', exc_info=True)
- if not DEBUG:
+ if not args.dry_run:
# acknowledge the message
ch.basic_ack(delivery_tag=method.delivery_tag)
@@ -80,11 +81,12 @@ def notify_user(ch, method, properties, body):
if __name__ == "__main__":
- print("Start listening to queue: {}".format(task))
+ logger.info(f'Start listening to queue: {task}')
rc_rmq.start_consume({
'queue': task,
'routing_key': "notify.*",
'cb': notify_user
})
+ logger.info('Disconnected')
rc_rmq.disconnect()
--
GitLab
From 67105051bd22ec7f0b89c10664aeee35b5682833 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 14:48:28 -0500
Subject: [PATCH 054/188] Rename mail.py to mail_config.py
---
mail.py => mail_config.py | 4 ++--
notify_user.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
rename mail.py => mail_config.py (88%)
diff --git a/mail.py b/mail_config.py
similarity index 88%
rename from mail.py
rename to mail_config.py
index e34e862..1146e38 100644
--- a/mail.py
+++ b/mail_config.py
@@ -1,7 +1,7 @@
# Some variable for email
Server = 'localhost'
-My_email = 'admin@localhost'
-Sender = 'ADMIN@LISTSERV.LOCALHOST'
+My_email = 'root@localhost'
+Sender = 'ROOT@LOCALHOST'
Sender_alias = 'Services'
Subject = 'New User Account'
Info_url = 'https://www.google.com'
diff --git a/notify_user.py b/notify_user.py
index e6c45b2..a72feff 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -5,7 +5,7 @@ import rc_util
import smtplib
from rc_rmq import RCRMQ
from jinja2 import Template
-import mail
+import mail_config as mail_cfg
task = 'notify_user'
--
GitLab
From 6055dfe348a0b63fb44881973eff0b10dfad73f8 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 15:02:21 -0500
Subject: [PATCH 055/188] Merge send_email function into callback function
Avoiding unnecessary function call
---
notify_user.py | 42 ++++++------------------------------------
1 file changed, 6 insertions(+), 36 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index a72feff..9eb558d 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -15,28 +15,6 @@ logger = rc_util.get_logger(args)
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
-def send_email(to, opts=None):
- opts['server'] = opts.get('server', 'localhost')
- opts['from'] = opts.get('from', 'SUPPORT@LISTSERV.UAB.EDU')
- opts['from_alias'] = opts.get('from_alias', 'Research Computing Services')
- opts['subject'] = opts.get('subject', 'HPC New User Account')
-
- msg = f"""From: {opts['from_alias']} <{opts['from']}>
-To: <{to}>
-Subject: {opts['subject']}
-{opts['body']}
-"""
- if args.dry_run:
- print(msg)
- return
-
- smtp = smtplib.SMTP(opts['server'])
- if 'bcc' in opts:
- smtp.sendmail(opts['from'], [to, opts['bcc']], msg)
- else:
- smtp.sendmail(opts['from'], [to], msg)
-
-
# Email instruction to user
def notify_user(ch, method, properties, body):
msg = json.loads(body)
@@ -45,25 +23,17 @@ def notify_user(ch, method, properties, body):
msg['task'] = task
msg['success'] = False
- email_body = Template(mail.body).render(username=username)
-
try:
#Send email to user
+ receiver = [user_mail, mail_cfg.My_mail]
+ message = Template(mail_cfg.Whole_mail).render(username=username, to=user_email)
+
if args.dry_run:
- send_email('louistw@uab.edu', {
- 'from': support_email,
- 'from_alias': 'Research Computing Services',
- 'subject': f'[TEST] New User Account on {hpcsrv}',
- 'body': email_body
- })
+ logger.info("smtp.sendmail(sender, receiver, message)")
else:
- send_email(user_email, {
- 'from': support_email,
- 'from_alias': 'Research Computing Services',
- 'subject': f'New User Account on {hpcsrv}',
- 'body': email_body
- })
+ smtp = smtplib.SMTP(mail_cfg.Server)
+ smtp.sendmail(sender, receiver, message)
msg['success'] = True
except Exception as exception:
--
GitLab
From 63389215412d8ef49e0f0f1946beebed7f61e65f Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 15:20:39 -0500
Subject: [PATCH 056/188] Send confirmation even dry-run mode
---
notify_user.py | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index 9eb558d..0d0d4c3 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -39,15 +39,14 @@ def notify_user(ch, method, properties, body):
except Exception as exception:
logger.error('', exc_info=True)
- if not args.dry_run:
- # acknowledge the message
- ch.basic_ack(delivery_tag=method.delivery_tag)
-
- # send confirm message
- rc_rmq.publish_msg({
- 'routing_key': 'confirm.' + username,
- 'msg': msg
- })
+ # acknowledge the message
+ ch.basic_ack(delivery_tag=method.delivery_tag)
+
+ # send confirm message
+ rc_rmq.publish_msg({
+ 'routing_key': 'confirm.' + username,
+ 'msg': msg
+ })
if __name__ == "__main__":
--
GitLab
From 3a0474f418c5360bf36f12d7aabdc5a07edb0b0e Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 15:21:40 -0500
Subject: [PATCH 057/188] Send confirmation at the end of task
---
notify_user.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index 0d0d4c3..f324782 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -39,15 +39,14 @@ def notify_user(ch, method, properties, body):
except Exception as exception:
logger.error('', exc_info=True)
- # acknowledge the message
- ch.basic_ack(delivery_tag=method.delivery_tag)
-
# send confirm message
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
'msg': msg
})
+ # acknowledge the message
+ ch.basic_ack(delivery_tag=method.delivery_tag)
if __name__ == "__main__":
logger.info(f'Start listening to queue: {task}')
--
GitLab
From eb53c160aa4a376da540a1d60603f2e0844051ee Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 23 Apr 2020 20:22:03 +0000
Subject: [PATCH 058/188] Add uid, gid to the msg obj if user exists else get
next uid,gid.
---
get-next-uid-gid.py | 24 ++++++++++--------------
1 file changed, 10 insertions(+), 14 deletions(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index b4b5d5e..05ded78 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -42,25 +42,21 @@ def get_next_uid_gid(ch, method, properties, body):
# Determine next available UID
try:
- #if user_exists(username):
- if False:
+ if user_exists(username):
logger.info("The user, {} already exists".format(username))
- sys.exit(1)
+ msg['uid'] = result[0][1]['uidNumber'][0].decode('utf-8')
+ msg['gid'] = result[0][1]['gidNumber'][0].decode('utf-8')
- cmd_uid = "/usr/bin/getent passwd | \
- awk -F: '($3>10000) && ($3<20000) && ($3>maxuid) { maxuid=$3; } END { print maxuid+1; }'"
- if not args.dry_run:
+ else:
+ cmd_uid = "/usr/bin/getent passwd | \
+ awk -F: '($3>10000) && ($3<20000) && ($3>maxuid) { maxuid=$3; } END { print maxuid+1; }'"
msg['uid'] = popen(cmd_uid).read().rstrip()
+ logger.info(f"UID query: {cmd_uid}")
- logger.info(f"UID query: {cmd_uid}")
-
- cmd_gid = "/usr/bin/getent group | \
- awk -F: '($3>10000) && ($3<20000) && ($3>maxgid) { maxgid=$3; } END { print maxgid+1; }'"
- if not args.dry_run:
+ cmd_gid = "/usr/bin/getent group | \
+ awk -F: '($3>10000) && ($3<20000) && ($3>maxgid) { maxgid=$3; } END { print maxgid+1; }'"
msg['gid'] = popen(cmd_gid).read().rstrip()
-
- logger.info(f"GID query: {cmd_gid}")
- success = True
+ logger.info(f"GID query: {cmd_gid}")
except Exception:
logger.exception("Fatal error:")
--
GitLab
From da7225e78e6b637e5e0aef4d8ff612e72a69a974 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 23 Apr 2020 20:25:17 +0000
Subject: [PATCH 059/188] Adapt msg passing to fit workflow/task manager agent.
Conflicts:
get-next-uid-gid.py
---
get-next-uid-gid.py | 15 +++------------
1 file changed, 3 insertions(+), 12 deletions(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index 05ded78..49eb977 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -57,6 +57,8 @@ def get_next_uid_gid(ch, method, properties, body):
awk -F: '($3>10000) && ($3<20000) && ($3>maxgid) { maxgid=$3; } END { print maxgid+1; }'"
msg['gid'] = popen(cmd_gid).read().rstrip()
logger.info(f"GID query: {cmd_gid}")
+ msg['task'] = task
+ msg['success'] = True
except Exception:
logger.exception("Fatal error:")
@@ -67,21 +69,10 @@ def get_next_uid_gid(ch, method, properties, body):
logger.debug('rc_rmq.publish_msg()')
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
- 'msg': {
- 'task': task,
- 'success': success
- }
+ 'msg': msg
})
logger.info('confirmation sent')
- if success:
- # Send create message to BrightCM agent
- logger.info(f'The task {task} finished, sending create msg to next queue')
- rc_rmq.publish_msg({
- 'routing_key': 'create.' + username,
- 'msg': msg
- })
-
logger.info("Start listening to queue: {}".format(task))
rc_rmq.start_consume({
'queue': task,
--
GitLab
From 550a4761cee5a04565637532f38698644bcc23d5 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 23 Apr 2020 20:29:55 +0000
Subject: [PATCH 060/188] Adapt msg passing in bright account create to fit
task manager
---
brightcm_account_create.py | 17 +++--------------
1 file changed, 3 insertions(+), 14 deletions(-)
diff --git a/brightcm_account_create.py b/brightcm_account_create.py
index ed6c71b..2c57b30 100644
--- a/brightcm_account_create.py
+++ b/brightcm_account_create.py
@@ -37,7 +37,8 @@ def bright_account_create(ch, method, properties, body):
if not args.dry_run:
popen(cmd)
logger.info(f'Bright command to create user:{cmd}')
- success = True
+ msg['task'] = task
+ msg['success'] = True
except Exception:
logger.exception("Fatal error:")
@@ -48,22 +49,10 @@ def bright_account_create(ch, method, properties, body):
logger.debug('rc_rmq.publish_msg()')
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
- 'msg': {
- 'task': task,
- 'success': success
- }
+ 'msg': msg
})
logger.info('confirmation sent')
- if success:
- # send create message to verify dir permissions agent
- logger.debug(f'The task {task} finished successfully')
- rc_rmq.publish_msg({
- 'routing_key': 'verify.' + username,
- 'msg': msg
- })
- logger.info('verify msg sent to next agent')
-
logger.info("Start listening to queue: {}".format(task))
rc_rmq.start_consume({
'queue': task,
--
GitLab
From 1abd8b72c7514ae3a0f3e8de601c23fabb31f030 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 23 Apr 2020 20:31:20 +0000
Subject: [PATCH 061/188] Change the mail list admin var.
---
subscribe_mail_lists.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index 98d6097..4867287 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -28,7 +28,7 @@ def mail_list_subscription(ch, method, properties, body):
fullname = msg['fullname']
email = msg['email']
- mail_list_admin = 'admin@uab.edu' #change this during deploy
+ mail_list_admin = 'root@localhost' #change this during deploy
mail_list = 'LISTSERV@LISTSERV.UAB.EDU'
listserv_cmd = {}
--
GitLab
From 86e45dc89a1d3c326a106487e8b549f94b347a1e Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 23 Apr 2020 20:33:38 +0000
Subject: [PATCH 062/188] Adapt msg passing to fit task manager
---
subscribe_mail_lists.py | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index 4867287..8304305 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -55,7 +55,8 @@ def mail_list_subscription(ch, method, properties, body):
logging.info(f'This email will add user {username} to {key}\n{email_msg}')
s.quit()
- success = True
+ msg['task'] = task
+ msg['success'] = True
except Exception:
logger.exception("Fatal error:")
@@ -66,10 +67,7 @@ def mail_list_subscription(ch, method, properties, body):
logger.debug('rc_rmq.publish_msg()')
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
- 'msg': {
- 'task': task,
- 'success': success
- }
+ 'msg': msg
})
logger.info('confirmation sent')
--
GitLab
From 9db8544a32dc59d8794a1dbc5c62b74316a4979e Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 16:15:16 -0500
Subject: [PATCH 063/188] Add database logic
---
notify_user.py | 43 ++++++++++++++++++++++++++++++++++++-------
1 file changed, 36 insertions(+), 7 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index f324782..e2cc28b 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -3,8 +3,10 @@ import sys
import json
import rc_util
import smtplib
+import dataset
from rc_rmq import RCRMQ
from jinja2 import Template
+from datetime import datetime
import mail_config as mail_cfg
task = 'notify_user'
@@ -12,6 +14,9 @@ task = 'notify_user'
args = rc_util.get_args()
logger = rc_util.get_logger(args)
+db = dataset.connect(f'sqlite:///.agent_db/{task}.db')
+table = db['notified_user']
+
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
@@ -24,16 +29,40 @@ def notify_user(ch, method, properties, body):
msg['success'] = False
try:
- #Send email to user
- receiver = [user_mail, mail_cfg.My_mail]
- message = Template(mail_cfg.Whole_mail).render(username=username, to=user_email)
- if args.dry_run:
- logger.info("smtp.sendmail(sender, receiver, message)")
+ # Search username in database
+ record = table.find_one(username=username)
+
+ if record:
+ # Update counter
+ count = record['count']
+ table.update({'username': username, 'count': count + 1}, ['username'])
+
+ logger.debug(f'User {username} counter updated to {count + 1}')
else:
- smtp = smtplib.SMTP(mail_cfg.Server)
- smtp.sendmail(sender, receiver, message)
+ # Send email to user
+ receiver = [user_mail, mail_cfg.My_mail]
+ message = Template(mail_cfg.Whole_mail).render(username=username, to=user_email)
+
+ if args.dry_run:
+ logger.info(f'smtp = smtplib.SMTP({mail_cfg.Server})')
+ logger.info(f'smtp.sendmail({sender}, {receiver}, message)')
+ logger.info(f"table.insert({'username': {username}, 'count': 1, 'sent_at': {datetime.now()}})")
+
+ else:
+ smtp = smtplib.SMTP(mail_cfg.Server)
+ smtp.sendmail(sender, receiver, message)
+
+ logger.debug(f'Email sent to: {user_email}')
+
+ table.insert({
+ 'username': username,
+ 'count': 1,
+ 'sent_at': datetime.now()
+ })
+
+ logger.debug(f'User {username} inserted into database')
msg['success'] = True
except Exception as exception:
--
GitLab
From ba2041de6b0514a5bf5c2a1b8cccc43e7cebd62e Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 23 Apr 2020 21:12:47 +0000
Subject: [PATCH 064/188] Adapt to new msg passing model to see all event msgs.
---
user_reg_event_logger.py | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/user_reg_event_logger.py b/user_reg_event_logger.py
index d762e06..02c1248 100644
--- a/user_reg_event_logger.py
+++ b/user_reg_event_logger.py
@@ -13,18 +13,13 @@ def log_user_reg_events(ch, method, properties, body):
# Retrieve message
msg = json.loads(body)
- #print(msg)
# Retrieve routing key
routing_key = method.routing_key
action = routing_key.split(".")[0]
user = routing_key.split(".")[1]
- if action != 'confirm':
- print(f'Got a message for {user}: {routing_key}')
- else:
- task = msg['task']
- status = msg['success']
- print(f'Task {task} completed?: {status}')
+ print(f'Got a {action} message for {user} with routing key: {routing_key}')
+ print(msg)
# Acknowledge message
ch.basic_ack(delivery_tag=method.delivery_tag)
--
GitLab
From c353b6f308d9a1720a805aa1c16b6aff4a6debd5 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 16:18:59 -0500
Subject: [PATCH 065/188] Update comment
---
notify_user.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index e2cc28b..51fa598 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -68,15 +68,16 @@ def notify_user(ch, method, properties, body):
except Exception as exception:
logger.error('', exc_info=True)
- # send confirm message
+ # Send confirm message
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
'msg': msg
})
- # acknowledge the message
+ # Acknowledge the message
ch.basic_ack(delivery_tag=method.delivery_tag)
+
if __name__ == "__main__":
logger.info(f'Start listening to queue: {task}')
rc_rmq.start_consume({
--
GitLab
From 1f625e0c95fb1f93115313b39f508635da12f385 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 16:20:01 -0500
Subject: [PATCH 066/188] Add logger message after confirmation sent
---
notify_user.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/notify_user.py b/notify_user.py
index 51fa598..67c5b0d 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -74,6 +74,8 @@ def notify_user(ch, method, properties, body):
'msg': msg
})
+ logger.debug(f'User {username} confirmation sent')
+
# Acknowledge the message
ch.basic_ack(delivery_tag=method.delivery_tag)
--
GitLab
From 66d7dbf27bbdc3aab525d17581e67bec2aed0a1d Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 17:24:17 -0500
Subject: [PATCH 067/188] Utilize rc_util
---
dir_verify.py | 32 +++++++++++++++++++++++---------
1 file changed, 23 insertions(+), 9 deletions(-)
diff --git a/dir_verify.py b/dir_verify.py
index 9491bce..e38d0b4 100644
--- a/dir_verify.py
+++ b/dir_verify.py
@@ -2,12 +2,16 @@
import sys
import json
import shutil
+import rc_util
from pathlib import Path
from rc_rmq import RCRMQ
task = 'dir_verify'
dirs = ['/home', '/data/user', '/data/scratch']
+args = rc_util.get_args()
+logger = rc_util.get_logger(args)
+
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
@@ -20,16 +24,23 @@ def dir_verify(ch, method, properties, body):
for d in dirs:
path = Path(d) / msg['username']
- if not path.exists():
- # Make sure folder exists and with right permission
- path.mkdir(mode=0o700)
+ if args.dry_run:
+ logger.info(f'Checking dirs: {path}')
+
+ else:
+ if not path.exists():
+ # Make sure folder exists and with right permission
+ path.mkdir(mode=0o700)
+
+ # Make sure ownership is correct
+ shutil.chown(path, msg['uid'], msg['gid'])
+
+ logger.debug(f'{path} created')
- # Make sure ownership is correct
- shutil.chown(path, msg['uid'], msg['gid'])
success = True
- except:
- e = sys.exc_info()[0]
- print("[{}]: Error: {}".format(task, e))
+
+ except Exception as exception:
+ logger.error('', exc_info=True)
# send confirm message
rc_rmq.publish_msg({
@@ -40,12 +51,15 @@ def dir_verify(ch, method, properties, body):
}
})
+ logger.debug(f'User {username} confirmation sent')
+
-print("Start listening to queue: {}".format(task))
+logger.info(f'Start listening to queue: {task}')
rc_rmq.start_consume({
'queue': task,
'routing_key': "verify.*",
'cb': dir_verify
})
+logger.info('Disconnected')
rc_rmq.disconnect()
--
GitLab
From 550583cef4a24c242cbfb3872dd774903cc5d318 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 17:25:29 -0500
Subject: [PATCH 068/188] Update confirmation message
---
dir_verify.py | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/dir_verify.py b/dir_verify.py
index e38d0b4..4486f28 100644
--- a/dir_verify.py
+++ b/dir_verify.py
@@ -18,7 +18,8 @@ rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
def dir_verify(ch, method, properties, body):
msg = json.loads(body)
username = msg['username']
- success = False
+ msg['task'] = task
+ msg['success'] = False
try:
for d in dirs:
@@ -37,7 +38,7 @@ def dir_verify(ch, method, properties, body):
logger.debug(f'{path} created')
- success = True
+ msg['success'] = True
except Exception as exception:
logger.error('', exc_info=True)
@@ -45,10 +46,7 @@ def dir_verify(ch, method, properties, body):
# send confirm message
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
- 'msg': {
- 'task': task,
- 'success': success
- }
+ 'msg': msg
})
logger.debug(f'User {username} confirmation sent')
--
GitLab
From 8b82d89df01fbc984eba86b0c932d4aa18b80b05 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 17:26:31 -0500
Subject: [PATCH 069/188] Acknowledge after confirmation sent
---
dir_verify.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/dir_verify.py b/dir_verify.py
index 4486f28..904c6fe 100644
--- a/dir_verify.py
+++ b/dir_verify.py
@@ -51,6 +51,8 @@ def dir_verify(ch, method, properties, body):
logger.debug(f'User {username} confirmation sent')
+ ch.basic_ack(delivery_tag=method.delivery_tag)
+
logger.info(f'Start listening to queue: {task}')
rc_rmq.start_consume({
--
GitLab
From 5d6045e8110212bae237c0822623216720a56dd1 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 17:39:09 -0500
Subject: [PATCH 070/188] Update confirmation message
---
git_commit.py | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index c7391e9..3cd197f 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -31,7 +31,8 @@ def git_commit(ch, method, properties, body):
msg = json.loads(body)
username = msg['username']
ticketnum = msg.get('ticketnum', 'add-users-' + username.lower())
- success = False
+ msg['task'] = task
+ msg['success'] = False
branch_name = 'issue-' + ticketnum
user_ldif = users_dir + f'/{username}.ldif'
group_ldif = groups_dir + f'/{username}.ldif'
@@ -75,7 +76,7 @@ def git_commit(ch, method, properties, body):
logger.info('Added ldif files and committed to git repo')
- success = True
+ msg['success'] = True
except Exception as exception:
logger.error('', exc_info=True)
@@ -83,10 +84,7 @@ def git_commit(ch, method, properties, body):
logger.debug('rc_rmq.publish_msge()')
rc_rmq.publish_msg({
'routing_key': 'confirm.' + username,
- 'msg': {
- 'task': task,
- 'success': success
- }
+ 'msg': msg
})
logger.info('confirmation sent')
--
GitLab
From 9f499aece5f6e7c6e55bbe37f5030e1b0dac3dae Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 17:58:18 -0500
Subject: [PATCH 071/188] Utilize rc_util
---
task_manager.py | 27 ++++++++++++++++++++++-----
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index b7542de..1daa215 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -1,11 +1,15 @@
#!/usr/bin/env python
import sys
import json
+import rc_util
from rc_rmq import RCRMQ
from datetime import datetime
task = 'task_manager'
+args = rc_util.get_args()
+logger = rc_util.get_logger(args)
+
record = {
'uid': -1,
'gid': -1,
@@ -46,6 +50,8 @@ def task_manager(ch, method, properties, body):
current['uid'] = msg.get('uid', -1)
current['gid'] = msg.get('gid', -1)
current['email'] = msg.get('email', '')
+
+ logger.debug(f'Tracking user {username}')
else:
current = tracking[username]
@@ -61,6 +67,8 @@ def task_manager(ch, method, properties, body):
routing_key = 'create.' + username
done = success
+ logger.debug(f'Request level task(s) done?{done}')
+
elif task_name in current['create']:
current['create'][task_name] = success
routing_key = 'verify.' + username
@@ -69,6 +77,8 @@ def task_manager(ch, method, properties, body):
if status is not True:
done = False
+ logger.debug(f'Create level task(s) done?{done}')
+
elif task_name in current['verify']:
current['verify'][task_name] = success
routing_key = 'notify.' + username
@@ -77,14 +87,17 @@ def task_manager(ch, method, properties, body):
if status is not True:
done = False
+ logger.debug(f'Verify level task(s) done?{done}')
+
elif task_name in current['notify']:
current['verify'][task_name] = success
routing_key = 'complete.' + username
done = success
- except:
- e = sys.exc_info()[0]
- print("[{}]: Error: {}".format(task, e))
+ logger.debug(f'Notify level task(s) done?{done}')
+
+ except Exception as exception:
+ logger.error('', exc_info=True)
if done:
# acknowledge all message from last level
@@ -92,6 +105,8 @@ def task_manager(ch, method, properties, body):
ch.basic_ack(tag)
current['delivery_tags'] = []
+ logger.debug('Previous level messages acknowledged')
+
# send trigger message
rc_rmq.publish_msg({
'routing_key': routing_key,
@@ -103,13 +118,15 @@ def task_manager(ch, method, properties, body):
}
})
+ logger.debug(f"Trigger message '{routing_key}' sent")
+
-print("Start listening to queue: {}".format(task))
+logger.info(f'Start listening to queue: {task}')
rc_rmq.start_consume({
'queue': task,
'routing_key': "confirm.*",
'cb': task_manager
})
-print("Disconnected")
+logger.info('Disconnected')
rc_rmq.disconnect()
--
GitLab
From 787c960e4f808f009d275420f2130d3319560962 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 18:00:56 -0500
Subject: [PATCH 072/188] Update and remove unnecessary comment
---
task_manager.py | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index 1daa215..ace0087 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -55,13 +55,10 @@ def task_manager(ch, method, properties, body):
else:
current = tracking[username]
- # save the delivery tags for future use
+ # Save the delivery tags for future use
current['delivery_tags'].append(method.delivery_tag)
try:
- # Check each level
- # task timeout
- # failure agent(?
if task_name in current['request']:
current['request'][task_name] = success
routing_key = 'create.' + username
@@ -100,14 +97,14 @@ def task_manager(ch, method, properties, body):
logger.error('', exc_info=True)
if done:
- # acknowledge all message from last level
+ # Acknowledge all message from last level
for tag in current['delivery_tags']:
ch.basic_ack(tag)
current['delivery_tags'] = []
logger.debug('Previous level messages acknowledged')
- # send trigger message
+ # Send trigger message
rc_rmq.publish_msg({
'routing_key': routing_key,
'msg': {
--
GitLab
From 26d98fa6658159b81092e4fe77eb407855b77692 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 18:01:10 -0500
Subject: [PATCH 073/188] Move message acknowledge to the end
---
task_manager.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index ace0087..b7db0e8 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -97,13 +97,6 @@ def task_manager(ch, method, properties, body):
logger.error('', exc_info=True)
if done:
- # Acknowledge all message from last level
- for tag in current['delivery_tags']:
- ch.basic_ack(tag)
- current['delivery_tags'] = []
-
- logger.debug('Previous level messages acknowledged')
-
# Send trigger message
rc_rmq.publish_msg({
'routing_key': routing_key,
@@ -117,6 +110,13 @@ def task_manager(ch, method, properties, body):
logger.debug(f"Trigger message '{routing_key}' sent")
+ # Acknowledge all message from last level
+ for tag in current['delivery_tags']:
+ ch.basic_ack(tag)
+ current['delivery_tags'] = []
+
+ logger.debug('Previous level messages acknowledged')
+
logger.info(f'Start listening to queue: {task}')
rc_rmq.start_consume({
--
GitLab
From daf84b2e91718177eb99d3a1632f428198898ae7 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 23 Apr 2020 18:11:53 -0500
Subject: [PATCH 074/188] Update requirements.txt
---
requirements.txt | 2 ++
1 file changed, 2 insertions(+)
diff --git a/requirements.txt b/requirements.txt
index becc2ce..79661ef 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,3 @@
pika==1.1.0
+dataset==1.3.1
+Jinja2==2.11.2
--
GitLab
From 9de5e430bb27df8017d77c3a9a65c1249e021b51 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 24 Apr 2020 10:07:01 -0500
Subject: [PATCH 075/188] Remove logging level set in agent
---
git_commit.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index 3cd197f..b16fbfc 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -23,7 +23,6 @@ if not args.dry_run:
git = sh.git.bake('-C', repo_location)
ldapsearch = sh.Command('ldapsearch')
else:
- logger_lvl = logging.INFO
git = sh.echo.bake('git', '-C', repo_location)
ldapsearch = sh.echo.bake('ldapsearch')
--
GitLab
From 1fe0507e2349a318f4e2a3bc6a740e84a7bed546 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 24 Apr 2020 15:35:39 +0000
Subject: [PATCH 076/188] Use logging from rc_util
---
user_reg_logger.py | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/user_reg_logger.py b/user_reg_logger.py
index dec5c18..0f890a9 100755
--- a/user_reg_logger.py
+++ b/user_reg_logger.py
@@ -2,7 +2,7 @@
import json
import sys
import dataset
-import logging
+import rc_util
from rc_rmq import RCRMQ
from datetime import datetime
@@ -10,7 +10,13 @@ from datetime import datetime
task = 'reg_logger'
# Instantiate rabbitmq object
-reg_logger = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+
+# Parse arguments
+args = rc_util.get_args()
+
+# Logger
+logger = rc_util.get_logger()# Define your callback function
# Parse arguments
args = rc_util.get_args()
@@ -35,7 +41,7 @@ def log_registration(ch, method, properties, body):
logger.info("Start listening to queue: {}".format(task))
# Start consuming messages from queue with callback function
-reg_logger.start_consume({
+rc_rmq.start_consume({
'queue': task,
'routing_key': "create.*",
'cb': log_registration
--
GitLab
From cc41adacf2a842aca32512e97aad172fc89120e5 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 24 Apr 2020 15:47:11 +0000
Subject: [PATCH 077/188] Remove the comment
---
user_reg_logger.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/user_reg_logger.py b/user_reg_logger.py
index 0f890a9..3230e76 100755
--- a/user_reg_logger.py
+++ b/user_reg_logger.py
@@ -16,7 +16,7 @@ rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
args = rc_util.get_args()
# Logger
-logger = rc_util.get_logger()# Define your callback function
+logger = rc_util.get_logger()
# Parse arguments
args = rc_util.get_args()
--
GitLab
From 7a9075246c6c646cb5ffb9a2403853a3680e65d4 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 24 Apr 2020 16:08:42 +0000
Subject: [PATCH 078/188] Remove duplicate lines
---
user_reg_logger.py | 6 ------
1 file changed, 6 deletions(-)
diff --git a/user_reg_logger.py b/user_reg_logger.py
index 3230e76..402c9e3 100755
--- a/user_reg_logger.py
+++ b/user_reg_logger.py
@@ -18,12 +18,6 @@ args = rc_util.get_args()
# Logger
logger = rc_util.get_logger()
-# Parse arguments
-args = rc_util.get_args()
-
-# Logger
-logger = rc_util.get_logger()
-
# Open registry table in DB
db = dataset.connect('sqlite:///reg_logger.db')
account_req_table = db['registry']
--
GitLab
From 8a7f1f0b83c47180cdca11b4f6c5d152e56fc674 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 24 Apr 2020 14:12:27 -0500
Subject: [PATCH 079/188] Solve referenced before assignment
---
task_manager.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index b7db0e8..78c71f7 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -42,7 +42,7 @@ def task_manager(ch, method, properties, body):
msg = json.loads(body)
username = method.routing_key.split('.')[1]
task_name = msg['task']
- success = msg['success']
+ done = success = msg['success']
if username not in tracking:
current = tracking[username] = record.copy()
--
GitLab
From eeb9566eea4cf187b9272bc5f3bad2ec832333bb Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Mon, 27 Apr 2020 19:00:41 +0000
Subject: [PATCH 080/188] send listserv cmds in one email rather than separate
emails
---
subscribe_mail_lists.py | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index 8304305..bf0afc9 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -31,9 +31,8 @@ def mail_list_subscription(ch, method, properties, body):
mail_list_admin = 'root@localhost' #change this during deploy
mail_list = 'LISTSERV@LISTSERV.UAB.EDU'
- listserv_cmd = {}
- listserv_cmd['hpc_announce'] = f'QUIET ADD hpc-announce {email} {fullname}'
- listserv_cmd['hpc_users'] = f'QUIET ADD hpc-users {email} {fullname}'
+ listserv_cmd = f'QUIET ADD hpc-announce {email} {fullname} \
+ \nQUIET ADD hpc-users {email} {fullname}'
logger.info("Adding user{} to mail list".format(username))
success = False
@@ -48,11 +47,10 @@ def mail_list_subscription(ch, method, properties, body):
# Create an smtp object and send email
s = smtplib.SMTP('localhost')
- for key,value in listserv_cmd.items():
- email_msg.set_content(value)
- if not args.dry_run:
- s.send_message(email_msg)
- logging.info(f'This email will add user {username} to {key}\n{email_msg}')
+ email_msg.set_content(listserv_cmd)
+ if not args.dry_run:
+ s.send_message(email_msg)
+ logging.info(f'This email will add user {username} to {key}\n{email_msg}')
s.quit()
msg['task'] = task
--
GitLab
From 5e5cba3a5555510790dc7e1060b88480d8b58307 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 28 Apr 2020 16:58:19 -0500
Subject: [PATCH 081/188] Use python3
---
create_account.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/create_account.py b/create_account.py
index b316cd0..ef77d78 100755
--- a/create_account.py
+++ b/create_account.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import sys
import rc_util
--
GitLab
From 1b5c98e97bd99a85966c0a2678ee08fa4d41ed71 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 28 Apr 2020 17:25:57 -0500
Subject: [PATCH 082/188] Utilize argparse library
---
create_account.py | 33 +++++++++++++++------------------
1 file changed, 15 insertions(+), 18 deletions(-)
diff --git a/create_account.py b/create_account.py
index ef77d78..bf45fb9 100755
--- a/create_account.py
+++ b/create_account.py
@@ -1,25 +1,22 @@
#!/usr/bin/env python3
-import sys
import rc_util
+import argparse
-if len(sys.argv) < 2:
- print("Usage: {} USERNAME [EMAIL] [FULL_NAME] [REASON]".format(sys.argv[0]), file=sys.stderr)
- exit(1)
+parser = argparse.ArgumentParser()
+parser.add_argument('username', help='username that will be created')
+parser.add_argument('email', nargs='?', default='', help="User's email")
+parser.add_argument('full_name', nargs='?', default='', help="User's full name")
+parser.add_argument('reason', nargs='?', default='', help='Reason of requesting')
+parser.add_argument('--domain', default='localhost', help='domain of email')
+args = parser.parse_args()
-domain = 'uab.edu'
-user_name = sys.argv[1]
-email = sys.argv[2] if len(sys.argv) >= 3 else ''
-full_name = sys.argv[3] if len(sys.argv) >= 4 else ''
-reason = sys.argv[4] if len(sys.argv) >= 5 else ''
+if args.email == '':
+ args.email = args.username
+ if '@' not in args.email:
+ args.email = args.username + '@' + args.domain
-if email == '':
- if '@' in user_name:
- email = user_name
- else:
- email = user_name + '@' + domain
-
-rc_util.add_account(user_name, email=email, full=full_name, reason=reason)
-print("Account requested for user: {}".format(user_name))
+rc_util.add_account(args.username, email=args.email, full=args.full_name, reason=args.reason)
+print(f'Account for {args.username} requested.')
print("Waiting for confirmation...")
-rc_util.consume(user_name)
+rc_util.consume(args.username)
--
GitLab
From 58a0f8cb32cb1f1a7f3946d91f711d1683f1e3e8 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 28 Apr 2020 17:28:37 -0500
Subject: [PATCH 083/188] Utilize logger from rc_util
---
create_account.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/create_account.py b/create_account.py
index bf45fb9..3a41fe5 100755
--- a/create_account.py
+++ b/create_account.py
@@ -2,6 +2,8 @@
import rc_util
import argparse
+logger = rc_util.get_logger()
+
parser = argparse.ArgumentParser()
parser.add_argument('username', help='username that will be created')
parser.add_argument('email', nargs='?', default='', help="User's email")
@@ -16,7 +18,7 @@ if args.email == '':
args.email = args.username + '@' + args.domain
rc_util.add_account(args.username, email=args.email, full=args.full_name, reason=args.reason)
-print(f'Account for {args.username} requested.')
+logger.info(f'Account for {args.username} requested.')
-print("Waiting for confirmation...")
+logger.info('Waiting for completion...')
rc_util.consume(args.username)
--
GitLab
From 9f9dc909e375ff3a3028aa6ae6f987377ff6f298 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 28 Apr 2020 17:29:29 -0500
Subject: [PATCH 084/188] Implement its own callback function
---
create_account.py | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/create_account.py b/create_account.py
index 3a41fe5..6db95a0 100755
--- a/create_account.py
+++ b/create_account.py
@@ -17,8 +17,18 @@ if args.email == '':
if '@' not in args.email:
args.email = args.username + '@' + args.domain
+def callback(channel, method, properties, body):
+ msg = json.loads(body)
+ username = msg['username']
+
+ logger.info(f'Account for {username} has been created.')
+
+ rc_rmq.stop_consume()
+ rc_rmq.delete_queue()
+
+
rc_util.add_account(args.username, email=args.email, full=args.full_name, reason=args.reason)
logger.info(f'Account for {args.username} requested.')
logger.info('Waiting for completion...')
-rc_util.consume(args.username)
+rc_util.consume(args.username, callback=callback)
--
GitLab
From 04326c0bd810b9627cf213440937bc3d0e54ffb6 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 28 Apr 2020 17:38:31 -0500
Subject: [PATCH 085/188] Update consume in rc_util
Now consume can take routing key as argument
---
rc_util.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/rc_util.py b/rc_util.py
index 0e7c4c1..6519734 100644
--- a/rc_util.py
+++ b/rc_util.py
@@ -35,13 +35,16 @@ def worker(ch, method, properties, body):
rc_rmq.stop_consume()
rc_rmq.delete_queue()
-def consume(username, callback=worker, debug=False):
+def consume(username, routing_key='', callback=worker, debug=False):
+ if routing_key == '':
+ routing_key = 'confirm.' + username
+
if debug:
sleep(5)
else:
rc_rmq.start_consume({
'queue': username,
- 'routing_key': 'confirm.' + username,
+ 'routing_key': routing_key,
'cb': callback
})
rc_rmq.disconnect()
--
GitLab
From c963c27736ebda160147f4c16e9956529c47432c Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 28 Apr 2020 17:43:10 -0500
Subject: [PATCH 086/188] Update routing_key used in create_account
---
create_account.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/create_account.py b/create_account.py
index 6db95a0..d7862c2 100755
--- a/create_account.py
+++ b/create_account.py
@@ -31,4 +31,4 @@ rc_util.add_account(args.username, email=args.email, full=args.full_name, reason
logger.info(f'Account for {args.username} requested.')
logger.info('Waiting for completion...')
-rc_util.consume(args.username, callback=callback)
+rc_util.consume(args.username, routing_key=f'complete.{args.username}', callback=callback)
--
GitLab
From 1c0315c8e839ddbf9fff90b69579a12b07580cc1 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 30 Apr 2020 21:56:54 -0500
Subject: [PATCH 087/188] Passing args into logger
---
create_account.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/create_account.py b/create_account.py
index d7862c2..cb00b76 100755
--- a/create_account.py
+++ b/create_account.py
@@ -2,16 +2,18 @@
import rc_util
import argparse
-logger = rc_util.get_logger()
-
parser = argparse.ArgumentParser()
parser.add_argument('username', help='username that will be created')
parser.add_argument('email', nargs='?', default='', help="User's email")
parser.add_argument('full_name', nargs='?', default='', help="User's full name")
parser.add_argument('reason', nargs='?', default='', help='Reason of requesting')
parser.add_argument('--domain', default='localhost', help='domain of email')
+parser.add_argument('-v', '--verbose', action='store_true', help='verbose output')
+parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
args = parser.parse_args()
+logger = rc_util.get_logger(args)
+
if args.email == '':
args.email = args.username
if '@' not in args.email:
--
GitLab
From 60e7ead16102069bd5fe83e1fcdd9804f4de55fd Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 1 May 2020 10:12:33 -0500
Subject: [PATCH 088/188] Solve routing_key referenced before assignment
---
task_manager.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/task_manager.py b/task_manager.py
index 78c71f7..d91902a 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -43,6 +43,7 @@ def task_manager(ch, method, properties, body):
username = method.routing_key.split('.')[1]
task_name = msg['task']
done = success = msg['success']
+ routing_key = ""
if username not in tracking:
current = tracking[username] = record.copy()
--
GitLab
From 39f3f082c154217d7d9f15e1f02ef4bb4587dc39 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 4 May 2020 17:50:24 -0500
Subject: [PATCH 089/188] Update task name
---
task_manager.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index d91902a..6921a9a 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -16,7 +16,7 @@ record = {
'email': '',
'last_update': datetime.now(),
'request': {
- 'uid_resolve': None
+ 'get_next_uid_gid': None
},
'create': {
'join_list': None,
--
GitLab
From 3c54b187b580d3d5d685b78f6ff7dbffc1d3b399 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 4 May 2020 17:50:46 -0500
Subject: [PATCH 090/188] Add fullname field
---
task_manager.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index 6921a9a..798781c 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -14,6 +14,7 @@ record = {
'uid': -1,
'gid': -1,
'email': '',
+ 'fullname': '',
'last_update': datetime.now(),
'request': {
'get_next_uid_gid': None
@@ -51,6 +52,7 @@ def task_manager(ch, method, properties, body):
current['uid'] = msg.get('uid', -1)
current['gid'] = msg.get('gid', -1)
current['email'] = msg.get('email', '')
+ current['fullname'] = msg.get('fullname', '')
logger.debug(f'Tracking user {username}')
else:
@@ -103,6 +105,7 @@ def task_manager(ch, method, properties, body):
'routing_key': routing_key,
'msg': {
'username': username,
+ 'fullname': current['fullname'],
'email': current['email'],
'uid': current['uid'],
'gid': current['gid']
--
GitLab
From f61e4e738ffe2f789fd301c304121e05a0cbc6c1 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 5 May 2020 11:02:06 -0500
Subject: [PATCH 091/188] Update task name
---
task_manager.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index 798781c..604c1ec 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -20,8 +20,8 @@ record = {
'get_next_uid_gid': None
},
'create': {
- 'join_list': None,
- 'create_account': None
+ 'subscribe_mail_list': None,
+ 'bright_account': None
},
'verify': {
'git_commit': None,
--
GitLab
From af42f4292c8baf37f998ee67d66d5e9c37ac7f68 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 8 May 2020 02:09:27 +0000
Subject: [PATCH 092/188] replace LDAP check with getent & move it to callback
function.
---
get-next-uid-gid.py | 22 ++++++----------------
1 file changed, 6 insertions(+), 16 deletions(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index 49eb977..f2a0b8b 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -18,19 +18,6 @@ args = rc_util.get_args()
# Logger
logger = rc_util.get_logger()
-#Check if the username already exists via LDAP
-def user_exists(username):
- try:
- logger.info(f"Searching LDAP for the user: {username}")
- con = ldap.initialize('ldap://ldapserver')
- ldap_base = "dc=cm,dc=cluster"
- query = "(uid={})".format(username)
- result = con.search_s(ldap_base, ldap.SCOPE_SUBTREE, query)
- logging.debug(f"The search result is: {result}")
- return result
- except ldap.LDAPError:
- logger.exception("Fatal LDAP error:")
-
# Define your callback function
def get_next_uid_gid(ch, method, properties, body):
@@ -42,10 +29,13 @@ def get_next_uid_gid(ch, method, properties, body):
# Determine next available UID
try:
- if user_exists(username):
+ user_exists_cmd = "/usr/bin/getent passwd {username}"
+ user_exists = popen(user_exists_cmd).read().rstrip()
+
+ if user_exists:
logger.info("The user, {} already exists".format(username))
- msg['uid'] = result[0][1]['uidNumber'][0].decode('utf-8')
- msg['gid'] = result[0][1]['gidNumber'][0].decode('utf-8')
+ msg['uid'] = user_exists.split(':')[2]
+ msg['gid'] = user_exists.split(':')[3]
else:
cmd_uid = "/usr/bin/getent passwd | \
--
GitLab
From e44fcf822c301bdfd36d276dfef48f7d5e8f3ba7 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 8 May 2020 02:28:23 +0000
Subject: [PATCH 093/188] Add account creation function.
---
get-next-uid-gid.py | 26 +++++++++++++++++++++++++-
1 file changed, 25 insertions(+), 1 deletion(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index f2a0b8b..87a47cc 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -18,6 +18,28 @@ args = rc_util.get_args()
# Logger
logger = rc_util.get_logger()
+#Account creation
+def create_account(msg):
+
+ logger.info(f'Account creation request received: {msg}')
+ username = msg['username']
+ uid = msg['uid']
+ email = msg['email']
+ fullname = msg['fullname']
+ success = False
+
+ try:
+ # Bright command to create user
+ cmd = '/cm/local/apps/cmd/bin/cmsh -c '
+ cmd += f'"user; add {username}; set userid {uid}; set email {email}; set commonname \\"{fullname}\\"; '
+ cmd += 'commit;"'
+
+ if not args.dry_run:
+ popen(cmd)
+ logger.info(f'Bright command to create user:{cmd}')
+ except Exception:
+ logger.exception("Fatal cmsh error:")
+
# Define your callback function
def get_next_uid_gid(ch, method, properties, body):
@@ -47,10 +69,12 @@ def get_next_uid_gid(ch, method, properties, body):
awk -F: '($3>10000) && ($3<20000) && ($3>maxgid) { maxgid=$3; } END { print maxgid+1; }'"
msg['gid'] = popen(cmd_gid).read().rstrip()
logger.info(f"GID query: {cmd_gid}")
+
+ create_account(msg)
msg['task'] = task
msg['success'] = True
except Exception:
- logger.exception("Fatal error:")
+ logger.exception("Fatal UID resolution error:")
# Acknowledge message
ch.basic_ack(delivery_tag=method.delivery_tag)
--
GitLab
From 50883756b71c66117eb86b227434ab042597d400 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 8 May 2020 02:31:01 +0000
Subject: [PATCH 094/188] Change task name
---
get-next-uid-gid.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index 87a47cc..3818be6 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -8,7 +8,7 @@ import rc_util
from os import popen
from rc_rmq import RCRMQ
-task = 'get_next_uid_gid'
+task = 'create_account'
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
--
GitLab
From cdf49d2437695dfbbb5ff027cfbec25d02fc8259 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 8 May 2020 02:31:33 +0000
Subject: [PATCH 095/188] Change call back func name
---
get-next-uid-gid.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index 3818be6..54342df 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -41,7 +41,7 @@ def create_account(msg):
logger.exception("Fatal cmsh error:")
# Define your callback function
-def get_next_uid_gid(ch, method, properties, body):
+def resolve_uid_gid(ch, method, properties, body):
# Retrieve message
msg = json.loads(body)
@@ -91,7 +91,7 @@ logger.info("Start listening to queue: {}".format(task))
rc_rmq.start_consume({
'queue': task,
'routing_key': "request.*",
- 'cb': get_next_uid_gid
+ 'cb': resolve_uid_gid
})
logger.info("Disconnected")
--
GitLab
From 220f55b6b65dd8fd3ab05c18e485d34cb8278dd8 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 8 May 2020 03:20:29 +0000
Subject: [PATCH 096/188] Change the logging msg to remove unused var
---
subscribe_mail_lists.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index bf0afc9..483e1dd 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -50,7 +50,7 @@ def mail_list_subscription(ch, method, properties, body):
email_msg.set_content(listserv_cmd)
if not args.dry_run:
s.send_message(email_msg)
- logging.info(f'This email will add user {username} to {key}\n{email_msg}')
+ logging.info(f'This email will add user {username} to listserv \n{email_msg}')
s.quit()
msg['task'] = task
--
GitLab
From 366634027b45057b51b58f2a3d336e6fae19d273 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 8 May 2020 03:20:54 +0000
Subject: [PATCH 097/188] Change routing key to listen to verify.
---
subscribe_mail_lists.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index 483e1dd..49ac1d7 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -72,7 +72,7 @@ def mail_list_subscription(ch, method, properties, body):
logger.info("Start listening to queue: {}".format(task))
rc_rmq.start_consume({
'queue': task, # Define your Queue name
- 'routing_key': "create.*", # Define your routing key
+ 'routing_key': "verify.*", # Define your routing key
'cb': mail_list_subscription # Pass in callback function you just define
})
--
GitLab
From 079f5e6b913706b643d1b583f27b352ed850d541 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 12 May 2020 11:18:05 -0500
Subject: [PATCH 098/188] Fix import and rc_rmq object
---
create_account.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/create_account.py b/create_account.py
index cb00b76..161c29d 100755
--- a/create_account.py
+++ b/create_account.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python3
+import json
import rc_util
import argparse
@@ -25,8 +26,8 @@ def callback(channel, method, properties, body):
logger.info(f'Account for {username} has been created.')
- rc_rmq.stop_consume()
- rc_rmq.delete_queue()
+ rc_util.rc_rmq.stop_consume()
+ rc_util.rc_rmq.delete_queue()
rc_util.add_account(args.username, email=args.email, full=args.full_name, reason=args.reason)
--
GitLab
From 5ecfce58203e769cb77a70d70f7fe6685f134e84 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 12 May 2020 15:14:31 -0500
Subject: [PATCH 099/188] Fix sender is not defined
---
notify_user.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index 67c5b0d..51c95d9 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -47,12 +47,12 @@ def notify_user(ch, method, properties, body):
if args.dry_run:
logger.info(f'smtp = smtplib.SMTP({mail_cfg.Server})')
- logger.info(f'smtp.sendmail({sender}, {receiver}, message)')
+ logger.info(f'smtp.sendmail({mail_cfg.Sender}, {receiver}, message)')
logger.info(f"table.insert({'username': {username}, 'count': 1, 'sent_at': {datetime.now()}})")
else:
smtp = smtplib.SMTP(mail_cfg.Server)
- smtp.sendmail(sender, receiver, message)
+ smtp.sendmail(mail_cfg.Sender, receiver, message)
logger.debug(f'Email sent to: {user_email}')
--
GitLab
From 86d9569e2cc2fb7c2b934b2a201672f5ef7ac837 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 12 May 2020 15:15:49 -0500
Subject: [PATCH 100/188] Fix invalid format specifier
---
notify_user.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/notify_user.py b/notify_user.py
index 51c95d9..0d5cd90 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -48,7 +48,7 @@ def notify_user(ch, method, properties, body):
if args.dry_run:
logger.info(f'smtp = smtplib.SMTP({mail_cfg.Server})')
logger.info(f'smtp.sendmail({mail_cfg.Sender}, {receiver}, message)')
- logger.info(f"table.insert({'username': {username}, 'count': 1, 'sent_at': {datetime.now()}})")
+ logger.info(f"table.insert({{'username': {username}, 'count': 1, 'sent_at': datetime.now()}})")
else:
smtp = smtplib.SMTP(mail_cfg.Server)
--
GitLab
From b4d96631e38db3c045ec60e7781a2fb54ca8ced7 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 12 May 2020 15:16:13 -0500
Subject: [PATCH 101/188] Rename variable
receiver to receivers
---
notify_user.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index 0d5cd90..4f820b3 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -42,17 +42,17 @@ def notify_user(ch, method, properties, body):
else:
# Send email to user
- receiver = [user_mail, mail_cfg.My_mail]
+ receivers = [user_mail, mail_cfg.My_email]
message = Template(mail_cfg.Whole_mail).render(username=username, to=user_email)
if args.dry_run:
logger.info(f'smtp = smtplib.SMTP({mail_cfg.Server})')
- logger.info(f'smtp.sendmail({mail_cfg.Sender}, {receiver}, message)')
+ logger.info(f'smtp.sendmail({mail_cfg.Sender}, {receivers}, message)')
logger.info(f"table.insert({{'username': {username}, 'count': 1, 'sent_at': datetime.now()}})")
else:
smtp = smtplib.SMTP(mail_cfg.Server)
- smtp.sendmail(mail_cfg.Sender, receiver, message)
+ smtp.sendmail(mail_cfg.Sender, receivers, message)
logger.debug(f'Email sent to: {user_email}')
--
GitLab
From d75c72d83d234be66cb0180a5283bc17575aed47 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 20 May 2020 17:02:55 -0500
Subject: [PATCH 102/188] Remove double quoting for ladpsearch
---
git_commit.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index b16fbfc..3ea965d 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -53,9 +53,9 @@ def git_commit(ch, method, properties, body):
with open(user_ldif, 'w') as ldif_u,\
open(group_ldif, 'w') as ldif_g:
logger.debug(f"ldapsearch -LLL -x -h ldapserver -b 'dc=cm,dc=cluster' uid={username} > {user_ldif}")
- ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'dc=cm,dc=cluster'", f"uid={username}", _out=ldif_u)
+ ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "dc=cm,dc=cluster", f"uid={username}", _out=ldif_u)
logger.debug(f"ldapsearch -LLL -x -h ldapserver -b 'ou=Group,dc=cm,dc=cluster' cn={username} > {group_ldif}")
- ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "'ou=Group,dc=cm,dc=cluster'", f"cn={username}", _out=ldif_g)
+ ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "ou=Group,dc=cm,dc=cluster", f"cn={username}", _out=ldif_g)
logger.info('user ldif files generated.')
logger.debug('git add %s', user_ldif)
--
GitLab
From 5ae8a9ad16b5bef80c470bcb1f5501054cf151aa Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 21 Sep 2020 16:13:21 -0500
Subject: [PATCH 103/188] Fixes #40
---
dir_verify.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dir_verify.py b/dir_verify.py
index 904c6fe..0f0127e 100644
--- a/dir_verify.py
+++ b/dir_verify.py
@@ -34,7 +34,7 @@ def dir_verify(ch, method, properties, body):
path.mkdir(mode=0o700)
# Make sure ownership is correct
- shutil.chown(path, msg['uid'], msg['gid'])
+ shutil.chown(path, int(msg['uid']), int(msg['gid']))
logger.debug(f'{path} created')
--
GitLab
From 4be0752fe432a11c51a5ab9943f31efe472597d8 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 21 Sep 2020 16:15:10 -0500
Subject: [PATCH 104/188] Fix typo
---
notify_user.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/notify_user.py b/notify_user.py
index 4f820b3..753d14d 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -42,7 +42,7 @@ def notify_user(ch, method, properties, body):
else:
# Send email to user
- receivers = [user_mail, mail_cfg.My_email]
+ receivers = [user_email, mail_cfg.My_email]
message = Template(mail_cfg.Whole_mail).render(username=username, to=user_email)
if args.dry_run:
--
GitLab
From c44469c3031d06b72619b0b0424e9edc997740e7 Mon Sep 17 00:00:00 2001
From: rtripath89 <ravi89@uab.edu>
Date: Fri, 25 Sep 2020 16:14:02 -0400
Subject: [PATCH 105/188] Revert "Feat add user bright cm"
---
brightcm_account_create.py | 64 --------------------------------------
1 file changed, 64 deletions(-)
delete mode 100644 brightcm_account_create.py
diff --git a/brightcm_account_create.py b/brightcm_account_create.py
deleted file mode 100644
index 2c57b30..0000000
--- a/brightcm_account_create.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env python
-import sys
-import json
-import logging
-import argparse
-import rc_util
-from os import popen
-from rc_rmq import RCRMQ
-
-task = 'bright_account'
-
-# Instantiate rabbitmq object
-rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
-
-# Parse arguments
-args = rc_util.get_args()
-
-# Logger
-logger = rc_util.get_logger()
-
-# Define your callback function
-def bright_account_create(ch, method, properties, body):
- # Retrieve message
- msg = json.loads(body)
- logger.info("Received {}".format(msg))
- username = msg['username']
- uid = msg['uid']
- email = msg['email']
- fullname = msg['fullname']
- success = False
- try:
- # Bright command to create user
- cmd = '/cm/local/apps/cmd/bin/cmsh -c '
- cmd += f'"user; add {username}; set userid {uid}; set email {email}; set commonname \\"{fullname}\\"; '
- cmd += 'commit;"'
-
- if not args.dry_run:
- popen(cmd)
- logger.info(f'Bright command to create user:{cmd}')
- msg['task'] = task
- msg['success'] = True
- except Exception:
- logger.exception("Fatal error:")
-
- # Acknowledge message
- ch.basic_ack(delivery_tag=method.delivery_tag)
-
- # send confirm message
- logger.debug('rc_rmq.publish_msg()')
- rc_rmq.publish_msg({
- 'routing_key': 'confirm.' + username,
- 'msg': msg
- })
- logger.info('confirmation sent')
-
-logger.info("Start listening to queue: {}".format(task))
-rc_rmq.start_consume({
- 'queue': task,
- 'routing_key': "create.*",
- 'cb': bright_account_create
-})
-
-logger.info("Disconnected")
-rc_rmq.disconnect()
--
GitLab
From 83dc8c30ae4f62fe76480042cd8a57daa556e983 Mon Sep 17 00:00:00 2001
From: atlurie <atlurie@uab.edu>
Date: Fri, 2 Oct 2020 11:11:29 -0500
Subject: [PATCH 106/188] Syntax change in brightCM 9.0 for creating a user
account.
---
get-next-uid-gid.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index 54342df..ef02b66 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -31,7 +31,7 @@ def create_account(msg):
try:
# Bright command to create user
cmd = '/cm/local/apps/cmd/bin/cmsh -c '
- cmd += f'"user; add {username}; set userid {uid}; set email {email}; set commonname \\"{fullname}\\"; '
+ cmd += f'"user; add {username}; set id {uid}; set email {email}; set commonname \\"{fullname}\\"; '
cmd += 'commit;"'
if not args.dry_run:
--
GitLab
From ff8fb410877c91f77a3af6591c19668de2e1b684 Mon Sep 17 00:00:00 2001
From: atlurie <atlurie@uab.edu>
Date: Fri, 2 Oct 2020 17:26:13 -0500
Subject: [PATCH 107/188] Change verify level tasks check in the Notify level
to notify level task check.
---
task_manager.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index 604c1ec..d661526 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -90,7 +90,7 @@ def task_manager(ch, method, properties, body):
logger.debug(f'Verify level task(s) done?{done}')
elif task_name in current['notify']:
- current['verify'][task_name] = success
+ current['notify'][task_name] = success
routing_key = 'complete.' + username
done = success
--
GitLab
From 2aff99bf232b98e4ab270af1cd8150cf293028a5 Mon Sep 17 00:00:00 2001
From: atlurie <atlurie@uab.edu>
Date: Fri, 2 Oct 2020 17:44:28 -0500
Subject: [PATCH 108/188] Change the deprecated -h option in favor of -H to
specify ldapuri
---
git_commit.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 3ea965d..0530cef 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -52,10 +52,10 @@ def git_commit(ch, method, properties, body):
logger.debug("open(%s, 'w'), open(%s, 'w')", user_ldif, group_ldif)
with open(user_ldif, 'w') as ldif_u,\
open(group_ldif, 'w') as ldif_g:
- logger.debug(f"ldapsearch -LLL -x -h ldapserver -b 'dc=cm,dc=cluster' uid={username} > {user_ldif}")
- ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "dc=cm,dc=cluster", f"uid={username}", _out=ldif_u)
- logger.debug(f"ldapsearch -LLL -x -h ldapserver -b 'ou=Group,dc=cm,dc=cluster' cn={username} > {group_ldif}")
- ldapsearch('-LLL', '-x', '-h', 'ldapserver', '-b', "ou=Group,dc=cm,dc=cluster", f"cn={username}", _out=ldif_g)
+ logger.debug(f"ldapsearch -LLL -x -H ldaps://ldapserver -b 'dc=cm,dc=cluster' uid={username} > {user_ldif}")
+ ldapsearch('-LLL', '-x', '-H', 'ldaps://ldapserver', '-b', "dc=cm,dc=cluster", f"uid={username}", _out=ldif_u)
+ logger.debug(f"ldapsearch -LLL -x -H ldapserver -b 'ou=Group,dc=cm,dc=cluster' cn={username} > {group_ldif}")
+ ldapsearch('-LLL', '-x', '-H', 'ldaps://ldapserver', '-b', "ou=Group,dc=cm,dc=cluster", f"cn={username}", _out=ldif_g)
logger.info('user ldif files generated.')
logger.debug('git add %s', user_ldif)
--
GitLab
From 7f80645fb27b7ff74b42853c36cc63aad0d03630 Mon Sep 17 00:00:00 2001
From: atlurie <atlurie@uab.edu>
Date: Fri, 2 Oct 2020 17:47:50 -0500
Subject: [PATCH 109/188] Add a missing ldap import
---
git_commit.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/git_commit.py b/git_commit.py
index 0530cef..bc20dd2 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -3,6 +3,7 @@ import os
import sh
import sys
import json
+import ldap
import rc_util
from rc_rmq import RCRMQ
--
GitLab
From db4f4a1293393dc9451a83676c0fc32ba9d01fc6 Mon Sep 17 00:00:00 2001
From: atlurie <atlurie@uab.edu>
Date: Fri, 2 Oct 2020 17:50:50 -0500
Subject: [PATCH 110/188] For backwards compatibility with git 1.8
---
git_commit.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index bc20dd2..a7d7b3b 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -24,7 +24,7 @@ if not args.dry_run:
git = sh.git.bake('-C', repo_location)
ldapsearch = sh.Command('ldapsearch')
else:
- git = sh.echo.bake('git', '-C', repo_location)
+ git = sh.git.bake('--git-dir', repo_location+'/.git', '--work-tree', repo_location)
ldapsearch = sh.echo.bake('ldapsearch')
def git_commit(ch, method, properties, body):
--
GitLab
From 5142f11d8a2266998e9d98331eabc9ff0b61cab9 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 8 May 2020 03:02:54 +0000
Subject: [PATCH 111/188] Change task mgr to match modifications in agent
structure.
Decided to move UID resolve and create account steps into one and
Move subscribe mail list to verify step and eliminate the create
level because we moved account creation to request level.
---
task_manager.py | 24 ++++++------------------
1 file changed, 6 insertions(+), 18 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index d661526..680ba52 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -17,15 +17,12 @@ record = {
'fullname': '',
'last_update': datetime.now(),
'request': {
- 'get_next_uid_gid': None
- },
- 'create': {
- 'subscribe_mail_list': None,
- 'bright_account': None
+ 'create_account': None
},
'verify': {
'git_commit': None,
- 'dir_verify': None
+ 'dir_verify': None,
+ 'subscribe_mail_list': None
},
'notify': {
'notify_user': None
@@ -64,21 +61,11 @@ def task_manager(ch, method, properties, body):
try:
if task_name in current['request']:
current['request'][task_name] = success
- routing_key = 'create.' + username
+ routing_key = 'verify.' + username
done = success
logger.debug(f'Request level task(s) done?{done}')
- elif task_name in current['create']:
- current['create'][task_name] = success
- routing_key = 'verify.' + username
- done = True
- for status in current['create'].values():
- if status is not True:
- done = False
-
- logger.debug(f'Create level task(s) done?{done}')
-
elif task_name in current['verify']:
current['verify'][task_name] = success
routing_key = 'notify.' + username
@@ -108,7 +95,8 @@ def task_manager(ch, method, properties, body):
'fullname': current['fullname'],
'email': current['email'],
'uid': current['uid'],
- 'gid': current['gid']
+ 'gid': current['gid'],
+ 'fullname': current['fullname']
}
})
--
GitLab
From f0ece12a7030921f2d568cc9fb10da57a21f9303 Mon Sep 17 00:00:00 2001
From: atlurie <atlurie@uab.edu>
Date: Fri, 2 Oct 2020 20:28:09 -0500
Subject: [PATCH 112/188] Fix dry-run and git command bake statements for back
compatibility.
---
git_commit.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index a7d7b3b..5be0199 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -21,10 +21,10 @@ args = rc_util.get_args()
logger = rc_util.get_logger(args)
if not args.dry_run:
- git = sh.git.bake('-C', repo_location)
+ git = sh.git.bake('--git-dir', repo_location+'/.git', '--work-tree', repo_location)
ldapsearch = sh.Command('ldapsearch')
else:
- git = sh.git.bake('--git-dir', repo_location+'/.git', '--work-tree', repo_location)
+ git = sh.echo.bake('--git-dir', repo_location+'/.git', '--work-tree', repo_location)
ldapsearch = sh.echo.bake('ldapsearch')
def git_commit(ch, method, properties, body):
--
GitLab
From af8133560f3b66fdcf780ad900b8050f6530893a Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Mon, 21 Dec 2020 22:09:14 -0600
Subject: [PATCH 113/188] Create /data/user /data/scratch dirs if they don't
exist
If they already exist then it would just ignore.
---
dir_verify.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dir_verify.py b/dir_verify.py
index 0f0127e..23dd341 100644
--- a/dir_verify.py
+++ b/dir_verify.py
@@ -31,7 +31,7 @@ def dir_verify(ch, method, properties, body):
else:
if not path.exists():
# Make sure folder exists and with right permission
- path.mkdir(mode=0o700)
+ path.mkdir(mode=0o700, parents=True, exist_ok=True)
# Make sure ownership is correct
shutil.chown(path, int(msg['uid']), int(msg['gid']))
--
GitLab
From cae88eee53b5d62f0e5c5ec2522ca833762ddbe5 Mon Sep 17 00:00:00 2001
From: root <root@bu-c-10-22-b90-c7u7.cm.cluster>
Date: Wed, 30 Dec 2020 07:29:31 -0600
Subject: [PATCH 114/188] changed account create tasks for cod cluster
---
rc_util.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/rc_util.py b/rc_util.py
index 6519734..9fe7c73 100644
--- a/rc_util.py
+++ b/rc_util.py
@@ -4,7 +4,7 @@ from rc_rmq import RCRMQ
import json
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
-tasks = {'ohpc_account': None, 'ood_account': None, 'slurm_account': None}
+tasks = {'create_account': None, 'git_commit': None, 'dir_verify': None, 'subscribe_mail_list': None, 'notify_user': None}
logger_fmt = '%(asctime)s [%(module)s] - %(message)s'
def add_account(username, email, full='', reason=''):
--
GitLab
From befc825e2695806e263b63f54f5175edc8112427 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 13 Jan 2021 13:53:59 -0600
Subject: [PATCH 115/188] Wait for the process to return before moving on.
---
get-next-uid-gid.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index ef02b66..d80e13d 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -2,6 +2,7 @@
import sys
import json
import ldap
+import time
import logging
import argparse
import rc_util
@@ -36,6 +37,7 @@ def create_account(msg):
if not args.dry_run:
popen(cmd)
+ time.sleep(1)
logger.info(f'Bright command to create user:{cmd}')
except Exception:
logger.exception("Fatal cmsh error:")
--
GitLab
From 9945b1f09547e68902d113b41d7ef1711150cd06 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 21 Jan 2021 16:24:48 -0600
Subject: [PATCH 116/188] Remove unnecessary ldap import as we use system
ldapsearch.
---
git_commit.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index 5be0199..0c99194 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -3,7 +3,6 @@ import os
import sh
import sys
import json
-import ldap
import rc_util
from rc_rmq import RCRMQ
--
GitLab
From 821a8fb47461b6efa3cd0fce9e280474484d7471 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 28 Jan 2021 16:53:34 -0600
Subject: [PATCH 117/188] Revert "Create /data/user /data/scratch dirs if they
don't exist"
This reverts commit af8133560f3b66fdcf780ad900b8050f6530893a.
---
dir_verify.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dir_verify.py b/dir_verify.py
index 23dd341..0f0127e 100644
--- a/dir_verify.py
+++ b/dir_verify.py
@@ -31,7 +31,7 @@ def dir_verify(ch, method, properties, body):
else:
if not path.exists():
# Make sure folder exists and with right permission
- path.mkdir(mode=0o700, parents=True, exist_ok=True)
+ path.mkdir(mode=0o700)
# Make sure ownership is correct
shutil.chown(path, int(msg['uid']), int(msg['gid']))
--
GitLab
From 744b1882aafa91df07a8f92759f4de7525c97065 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 2 Feb 2021 05:53:05 -0600
Subject: [PATCH 118/188] Check if the dirs exist & have correct permissions
and ownerships
If they don't then send a fail message, but do not create them
---
dir_verify.py | 30 ++++++++++++++++++++----------
1 file changed, 20 insertions(+), 10 deletions(-)
diff --git a/dir_verify.py b/dir_verify.py
index 0f0127e..f340325 100644
--- a/dir_verify.py
+++ b/dir_verify.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+import os
import sys
import json
import shutil
@@ -19,7 +20,9 @@ def dir_verify(ch, method, properties, body):
msg = json.loads(body)
username = msg['username']
msg['task'] = task
- msg['success'] = False
+ msg['success'] = True
+
+ missing_dirs = []
try:
for d in dirs:
@@ -30,17 +33,24 @@ def dir_verify(ch, method, properties, body):
else:
if not path.exists():
- # Make sure folder exists and with right permission
- path.mkdir(mode=0o700)
-
- # Make sure ownership is correct
- shutil.chown(path, int(msg['uid']), int(msg['gid']))
-
- logger.debug(f'{path} created')
-
- msg['success'] = True
+ # check if dirs exist and record any missing dirs
+ missing_dirs.append(path)
+ msg['success'] = False
+ msg['errmsg'] = f"Error: missing dirs {missing_dirs}"
+ logger.info(f'{path} does not exist')
+ else:
+ # check existing dirs for correct ownership and permissions
+ status = os.stat(path)
+ mask = oct(status.st_mode)[-3:]
+ uid = str(status.st_uid)
+ gid = str(status.st_gid)
+ if mask!='700' or uid!=msg['uid'] or gid!=msg['gid']:
+ msg['success'] = False
+ msg['errmsg'] = f"Error: dir {path} permissions or ownership are wrong"
except Exception as exception:
+ msg['success'] = False
+ msg['errmsg'] = "Exception raised, check the logs for stack trace"
logger.error('', exc_info=True)
# send confirm message
--
GitLab
From 6f721992e5786740cf7424e2318e9ff82819f0cb Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 2 Feb 2021 17:08:01 -0600
Subject: [PATCH 119/188] Pass fail msg to task_manger to handle failure and
report to admin
---
get-next-uid-gid.py | 23 +++++++++++------------
1 file changed, 11 insertions(+), 12 deletions(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index d80e13d..a58ddea 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -29,18 +29,15 @@ def create_account(msg):
fullname = msg['fullname']
success = False
- try:
# Bright command to create user
- cmd = '/cm/local/apps/cmd/bin/cmsh -c '
- cmd += f'"user; add {username}; set id {uid}; set email {email}; set commonname \\"{fullname}\\"; '
- cmd += 'commit;"'
+ cmd = '/cm/local/apps/cmd/bin/cmsh -c '
+ cmd += f'"user; add {username}; set id {uid}; set email {email}; set commonname \\"{fullname}\\"; '
+ cmd += 'commit;"'
- if not args.dry_run:
- popen(cmd)
- time.sleep(1)
- logger.info(f'Bright command to create user:{cmd}')
- except Exception:
- logger.exception("Fatal cmsh error:")
+ if not args.dry_run:
+ popen(cmd)
+ time.sleep(1)
+ logger.info(f'Bright command to create user:{cmd}')
# Define your callback function
def resolve_uid_gid(ch, method, properties, body):
@@ -75,8 +72,10 @@ def resolve_uid_gid(ch, method, properties, body):
create_account(msg)
msg['task'] = task
msg['success'] = True
- except Exception:
- logger.exception("Fatal UID resolution error:")
+ except Exception as exception:
+ msg['success'] = False
+ msg['errmsg'] = f"Exception raised during account creation, check logs for stack trace"
+ logger.error('', exc_info=True)
# Acknowledge message
ch.basic_ack(delivery_tag=method.delivery_tag)
--
GitLab
From 7f9983275904e8487ddab207bf7c90ec401f4f6a Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 2 Feb 2021 17:30:59 -0600
Subject: [PATCH 120/188] Remove the var that is not used in forming branch
name
---
git_commit.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 5be0199..ba87e00 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -30,10 +30,9 @@ else:
def git_commit(ch, method, properties, body):
msg = json.loads(body)
username = msg['username']
- ticketnum = msg.get('ticketnum', 'add-users-' + username.lower())
msg['task'] = task
msg['success'] = False
- branch_name = 'issue-' + ticketnum
+ branch_name = 'issue-add-users-' + username.lower()
user_ldif = users_dir + f'/{username}.ldif'
group_ldif = groups_dir + f'/{username}.ldif'
--
GitLab
From a9a8774afbbf5fb3f39c9aa3bda18a04c85d8013 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 2 Feb 2021 17:51:35 -0600
Subject: [PATCH 121/188] Set task status in the msg dict
---
subscribe_mail_lists.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index 49ac1d7..9567464 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -35,7 +35,7 @@ def mail_list_subscription(ch, method, properties, body):
\nQUIET ADD hpc-users {email} {fullname}'
logger.info("Adding user{} to mail list".format(username))
- success = False
+ msg['success'] = False
try:
# Create a text/plain message
email_msg = EmailMessage()
--
GitLab
From e459af09c800c38a1e262f750af4e77b554f899f Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 2 Feb 2021 17:53:59 -0600
Subject: [PATCH 122/188] Redirect exception to logger error stream, to keep it
uniform with other agents
---
subscribe_mail_lists.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index 9567464..91d5a8e 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -55,8 +55,8 @@ def mail_list_subscription(ch, method, properties, body):
s.quit()
msg['task'] = task
msg['success'] = True
- except Exception:
- logger.exception("Fatal error:")
+ except Exception as exception:
+ logger.error('', exc_info=True)
# Acknowledge message
ch.basic_ack(delivery_tag=method.delivery_tag)
--
GitLab
From 9af396a352375441dd63e68b7955ff6aa2169602 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 3 Feb 2021 19:13:10 -0600
Subject: [PATCH 123/188] Parameterize the rc_users ldap repo location using
rmq config file
---
git_commit.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index 5be0199..3cf0f8e 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -6,6 +6,7 @@ import json
import ldap
import rc_util
from rc_rmq import RCRMQ
+import rabbit_config as rmq_cfg
task = 'git_commit'
@@ -13,7 +14,7 @@ task = 'git_commit'
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
# Define some location
-repo_location = os.path.expanduser('~/git/rc-users')
+repo_location = os.path.expanduser(rmq_cfg.rc_users_ldap_repo_loc)
users_dir = repo_location + '/users'
groups_dir = repo_location + '/groups'
--
GitLab
From 02bb215aeee7dd0f574369bda26b42b869c63570 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 3 Feb 2021 19:18:04 -0600
Subject: [PATCH 124/188] Use the new single merged config file in the import
for notify_user
---
notify_user.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/notify_user.py b/notify_user.py
index 753d14d..45e90d5 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -7,7 +7,7 @@ import dataset
from rc_rmq import RCRMQ
from jinja2 import Template
from datetime import datetime
-import mail_config as mail_cfg
+import rabbit_config as mail_cfg
task = 'notify_user'
--
GitLab
From 570f0fb736b779a5277d7663040669e45ac8c994 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 3 Feb 2021 19:20:36 -0600
Subject: [PATCH 125/188] Use parameters from the rabbit config file in
subscribe_mail_list
---
subscribe_mail_lists.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/subscribe_mail_lists.py b/subscribe_mail_lists.py
index 49ac1d7..2543a27 100644
--- a/subscribe_mail_lists.py
+++ b/subscribe_mail_lists.py
@@ -7,6 +7,7 @@ import argparse
import rc_util
from email.message import EmailMessage
from rc_rmq import RCRMQ
+import rabbit_config as rcfg
task = 'subscribe_mail_list'
@@ -28,8 +29,10 @@ def mail_list_subscription(ch, method, properties, body):
fullname = msg['fullname']
email = msg['email']
- mail_list_admin = 'root@localhost' #change this during deploy
- mail_list = 'LISTSERV@LISTSERV.UAB.EDU'
+ mail_list_admin = rcfg.Sender
+ mail_list = rcfg.Mail_list
+ mail_list_bcc = rcfg.Mail_list_bcc
+ server = rcfg.Server
listserv_cmd = f'QUIET ADD hpc-announce {email} {fullname} \
\nQUIET ADD hpc-users {email} {fullname}'
@@ -43,9 +46,10 @@ def mail_list_subscription(ch, method, properties, body):
email_msg['From'] = mail_list_admin
email_msg['To'] = mail_list
email_msg['Subject'] = ''
+ email_msg['Bcc'] = mail_list_bcc
# Create an smtp object and send email
- s = smtplib.SMTP('localhost')
+ s = smtplib.SMTP(server)
email_msg.set_content(listserv_cmd)
if not args.dry_run:
--
GitLab
From 1f080f3a03413f7e3decfefb74796d7c619d0936 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 5 Feb 2021 14:41:28 -0600
Subject: [PATCH 126/188] Add an example RMQ agent related config file
---
rabbit_config.py.example | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/rabbit_config.py.example b/rabbit_config.py.example
index 5643bb1..d929c5d 100644
--- a/rabbit_config.py.example
+++ b/rabbit_config.py.example
@@ -4,3 +4,39 @@ Password = 'CHANGE_IT_TO_YOUR_OWN_PASSWORD'
VHost = '/'
Server = 'ohpc'
Port = 5672
+
+# git_commit agent config
+rc_users_ldap_repo_loc = "~/git/rc-users"
+
+# Config related to email
+Server = 'localhost'
+My_email = 'root@localhost'
+Sender = 'ROOT@LOCALHOST'
+Sender_alias = 'Services'
+Subject = 'New User Account'
+Info_url = 'https://www.google.com'
+Mail_list = 'root@localhost'
+Mail_list_bcc = 'cmsupport@localhost'
+
+Head = f"""From: {Sender_alias} <{Sender}>
+To: <{{{{ to }}}}>
+Subject: {Subject}
+"""
+
+Body = f"""
+Hi {{{{ username }}}}
+Your account has been set up with:
+
+============================
+User ID: {{{{ username }}}}
+============================
+
+If you have any questions, please visit:
+{Info_url}
+
+or email at {My_email}
+
+Cheers,
+"""
+
+Whole_mail = Head + Body
--
GitLab
From acbd96b478a846565f8939c787764b55739962aa Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 9 Feb 2021 10:49:54 -0600
Subject: [PATCH 127/188] Init task status to false
---
get-next-uid-gid.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index d80e13d..c78509c 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -27,7 +27,7 @@ def create_account(msg):
uid = msg['uid']
email = msg['email']
fullname = msg['fullname']
- success = False
+ msg['success'] = False
try:
# Bright command to create user
@@ -49,7 +49,7 @@ def resolve_uid_gid(ch, method, properties, body):
msg = json.loads(body)
logger.info("Received {}".format(msg))
username = msg['username']
- success = False
+ msg['success'] = False
# Determine next available UID
try:
--
GitLab
From dbc981fe79c54cb148b6f1a34d459f4d429e7127 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 9 Feb 2021 10:58:10 -0600
Subject: [PATCH 128/188] Make the user query a python formatted string
---
get-next-uid-gid.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index c78509c..8ccb86a 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -53,7 +53,7 @@ def resolve_uid_gid(ch, method, properties, body):
# Determine next available UID
try:
- user_exists_cmd = "/usr/bin/getent passwd {username}"
+ user_exists_cmd = f"/usr/bin/getent passwd {username}"
user_exists = popen(user_exists_cmd).read().rstrip()
if user_exists:
--
GitLab
From 1379bbdfc62422e1243c4220e1d443fc3f2ba435 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 9 Feb 2021 14:33:03 -0600
Subject: [PATCH 129/188] Parameterize the delay used to avoid dir_verify
config issue
---
get-next-uid-gid.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index d80e13d..f2c4fdf 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -8,6 +8,7 @@ import argparse
import rc_util
from os import popen
from rc_rmq import RCRMQ
+import rabbit_config as rcfg
task = 'create_account'
@@ -37,7 +38,7 @@ def create_account(msg):
if not args.dry_run:
popen(cmd)
- time.sleep(1)
+ time.sleep(rcfg.Delay)
logger.info(f'Bright command to create user:{cmd}')
except Exception:
logger.exception("Fatal cmsh error:")
--
GitLab
From 58c4aef0f12342983d5eb6b632f9bf87d1c09852 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 10 Feb 2021 00:28:05 -0600
Subject: [PATCH 130/188] Create new branch only if it doesn't exist, otherwise
it errors
The git_commit agent task status would be true if everything
goes well the first time but on a repeated request it would raise
an exception that the branch already exists and the task_manager
would set the task status false as it saw an exception. So the
admin would get a report with git_commit agent status set to
false even though it ran as expected the first time.
---
git_commit.py | 57 ++++++++++++++++++++++++++-------------------------
1 file changed, 29 insertions(+), 28 deletions(-)
diff --git a/git_commit.py b/git_commit.py
index 5be0199..e31d804 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -47,34 +47,35 @@ def git_commit(ch, method, properties, body):
git.checkout('master')
logger.debug('git pull')
git.pull()
- logger.debug('git checkout -b %s', branch_name)
- git.checkout('-b', branch_name)
-
- logger.debug("open(%s, 'w'), open(%s, 'w')", user_ldif, group_ldif)
- with open(user_ldif, 'w') as ldif_u,\
- open(group_ldif, 'w') as ldif_g:
- logger.debug(f"ldapsearch -LLL -x -H ldaps://ldapserver -b 'dc=cm,dc=cluster' uid={username} > {user_ldif}")
- ldapsearch('-LLL', '-x', '-H', 'ldaps://ldapserver', '-b', "dc=cm,dc=cluster", f"uid={username}", _out=ldif_u)
- logger.debug(f"ldapsearch -LLL -x -H ldapserver -b 'ou=Group,dc=cm,dc=cluster' cn={username} > {group_ldif}")
- ldapsearch('-LLL', '-x', '-H', 'ldaps://ldapserver', '-b', "ou=Group,dc=cm,dc=cluster", f"cn={username}", _out=ldif_g)
- logger.info('user ldif files generated.')
-
- logger.debug('git add %s', user_ldif)
- git.add(user_ldif)
- logger.debug('git add %s', group_ldif)
- git.add(group_ldif)
- logger.debug("git commit -m 'Added new cheaha user: %s'", username)
- git.commit(m="Added new cheaha user: " + username)
- logger.debug('git checkout master')
- git.checkout('master')
-
- logger.debug('git merge %s --no-ff --no-edit', branch_name)
- git.merge(branch_name, '--no-ff', '--no-edit')
- logger.debug('git push origin master')
- git.push('origin', 'master')
- # merge with gitlab api
-
- logger.info('Added ldif files and committed to git repo')
+ branch_exists = git.branch('--list', branch_name)
+ if not branch_exists:
+ logger.debug('git checkout -b %s', branch_name)
+ git.checkout('-b', branch_name)
+ logger.debug("open(%s, 'w'), open(%s, 'w')", user_ldif, group_ldif)
+ with open(user_ldif, 'w') as ldif_u,\
+ open(group_ldif, 'w') as ldif_g:
+ logger.debug(f"ldapsearch -LLL -x -H ldaps://ldapserver -b 'dc=cm,dc=cluster' uid={username} > {user_ldif}")
+ ldapsearch('-LLL', '-x', '-H', 'ldaps://ldapserver', '-b', "dc=cm,dc=cluster", f"uid={username}", _out=ldif_u)
+ logger.debug(f"ldapsearch -LLL -x -H ldapserver -b 'ou=Group,dc=cm,dc=cluster' cn={username} > {group_ldif}")
+ ldapsearch('-LLL', '-x', '-H', 'ldaps://ldapserver', '-b', "ou=Group,dc=cm,dc=cluster", f"cn={username}", _out=ldif_g)
+ logger.info('user ldif files generated.')
+
+ logger.debug('git add %s', user_ldif)
+ git.add(user_ldif)
+ logger.debug('git add %s', group_ldif)
+ git.add(group_ldif)
+ logger.debug("git commit -m 'Added new cheaha user: %s'", username)
+ git.commit(m="Added new cheaha user: " + username)
+ logger.debug('git checkout master')
+ git.checkout('master')
+
+ logger.debug('git merge %s --no-ff --no-edit', branch_name)
+ git.merge(branch_name, '--no-ff', '--no-edit')
+ logger.debug('git push origin master')
+ git.push('origin', 'master')
+ # merge with gitlab api
+
+ logger.info('Added ldif files and committed to git repo')
msg['success'] = True
except Exception as exception:
--
GitLab
From 113ff0e05aaae8ce11d74730a488e79ae28f4133 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 5 Feb 2021 16:03:00 -0600
Subject: [PATCH 131/188] Update default callback in rc_util
It acted like task manager when first design it.
Now we have proper task manager, update it.
The default behavior will be a caller waiting to complete msg
---
rc_util.py | 23 +++++++++++------------
1 file changed, 11 insertions(+), 12 deletions(-)
diff --git a/rc_util.py b/rc_util.py
index 9fe7c73..9268203 100644
--- a/rc_util.py
+++ b/rc_util.py
@@ -21,19 +21,18 @@ def add_account(username, email, full='', reason=''):
def worker(ch, method, properties, body):
msg = json.loads(body)
- task = msg['task']
- tasks[task] = msg['success']
- print("Got msg: {}({})".format(msg['task'], msg['success']))
+ username = msg['username']
- # Check if all tasks are done
- done = True
- for key, status in tasks.items():
- if not status:
- print("{} is not done yet.".format(key))
- done = False
- if done:
- rc_rmq.stop_consume()
- rc_rmq.delete_queue()
+ if msg['success']:
+ print(f'Account for {username} has been created.')
+ else:
+ print(f"There's some issue while creating account for {username}")
+ errmsg = msg.get('errmsg', [])
+ for err in errmsg:
+ print(err)
+
+ rc_rmq.stop_consume()
+ rc_rmq.delete_queue()
def consume(username, routing_key='', callback=worker, debug=False):
if routing_key == '':
--
GitLab
From 0172a8867d897e375eea6d37192493b83a2905c9 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Fri, 5 Feb 2021 16:06:00 -0600
Subject: [PATCH 132/188] Update routing key according to the callback change
---
rc_util.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/rc_util.py b/rc_util.py
index 9268203..e16fe30 100644
--- a/rc_util.py
+++ b/rc_util.py
@@ -36,7 +36,7 @@ def worker(ch, method, properties, body):
def consume(username, routing_key='', callback=worker, debug=False):
if routing_key == '':
- routing_key = 'confirm.' + username
+ routing_key = 'complete.' + username
if debug:
sleep(5)
--
GitLab
From 758627bfdd0011ee621b9f89bb6ba9d77477ed14 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 23:05:16 -0600
Subject: [PATCH 133/188] Check success from msg received
---
create_account.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/create_account.py b/create_account.py
index 60ce5fc..7590adc 100755
--- a/create_account.py
+++ b/create_account.py
@@ -25,7 +25,13 @@ def callback(channel, method, properties, body):
msg = json.loads(body)
username = msg['username']
- logger.info(f'Account for {username} has been created.')
+ if msg['success']:
+ print(f'Account for {username} has been created.')
+ else:
+ print(f"There's some issue while creating account for {username}")
+ errmsg = msg.get('errmsg', [])
+ for err in errmsg:
+ print(err)
rc_util.rc_rmq.stop_consume()
rc_util.rc_rmq.delete_queue()
--
GitLab
From 2a82f31af1dfb1871488bc66285c6eb52e941f25 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 4 Feb 2021 15:48:38 -0600
Subject: [PATCH 134/188] Replace logger with print
---
create_account.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/create_account.py b/create_account.py
index 7590adc..87a58bd 100755
--- a/create_account.py
+++ b/create_account.py
@@ -14,7 +14,6 @@ parser.add_argument('-v', '--verbose', action='store_true', help='verbose output
parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
args = parser.parse_args()
-logger = rc_util.get_logger(args)
if args.email == '':
args.email = args.username
@@ -38,7 +37,7 @@ def callback(channel, method, properties, body):
rc_util.add_account(args.username, email=args.email, full=args.full_name, reason=args.reason)
-logger.info(f'Account for {args.username} requested.')
+print(f'Account for {args.username} requested.')
-logger.info('Waiting for completion...')
+print('Waiting for completion...')
rc_util.consume(args.username, routing_key=f'complete.{args.username}', callback=callback)
--
GitLab
From baa1c6d2ebc3c57c9cbe041ce96eba14cb0a4d01 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 4 Feb 2021 16:16:29 -0600
Subject: [PATCH 135/188] Add timeout handler
---
create_account.py | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/create_account.py b/create_account.py
index 87a58bd..d46906f 100755
--- a/create_account.py
+++ b/create_account.py
@@ -3,6 +3,7 @@ import json
import sys
import rc_util
import argparse
+import signal
parser = argparse.ArgumentParser()
parser.add_argument('username', help='username that will be created')
@@ -14,12 +15,18 @@ parser.add_argument('-v', '--verbose', action='store_true', help='verbose output
parser.add_argument('-n', '--dry-run', action='store_true', help='enable dry run mode')
args = parser.parse_args()
+timeout = 60
if args.email == '':
args.email = args.username
if '@' not in args.email:
args.email = args.username + '@' + args.domain
+def timeout_handler(signum, frame):
+ print("Process timeout, there's might some issue with agents")
+ rc_util.rc_rmq.stop_consume()
+
+
def callback(channel, method, properties, body):
msg = json.loads(body)
username = msg['username']
@@ -39,5 +46,9 @@ def callback(channel, method, properties, body):
rc_util.add_account(args.username, email=args.email, full=args.full_name, reason=args.reason)
print(f'Account for {args.username} requested.')
+# Set initial timeout timer
+signal.signal(signal.SIGALRM, timeout_handler)
+signal.setitimer(signal.ITIMER_REAL, timeout)
+
print('Waiting for completion...')
rc_util.consume(args.username, routing_key=f'complete.{args.username}', callback=callback)
--
GitLab
From 073f1e37a46d67e5da47138167b0a45dd70e56b3 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 28 Jan 2021 21:30:03 -0600
Subject: [PATCH 136/188] Change My_email to Admin_email
---
mail_config.py | 4 ++--
notify_user.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/mail_config.py b/mail_config.py
index 1146e38..b7c5191 100644
--- a/mail_config.py
+++ b/mail_config.py
@@ -1,6 +1,6 @@
# Some variable for email
Server = 'localhost'
-My_email = 'root@localhost'
+Admin_email = 'root@localhost'
Sender = 'ROOT@LOCALHOST'
Sender_alias = 'Services'
Subject = 'New User Account'
@@ -22,7 +22,7 @@ User ID: {{{{ username }}}}
If you have any questions, please visit:
{Info_url}
-or email at {My_email}
+or email at {Admin_email}
Cheers,
"""
diff --git a/notify_user.py b/notify_user.py
index 753d14d..97f3fce 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -42,7 +42,7 @@ def notify_user(ch, method, properties, body):
else:
# Send email to user
- receivers = [user_email, mail_cfg.My_email]
+ receivers = [user_email, mail_cfg.Admin_email]
message = Template(mail_cfg.Whole_mail).render(username=username, to=user_email)
if args.dry_run:
--
GitLab
From 9b606b86fae08469eabebc35a1c42bbc9de74212 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 28 Jan 2021 22:23:48 -0600
Subject: [PATCH 137/188] Add User report email head
---
mail_config.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/mail_config.py b/mail_config.py
index b7c5191..911ba23 100644
--- a/mail_config.py
+++ b/mail_config.py
@@ -28,3 +28,9 @@ Cheers,
"""
Whole_mail = Head + Body
+
+UserReportHead = f"""From: {Sender_alias} <{Sender}>
+To: <{Admin_email}>
+Subject: User Creation Report:
+"""
+
--
GitLab
From f604d358df8a1e90e11ef08ac9e8d608d771a1cc Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 28 Jan 2021 22:34:31 -0600
Subject: [PATCH 138/188] Add notify admin logic
When complete or terminate, sent out mail to admin
---
task_manager.py | 42 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 42 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index 680ba52..5c7982f 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -2,8 +2,10 @@
import sys
import json
import rc_util
+import smtplib
from rc_rmq import RCRMQ
from datetime import datetime
+import mail_config as mail_cfg
task = 'task_manager'
@@ -36,11 +38,37 @@ tracking = {}
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
+def notify_admin(username, user_record):
+ message = mail_cfg.UserReportHead
+ message += f"""
+ User Creation Report for user {username}
+ uid: {user_record["uid"]}, gid: {user_record["gid"]}
+ Tasks:
+ 'create_account': {user_record["request"]["create_account"]}
+ 'git_commit': {user_record["verify"]["git_commit"]}
+ 'dir_verify': {user_record["verify"]["dir_verify"]}
+ 'subscribe_mail_list': {user_record["verify"]["subscribe_mail_list"]}
+ 'notify_user': {user_record["notify"]["notify_user"]}
+ """
+
+ if args.dry_run:
+ logger.info(f'smtp = smtplib.SMTP({mail_cfg.Server})')
+ logger.info(f'smtp.sendmail({mail_cfg.Sender}, {mail_cfg.Admin_email}, message)')
+ logger.info(message)
+
+ else:
+ smtp = smtplib.SMTP(mail_cfg.Server)
+ smtp.sendmail(mail_cfg.Sender, receivers, message)
+
+ logger.debug(f'User report sent to: {mail_cfg.Admin_email}')
+
+
def task_manager(ch, method, properties, body):
msg = json.loads(body)
username = method.routing_key.split('.')[1]
task_name = msg['task']
done = success = msg['success']
+ completed = terminated = False
routing_key = ""
if username not in tracking:
@@ -63,6 +91,9 @@ def task_manager(ch, method, properties, body):
current['request'][task_name] = success
routing_key = 'verify.' + username
done = success
+ # Terminate the process if failed
+ if not success:
+ terminated = True
logger.debug(f'Request level task(s) done?{done}')
@@ -74,12 +105,19 @@ def task_manager(ch, method, properties, body):
if status is not True:
done = False
+ # Terminate the process if dir_verify failed
+ if task_name == "dir_verify":
+ if not success:
+ terminated = True
+
logger.debug(f'Verify level task(s) done?{done}')
elif task_name in current['notify']:
current['notify'][task_name] = success
routing_key = 'complete.' + username
done = success
+ # The whole creation process has completed
+ completed = True
logger.debug(f'Notify level task(s) done?{done}')
@@ -109,6 +147,10 @@ def task_manager(ch, method, properties, body):
logger.debug('Previous level messages acknowledged')
+ # Send report to admin after all tasks confirmed or terminated
+ if completed or terminated:
+ notify_admin(username, current)
+
logger.info(f'Start listening to queue: {task}')
rc_rmq.start_consume({
--
GitLab
From 47fb91d0b929fd2fb0158fc72633808f562f171b Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 2 Feb 2021 23:52:35 -0600
Subject: [PATCH 139/188] Include reason in message
---
task_manager.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index 5c7982f..b7bd592 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -16,6 +16,7 @@ record = {
'uid': -1,
'gid': -1,
'email': '',
+ 'reason': '',
'fullname': '',
'last_update': datetime.now(),
'request': {
@@ -77,6 +78,7 @@ def task_manager(ch, method, properties, body):
current['uid'] = msg.get('uid', -1)
current['gid'] = msg.get('gid', -1)
current['email'] = msg.get('email', '')
+ current['reason'] = msg.get('reason', '')
current['fullname'] = msg.get('fullname', '')
logger.debug(f'Tracking user {username}')
@@ -134,7 +136,7 @@ def task_manager(ch, method, properties, body):
'email': current['email'],
'uid': current['uid'],
'gid': current['gid'],
- 'fullname': current['fullname']
+ 'reason': current['reason']
}
})
--
GitLab
From bc830d2c2ce8ff053c9e356656485c0703634ff0 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 2 Feb 2021 23:55:46 -0600
Subject: [PATCH 140/188] Update log message
---
task_manager.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index b7bd592..c07a9ef 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -97,7 +97,7 @@ def task_manager(ch, method, properties, body):
if not success:
terminated = True
- logger.debug(f'Request level task(s) done?{done}')
+ logger.debug(f'Request level {task_name}? {success}')
elif task_name in current['verify']:
current['verify'][task_name] = success
@@ -112,7 +112,7 @@ def task_manager(ch, method, properties, body):
if not success:
terminated = True
- logger.debug(f'Verify level task(s) done?{done}')
+ logger.debug(f'Verify level {task_name}? {success}')
elif task_name in current['notify']:
current['notify'][task_name] = success
@@ -121,7 +121,7 @@ def task_manager(ch, method, properties, body):
# The whole creation process has completed
completed = True
- logger.debug(f'Notify level task(s) done?{done}')
+ logger.debug(f'Notify level {task_name}? {success}')
except Exception as exception:
logger.error('', exc_info=True)
--
GitLab
From 8d5f777cbe39863d16c14038857f5c96869128d1 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 00:01:59 -0600
Subject: [PATCH 141/188] Save error message
---
task_manager.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index c07a9ef..654a578 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -19,6 +19,7 @@ record = {
'reason': '',
'fullname': '',
'last_update': datetime.now(),
+ 'errmsg': [],
'request': {
'create_account': None
},
@@ -88,6 +89,12 @@ def task_manager(ch, method, properties, body):
# Save the delivery tags for future use
current['delivery_tags'].append(method.delivery_tag)
+ # Save error message if the task was failed
+ if not success:
+ errmsg = msg.get('errmsg', '')
+ if errmsg:
+ current['errmsg'].append(f"{task_name}: {errmsg}")
+
try:
if task_name in current['request']:
current['request'][task_name] = success
--
GitLab
From 56cf1325b89ba58e7bca02d0bf8d3ffa8dcce183 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 00:03:00 -0600
Subject: [PATCH 142/188] Add error message list into report
---
task_manager.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index 654a578..a002f4c 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -52,6 +52,13 @@ def notify_admin(username, user_record):
'subscribe_mail_list': {user_record["verify"]["subscribe_mail_list"]}
'notify_user': {user_record["notify"]["notify_user"]}
"""
+ if user_record['errmsg']:
+ message += """
+
+ Error(s):
+ """
+ for msg in user_record['errmsg']:
+ message += msg + "\n"
if args.dry_run:
logger.info(f'smtp = smtplib.SMTP({mail_cfg.Server})')
--
GitLab
From 82ba2e8fb9b5a920c91cae150bd04af051cdfffc Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 22:11:17 -0600
Subject: [PATCH 143/188] Change variable name
done to send
---
task_manager.py | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index a002f4c..508d19d 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -76,8 +76,8 @@ def task_manager(ch, method, properties, body):
msg = json.loads(body)
username = method.routing_key.split('.')[1]
task_name = msg['task']
- done = success = msg['success']
- completed = terminated = False
+ success = msg['success']
+ send = completed = terminated = False
routing_key = ""
if username not in tracking:
@@ -106,20 +106,21 @@ def task_manager(ch, method, properties, body):
if task_name in current['request']:
current['request'][task_name] = success
routing_key = 'verify.' + username
- done = success
+
# Terminate the process if failed
if not success:
terminated = True
+ send = True
logger.debug(f'Request level {task_name}? {success}')
elif task_name in current['verify']:
current['verify'][task_name] = success
routing_key = 'notify.' + username
- done = True
+ send = True
for status in current['verify'].values():
if status is not True:
- done = False
+ send = False
# Terminate the process if dir_verify failed
if task_name == "dir_verify":
@@ -131,7 +132,8 @@ def task_manager(ch, method, properties, body):
elif task_name in current['notify']:
current['notify'][task_name] = success
routing_key = 'complete.' + username
- done = success
+ send = True
+
# The whole creation process has completed
completed = True
@@ -140,7 +142,7 @@ def task_manager(ch, method, properties, body):
except Exception as exception:
logger.error('', exc_info=True)
- if done:
+ if send:
# Send trigger message
rc_rmq.publish_msg({
'routing_key': routing_key,
--
GitLab
From 51e5bfe314a4a0fc9492809258dbe23de580f5ca Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 22:13:07 -0600
Subject: [PATCH 144/188] Add missing email receivers
---
task_manager.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/task_manager.py b/task_manager.py
index 508d19d..422e090 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -41,6 +41,7 @@ tracking = {}
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
def notify_admin(username, user_record):
+ receivers = [user_record['email'], mail_cfg.Admin_email]
message = mail_cfg.UserReportHead
message += f"""
User Creation Report for user {username}
--
GitLab
From 9a95c611325dc8ed41d39ba0eaeff9e650c8130f Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 22:13:58 -0600
Subject: [PATCH 145/188] Move message define to the top
---
task_manager.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index 422e090..ee8fa23 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -103,6 +103,16 @@ def task_manager(ch, method, properties, body):
if errmsg:
current['errmsg'].append(f"{task_name}: {errmsg}")
+ # Define message that's going to be published
+ message = {
+ 'username': username,
+ 'uid': current['uid'],
+ 'gid': current['gid'],
+ 'email': current['email'],
+ 'reason': current['reason'],
+ 'fullname': current['fullname']
+ }
+
try:
if task_name in current['request']:
current['request'][task_name] = success
@@ -147,14 +157,7 @@ def task_manager(ch, method, properties, body):
# Send trigger message
rc_rmq.publish_msg({
'routing_key': routing_key,
- 'msg': {
- 'username': username,
- 'fullname': current['fullname'],
- 'email': current['email'],
- 'uid': current['uid'],
- 'gid': current['gid'],
- 'reason': current['reason']
- }
+ 'msg': message
})
logger.debug(f"Trigger message '{routing_key}' sent")
--
GitLab
From 3dd3af9f23bd7038b6d4b3aace882e3cea0fa18a Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 22:20:30 -0600
Subject: [PATCH 146/188] Introduce waiting variable
---
task_manager.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index ee8fa23..c472da1 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -20,6 +20,7 @@ record = {
'fullname': '',
'last_update': datetime.now(),
'errmsg': [],
+ 'waiting': {},
'request': {
'create_account': None
},
@@ -123,15 +124,16 @@ def task_manager(ch, method, properties, body):
terminated = True
send = True
+ current['waiting'] = {'git_commit', 'dir_verify', 'subscribe_mail_list'}
logger.debug(f'Request level {task_name}? {success}')
elif task_name in current['verify']:
current['verify'][task_name] = success
+ current['waiting'].discard(task_name)
routing_key = 'notify.' + username
- send = True
- for status in current['verify'].values():
- if status is not True:
- send = False
+ if not current['waiting']:
+ send = True
+ current['waiting'] = {'notify_user'}
# Terminate the process if dir_verify failed
if task_name == "dir_verify":
@@ -142,6 +144,7 @@ def task_manager(ch, method, properties, body):
elif task_name in current['notify']:
current['notify'][task_name] = success
+ current['waiting'].discard(task_name)
routing_key = 'complete.' + username
send = True
--
GitLab
From 2e399d08fca8ecf7122506f303d8612c261a7395 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 22:22:06 -0600
Subject: [PATCH 147/188] Send out error message to caller
When complete or terminated
---
task_manager.py | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index c472da1..a7910a2 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -122,6 +122,9 @@ def task_manager(ch, method, properties, body):
# Terminate the process if failed
if not success:
terminated = True
+ routing_key = 'complete.' + username
+ message['success'] = False
+ message['errmsg'] = current['errmsg']
send = True
current['waiting'] = {'git_commit', 'dir_verify', 'subscribe_mail_list'}
@@ -131,14 +134,17 @@ def task_manager(ch, method, properties, body):
current['verify'][task_name] = success
current['waiting'].discard(task_name)
routing_key = 'notify.' + username
+
if not current['waiting']:
send = True
current['waiting'] = {'notify_user'}
- # Terminate the process if dir_verify failed
- if task_name == "dir_verify":
- if not success:
- terminated = True
+ # Terminate if dir_verify failed and all agents has responsed
+ if send and not current['verify']['dir_verify']:
+ terminated = True
+ routing_key = 'complete.' + username
+ message['success'] = False
+ message['errmsg'] = current['errmsg']
logger.debug(f'Verify level {task_name}? {success}')
@@ -146,6 +152,9 @@ def task_manager(ch, method, properties, body):
current['notify'][task_name] = success
current['waiting'].discard(task_name)
routing_key = 'complete.' + username
+ message['success'] = success
+ message['errmsg'] = current['errmsg']
+
send = True
# The whole creation process has completed
--
GitLab
From b6f10bb20b8c72c620b8ee21029a27447a8c4ffb Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 22:56:23 -0600
Subject: [PATCH 148/188] Update the way initial waiting
---
task_manager.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index a7910a2..4918cb5 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -20,7 +20,7 @@ record = {
'fullname': '',
'last_update': datetime.now(),
'errmsg': [],
- 'waiting': {},
+ 'waiting': set(),
'request': {
'create_account': None
},
--
GitLab
From 932758a602b2f611e8a9fd8c8d29c1911ce182a4 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 22:57:14 -0600
Subject: [PATCH 149/188] Update last update every time receive a message
---
task_manager.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index 4918cb5..a4e49ac 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -98,6 +98,8 @@ def task_manager(ch, method, properties, body):
# Save the delivery tags for future use
current['delivery_tags'].append(method.delivery_tag)
+ current['last_update'] = datetime.now()
+
# Save error message if the task was failed
if not success:
errmsg = msg.get('errmsg', '')
--
GitLab
From 99fcad206190452bf2af126e06fe14ba2951f273 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 22:59:45 -0600
Subject: [PATCH 150/188] Remove from tracking after report sent
---
task_manager.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index a4e49ac..526bc6b 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -183,10 +183,15 @@ def task_manager(ch, method, properties, body):
logger.debug('Previous level messages acknowledged')
- # Send report to admin after all tasks confirmed or terminated
+ # Send report to admin
if completed or terminated:
+
notify_admin(username, current)
+ tracking.pop(username)
+
+ logger.debug('Admin report sent')
+
logger.info(f'Start listening to queue: {task}')
rc_rmq.start_consume({
--
GitLab
From 3acf62910366e2c85d108ad7ed7038aa1d29d0e1 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Wed, 3 Feb 2021 23:01:01 -0600
Subject: [PATCH 151/188] Add timeout checker
Default to check every 30 seconds
Will inform caller and remove from tracking
---
task_manager.py | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index 526bc6b..83c7bfe 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
import sys
import json
+import signal
import rc_util
import smtplib
from rc_rmq import RCRMQ
@@ -8,6 +9,7 @@ from datetime import datetime
import mail_config as mail_cfg
task = 'task_manager'
+timeout = 30
args = rc_util.get_args()
logger = rc_util.get_logger(args)
@@ -193,6 +195,32 @@ def task_manager(ch, method, properties, body):
logger.debug('Admin report sent')
+def timeout_handler(signum, frame):
+ current_time = datetime.now()
+ for user in tuple(tracking):
+ print(tracking[user])
+ delta = tracking[user]['last_update'] - current_time
+
+ if delta.seconds > timeout:
+
+ rc_rmq.publish_msg({
+ 'routing_key': 'complete.' + user,
+ 'msg': {
+ 'username': user,
+ 'success': False,
+ 'errmsg': ["Timeout on " + ', '.join(tracking[user]['waiting'])]
+ }
+ })
+
+ notify_admin(user, tracking[user])
+
+ tracking.pop(user)
+
+
+# Set initial timeout timer
+signal.signal(signal.SIGALRM, timeout_handler)
+signal.setitimer(signal.ITIMER_REAL, timeout, timeout)
+
logger.info(f'Start listening to queue: {task}')
rc_rmq.start_consume({
'queue': task,
--
GitLab
From e1739957a719700bd6560dde5a7b5fe092c91b59 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 8 Feb 2021 20:02:12 -0600
Subject: [PATCH 152/188] Create new array for errmsg
avoiding referencing to the same list
---
task_manager.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index 83c7bfe..36f1dff 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -21,7 +21,7 @@ record = {
'reason': '',
'fullname': '',
'last_update': datetime.now(),
- 'errmsg': [],
+ 'errmsg': None,
'waiting': set(),
'request': {
'create_account': None
@@ -87,6 +87,7 @@ def task_manager(ch, method, properties, body):
if username not in tracking:
current = tracking[username] = record.copy()
current['delivery_tags'] = []
+ current['errmsg'] = []
current['uid'] = msg.get('uid', -1)
current['gid'] = msg.get('gid', -1)
current['email'] = msg.get('email', '')
--
GitLab
From f2a04d5952dca2e68214d5110f625d87cab243cd Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 8 Feb 2021 21:40:01 -0600
Subject: [PATCH 153/188] Fix timeout calculation error
---
task_manager.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index 36f1dff..2800b00 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -200,7 +200,7 @@ def timeout_handler(signum, frame):
current_time = datetime.now()
for user in tuple(tracking):
print(tracking[user])
- delta = tracking[user]['last_update'] - current_time
+ delta = current_time - tracking[user]['last_update']
if delta.seconds > timeout:
--
GitLab
From 4230c807156e6ae69f60445ae15237ccd63c9e9e Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 8 Feb 2021 21:41:16 -0600
Subject: [PATCH 154/188] Remove unnecessary print
---
task_manager.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index 2800b00..97fe6f9 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -199,7 +199,6 @@ def task_manager(ch, method, properties, body):
def timeout_handler(signum, frame):
current_time = datetime.now()
for user in tuple(tracking):
- print(tracking[user])
delta = current_time - tracking[user]['last_update']
if delta.seconds > timeout:
--
GitLab
From 38251670e77621e774b6db169c5c5d41eadf7f22 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 11 Feb 2021 21:09:59 -0600
Subject: [PATCH 155/188] Use deepcopy to avoid change skeleton
---
task_manager.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index 97fe6f9..e3bc566 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
import sys
+import copy
import json
import signal
import rc_util
@@ -85,7 +86,7 @@ def task_manager(ch, method, properties, body):
routing_key = ""
if username not in tracking:
- current = tracking[username] = record.copy()
+ current = tracking[username] = copy.deepcopy(record)
current['delivery_tags'] = []
current['errmsg'] = []
current['uid'] = msg.get('uid', -1)
--
GitLab
From e6537d38e52c3cd682d6453871e2c16e883f6f39 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 4 Feb 2021 21:21:25 -0600
Subject: [PATCH 156/188] Add database settings
---
task_manager.py | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index e3bc566..79c2a66 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -3,6 +3,7 @@ import sys
import copy
import json
import signal
+import dataset
import rc_util
import smtplib
from rc_rmq import RCRMQ
@@ -15,6 +16,9 @@ timeout = 30
args = rc_util.get_args()
logger = rc_util.get_logger(args)
+db = dataset.connect(f'sqlite:///.agent_db/user_reg.db')
+table = db['users']
+
record = {
'uid': -1,
'gid': -1,
@@ -77,6 +81,29 @@ def notify_admin(username, user_record):
logger.debug(f'User report sent to: {mail_cfg.Admin_email}')
+def insert_db(username, msg):
+ # Search username in db
+ record = table.find_one(username=username)
+
+ if not record:
+ # SQL insert
+ table.insert({
+ 'username': username,
+ 'uid': msg.get('uid', -1),
+ 'gid': msg.get('gid', -1),
+ 'email': msg.get('email', ''),
+ 'reason': msg.get('reason', ''),
+ 'fullname': msg.get('fullname', ''),
+ 'sent': None,
+ 'last_update': datetime.now()
+ })
+
+
+def update_db(username, data):
+ obj = { 'username': username, **data }
+ table.update(obj, ['username'])
+
+
def task_manager(ch, method, properties, body):
msg = json.loads(body)
username = method.routing_key.split('.')[1]
--
GitLab
From a4bafbc3277337e0226287cba608a7b0faed5f34 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 4 Feb 2021 21:26:10 -0600
Subject: [PATCH 157/188] Create new record when not found
---
task_manager.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index 79c2a66..374cffa 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -122,6 +122,8 @@ def task_manager(ch, method, properties, body):
current['reason'] = msg.get('reason', '')
current['fullname'] = msg.get('fullname', '')
+ insert_db(username, msg)
+
logger.debug(f'Tracking user {username}')
else:
current = tracking[username]
--
GitLab
From a8d20e689a28973d5613ab487e19bb7af553239f Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 4 Feb 2021 21:27:28 -0600
Subject: [PATCH 158/188] Update database every time receive msg
---
task_manager.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index 374cffa..8312363 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -133,6 +133,12 @@ def task_manager(ch, method, properties, body):
current['last_update'] = datetime.now()
+ # Update Database
+ update_db(username, {
+ task_name: success,
+ 'last_update': current['last_update']}
+ )
+
# Save error message if the task was failed
if not success:
errmsg = msg.get('errmsg', '')
--
GitLab
From 47a2a78ec4daf82b1320402e051baf5b4ba2a899 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 4 Feb 2021 21:28:38 -0600
Subject: [PATCH 159/188] Update database after notify admin
---
task_manager.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index 8312363..321ff1b 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -227,6 +227,8 @@ def task_manager(ch, method, properties, body):
notify_admin(username, current)
+ update_db(username, {'reported': True})
+
tracking.pop(username)
logger.debug('Admin report sent')
@@ -250,6 +252,8 @@ def timeout_handler(signum, frame):
notify_admin(user, tracking[user])
+ update_db(user, {'reported': True})
+
tracking.pop(user)
--
GitLab
From ac7100d94fbaaa8b323691aea849fe12676b1ff7 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 4 Feb 2021 21:33:18 -0600
Subject: [PATCH 160/188] Update database settings
Use the same database and table that task manager use
Change operation to update only, leave create to task manager
---
notify_user.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index 97f3fce..4c68a27 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -14,8 +14,8 @@ task = 'notify_user'
args = rc_util.get_args()
logger = rc_util.get_logger(args)
-db = dataset.connect(f'sqlite:///.agent_db/{task}.db')
-table = db['notified_user']
+db = dataset.connect(f'sqlite:///.agent_db/user_reg.db')
+table = db['users']
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
@@ -33,7 +33,7 @@ def notify_user(ch, method, properties, body):
# Search username in database
record = table.find_one(username=username)
- if record:
+ if record['sent'] is not None:
# Update counter
count = record['count']
table.update({'username': username, 'count': count + 1}, ['username'])
@@ -48,7 +48,7 @@ def notify_user(ch, method, properties, body):
if args.dry_run:
logger.info(f'smtp = smtplib.SMTP({mail_cfg.Server})')
logger.info(f'smtp.sendmail({mail_cfg.Sender}, {receivers}, message)')
- logger.info(f"table.insert({{'username': {username}, 'count': 1, 'sent_at': datetime.now()}})")
+ logger.info(f"table.update({{'username': {username}, 'count': 1, 'sent_at': datetime.now()}}, ['username'])")
else:
smtp = smtplib.SMTP(mail_cfg.Server)
@@ -56,11 +56,11 @@ def notify_user(ch, method, properties, body):
logger.debug(f'Email sent to: {user_email}')
- table.insert({
+ table.update({
'username': username,
'count': 1,
'sent_at': datetime.now()
- })
+ }, ['username'])
logger.debug(f'User {username} inserted into database')
--
GitLab
From d497d60f6f72afb813ffc4582e461d00e6393715 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 4 Feb 2021 21:40:42 -0600
Subject: [PATCH 161/188] Add dryrun condition
---
notify_user.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/notify_user.py b/notify_user.py
index 4c68a27..00323bc 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -36,7 +36,14 @@ def notify_user(ch, method, properties, body):
if record['sent'] is not None:
# Update counter
count = record['count']
- table.update({'username': username, 'count': count + 1}, ['username'])
+ if args.dry_run:
+ logger.info('Update counter in database')
+
+ else:
+ table.update({
+ 'username': username,
+ 'count': count + 1
+ }, ['username'])
logger.debug(f'User {username} counter updated to {count + 1}')
--
GitLab
From 98b8c5120131c064bbb45ddad5268c83cc5e54a1 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 9 Feb 2021 22:08:44 -0600
Subject: [PATCH 162/188] Add reported field when create new user
---
task_manager.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/task_manager.py b/task_manager.py
index 321ff1b..46728e2 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -95,6 +95,7 @@ def insert_db(username, msg):
'reason': msg.get('reason', ''),
'fullname': msg.get('fullname', ''),
'sent': None,
+ 'reported': False,
'last_update': datetime.now()
})
--
GitLab
From 03e0df2d409a96be1ff8278646cf63535328f8d6 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 9 Feb 2021 22:12:01 -0600
Subject: [PATCH 163/188] Reorder if else block
---
task_manager.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index 46728e2..a7ef9c9 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -113,7 +113,10 @@ def task_manager(ch, method, properties, body):
send = completed = terminated = False
routing_key = ""
- if username not in tracking:
+ if username in tracking:
+ current = tracking[username]
+
+ else:
current = tracking[username] = copy.deepcopy(record)
current['delivery_tags'] = []
current['errmsg'] = []
@@ -126,8 +129,6 @@ def task_manager(ch, method, properties, body):
insert_db(username, msg)
logger.debug(f'Tracking user {username}')
- else:
- current = tracking[username]
# Save the delivery tags for future use
current['delivery_tags'].append(method.delivery_tag)
--
GitLab
From bf60610c5d89670c91df8c79c23090194629ff46 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Tue, 9 Feb 2021 22:12:36 -0600
Subject: [PATCH 164/188] Search DB when tracking not hit
If found the user in DB, restore task status
---
task_manager.py | 31 ++++++++++++++++++++++---------
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index a7ef9c9..f44e593 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -117,18 +117,31 @@ def task_manager(ch, method, properties, body):
current = tracking[username]
else:
+ user_db = table.find_one(username=username)
+
current = tracking[username] = copy.deepcopy(record)
current['delivery_tags'] = []
current['errmsg'] = []
- current['uid'] = msg.get('uid', -1)
- current['gid'] = msg.get('gid', -1)
- current['email'] = msg.get('email', '')
- current['reason'] = msg.get('reason', '')
- current['fullname'] = msg.get('fullname', '')
-
- insert_db(username, msg)
-
- logger.debug(f'Tracking user {username}')
+ current['uid'] = user_db['uid'] if user_db else msg['uid']
+ current['gid'] = user_db['gid'] if user_db else msg['gid']
+ current['email'] = user_db['email'] if user_db else msg['email']
+ current['reason'] = user_db['reason'] if user_db else msg['reason']
+ current['fullname'] = user_db['fullname'] if user_db else msg['fullname']
+
+ if user_db:
+ # Restore task status
+ current['request']['create_account'] = user_db['create_account']
+ current['verify']['git_commit'] = user_db['git_commit']
+ current['verify']['dir_verify'] = user_db['dir_verify']
+ current['verify']['subscribe_mail_list'] = user_db['subscribe_mail_list']
+ current['notify']['notify_user'] = user_db['notify_user']
+
+ logger.debug(f'Loaded user {username} from DB')
+
+ else:
+ insert_db(username, msg)
+
+ logger.debug(f'Tracking user {username}')
# Save the delivery tags for future use
current['delivery_tags'].append(method.delivery_tag)
--
GitLab
From 74459795140972195348cb04d79eb483a2f2f633 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 11 Feb 2021 13:00:45 -0600
Subject: [PATCH 165/188] Update field name for sent time
---
notify_user.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/notify_user.py b/notify_user.py
index 00323bc..a193b3d 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -66,7 +66,7 @@ def notify_user(ch, method, properties, body):
table.update({
'username': username,
'count': 1,
- 'sent_at': datetime.now()
+ 'sent': datetime.now()
}, ['username'])
logger.debug(f'User {username} inserted into database')
--
GitLab
From bff83ba19c2667be45b7fd2e73eb79ce1c9c2810 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 11 Feb 2021 14:32:51 -0600
Subject: [PATCH 166/188] Update default field in db
---
task_manager.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index f44e593..0e44442 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -94,6 +94,11 @@ def insert_db(username, msg):
'email': msg.get('email', ''),
'reason': msg.get('reason', ''),
'fullname': msg.get('fullname', ''),
+ 'create_account': None,
+ 'git_commit': None,
+ 'dir_verify': None,
+ 'subscribe_mail_list': None,
+ 'notify_user': None,
'sent': None,
'reported': False,
'last_update': datetime.now()
--
GitLab
From 6130180f4ad78fb776ab96ed6db790b543f953b6 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 11 Feb 2021 14:33:27 -0600
Subject: [PATCH 167/188] Acknowledge right after process it
was saved before, but will cause message remind in queue if timeout
---
task_manager.py | 15 ++++-----------
1 file changed, 4 insertions(+), 11 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index 0e44442..ff506a2 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -38,8 +38,7 @@ record = {
},
'notify': {
'notify_user': None
- },
- 'delivery_tags': None
+ }
}
# Currently tracking users
@@ -125,7 +124,6 @@ def task_manager(ch, method, properties, body):
user_db = table.find_one(username=username)
current = tracking[username] = copy.deepcopy(record)
- current['delivery_tags'] = []
current['errmsg'] = []
current['uid'] = user_db['uid'] if user_db else msg['uid']
current['gid'] = user_db['gid'] if user_db else msg['gid']
@@ -148,9 +146,6 @@ def task_manager(ch, method, properties, body):
logger.debug(f'Tracking user {username}')
- # Save the delivery tags for future use
- current['delivery_tags'].append(method.delivery_tag)
-
current['last_update'] = datetime.now()
# Update Database
@@ -235,11 +230,6 @@ def task_manager(ch, method, properties, body):
logger.debug(f"Trigger message '{routing_key}' sent")
- # Acknowledge all message from last level
- for tag in current['delivery_tags']:
- ch.basic_ack(tag)
- current['delivery_tags'] = []
-
logger.debug('Previous level messages acknowledged')
# Send report to admin
@@ -253,6 +243,9 @@ def task_manager(ch, method, properties, body):
logger.debug('Admin report sent')
+ # Acknowledge message
+ ch.basic_ack(method.delivery_tag)
+
def timeout_handler(signum, frame):
current_time = datetime.now()
--
GitLab
From 7c5ff0d1fb325b120cacc51c39a2f7004f8e17e8 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 11 Feb 2021 20:53:57 -0600
Subject: [PATCH 168/188] Update waiting list when restore from DB
---
task_manager.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index ff506a2..a7a6eed 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -139,6 +139,13 @@ def task_manager(ch, method, properties, body):
current['verify']['subscribe_mail_list'] = user_db['subscribe_mail_list']
current['notify']['notify_user'] = user_db['notify_user']
+ for t in ['git_commit', 'dir_verify', 'subscribe_mail_list']:
+ if user_db[t] is None:
+ current['waiting'].add(t)
+
+ if not current['waiting'] and user_db['notify_user'] is None:
+ current['waiting'].add('notify_user')
+
logger.debug(f'Loaded user {username} from DB')
else:
--
GitLab
From 79c32878c243a3149c4644780af8555b3e1e8ca5 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 4 Feb 2021 21:37:10 -0600
Subject: [PATCH 169/188] Send errmsg if error occur
---
notify_user.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/notify_user.py b/notify_user.py
index a193b3d..5bde208 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -27,6 +27,7 @@ def notify_user(ch, method, properties, body):
user_email = msg['email']
msg['task'] = task
msg['success'] = False
+ errmsg = ""
try:
@@ -34,6 +35,7 @@ def notify_user(ch, method, properties, body):
record = table.find_one(username=username)
if record['sent'] is not None:
+ errmsg = 'Updating database counter'
# Update counter
count = record['count']
if args.dry_run:
@@ -58,11 +60,13 @@ def notify_user(ch, method, properties, body):
logger.info(f"table.update({{'username': {username}, 'count': 1, 'sent_at': datetime.now()}}, ['username'])")
else:
+ errmsg = 'Sending email to user'
smtp = smtplib.SMTP(mail_cfg.Server)
smtp.sendmail(mail_cfg.Sender, receivers, message)
logger.debug(f'Email sent to: {user_email}')
+ errmsg = 'Updating database email sent time'
table.update({
'username': username,
'count': 1,
@@ -74,6 +78,7 @@ def notify_user(ch, method, properties, body):
msg['success'] = True
except Exception as exception:
logger.error('', exc_info=True)
+ msg['errmsg'] = errmsg if errmsg else 'Unexpected error'
# Send confirm message
rc_rmq.publish_msg({
--
GitLab
From 154bbef033e7e6156b4595c10462a996b1d2f818 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 12 Feb 2021 00:36:59 -0600
Subject: [PATCH 170/188] Add delay parameter to example config
---
rabbit_config.py.example | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/rabbit_config.py.example b/rabbit_config.py.example
index d929c5d..57d632f 100644
--- a/rabbit_config.py.example
+++ b/rabbit_config.py.example
@@ -5,6 +5,10 @@ VHost = '/'
Server = 'ohpc'
Port = 5672
+# time delay to let account creation finish
+# to avoid concurrency with downstream agents
+Delay = 5
+
# git_commit agent config
rc_users_ldap_repo_loc = "~/git/rc-users"
--
GitLab
From b2570298fd3ffc4b1e941da26fd8942b08c81ad5 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 12 Feb 2021 00:57:25 -0600
Subject: [PATCH 171/188] Change the var name to admin_email in example config
---
rabbit_config.py.example | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/rabbit_config.py.example b/rabbit_config.py.example
index 57d632f..316d253 100644
--- a/rabbit_config.py.example
+++ b/rabbit_config.py.example
@@ -14,7 +14,7 @@ rc_users_ldap_repo_loc = "~/git/rc-users"
# Config related to email
Server = 'localhost'
-My_email = 'root@localhost'
+Admin_email = 'root@localhost'
Sender = 'ROOT@LOCALHOST'
Sender_alias = 'Services'
Subject = 'New User Account'
@@ -38,7 +38,7 @@ User ID: {{{{ username }}}}
If you have any questions, please visit:
{Info_url}
-or email at {My_email}
+or email at {Admin_email}
Cheers,
"""
--
GitLab
From 38f40524d00617116cc502404374ed196e27e549 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 12 Feb 2021 01:22:17 -0600
Subject: [PATCH 172/188] Remove exception block, it is handled in the callback
function
---
get-next-uid-gid.py | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/get-next-uid-gid.py b/get-next-uid-gid.py
index 08eb83f..565501b 100644
--- a/get-next-uid-gid.py
+++ b/get-next-uid-gid.py
@@ -35,13 +35,10 @@ def create_account(msg):
cmd += f'"user; add {username}; set id {uid}; set email {email}; set commonname \\"{fullname}\\"; '
cmd += 'commit;"'
- if not args.dry_run:
- popen(cmd)
- time.sleep(rcfg.Delay)
- logger.info(f'Bright command to create user:{cmd}')
- except Exception:
- logger.exception("Fatal cmsh error:")
-
+ if not args.dry_run:
+ popen(cmd)
+ time.sleep(rcfg.Delay)
+ logger.info(f'Bright command to create user:{cmd}')
# Define your callback function
def resolve_uid_gid(ch, method, properties, body):
--
GitLab
From 2c72035898092d475ed7761090e1ee55d331427d Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 12 Feb 2021 01:30:38 -0600
Subject: [PATCH 173/188] Remove the line using deleted var
---
git_commit.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/git_commit.py b/git_commit.py
index c1913ad..5c0de34 100644
--- a/git_commit.py
+++ b/git_commit.py
@@ -37,7 +37,6 @@ def git_commit(ch, method, properties, body):
group_ldif = groups_dir + f'/{username}.ldif'
logger.info("Received: %s", msg)
- logger.debug("ticketnum: %s", ticketnum)
logger.debug("branch_name: %s", branch_name)
try:
--
GitLab
From 71e0ae3bb9b2155e2d42a80b73ce896e5120b482 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 12 Feb 2021 15:00:01 -0600
Subject: [PATCH 174/188] Move dir list to be checked to config file
---
dir_verify.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/dir_verify.py b/dir_verify.py
index f340325..156db1c 100644
--- a/dir_verify.py
+++ b/dir_verify.py
@@ -6,9 +6,10 @@ import shutil
import rc_util
from pathlib import Path
from rc_rmq import RCRMQ
+import rabbit_config as rcfg
task = 'dir_verify'
-dirs = ['/home', '/data/user', '/data/scratch']
+dirs = rcfg.User_dirs
args = rc_util.get_args()
logger = rc_util.get_logger(args)
--
GitLab
From f5450f49fd8f7b3212745f8f3485577cf70b341e Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 12 Feb 2021 15:02:30 -0600
Subject: [PATCH 175/188] Add the user dir list to be verified in eg config
file
---
rabbit_config.py.example | 3 +++
1 file changed, 3 insertions(+)
diff --git a/rabbit_config.py.example b/rabbit_config.py.example
index 316d253..acd1535 100644
--- a/rabbit_config.py.example
+++ b/rabbit_config.py.example
@@ -9,6 +9,9 @@ Port = 5672
# to avoid concurrency with downstream agents
Delay = 5
+# dir_verify agent config
+User_dirs = ['/home', '/data/user', '/data/scratch']
+
# git_commit agent config
rc_users_ldap_repo_loc = "~/git/rc-users"
--
GitLab
From d5fda70602c2320835a5aac29fde96ee0ede0b07 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Mon, 15 Feb 2021 18:02:54 -0600
Subject: [PATCH 176/188] Add fullname and blazerID to the subj line of admin
report email.
---
mail_config.py | 2 +-
task_manager.py | 26 ++++++++++++++------------
2 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/mail_config.py b/mail_config.py
index 911ba23..e262696 100644
--- a/mail_config.py
+++ b/mail_config.py
@@ -31,6 +31,6 @@ Whole_mail = Head + Body
UserReportHead = f"""From: {Sender_alias} <{Sender}>
To: <{Admin_email}>
-Subject: User Creation Report:
+Subject: RC Account Creation Report: {{{{ fullname }}}}, {{{{ username }}}}
"""
diff --git a/task_manager.py b/task_manager.py
index a7a6eed..fea5e9c 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -7,6 +7,7 @@ import dataset
import rc_util
import smtplib
from rc_rmq import RCRMQ
+from jinja2 import Template
from datetime import datetime
import mail_config as mail_cfg
@@ -47,18 +48,18 @@ tracking = {}
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
-def notify_admin(username, user_record):
+def notify_admin(username, fullname, user_record):
receivers = [user_record['email'], mail_cfg.Admin_email]
- message = mail_cfg.UserReportHead
- message += f"""
- User Creation Report for user {username}
- uid: {user_record["uid"]}, gid: {user_record["gid"]}
- Tasks:
- 'create_account': {user_record["request"]["create_account"]}
- 'git_commit': {user_record["verify"]["git_commit"]}
- 'dir_verify': {user_record["verify"]["dir_verify"]}
- 'subscribe_mail_list': {user_record["verify"]["subscribe_mail_list"]}
- 'notify_user': {user_record["notify"]["notify_user"]}
+ message = Template(mail_cfg.UserReportHead).render(username=username, fullname=fullname)
+ message += f""" \n
+ User Creation Report for user {username}
+ uid: {user_record["uid"]}, gid: {user_record["gid"]}
+ Tasks:
+ 'create_account': {user_record["request"]["create_account"]}
+ 'git_commit': {user_record["verify"]["git_commit"]}
+ 'dir_verify': {user_record["verify"]["dir_verify"]}
+ 'subscribe_mail_list': {user_record["verify"]["subscribe_mail_list"]}
+ 'notify_user': {user_record["notify"]["notify_user"]}
"""
if user_record['errmsg']:
message += """
@@ -112,6 +113,7 @@ def update_db(username, data):
def task_manager(ch, method, properties, body):
msg = json.loads(body)
username = method.routing_key.split('.')[1]
+ fullname = msg['fullname']
task_name = msg['task']
success = msg['success']
send = completed = terminated = False
@@ -242,7 +244,7 @@ def task_manager(ch, method, properties, body):
# Send report to admin
if completed or terminated:
- notify_admin(username, current)
+ notify_admin(username, fullname, current)
update_db(username, {'reported': True})
--
GitLab
From 9fb10be14c81d56f1e531d7651aea3d5b341d756 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 18 Feb 2021 14:34:26 -0600
Subject: [PATCH 177/188] Use full name from the current user_record
---
mail_config.py | 2 +-
task_manager.py | 7 +++----
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/mail_config.py b/mail_config.py
index e262696..e01f1f8 100644
--- a/mail_config.py
+++ b/mail_config.py
@@ -31,6 +31,6 @@ Whole_mail = Head + Body
UserReportHead = f"""From: {Sender_alias} <{Sender}>
To: <{Admin_email}>
-Subject: RC Account Creation Report: {{{{ fullname }}}}, {{{{ username }}}}
+Subject: RC Account Creation Report: {{{{ fullname }}}}, {{{{ username }}}}
"""
diff --git a/task_manager.py b/task_manager.py
index fea5e9c..5498a24 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -48,9 +48,9 @@ tracking = {}
# Instantiate rabbitmq object
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
-def notify_admin(username, fullname, user_record):
+def notify_admin(username, user_record):
receivers = [user_record['email'], mail_cfg.Admin_email]
- message = Template(mail_cfg.UserReportHead).render(username=username, fullname=fullname)
+ message = Template(mail_cfg.UserReportHead).render(username=username, fullname=user_record['fullname'])
message += f""" \n
User Creation Report for user {username}
uid: {user_record["uid"]}, gid: {user_record["gid"]}
@@ -113,7 +113,6 @@ def update_db(username, data):
def task_manager(ch, method, properties, body):
msg = json.loads(body)
username = method.routing_key.split('.')[1]
- fullname = msg['fullname']
task_name = msg['task']
success = msg['success']
send = completed = terminated = False
@@ -244,7 +243,7 @@ def task_manager(ch, method, properties, body):
# Send report to admin
if completed or terminated:
- notify_admin(username, fullname, current)
+ notify_admin(username, current)
update_db(username, {'reported': True})
--
GitLab
From dce345bbafffc2b933e531dde74b52ba08f52d3f Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Mon, 15 Feb 2021 20:45:16 -0600
Subject: [PATCH 178/188] Add reported to default and from DB
---
task_manager.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index 5498a24..427136b 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -39,7 +39,8 @@ record = {
},
'notify': {
'notify_user': None
- }
+ },
+ 'reported': False
}
# Currently tracking users
@@ -140,6 +141,8 @@ def task_manager(ch, method, properties, body):
current['verify']['subscribe_mail_list'] = user_db['subscribe_mail_list']
current['notify']['notify_user'] = user_db['notify_user']
+ current['reported'] = user_db['reported']
+
for t in ['git_commit', 'dir_verify', 'subscribe_mail_list']:
if user_db[t] is None:
current['waiting'].add(t)
--
GitLab
From 69cf398844ae4adcae3c4bb1f666930adeb62c81 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 18 Feb 2021 13:47:31 -0600
Subject: [PATCH 179/188] Remove newline in UserReportHead
---
mail_config.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/mail_config.py b/mail_config.py
index e01f1f8..92172bb 100644
--- a/mail_config.py
+++ b/mail_config.py
@@ -31,6 +31,5 @@ Whole_mail = Head + Body
UserReportHead = f"""From: {Sender_alias} <{Sender}>
To: <{Admin_email}>
-Subject: RC Account Creation Report: {{{{ fullname }}}}, {{{{ username }}}}
-"""
+Subject: RC Account Creation Report: {{{{ fullname }}}}, {{{{ username }}}} """
--
GitLab
From c8b909288dbe588d503478311853b88281b19a49 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 18 Feb 2021 13:48:04 -0600
Subject: [PATCH 180/188] Add Duplicate into mail subject
---
task_manager.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/task_manager.py b/task_manager.py
index 427136b..061bd58 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -52,6 +52,8 @@ rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
def notify_admin(username, user_record):
receivers = [user_record['email'], mail_cfg.Admin_email]
message = Template(mail_cfg.UserReportHead).render(username=username, fullname=user_record['fullname'])
+ if user_record['reported']:
+ message += ' (Duplicate)'
message += f""" \n
User Creation Report for user {username}
uid: {user_record["uid"]}, gid: {user_record["gid"]}
--
GitLab
From 6ecabd9e0274d1dfe7ab11273309390842bb0166 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 19 Feb 2021 14:25:26 -0600
Subject: [PATCH 181/188] Move mail cfg vars to rabbit config, keep only mail
template.
---
mail_config.py | 22 ++++++++--------------
1 file changed, 8 insertions(+), 14 deletions(-)
diff --git a/mail_config.py b/mail_config.py
index 92172bb..434cb44 100644
--- a/mail_config.py
+++ b/mail_config.py
@@ -1,14 +1,8 @@
-# Some variable for email
-Server = 'localhost'
-Admin_email = 'root@localhost'
-Sender = 'ROOT@LOCALHOST'
-Sender_alias = 'Services'
-Subject = 'New User Account'
-Info_url = 'https://www.google.com'
-
-Head = f"""From: {Sender_alias} <{Sender}>
+import rabbit_config as rcfg
+
+Head = f"""From: {rcfg.Sender_alias} <{rcfg.Sender}>
To: <{{{{ to }}}}>
-Subject: {Subject}
+Subject: {rcfg.Subject}
"""
Body = f"""
@@ -20,16 +14,16 @@ User ID: {{{{ username }}}}
============================
If you have any questions, please visit:
-{Info_url}
+{rcfg.Info_url}
-or email at {Admin_email}
+or email at {rcfg.Admin_email}
Cheers,
"""
Whole_mail = Head + Body
-UserReportHead = f"""From: {Sender_alias} <{Sender}>
-To: <{Admin_email}>
+UserReportHead = f"""From: {rcfg.Sender_alias} <{rcfg.Sender}>
+To: <{rcfg.Admin_email}>
Subject: RC Account Creation Report: {{{{ fullname }}}}, {{{{ username }}}} """
--
GitLab
From d9e6789a37024da129c353b6cf5f288ad0177d53 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 19 Feb 2021 14:27:01 -0600
Subject: [PATCH 182/188] Use mail config template for notify user agent
---
notify_user.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/notify_user.py b/notify_user.py
index 9b355f3..b7ed948 100644
--- a/notify_user.py
+++ b/notify_user.py
@@ -7,8 +7,8 @@ import dataset
from rc_rmq import RCRMQ
from jinja2 import Template
from datetime import datetime
-import rabbit_config as mail_cfg
-
+import rabbit_config as rcfg
+import mail_config as mail_cfg
task = 'notify_user'
args = rc_util.get_args()
@@ -51,18 +51,18 @@ def notify_user(ch, method, properties, body):
else:
# Send email to user
- receivers = [user_email, mail_cfg.Admin_email]
+ receivers = [user_email, rcfg.Admin_email]
message = Template(mail_cfg.Whole_mail).render(username=username, to=user_email)
if args.dry_run:
- logger.info(f'smtp = smtplib.SMTP({mail_cfg.Server})')
- logger.info(f'smtp.sendmail({mail_cfg.Sender}, {receivers}, message)')
+ logger.info(f'smtp = smtplib.SMTP({rcfg.Server})')
+ logger.info(f'smtp.sendmail({rcfg.Sender}, {receivers}, message)')
logger.info(f"table.update({{'username': {username}, 'count': 1, 'sent_at': datetime.now()}}, ['username'])")
else:
errmsg = 'Sending email to user'
- smtp = smtplib.SMTP(mail_cfg.Server)
- smtp.sendmail(mail_cfg.Sender, receivers, message)
+ smtp = smtplib.SMTP(rcfg.Server)
+ smtp.sendmail(rcfg.Sender, receivers, message)
logger.debug(f'Email sent to: {user_email}')
--
GitLab
From c37743d05ed5b66c4fcb78e797a8a2c48b439f88 Mon Sep 17 00:00:00 2001
From: Ravi Tripathi <ravi89@uab.edu>
Date: Wed, 24 Feb 2021 01:30:01 -0500
Subject: [PATCH 183/188] Modifying task_manager to differnetiate variables
between mail_server and rabbit_config file
---
task_manager.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/task_manager.py b/task_manager.py
index 061bd58..9fa4b12 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -10,6 +10,7 @@ from rc_rmq import RCRMQ
from jinja2 import Template
from datetime import datetime
import mail_config as mail_cfg
+import rabbitmq_config as rcfg
task = 'task_manager'
timeout = 30
@@ -50,7 +51,7 @@ tracking = {}
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
def notify_admin(username, user_record):
- receivers = [user_record['email'], mail_cfg.Admin_email]
+ receivers = [user_record['email'], rcfg.Admin_email]
message = Template(mail_cfg.UserReportHead).render(username=username, fullname=user_record['fullname'])
if user_record['reported']:
message += ' (Duplicate)'
@@ -73,15 +74,15 @@ def notify_admin(username, user_record):
message += msg + "\n"
if args.dry_run:
- logger.info(f'smtp = smtplib.SMTP({mail_cfg.Server})')
- logger.info(f'smtp.sendmail({mail_cfg.Sender}, {mail_cfg.Admin_email}, message)')
+ logger.info(f'smtp = smtplib.SMTP({rcfg.Server})')
+ logger.info(f'smtp.sendmail({rcfg.Sender}, {rcfg.Admin_email}, message)')
logger.info(message)
else:
- smtp = smtplib.SMTP(mail_cfg.Server)
- smtp.sendmail(mail_cfg.Sender, receivers, message)
+ smtp = smtplib.SMTP(rcfg.Server)
+ smtp.sendmail(rcfg.Sender, receivers, message)
- logger.debug(f'User report sent to: {mail_cfg.Admin_email}')
+ logger.debug(f'User report sent to: {rcfg.Admin_email}')
def insert_db(username, msg):
--
GitLab
From 1bdf5a0cdbb4687ca26ea027de465d3a1d755089 Mon Sep 17 00:00:00 2001
From: Ravi Tripathi <ravi89@uab.edu>
Date: Wed, 24 Feb 2021 01:30:47 -0500
Subject: [PATCH 184/188] Renaming mail_config.pyto mail_config.py.example
---
mail_config.py => mail_config.py.example | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename mail_config.py => mail_config.py.example (100%)
diff --git a/mail_config.py b/mail_config.py.example
similarity index 100%
rename from mail_config.py
rename to mail_config.py.example
--
GitLab
From 4b224271566959988f5e74dc9bb4ee14f4c34289 Mon Sep 17 00:00:00 2001
From: Ravi Tripathi <ravi89@uab.edu>
Date: Wed, 24 Feb 2021 01:51:59 -0500
Subject: [PATCH 185/188] Correcting the name of rabbit_config while importing
---
task_manager.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/task_manager.py b/task_manager.py
index 9fa4b12..988b0cf 100644
--- a/task_manager.py
+++ b/task_manager.py
@@ -10,7 +10,7 @@ from rc_rmq import RCRMQ
from jinja2 import Template
from datetime import datetime
import mail_config as mail_cfg
-import rabbitmq_config as rcfg
+import rabbit_config as rcfg
task = 'task_manager'
timeout = 30
--
GitLab
From d88c39bd79d1a3c716cfd78bcb31f482033117a9 Mon Sep 17 00:00:00 2001
From: Ravi Tripathi <ravi89@uab.edu>
Date: Wed, 24 Feb 2021 18:54:39 -0500
Subject: [PATCH 186/188] Creating a directory structure for prod and dev rmq
agents
---
ohpc_account_create.py => dev_rmq_agents/ohpc_account_create.py | 0
ood_account_create.py => dev_rmq_agents/ood_account_create.py | 0
slurm_agent.py => dev_rmq_agents/slurm_agent.py | 0
dir_verify.py => prod_rmq_agents/dir_verify.py | 0
get-next-uid-gid.py => prod_rmq_agents/get-next-uid-gid.py | 0
git_commit.py => prod_rmq_agents/git_commit.py | 0
notify_user.py => prod_rmq_agents/notify_user.py | 0
.../subscribe_mail_lists.py | 0
task_manager.py => prod_rmq_agents/task_manager.py | 0
.../user_reg_event_logger.py | 0
user_reg_logger.py => prod_rmq_agents/user_reg_logger.py | 0
11 files changed, 0 insertions(+), 0 deletions(-)
rename ohpc_account_create.py => dev_rmq_agents/ohpc_account_create.py (100%)
rename ood_account_create.py => dev_rmq_agents/ood_account_create.py (100%)
rename slurm_agent.py => dev_rmq_agents/slurm_agent.py (100%)
rename dir_verify.py => prod_rmq_agents/dir_verify.py (100%)
rename get-next-uid-gid.py => prod_rmq_agents/get-next-uid-gid.py (100%)
rename git_commit.py => prod_rmq_agents/git_commit.py (100%)
rename notify_user.py => prod_rmq_agents/notify_user.py (100%)
rename subscribe_mail_lists.py => prod_rmq_agents/subscribe_mail_lists.py (100%)
rename task_manager.py => prod_rmq_agents/task_manager.py (100%)
rename user_reg_event_logger.py => prod_rmq_agents/user_reg_event_logger.py (100%)
rename user_reg_logger.py => prod_rmq_agents/user_reg_logger.py (100%)
diff --git a/ohpc_account_create.py b/dev_rmq_agents/ohpc_account_create.py
similarity index 100%
rename from ohpc_account_create.py
rename to dev_rmq_agents/ohpc_account_create.py
diff --git a/ood_account_create.py b/dev_rmq_agents/ood_account_create.py
similarity index 100%
rename from ood_account_create.py
rename to dev_rmq_agents/ood_account_create.py
diff --git a/slurm_agent.py b/dev_rmq_agents/slurm_agent.py
similarity index 100%
rename from slurm_agent.py
rename to dev_rmq_agents/slurm_agent.py
diff --git a/dir_verify.py b/prod_rmq_agents/dir_verify.py
similarity index 100%
rename from dir_verify.py
rename to prod_rmq_agents/dir_verify.py
diff --git a/get-next-uid-gid.py b/prod_rmq_agents/get-next-uid-gid.py
similarity index 100%
rename from get-next-uid-gid.py
rename to prod_rmq_agents/get-next-uid-gid.py
diff --git a/git_commit.py b/prod_rmq_agents/git_commit.py
similarity index 100%
rename from git_commit.py
rename to prod_rmq_agents/git_commit.py
diff --git a/notify_user.py b/prod_rmq_agents/notify_user.py
similarity index 100%
rename from notify_user.py
rename to prod_rmq_agents/notify_user.py
diff --git a/subscribe_mail_lists.py b/prod_rmq_agents/subscribe_mail_lists.py
similarity index 100%
rename from subscribe_mail_lists.py
rename to prod_rmq_agents/subscribe_mail_lists.py
diff --git a/task_manager.py b/prod_rmq_agents/task_manager.py
similarity index 100%
rename from task_manager.py
rename to prod_rmq_agents/task_manager.py
diff --git a/user_reg_event_logger.py b/prod_rmq_agents/user_reg_event_logger.py
similarity index 100%
rename from user_reg_event_logger.py
rename to prod_rmq_agents/user_reg_event_logger.py
diff --git a/user_reg_logger.py b/prod_rmq_agents/user_reg_logger.py
similarity index 100%
rename from user_reg_logger.py
rename to prod_rmq_agents/user_reg_logger.py
--
GitLab
From b15f8a6548bb11009332c3885b2139fff27305e9 Mon Sep 17 00:00:00 2001
From: Bo-Chun Louis Chen <louistw@uab.edu>
Date: Thu, 25 Feb 2021 19:45:00 -0600
Subject: [PATCH 187/188] Remove user from receiving creation report
---
prod_rmq_agents/task_manager.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/prod_rmq_agents/task_manager.py b/prod_rmq_agents/task_manager.py
index 988b0cf..795f7c3 100644
--- a/prod_rmq_agents/task_manager.py
+++ b/prod_rmq_agents/task_manager.py
@@ -51,7 +51,7 @@ tracking = {}
rc_rmq = RCRMQ({'exchange': 'RegUsr', 'exchange_type': 'topic'})
def notify_admin(username, user_record):
- receivers = [user_record['email'], rcfg.Admin_email]
+ receivers = [rcfg.Admin_email]
message = Template(mail_cfg.UserReportHead).render(username=username, fullname=user_record['fullname'])
if user_record['reported']:
message += ' (Duplicate)'
--
GitLab
From ada257b78cad3c121cd035b7632ecfa6ec1b768e Mon Sep 17 00:00:00 2001
From: root <root@cluster-on-demand.cm.cluster>
Date: Fri, 19 Mar 2021 15:57:25 -0500
Subject: [PATCH 188/188] kill nginx process running under user from login node
---
remove_user_login_node.sh | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
create mode 100644 remove_user_login_node.sh
diff --git a/remove_user_login_node.sh b/remove_user_login_node.sh
new file mode 100644
index 0000000..ef98454
--- /dev/null
+++ b/remove_user_login_node.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+username="$1"
+usage() {
+ echo "Usage: $0 USERNAME"
+}
+if [[ "$EUID" -ne 0 ]]; then
+ echo "This script must be run as root!"
+ exit 1
+fi
+if [ -z "$username" ]; then
+ usage
+ exit 1
+fi
+if id "$username" &>/dev/null; then
+ echo "Deleting nginx process running under user: ${username}"
+ kill -9 ` ps -ef | grep 'nginx' | grep ${username} | awk '{print $2}'`
+ echo "Deleted process"
+else
+ echo "user: ${username} not found."
+ exit 1
+fi
--
GitLab