Skip to content
Snippets Groups Projects
Commit 3acf6291 authored by Bo-Chun Chen's avatar Bo-Chun Chen
Browse files

Add timeout checker

Default to check every 30 seconds
Will inform caller and remove from tracking
parent 99fcad20
No related branches found
No related tags found
6 merge requests!147Merge previous default branch feat-cod-rmq into main,!85kill nginx process running under user from login node,!78Feat cod rmq,!62Feat notify user errmsg,!63Feat db handle,!55task manager fail handling
#!/usr/bin/env python
import sys
import json
import signal
import rc_util
import smtplib
from rc_rmq import RCRMQ
......@@ -8,6 +9,7 @@ from datetime import datetime
import mail_config as mail_cfg
task = 'task_manager'
timeout = 30
args = rc_util.get_args()
logger = rc_util.get_logger(args)
......@@ -193,6 +195,32 @@ def task_manager(ch, method, properties, body):
logger.debug('Admin report sent')
def timeout_handler(signum, frame):
current_time = datetime.now()
for user in tuple(tracking):
print(tracking[user])
delta = tracking[user]['last_update'] - current_time
if delta.seconds > timeout:
rc_rmq.publish_msg({
'routing_key': 'complete.' + user,
'msg': {
'username': user,
'success': False,
'errmsg': ["Timeout on " + ', '.join(tracking[user]['waiting'])]
}
})
notify_admin(user, tracking[user])
tracking.pop(user)
# Set initial timeout timer
signal.signal(signal.SIGALRM, timeout_handler)
signal.setitimer(signal.ITIMER_REAL, timeout, timeout)
logger.info(f'Start listening to queue: {task}')
rc_rmq.start_consume({
'queue': task,
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment