multithreading - Python Split paramiko logging logs to separate files in multithreaded program -
i'm having issues understanding logic of how split paramiko logs separate files in threads.
reasoning: have main logfile i've been using, don't want polluted out of order nature of each individual thread. (assume each thread has multiple commands , outputs, etc.) want log these multiple files before aggregating in. found code push logs separate files based on thread names, don't know how force paramiko log thread's instance of paramiko file. appreciated.
import paramiko import logging import threading import os _l = logging.getlogger("demo") class multihandler(logging.handler): def __init__(self, dirname): super(multihandler, self).__init__() self.files = {} self.dirname = dirname if not os.access(dirname, os.w_ok): raise exception("directory %s not writeable" % dirname) def flush(self): self.acquire() try: fp in self.files.values(): fp.flush() finally: self.release() def _get_or_open(self, key): "get file pointer given key, or else open file" self.acquire() try: if self.files.has_key(key): return self.files[key] else: fp = open(os.path.join(self.dirname, "%s.log" % key), "a") self.files[key] = fp return fp finally: self.release() def emit(self, record): # no lock here; following code streamhandler , filehandler try: fp = self._get_or_open(record.threadname) msg = self.format(record) fp.write('%s\n' % msg.encode("utf-8")) except (keyboardinterrupt, systemexit): raise except: self.handleerror(record) def do_ssh_stuff(host, ip): cli = paramiko.client.sshclient(); cli.set_missing_host_key_policy(paramiko.client.autoaddpolicy()); cli.connect(hostname=ip, username="usr"); stdin_, stdout_, stderr_ = cli.exec_command("ls -l ~"); ls_out = stdout_.readlines() print ls_out _l.info("finished ls") ### set logger creates 1 file per thread multi_handler = multihandler("/tmp") logging.getlogger().addhandler(multi_handler) logging.getlogger("paramiko").setlevel(logging.warning) # example ### set default log level, log message _l.setlevel(logging.debug) _l.info("run initiated") hosts = {"iad":"10.10.10.10", "dca":"10.10.10.11"} #assume legit hosts host, ip in hosts: thread = threading.thread(target=do_ssh_stuff, args=(host, ip), name=host) thread.start() thread.join()
Comments
Post a Comment