00001
00002
00003 '''
00004 This module contains all necessary classes and functions for the gz_parser.py and srb log file parser -> start_server.py .
00005
00006 Reading University
00007 MSc in Network Centered Computing
00008 a.weise - a.weise@reading.ac.uk - December 2005
00009 '''
00010
00011
00012 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
00013 from M2Crypto import SSL
00014
00015
00016 import os, time, stat
00017
00018
00019 import re
00020
00021
00022 from utils_server import delete_file, list_to_string, get_keywords, LoadConfig, find
00023
00024
00025 import thread
00026 import threading
00027
00028 import socket
00029
00030
00031
00032 class SimpleSSLXMLRPCServer(SSL.SSLServer, SimpleXMLRPCServer):
00033 '''
00034 This class is derived from SSL.SSLServer and SimpleXMLRPCServer.
00035 '''
00036 def __init__(self, ssl_context, address, verbose, handler=SimpleXMLRPCRequestHandler):
00037 '''
00038 Constructor overwrites the init function of the SimpleXMLRPCServer and replace it with the secure SSLServer.
00039 '''
00040 SSL.SSLServer.__init__(self, address, handler, ssl_context)
00041 self.funcs = {}
00042 self.logRequests = 0
00043 self.instance = None
00044 self._verbose = verbose
00045
00046 def handle_request(self, serv):
00047 '''
00048 Handle one request by passing it on the a thread.
00049 '''
00050 try:
00051 request, client_address = self.get_request()
00052 if self._verbose == 1:
00053 print "%s -> request accepted from %s....." % (time.ctime(), client_address[0])
00054
00055 except socket.error:
00056 return
00057 if self.verify_request(request, client_address):
00058 thd = MyClientThread(request, client_address, serv)
00059 thd.start()
00060
00061
00062
00063 class My_SSL_Server:
00064 '''
00065 provide functions for the server class
00066 '''
00067
00068 def __init__(self, server_cert, ca_cert, verbose):
00069 '''
00070 Constructor
00071 '''
00072 self._server_certificate = server_cert
00073 self._ca_certificate = ca_cert
00074 self._verbose = verbose
00075
00076 def start_server(self, address, port, xmlrpcserver=SimpleSSLXMLRPCServer):
00077 '''
00078 Start the actual server using SSL:
00079
00080 sslv23 -> compatibility mode, can handle any of the three SSL/TLS protocol versions
00081 server.pem -> server certificate including server RSA private key
00082 ca.pem -> root certificate
00083 SSL.verify_none -> no request that the client has to send his certificate as well
00084 '''
00085
00086 ctx = self.init_context('sslv3', self._verbose, self._server_certificate, self._ca_certificate, SSL.verify_none)
00087
00088 server = xmlrpcserver(ctx, (address, port), self._verbose)
00089
00090 return server
00091
00092 def init_context(self, protocol, verbose, certfile, cafile, verify, verify_depth=10):
00093 '''
00094 This function is used to generate the SSL context:
00095 - verify_depth -> chain depth
00096 '''
00097 ctx = SSL.Context(protocol)
00098 ctx.load_cert_chain(certfile)
00099 ctx.load_verify_locations(cafile)
00100 ctx.set_client_CA_list_from_file(cafile)
00101 ctx.set_verify(verify, verify_depth)
00102 ctx.set_session_id_ctx('server')
00103
00104
00105 return ctx
00106
00107
00108
00109 class MyClientThread(threading.Thread):
00110 '''
00111 This class presents a client, which connects to the server.
00112 '''
00113
00114 def __init__(self, request, address, serv_object):
00115 '''
00116 Constructor
00117 '''
00118 self._request = request
00119 self._client = address
00120 self._serv = serv_object
00121 threading.Thread.__init__(self)
00122
00123 def run(self):
00124 '''
00125 This function overrides the standard run method.
00126 '''
00127 try:
00128 self._serv.process_request(self._request, self._client)
00129 self._serv.close_request(self._request)
00130 except:
00131 self._serv.handle_error(self._request, self._client)
00132 self._serv.close_request(self._request)
00133
00134
00135
00136 class LogFileParser:
00137 '''
00138 This class provides all the neccessarey tools to parse and work up to logfile of the SRB-System.
00139 '''
00140
00141 def __init__(self, logfilepath, keywords, ignore_error, xml_file_path, xml_file_name, verbose):
00142 '''
00143 Constructor
00144 '''
00145 self._verbose = verbose
00146 self._ignore_error = ignore_error
00147 self._keywords = keywords
00148 self._logfile_path = logfilepath
00149 self._client_log_file = "%s/%s" % (xml_file_path, xml_file_name)
00150 self._client_log_file_fd = -1
00151 self._first_line = range(15)
00152 self._last_byte_number = 0
00153 self._line_number = 1
00154
00155 if(0 == os.path.exists(self._logfile_path)):
00156 print "Could not locate log file path under %s !\nMaybe change configuration file and try again!\n\n" % self._logfile_path
00157 os._exit(-1)
00158
00159 def _fetch_first_lines(self, file_name):
00160 '''
00161 This function returns the first 15 lines from current logfile without saving them anywhere.
00162 '''
00163 try:
00164 log_file_fd = open(file_name, "r")
00165
00166 listline = range(15)
00167 log_file_fd.seek(0)
00168 for i in range(15):
00169 listline[i] = log_file_fd.readline()
00170 log_file_fd.close()
00171 return listline
00172 except IOError:
00173 print "Could not open file -> ", file_name
00174
00175 def set_first_lines(self, file_name):
00176 '''
00177 This function saves the first 15 lines of the log file into the member variable.
00178 '''
00179 try:
00180 log_file_fd = open(file_name, 'r')
00181 log_file_fd.seek(0)
00182 for i in range(15):
00183 self._first_line[i] = log_file_fd.readline()
00184 log_file_fd.close()
00185 except IOError:
00186 if self._verbose == 1:
00187 print "%s -> Could not open file -> \"%s\"" % (time.ctime(), file_name)
00188
00189 def get_first_lines(self):
00190 '''
00191 This function returns the member variable _first_lines.
00192 '''
00193 return self._first_line
00194
00195 def test_first_lines(self, file_name):
00196 '''
00197 This function compares the first 15 lines of a log file and return 0 if they are the same, otherwise -1.
00198 '''
00199 listline = self._fetch_first_lines(file_name)
00200 z = 0
00201 while(z<15):
00202 if(listline == self._first_line):
00203 z += 1
00204 else:
00205 return -1
00206
00207 return 0
00208
00209 def find_size_last_message_tag(self, fd, msg_tag):
00210 '''
00211 This function find out, how many bytes the last message tag needs. This function was necessary, because this new xml messages have to be added into the client xml file. Since the creating of this file is not very straightforward using known techniques like sax or dom, the last tag gets deleted, the new messages added and the last tag writen again.
00212
00213 fd = file descriptor of the file
00214 msg_tag = message tag to search for
00215 '''
00216
00217 z = -1
00218 fd.seek(0, 2)
00219
00220 while(1):
00221 fd.seek(z, 2)
00222 tag = fd.read()
00223 if (-1 != tag.rfind(msg_tag)):
00224 return z
00225 z -= 1
00226 if -15 == z:
00227
00228 return 0
00229
00230 def analyse_log_file(self, parser_file_name, file_time=None):
00231 '''
00232 takes the templog file and goes through each lines and searches for keywords, if keywords are found, the line and the two lines before and after are dumped into a xml file, which the client can collect. This function uses the system function write to create the xml file. The dom function were to ineffectiv and sax unflexible.
00233
00234 parser_file_name = file name of the file, which needs to be parsed
00235 '''
00236
00237 if file_time != None:
00238 tupel = time.gmtime(file_time)
00239 else:
00240 status = os.stat(parser_file_name)
00241 file_time = status[8]
00242 tupel = time.gmtime(file_time)
00243
00244 pf_year = time.strftime("%Y ", tupel)
00245
00246 byte_count = 0
00247 interrupt = 0
00248 z = 0
00249
00250 try:
00251 self._client_log_file_fd = file(self._client_log_file, 'r+')
00252
00253 except IOError:
00254
00255 self._client_log_file_fd = open(self._client_log_file, 'w')
00256
00257 xml_header = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?>\n"
00258 self._client_log_file_fd.writelines(xml_header)
00259 self._client_log_file_fd.writelines("<message>\n")
00260 self._client_log_file_fd.writelines("</message>\n")
00261 self._client_log_file_fd.close()
00262 self._client_log_file_fd = file(self._client_log_file, 'r+')
00263
00264
00265
00266 x = self.find_size_last_message_tag(self._client_log_file_fd, "</message>")
00267
00268 self._client_log_file_fd.seek(x, 2)
00269 shorten = self._client_log_file_fd.tell()
00270
00271 self._client_log_file_fd.truncate(shorten)
00272 self._client_log_file_fd.seek(0, 2)
00273
00274
00275 log_file = parser_file_name
00276
00277 try:
00278 log_file_fd = open(log_file, "r")
00279 log_file_fd.seek(self._last_byte_number)
00280
00281 except IOError:
00282 if self._verbose == 1:
00283 print "%s -> could not open srb log file -> %s" % (time.ctime(), log_file)
00284 return -1
00285
00286 if self._verbose == 1:
00287 print "%s -> start parsing " % (time.ctime())
00288 starttime = time.time()
00289 while(interrupt == 0):
00290
00291 content = log_file_fd.readline()
00292 if(content == ''):
00293 interrupt = 1
00294 break
00295
00296 if 0 == len(self._keywords):
00297 check = 0
00298 else:
00299 check = self._test_keywords(self._keywords, len(self._keywords)-1, content)
00300
00301 if(0 == check):
00302
00303 error_number = self._extract_error_number(content)
00304
00305 if (None == error_number):
00306 error_number = "-"
00307 temp = ""
00308 else:
00309 temp = int(error_number)
00310
00311 if ( None == find(temp, self._ignore_error)):
00312
00313 line_number_string = "%d" % self._line_number
00314
00315 content_ = content.rstrip()
00316
00317 date_time = self._extract_time(content, pf_year)
00318
00319 if (date_time == -1):
00320 date_time = ["", ""]
00321
00322 byte_count = log_file_fd.tell()
00323 back = -1
00324 read = 0
00325 while(1):
00326 try:
00327
00328 log_file_fd.seek(back, 1)
00329 read = back*(-1)
00330 tag = log_file_fd.read(read)
00331 if (None != re.search('^NOTICE: *[A-Z][a-z]{2} +[0-9]{1,2} +[0-9]{2}:[0-9]{2}:[0-9]{2}:', tag)):
00332 date_time = self._extract_time(tag, pf_year)
00333 break
00334 back = back-1
00335 except IOError:
00336
00337 date_time[0] = time.strftime("%Y-%m-%d", tupel)
00338 date_time[1] = time.strftime("%H:%M:%S", tupel)
00339
00340 break
00341
00342
00343 log_file_fd.seek(byte_count)
00344
00345 z += 1
00346
00347 if -1 == self.start_entry('entry'):
00348 interrupt = 1
00349 break
00350 if -1 == self.write_entry('date', date_time[0]):
00351 interrupt = 1
00352 break
00353 if -1 == self.write_entry('time', date_time[1]):
00354 interrupt = 1
00355 break
00356 if -1 == self.write_entry('error_number', error_number):
00357 interrupt = 1
00358 break
00359 if -1 == self.write_entry('error_string', content_):
00360 interrupt = 1
00361 break
00362 if -1 == self.write_entry('linenumber', line_number_string):
00363 interrupt = 1
00364 break
00365 if -1 == self.end_entry('entry'):
00366 interrupt = 1
00367 break
00368
00369 self._line_number += 1
00370 if self._verbose == 1:
00371 print "%s -> end parsing " % (time.ctime())
00372 endtime = time.time()
00373 if self._verbose == 1:
00374 print "%s -> parsing time: %s" % (time.ctime(), (endtime-starttime))
00375 self.end_entry('message')
00376 if self._verbose == 1:
00377 print "%s -> %d errors found\n" % (time.ctime(), z)
00378
00379 try:
00380 self._client_log_file_fd.close()
00381 except IOError, e:
00382 if self._verbose == 1:
00383 print "%s -> Problem closing XML file: \"%s\" !" % (time.ctime(), e)
00384 self._last_byte_number = log_file_fd.tell()
00385
00386 def start_entry(self, name):
00387 '''
00388 This function inserts a start tag into the XML file. (name = tag name)
00389 '''
00390 start_tag = "<%s>\n" % name
00391 try:
00392 self._client_log_file_fd.write(start_tag)
00393 return 0
00394 except IOError, e:
00395 if self._verbose == 1:
00396 print "%s -> Problem writing XML file: \"%s\" !" % (time.ctime(), e)
00397 return -1
00398
00399 def write_entry(self, tagname, content):
00400 '''
00401 This function inserts an entry into the xml file.
00402
00403 tagname = tag name
00404 content = message between start and end tag
00405 '''
00406
00407 if len(content) < 50000000:
00408
00409 bad_character = re.sub('[\x09\x0a\x0d\x20-\x25\x27-\xd7]*', "", content)
00410
00411 for i in range(len(bad_character)):
00412 if bad_character[i] == '\x00':
00413
00414 content = content.replace(bad_character[i], '')
00415 else:
00416 content = content.replace(bad_character[i], "?")
00417
00418 entry = "<%s>%s</%s>\n" % (tagname, content, tagname)
00419 try:
00420 self._client_log_file_fd.write(entry)
00421 return 0
00422 except IOError, e:
00423 if self._verbose == 1:
00424 print "%s -> Problem writing XML file: \"%s\" !" % (time.ctime(), e)
00425 return -1
00426
00427 else:
00428 entry = "<%s>LOGFILE ENTRY TO LONG !!!</%s>\n" % (tagname, tagname)
00429 try:
00430 self._client_log_file_fd.write(entry)
00431 return 0
00432 except IOError, e:
00433 if self._verbose == 1:
00434 print "%s -> Problem writing XML file: \"%s\" !" % (time.ctime(), e)
00435 return -1
00436
00437 def end_entry(self, name):
00438 '''
00439 This function inserts an end tag into the XML file. (name = tag name)
00440 '''
00441 endtag = "</%s>\n" % name
00442 try:
00443 self._client_log_file_fd.write(endtag)
00444 return 0
00445 except IOError, e:
00446 if self._verbose == 1:
00447 print "%s -> Problem writing XML file: \"%s\" !" % (time.ctime(), e)
00448 return -1
00449
00450 def reset(self):
00451 '''
00452 This function resets member variable, in case of a new log file.
00453 '''
00454 self._line_number = 1
00455 self._last_byte_number = 0
00456
00457 def _test_keywords(self, keywordlist, amount_of_keywords, teststring):
00458 '''
00459 This is a recursive function, which tests if a list of keywords is part of a string (AND relation). If all keywords found 0 is returned, otherwise -1
00460
00461 keywordlist = list of all keywords
00462 amount_of_keywords = number of keywords in list
00463 teststring = string, which needs to be investigated
00464
00465 return -1 if line is not interesting
00466 return 0 if line is taken
00467 '''
00468 if (amount_of_keywords == 0):
00469
00470 if( 2 == len(keywordlist[amount_of_keywords])):
00471 if (-1 == teststring.rfind(keywordlist[amount_of_keywords][0])):
00472
00473 return 0
00474 else:
00475 if( -1 == keywordlist[amount_of_keywords][1].rfind("!")):
00476
00477 temp = keywordlist[amount_of_keywords][1].strip("!")
00478 if ( -1 == teststring.rfind(temp)):
00479
00480 return 0
00481 else:
00482 return -1
00483 else:
00484
00485 if ( -1 != teststring.rfind(keywordlist[amount_of_keywords][1])):
00486
00487 return 0
00488 else:
00489 return -1
00490 else:
00491 if (-1 == teststring.rfind(keywordlist[amount_of_keywords][0])):
00492
00493 return 0
00494 else:
00495 return -1
00496 else:
00497 if( 2 == len(keywordlist[amount_of_keywords])):
00498 if (-1 == teststring.rfind(keywordlist[amount_of_keywords][0])):
00499
00500 return self._test_keywords(keywordlist, amount_of_keywords-1, teststring)
00501 else:
00502 if( -1 == keywordlist[amount_of_keywords][1].rfind("!")):
00503
00504 temp = keywordlist[amount_of_keywords][1].strip("!")
00505 if ( -1 == teststring.rfind(temp)):
00506
00507 return self._test_keywords(keywordlist, amount_of_keywords-1, teststring)
00508 else:
00509 return -1
00510 else:
00511
00512 if ( -1 != teststring.rfind(keywordlist[amount_of_keywords][1])):
00513
00514 return self._test_keywords(keywordlist, amount_of_keywords-1, teststring)
00515 else:
00516 return -1
00517 else:
00518 if (-1 == teststring.rfind(keywordlist[amount_of_keywords][0])):
00519
00520 return self._test_keywords(keywordlist, amount_of_keywords-1, teststring)
00521 else:
00522 return -1
00523
00524 def update_keywords(self, keys):
00525 '''
00526 This function updates the member variable keywords.
00527
00528 keys = new keyword list
00529 '''
00530 self._keywords = keys
00531
00532 def update_ignore_error(self, error):
00533 '''
00534 This function updates the ignore_error list.
00535
00536 error = new ignore list
00537 '''
00538 self._ignore_error = error
00539
00540 def _extract_time(self, time_string, year):
00541 '''
00542 This function takes a line from the logfile and extracts the time from there.
00543 '''
00544 if ( None == re.search('^NOTICE: *[A-Z][a-z]{2} +[0-9]{1,2} +[0-9]{2}:[0-9]{2}:[0-9]{2}:', time_string)):
00545 return -1
00546 else:
00547 listus = time_string.split(":")
00548 zeit = year+listus[1]+":"+listus[2]+":"+listus[3]
00549 time_tupel = time.strptime(zeit, "%Y %b %d %X")
00550 date_time = range(2)
00551 date_time[0] = time.strftime("%Y-%m-%d", time_tupel)
00552 date_time[1] = time.strftime("%H:%M:%S", time_tupel)
00553 return date_time
00554
00555 def _extract_error_number(self, text):
00556 '''
00557 Thhis function takes a line from the logfile and extract the error number.
00558 '''
00559 match = re.search('(status|errno) *= *-*\d{1,10}', text)
00560 if(None != match):
00561
00562 listus = match.string[match.start():match.end()]
00563 listus = re.findall('-*\d{1,10}', listus)
00564 if(1 == len(listus)):
00565 return listus[0]
00566 else:
00567 return None
00568 else:
00569 return None
00570
00571
00572
00573 class MyParserThread(threading.Thread):
00574 '''
00575 This class is used to create a thread, which is doing all the necessery work in the background.
00576 '''
00577 def __init__(self, shared, timus, gz_path, logfilepath, logfilename, keyword, ignore_error, xml_file_path, xml_file_name, configfile, keywordfile, verbose):
00578 '''
00579 Constructor
00580 '''
00581 self._keywordfile = keywordfile
00582 temp = os.stat(self._keywordfile)
00583 self._keywordfile_time = temp[8]
00584 self._configfile = configfile
00585 temp = os.stat(self._configfile)
00586 self._configfile_time = temp[8]
00587 self._verbose = verbose
00588 self._id = thread.get_ident()
00589 self._shared_obj = shared
00590 self._interval = timus
00591 self._stop = 0
00592 threading.Thread.__init__(self)
00593 self._parser = LogFileParser (logfilepath, keyword, ignore_error, xml_file_path, xml_file_name, self._verbose)
00594 self._logfilepath = logfilepath
00595 self._log_file_name = logfilepath+"/"+logfilename
00596 self._gz_diect = gz_path
00597 self._list = []
00598
00599 def run(self):
00600 '''
00601 This function overwrites the standard run method.
00602 '''
00603 block_counter = 0
00604 while(1):
00605
00606 if self._stop == 1:
00607 print "%s -> working thread stopped !!!" % time.ctime()
00608 os._exit(0)
00609
00610 if (self._configfile != ""):
00611 temp = os.stat(self._configfile)
00612 if self._configfile_time != temp[8]:
00613
00614 if self._verbose == 1:
00615 print "%s -> config file has changed, reading new values ..." % time.ctime()
00616 self._configfile_time = temp[8]
00617 self._refresh_configuration()
00618
00619
00620 if (self._keywordfile != ''):
00621 temp = os.stat(self._keywordfile)
00622 if self._keywordfile_time != temp[8]:
00623
00624 if self._verbose == 1:
00625 print "%s -> keyword file has changed, reading new values ..." % time.ctime()
00626 self._keywordfile_time = temp[8]
00627 self._refresh_keywords()
00628
00629 if(-1 == self._shared_obj.set_variable_parsing(1, self)):
00630
00631 block_counter += 1
00632 time.sleep(5*block_counter)
00633 if self._verbose == 1:
00634 print "%s -> client busy" % time.ctime()
00635 if block_counter > 5:
00636 if self._verbose == 1:
00637 print "%s -> client needs a long time, miss this parsing period" % time.ctime()
00638 block_counter = 0
00639 time.sleep(self._interval*60)
00640 else:
00641
00642 try:
00643
00644
00645 if(0 == self._parser.test_first_lines(self._log_file_name)):
00646
00647 if self._verbose == 1:
00648 print "%s -> no log file rotation" % time.ctime()
00649 self._parser.analyse_log_file(self._log_file_name)
00650 else:
00651 if self._verbose == 1:
00652 print "%s -> new log file" % time.ctime()
00653
00654
00655 self._list = []
00656
00657 os.path.walk(self._gz_diect, self.parse_directory, self._list)
00658 if (0 < len(self._list)):
00659 self.gunzip(self._list[0][1])
00660 d = os.getcwd()
00661 try:
00662 os.chdir(self._gz_diect)
00663 self.gunzip(self._list[0][1])
00664 self._parser.analyse_log_file(self._gz_diect+"/temp_srbLog")
00665 delete_file("temp_srbLog", self._verbose)
00666 os.chdir(d)
00667 except:
00668 if self._verbose == 1:
00669 print "%s -> could not find directory \"%s\"" % (time.ctime(), self._gz_diect)
00670 else:
00671 pass
00672
00673 self._parser.reset()
00674 self._parser.set_first_lines(self._log_file_name)
00675 self._parser.analyse_log_file(self._log_file_name)
00676
00677 finally:
00678
00679
00680 self._shared_obj.set_variable_parsing(0, self)
00681
00682 time.sleep(self._interval*60)
00683
00684 if self._verbose == 1:
00685 print "\n%s -> -------- parse ------" % time.ctime()
00686
00687 def _refresh_keywords(self):
00688 '''
00689 This function gets keywords from the keyword file !
00690 '''
00691 if(1 == os.path.exists(self._keywordfile)):
00692 keyword = get_keywords(self._keywordfile)
00693 self._parser.update_keywords(keyword)
00694 return 0
00695 else:
00696
00697 if self._verbose == 1:
00698 print "%s -> Sorry, the keyword file does NOT exist !\nUsing old configuration!\n\n" % time.ctime()
00699 return -1
00700
00701 def _refresh_configuration(self):
00702 '''
00703 This function gets the needed information from the configfile!
00704 '''
00705 if(1 == os.path.exists(self._configfile)):
00706 config = LoadConfig(self._configfile)
00707 error = config.get("misc.ignore_error")
00708 error = error.strip()
00709 error = error.strip(",")
00710 ignore_error = error.split(",")
00711 for i in range(len(ignore_error)):
00712
00713 if ignore_error[i] != '':
00714 try:
00715 ignore_error[i] = int(ignore_error[i].strip())
00716 except ValueError:
00717 if self._verbose == 1:
00718 print "%s -> \"ignore_error\" in the config file has NO valid values, use old configuration" % time.ctime()
00719 return -1
00720 else:
00721 del ignore_error[i]
00722
00723 self._parser.update_ignore_error(ignore_error)
00724
00725 return 0
00726 else:
00727
00728 if self._verbose == 1:
00729 print "%s -> Sorry, the given config file does NOT exist !\nUsing old configuration!\n\n" % time.ctime()
00730 return -1
00731
00732 def stop_thread(self):
00733 """
00734 Stop the thread
00735 """
00736 self._stop = 1
00737
00738 def parse_directory(self, arg, dirname, fnames):
00739 '''
00740 This function "walks" through a given directory and considers all srbLOG*.gz files. The name and last modified time are saved in a list (2 dimensional array). The function should be used with os.path.walk(path, function_name, arg)!
00741 '''
00742 d = os.getcwd()
00743
00744 try:
00745 os.chdir(dirname)
00746 except:
00747 if self._verbose == 1:
00748 print "%s -> could not find directory \"%s\"" % (time.ctime(), dirname)
00749 return -1
00750
00751 for f in fnames:
00752
00753 if (not os.path.isfile(f)) or (None == re.search('^srbLog[_0-9.-]*.gz', f)):
00754 continue
00755
00756 date = os.stat(f)[stat.ST_MTIME]
00757
00758 tupel = (date, f)
00759
00760 self._list.append(tupel)
00761
00762 os.chdir(d)
00763
00764 self._list.sort()
00765
00766 self._list.reverse()
00767 return 0
00768
00769 def gunzip(self, filus, name_temp_file="temp_srbLog"):
00770 '''
00771 This function unzips a *.gz file using the system tool gunzip. Make sure when calling the function the file exists in this directory. The function creates a temporary file and leave the orignal *.gz file untouched!
00772 '''
00773 if (not os.path.isfile(filus)):
00774 return -1
00775 else:
00776 command = "gunzip -c %s > %s" % (filus, name_temp_file)
00777 try:
00778 os.system(command)
00779 return 0
00780 except:
00781 return -1
00782
00783
00784
00785 class Mutex:
00786 '''
00787 This class makes sure that server and client are not accessing the same file at the same time.
00788 '''
00789
00790 _locked = threading.Lock()
00791
00792
00793 def __init__(self):
00794 '''
00795 Constructor
00796 '''
00797 self.parsing = 0
00798 self._parsing_thread_id = 0
00799 self.client = 0
00800
00801
00802 def set_variable_parsing(self, value, the_thread):
00803 '''
00804 set variable parsing
00805 '''
00806 Mutex._locked.acquire()
00807 self._parsing_thread_id = the_thread
00808
00809 if self.client == 0:
00810
00811 self.parsing = value
00812 Mutex._locked.release()
00813 time.sleep(1)
00814 if value == 0:
00815
00816 self._parsing_thread_id = 0
00817 return 0
00818 else:
00819 Mutex._locked.release()
00820 time.sleep(1)
00821 return -1
00822
00823 def set_variable_client(self, value):
00824 '''
00825 set variable client
00826 '''
00827 Mutex._locked.acquire()
00828
00829 if self.parsing == 0:
00830
00831 self.client = value
00832
00833 Mutex._locked.release()
00834 time.sleep(1)
00835 return 0
00836 else:
00837 if (0 != self._parsing_thread_id):
00838 if (1 != self._parsing_thread_id.isAlive()):
00839
00840 self.parsing = 0
00841 self._parsing_thread_id = 0
00842 Mutex._locked.release()
00843 time.sleep(1)
00844 return -1
00845
00846
00847
00848 class RPC:
00849 '''
00850 This class contains the RPC functions.
00851 '''
00852 def __init__(self, verbose, semaphore, config_file, interval, keyword_path, keyword_name, xml_file_path):
00853 '''
00854 constructor
00855 '''
00856 self._verbose = verbose
00857 self._client = True
00858 self._share = semaphore
00859 self._config_file = config_file
00860 self._interval = interval
00861 self._keyword_path = keyword_path
00862 self._keyword_name = keyword_name
00863 self._xml_file_path = xml_file_path
00864
00865 self._list = []
00866
00867 def rpc_stop_server(self):
00868 '''
00869 This function stops the server!
00870 '''
00871 command = "./stop_server"
00872
00873 answer = os.system(command)
00874 print answer
00875 return answer
00876
00877 def rpc_disable_rpc_calls(self):
00878 '''
00879 This function disables rpc.
00880 '''
00881 self._client = False
00882 if self._verbose == 1:
00883 print "%s -> RPC through \"admin tool\" disabled!" % time.ctime()
00884 return "RPC disabled"
00885
00886 def rpc_enable_rpc_calls(self):
00887 '''
00888 This function enables rpc.
00889 '''
00890 self._client = True
00891 if self._verbose == 1:
00892 print "%s -> RPC through \"admin tool\" enabled!" % time.ctime()
00893 return "RPC enabled"
00894
00895 def rpc_status(self):
00896 '''
00897 This function return the current status of the self._client variable.
00898 '''
00899 if self._client == True:
00900 return "RPC enabled"
00901 else:
00902 return "RPC disabled"
00903
00904 def rpc_interval_status(self):
00905 '''
00906 This function returns the current parsing interval time.
00907 '''
00908 if self._client == True:
00909 return self._interval
00910 else:
00911 return -2
00912
00913 def rpc_get_my_xml_file(self, filename):
00914 '''
00915 This functions gets the xml file from the server !
00916 '''
00917 if (self._client == True):
00918 if (0 == self._share.set_variable_client(1)):
00919
00920 try:
00921 filus = self._xml_file_path+"/"+filename
00922 client_xml_fd = open(filus, 'r')
00923 file_content = client_xml_fd.read()
00924 client_xml_fd.close()
00925 except IOError:
00926 self._share.set_variable_client(0)
00927 return "no file"
00928
00929 if (0 == delete_file((self._xml_file_path+"/"+filename), self._verbose)):
00930 self._share.set_variable_client(0)
00931 return file_content
00932 else:
00933 if self._verbose == 1:
00934 print "problems deleting file"
00935 self._share.set_variable_client(0)
00936 return -1
00937 else:
00938 return -3
00939 else:
00940 return -2
00941
00942 def rpc_check_availabitity(self):
00943 '''
00944 This function check if the server is still in the parsing process.
00945 '''
00946 if (self._client == True):
00947 if (0 == self._share.set_variable_client(1)):
00948 return 0
00949 else:
00950 return -3
00951 else:
00952 return -2
00953
00954 def rpc_get_file_list(self):
00955 '''
00956 This function walks through the *.xml directory and finds all files, which need to be fetched form the client.
00957 '''
00958 if (self._client == True):
00959 self._list = []
00960 try:
00961 os.path.walk(self._xml_file_path, self._parse_directory, self._list)
00962 if (0 < len(self._list)):
00963 return self._list
00964 else:
00965 return 0
00966 except:
00967 return -1
00968 else:
00969 return -2
00970
00971 def rpc_update_configuration(self, section, key, value, action):
00972 '''
00973 This functions adds or deletes values in the config.ini.
00974 action:
00975 0 = delete
00976 1 = add
00977 2 = exchange
00978 4 = info
00979 '''
00980 print "section: ", section
00981 print "key: ", key
00982 print "value: ", value
00983 print "action: ", action
00984 if(self._client == True):
00985 try:
00986 config_fd = file(self._config_file, 'r+')
00987 except IOError, e:
00988 return "Problem -> %s" % e
00989
00990 byte_count = 0
00991
00992 while(1):
00993 byte_count = config_fd.tell()
00994 line = config_fd.readline()
00995 if line == '':
00996 if self._verbose == 1:
00997 print "%s -> Section: \"%s\" and key: \"%s\" do not exist in config file!" % (time.ctime(), section, key)
00998 config_fd.close()
00999 return "Section: \"%s\" and key: \"%s\" do not exist in config file!" % (section, key)
01000 if (-1 != line.find(section)):
01001 while(1):
01002 byte_count = config_fd.tell()
01003 line = config_fd.readline()
01004 if line == '':
01005 if self._verbose == 1:
01006 print "%s -> Key: \"%s\" do not exist under section \"%s\" in config file!" % (time.ctime(), key, section)
01007 config_fd.close()
01008 return "Key \"%s\" do not exist under section \"%s\" in config file!" % (key, section)
01009 if (-1 != line.find(key) and -1 != line.find("=") and -1 == line.find("#", 0, 1)):
01010 if action == 0:
01011 if self._verbose == 1:
01012 print "%s -> Delete \"%s:%s\" value \"%s\"" % (time.ctime(), section, key, value)
01013 listus = line.split("=")
01014 listus[1] = listus[1].strip()
01015 listus[1] = listus[1].strip(",")
01016 listus = listus[1].split(",")
01017 for i in range(len(listus)):
01018 listus[i] = listus[i].strip()
01019 new_content = ''
01020 for i in range(len(listus)):
01021 if int(listus[i]) != value:
01022 new_content = new_content+"%s, " % listus[i]
01023
01024 new_content = new_content.strip()
01025 new_content = new_content.strip(",")
01026 new_content = "%s = %s\n" % (key, new_content)
01027 rest = config_fd.read()
01028 config_fd.truncate(byte_count)
01029 config_fd.seek(byte_count)
01030 config_fd.writelines(new_content)
01031 config_fd.write(rest)
01032 config_fd.close()
01033 return "Changes applied: %s" % new_content
01034 elif action == 1:
01035 if self._verbose == 1:
01036 print "%s -> Add \"%s:%s\" value \"%s\"" % (time.ctime(), section, key, value)
01037 listus = line.split("=")
01038 listus[1] = listus[1].strip()
01039 listus[1] = listus[1].strip(",")
01040 listus = listus[1].split(",")
01041 finish = 0
01042 while (finish == 0):
01043 if len(listus) == 0:
01044 finish = 1
01045 for i in range(len(listus)):
01046 finish = 1
01047 print listus[i]
01048 listus[i] = listus[i].strip()
01049
01050 try:
01051 print "test auf int"
01052 temp_value = int(listus[i])
01053 if value == temp_value:
01054 config_fd.close()
01055 return "Value %d already exists!" % value
01056 except ValueError, e:
01057 finish = 0
01058 print e, listus[i]
01059 del listus[i]
01060 break
01061
01062
01063 new_content = ''
01064 for i in range(len(listus)):
01065 new_content = new_content+"%s, " % listus[i]
01066 new_content = "%s = %s%s\n" % (key, new_content, value)
01067 rest = config_fd.read()
01068 config_fd.truncate(byte_count)
01069 config_fd.seek(byte_count)
01070 config_fd.writelines(new_content)
01071 config_fd.write(rest)
01072 config_fd.close()
01073 return "Changes applied: %s" % new_content
01074 elif action == 2:
01075
01076 if self._verbose == 1:
01077 print "%s -> Change \"%s:%s\" to value \"%s\"" % (time.ctime(), section, key, value)
01078 rest = config_fd.read()
01079 config_fd.truncate(byte_count)
01080 config_fd.seek(byte_count)
01081 new_content = "%s = %s\n" % (key, value)
01082 config_fd.writelines(new_content)
01083 config_fd.write(rest)
01084 config_fd.close()
01085 if section == 'misc' and key == 'minute':
01086 self._interval = int(value)
01087 return 0
01088 elif action == 4:
01089 config_fd.close()
01090 return line
01091 else:
01092 config_fd.close()
01093 return -1
01094 else:
01095 return "RPC disabled"
01096
01097 def rpc_update_keyword_file(self, keyword, action):
01098 '''
01099 This function updates the keyword file.
01100
01101 action:
01102 0 = delete
01103 1 = add
01104 2 = info
01105 '''
01106 if(self._client == True):
01107 byte_count = 0
01108 file_size = 0
01109 comments = ''
01110
01111 filus = self._keyword_path+"/"+self._keyword_name
01112
01113 try:
01114 key_fd = file(filus, 'r+')
01115 except IOError, e:
01116 if self._verbose == 1:
01117 print "%s -> Problem open keyword file -> %s !" % (time.ctime(), e)
01118 return "Problem -> %s" % e
01119
01120 key_fd.seek(0, 2)
01121 file_size = key_fd.tell()
01122 key_fd.seek(0)
01123
01124 while(1):
01125
01126 byte_count = key_fd.tell()
01127 line = key_fd.readline()
01128 if byte_count >= file_size:
01129 break
01130 if (-1 != line.find("#", 0, 1)):
01131 comments += line
01132
01133 key_fd.close()
01134 keyword_list = get_keywords(filus)
01135 keyword = keyword.split(":")
01136
01137 if action == 0:
01138
01139 for i in range(len(keyword_list)):
01140 if 1 == len(keyword) and 1 == len(keyword_list[i]):
01141 if keyword[0] == keyword_list[i][0]:
01142 del keyword_list[i]
01143 try:
01144 key_fd = file(filus, 'w+')
01145 print key_fd
01146 key_string = list_to_string(keyword_list)
01147 key_fd.write(comments)
01148 key_fd.write(key_string)
01149 key_fd.close()
01150 except IOError, e:
01151 if self._verbose == 1:
01152 print "%s -> Problem open keyword file -> %s !" % (time.ctime(), e)
01153 return "Problem -> %s" % e
01154 return "keyword %s from keyword list deleted" % keyword
01155 elif 2 == len(keyword) and 2 == len(keyword_list[i]):
01156 if keyword[0] == keyword_list[i][0] and keyword[1] == keyword_list[i][1]:
01157 del keyword_list[i]
01158 try:
01159 key_fd = file(filus, 'w+')
01160 key_string = list_to_string(keyword_list)
01161 key_fd.write(comments)
01162 key_fd.write(key_string)
01163 key_fd.close()
01164 except IOError, e:
01165 if self._verbose == 1:
01166 print "%s -> Problem open keyword file -> %s !" % (time.ctime(), e)
01167 return "Problem -> %s" % e
01168 return "keyword %s from keyword list deleted" % keyword
01169
01170 return "keyword %s was not part of keyword list" % keyword
01171
01172 if action == 1:
01173
01174 for i in range(len(keyword_list)):
01175 if 1 == len(keyword) and 1 == len(keyword_list[i]):
01176 if keyword[0] == keyword_list[i][0]:
01177 return "keyword %s already in keyword list" % keyword
01178 elif 2 == len(keyword) and 2 == len(keyword_list[i]):
01179 if keyword[0] == keyword_list[i][0] and keyword[1] == keyword_list[i][1]:
01180 return "keywords %s already in keyword list" % keyword
01181
01182 keyword_list.append(keyword)
01183 try:
01184 key_fd = file(filus, 'w+')
01185 except IOError, e:
01186 if self._verbose == 1:
01187 print "%s -> Problem open keyword file -> %s !" % (time.ctime(), e)
01188 return "Problem -> %s" % e
01189
01190 key_string = list_to_string(keyword_list)
01191 key_fd.write(comments)
01192 key_fd.write(key_string)
01193 key_fd.close()
01194
01195 return key_string
01196
01197 if action == 2:
01198 return list_to_string(keyword_list)
01199
01200 else:
01201 return "RPC disabled"
01202
01203 def _parse_directory(self, arg, dirname, fnames):
01204 '''
01205 This function "walks" through a given directory and looks for the client_log.xml file. The name and last modified time are saved in a list (2 dimensional array). The function should be used with os.path.walk(path, function_name, arg)!
01206
01207 dirname = directory which need to be pared
01208 fnames = files within dirname
01209 '''
01210 d = os.getcwd()
01211
01212 try:
01213 os.chdir(dirname)
01214 except:
01215 if self._verbose == 1:
01216 print "could not find directory \"%s\"" % dirname
01217 return -1
01218
01219 for f in fnames:
01220
01221 if (not os.path.isfile(f)) or (None == re.search('client_log.xml', f)):
01222 continue
01223 else:
01224
01225 self._list.append(f)
01226
01227 os.chdir(d)
01228
01229
01230
01231
01232
01233
01234
01235