F:/thesis_austausch/dissertation/code_docu_doxygen/Server/server_classes.py

Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 
00003 '''
00004 This module contains all necessary classes and functions for the gz_parser.py and srb log file parser -> start_server.py .
00005 
00006 Reading University
00007 MSc in Network Centered Computing
00008 a.weise - a.weise@reading.ac.uk - December 2005
00009 '''
00010 
00011 # ssl connection
00012 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
00013 from M2Crypto import SSL
00014 
00015 # misc
00016 import os, time, stat
00017 
00018 # regular expressions
00019 import re
00020 
00021 # utilities
00022 from utils_server import delete_file, list_to_string, get_keywords, LoadConfig, find
00023 
00024 # threads
00025 import thread
00026 import threading
00027 
00028 import socket
00029 
00030 ################## CLASS  SimpleSSLXMLRPCServer  ##################
00031 
00032 class SimpleSSLXMLRPCServer(SSL.SSLServer, SimpleXMLRPCServer):
00033     '''
00034     This class is derived from SSL.SSLServer and SimpleXMLRPCServer.
00035     '''
00036     def __init__(self, ssl_context, address, verbose, handler=SimpleXMLRPCRequestHandler):
00037         '''
00038         Constructor overwrites the init function of the SimpleXMLRPCServer and replace it with the secure SSLServer.        
00039         '''
00040         SSL.SSLServer.__init__(self, address, handler, ssl_context)
00041         self.funcs = {}
00042         self.logRequests = 0
00043         self.instance = None
00044         self._verbose = verbose
00045 
00046     def handle_request(self, serv):
00047         '''
00048         Handle one request by passing it on the a thread.
00049         '''
00050         try:
00051             request, client_address = self.get_request()
00052             if self._verbose == 1:
00053                 print "%s -> request accepted from %s....." % (time.ctime(), client_address[0])
00054 
00055         except socket.error:
00056             return
00057         if self.verify_request(request, client_address):
00058             thd = MyClientThread(request, client_address, serv)
00059             thd.start()
00060 
00061 ################## CLASS  My_SSL_Server  ##################
00062 
00063 class My_SSL_Server:
00064     '''
00065     provide functions for the server class
00066     '''
00067 
00068     def __init__(self, server_cert, ca_cert, verbose):
00069         '''
00070         Constructor
00071         '''
00072         self._server_certificate = server_cert
00073         self._ca_certificate = ca_cert
00074         self._verbose = verbose
00075     
00076     def start_server(self, address, port, xmlrpcserver=SimpleSSLXMLRPCServer):
00077         '''
00078         Start the actual server using SSL:
00079         
00080         sslv23 -> compatibility mode, can handle any of the three SSL/TLS protocol versions
00081         server.pem -> server certificate including server RSA private key
00082         ca.pem -> root certificate
00083         SSL.verify_none -> no request that the client has to send his certificate as well
00084         '''
00085         # create SSL context
00086         ctx = self.init_context('sslv3', self._verbose, self._server_certificate, self._ca_certificate, SSL.verify_none)
00087         # create server object
00088         server = xmlrpcserver(ctx, (address, port), self._verbose)
00089         # return server object
00090         return server
00091 
00092     def init_context(self, protocol, verbose, certfile, cafile, verify, verify_depth=10):
00093         '''
00094         This function is used to generate the SSL context:
00095         - verify_depth -> chain depth
00096         '''
00097         ctx = SSL.Context(protocol)             # create context object
00098         ctx.load_cert_chain(certfile)           # load server certificate chain
00099         ctx.load_verify_locations(cafile)
00100         ctx.set_client_CA_list_from_file(cafile)
00101         ctx.set_verify(verify, verify_depth)    # verfiy options
00102         ctx.set_session_id_ctx('server')        # set session id
00103         #if verbose == 1:
00104         #    ctx.set_info_callback()            # show handshake information --- debug
00105         return ctx
00106 
00107 ################## CLASS  MyClientThread  ##################
00108 
00109 class MyClientThread(threading.Thread):
00110     '''
00111     This class presents a client, which connects to the server.
00112     '''
00113     
00114     def __init__(self, request, address, serv_object):
00115         '''
00116         Constructor
00117         '''
00118         self._request = request
00119         self._client = address
00120         self._serv = serv_object
00121         threading.Thread.__init__(self)
00122        
00123     def run(self):
00124         '''
00125         This function overrides the standard run method.
00126         '''
00127         try:
00128             self._serv.process_request(self._request, self._client)
00129             self._serv.close_request(self._request)
00130         except:
00131             self._serv.handle_error(self._request, self._client)
00132             self._serv.close_request(self._request)
00133 
00134 ################## CLASS  LogFileParser  ##################
00135 
00136 class LogFileParser:
00137     '''
00138     This class provides all the neccessarey tools to parse and work up to logfile of the SRB-System.
00139     '''
00140     
00141     def __init__(self, logfilepath, keywords, ignore_error, xml_file_path, xml_file_name, verbose):
00142         '''
00143         Constructor
00144         '''
00145         self._verbose = verbose
00146         self._ignore_error = ignore_error
00147         self._keywords = keywords  # keywords
00148         self._logfile_path = logfilepath
00149         self._client_log_file =  "%s/%s" % (xml_file_path, xml_file_name)  # name and path of client log file
00150         self._client_log_file_fd = -1      # client log file - file descriptor
00151         self._first_line = range(15)   # save first 15 lines of log file
00152         self._last_byte_number = 0     # save last byte number which was parsed
00153         self._line_number     = 1         #save last line number which was parsed
00154         
00155         if(0 == os.path.exists(self._logfile_path)):
00156             print "Could not locate log file path under %s !\nMaybe change configuration file and try again!\n\n" % self._logfile_path
00157             os._exit(-1)
00158    
00159     def _fetch_first_lines(self, file_name):
00160         '''
00161         This function returns the first 15 lines from current logfile without saving them anywhere.
00162         '''
00163         try:
00164             log_file_fd = open(file_name, "r")
00165                 
00166             listline = range(15)
00167             log_file_fd.seek(0) #set cursor on first position
00168             for i in range(15):
00169                 listline[i] = log_file_fd.readline()
00170             log_file_fd.close()
00171             return listline
00172         except IOError:
00173             print "Could not open file -> ", file_name
00174         
00175     def set_first_lines(self, file_name):
00176         '''
00177         This function saves the first 15 lines of the log file into the member variable.
00178         '''
00179         try:
00180             log_file_fd = open(file_name, 'r')
00181             log_file_fd.seek(0) #set cursor on first position
00182             for i in range(15):
00183                 self._first_line[i] = log_file_fd.readline()
00184             log_file_fd.close()
00185         except IOError:
00186             if self._verbose == 1:
00187                 print "%s -> Could not open file -> \"%s\"" % (time.ctime(), file_name)
00188         
00189     def get_first_lines(self):
00190         '''
00191         This function returns the member variable _first_lines.
00192         '''
00193         return self._first_line
00194     
00195     def test_first_lines(self, file_name):
00196         '''
00197         This function compares the first 15 lines of a log file and return 0 if they are the same, otherwise -1.
00198         '''
00199         listline = self._fetch_first_lines(file_name)
00200         z = 0
00201         while(z<15):
00202             if(listline == self._first_line):
00203                 z += 1
00204             else:
00205                 return -1
00206             
00207         return 0
00208     
00209     def find_size_last_message_tag(self, fd, msg_tag):
00210         '''
00211         This function find out, how many bytes the last message tag needs. This function was necessary, because this new xml messages have to be added into the client xml file. Since the creating of this file is not very straightforward using known techniques like sax or dom, the last tag gets deleted, the new messages added and the last tag writen again.
00212         
00213         fd = file descriptor of the file
00214         msg_tag = message tag to search for
00215         '''
00216         #set cursor back
00217         z = -1
00218         fd.seek(0, 2)
00219 
00220         while(1):
00221             fd.seek(z, 2)
00222             tag = fd.read()
00223             if (-1 != tag.rfind(msg_tag)):
00224                 return z
00225             z -= 1
00226             if -15 == z:
00227                 # message tag was not part of the file
00228                 return 0
00229 
00230     def analyse_log_file(self, parser_file_name, file_time=None):
00231         '''
00232         takes the templog file and goes through each lines and searches for keywords, if keywords are found, the line and the two lines before and after are dumped into a xml file, which the client can collect. This function uses the system function write to create the xml file. The dom function were to ineffectiv and sax unflexible.
00233         
00234         parser_file_name = file name of the file, which needs to be parsed
00235         '''
00236         # determine year of paser file, needed for extract time, since log file content does not provide a year
00237         if file_time != None:    
00238             tupel = time.gmtime(file_time)
00239         else:
00240             status = os.stat(parser_file_name)
00241             file_time = status[8]
00242             tupel = time.gmtime(file_time)
00243         
00244         pf_year = time.strftime("%Y ", tupel)
00245         
00246         byte_count = 0
00247         interrupt = 0
00248         z = 0
00249         
00250         try:
00251             self._client_log_file_fd = file(self._client_log_file, 'r+')
00252 
00253         except IOError:
00254             # create new client log file
00255             self._client_log_file_fd = open(self._client_log_file, 'w')
00256            
00257             xml_header = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?>\n"
00258             self._client_log_file_fd.writelines(xml_header)
00259             self._client_log_file_fd.writelines("<message>\n")
00260             self._client_log_file_fd.writelines("</message>\n")
00261             self._client_log_file_fd.close()
00262             self._client_log_file_fd = file(self._client_log_file, 'r+')
00263             
00264         
00265         # set cursor in file
00266         x = self.find_size_last_message_tag(self._client_log_file_fd, "</message>")
00267         
00268         self._client_log_file_fd.seek(x, 2)
00269         shorten = self._client_log_file_fd.tell()
00270         # delete last </message>
00271         self._client_log_file_fd.truncate(shorten)
00272         self._client_log_file_fd.seek(0, 2)
00273 
00274         #open log file
00275         log_file = parser_file_name
00276         
00277         try:
00278             log_file_fd = open(log_file, "r")
00279             log_file_fd.seek(self._last_byte_number)
00280             
00281         except IOError:
00282             if self._verbose == 1:
00283                 print "%s -> could not open srb log file -> %s" % (time.ctime(), log_file)
00284             return -1
00285             
00286         if self._verbose == 1:
00287             print "%s -> start parsing " % (time.ctime())
00288         starttime = time.time()
00289         while(interrupt == 0):
00290             # read line
00291             content = log_file_fd.readline()
00292             if(content == ''):
00293                 interrupt = 1
00294                 break
00295             
00296             if 0 == len(self._keywords):
00297                 check = 0
00298             else:
00299                 check = self._test_keywords(self._keywords, len(self._keywords)-1, content)
00300             
00301             if(0 == check):
00302                 # extract error number
00303                 error_number = self._extract_error_number(content)
00304 
00305                 if (None == error_number):
00306                     error_number = "-"
00307                     temp = ""
00308                 else:
00309                     temp = int(error_number)
00310                     
00311                 if ( None == find(temp, self._ignore_error)):
00312 
00313                     line_number_string = "%d" % self._line_number
00314                     # delete whitespace
00315                     content_ = content.rstrip()
00316 
00317                     date_time = self._extract_time(content, pf_year)
00318                     
00319                     if (date_time == -1):
00320                         date_time = ["", ""]
00321                         # save current byte count
00322                         byte_count = log_file_fd.tell()
00323                         back = -1
00324                         read = 0
00325                         while(1):
00326                             try:
00327                                 # find time pattern by going back character by character
00328                                 log_file_fd.seek(back, 1)
00329                                 read = back*(-1)
00330                                 tag = log_file_fd.read(read)
00331                                 if (None !=  re.search('^NOTICE: *[A-Z][a-z]{2} +[0-9]{1,2} +[0-9]{2}:[0-9]{2}:[0-9]{2}:', tag)):
00332                                     date_time = self._extract_time(tag, pf_year)
00333                                     break
00334                                 back = back-1
00335                             except IOError:
00336                                 # no time available
00337                                 date_time[0] = time.strftime("%Y-%m-%d", tupel)
00338                                 date_time[1] = time.strftime("%H:%M:%S", tupel)
00339                                  
00340                                 break
00341                     
00342                         #restore byte count
00343                         log_file_fd.seek(byte_count)
00344                         
00345                     z += 1 # entry counter
00346 
00347                     if -1 == self.start_entry('entry'):
00348                         interrupt = 1
00349                         break
00350                     if -1 == self.write_entry('date', date_time[0]):
00351                         interrupt = 1
00352                         break                        
00353                     if -1 == self.write_entry('time', date_time[1]):
00354                         interrupt = 1
00355                         break                        
00356                     if -1 == self.write_entry('error_number', error_number):
00357                         interrupt = 1
00358                         break                        
00359                     if -1 == self.write_entry('error_string', content_):
00360                         interrupt = 1
00361                         break                        
00362                     if -1 == self.write_entry('linenumber', line_number_string):
00363                         interrupt = 1
00364                         break                        
00365                     if -1 == self.end_entry('entry'):
00366                         interrupt = 1
00367                         break                        
00368             
00369             self._line_number += 1
00370         if self._verbose == 1:
00371             print "%s -> end parsing " % (time.ctime())
00372         endtime = time.time()
00373         if self._verbose == 1:
00374             print "%s -> parsing time: %s" % (time.ctime(), (endtime-starttime))
00375         self.end_entry('message')
00376         if self._verbose == 1:
00377             print "%s -> %d errors found\n" % (time.ctime(), z) #--- debug ---
00378         # save last byte number
00379         try:
00380             self._client_log_file_fd.close()
00381         except IOError, e:
00382             if self._verbose == 1:
00383                 print "%s -> Problem closing XML file: \"%s\" !" % (time.ctime(), e)
00384         self._last_byte_number = log_file_fd.tell()
00385     
00386     def start_entry(self, name):
00387         '''
00388         This function inserts a start tag into the XML file. (name = tag name)
00389         '''
00390         start_tag = "<%s>\n" % name
00391         try:
00392             self._client_log_file_fd.write(start_tag)
00393             return 0
00394         except IOError, e:
00395             if self._verbose == 1:
00396                 print "%s -> Problem writing XML file: \"%s\" !" % (time.ctime(), e)
00397             return -1
00398         
00399     def write_entry(self, tagname, content):
00400         '''
00401         This function inserts an entry into the xml file.
00402         
00403         tagname = tag name
00404         content = message between start and end tag
00405         '''
00406        
00407         if len(content) < 50000000:
00408             #find all not allowed character  old: [\x09\x0a\x0d\x20-\xd7]*
00409             bad_character = re.sub('[\x09\x0a\x0d\x20-\x25\x27-\xd7]*', "", content)
00410             # replace each not allowed character with "?"
00411             for i in range(len(bad_character)):
00412                 if bad_character[i] == '\x00':
00413                     # delete NUL character
00414                     content = content.replace(bad_character[i], '')
00415                 else:
00416                     content = content.replace(bad_character[i], "?")
00417     
00418             entry = "<%s>%s</%s>\n" % (tagname, content, tagname)
00419             try:
00420                 self._client_log_file_fd.write(entry)
00421                 return 0
00422             except IOError, e:
00423                 if self._verbose == 1:
00424                     print "%s -> Problem writing XML file: \"%s\" !" % (time.ctime(), e)
00425                 return -1
00426             
00427         else:
00428             entry = "<%s>LOGFILE ENTRY TO LONG !!!</%s>\n" % (tagname, tagname)
00429             try:
00430                 self._client_log_file_fd.write(entry)
00431                 return 0
00432             except IOError, e:
00433                 if self._verbose == 1:
00434                     print "%s -> Problem writing XML file: \"%s\" !" % (time.ctime(), e)
00435                 return -1
00436         
00437     def end_entry(self, name):
00438         '''
00439         This function inserts an end tag into the XML file. (name = tag name)
00440         '''
00441         endtag = "</%s>\n" % name
00442         try:
00443             self._client_log_file_fd.write(endtag)
00444             return 0
00445         except IOError, e:
00446             if self._verbose == 1:
00447                 print "%s -> Problem writing XML file: \"%s\" !" % (time.ctime(), e)
00448             return -1
00449         
00450     def reset(self):
00451         '''
00452         This function resets member variable, in case of a new log file.
00453         '''
00454         self._line_number = 1
00455         self._last_byte_number = 0
00456         
00457     def _test_keywords(self, keywordlist, amount_of_keywords, teststring):
00458         '''
00459         This is a recursive function, which tests if a list of keywords is part of a string (AND relation). If all keywords found 0 is returned, otherwise -1
00460         
00461         keywordlist = list of all keywords
00462         amount_of_keywords = number of keywords in list
00463         teststring = string, which needs to be investigated
00464         
00465         return -1 if line is not interesting
00466         return 0 if line is taken
00467         '''
00468         if (amount_of_keywords == 0):
00469             #last keyword check   -1 != content.rfind("NOTICE")
00470             if( 2 == len(keywordlist[amount_of_keywords])):
00471                 if (-1 == teststring.rfind(keywordlist[amount_of_keywords][0])):
00472                     # not in string go to next keyword
00473                     return 0
00474                 else:
00475                     if( -1 == keywordlist[amount_of_keywords][1].rfind("!")):
00476                         # check for NO keyword
00477                         temp = keywordlist[amount_of_keywords][1].strip("!")
00478                         if ( -1 == teststring.rfind(temp)):
00479                             # go on to next keyword
00480                             return 0
00481                         else:
00482                             return -1
00483                     else:
00484                         # there is no "!"
00485                         if ( -1 != teststring.rfind(keywordlist[amount_of_keywords][1])):
00486                             # string is there, go on to next keyword
00487                             return 0
00488                         else:
00489                             return -1
00490             else:
00491                 if (-1 == teststring.rfind(keywordlist[amount_of_keywords][0])):
00492                     # not in string go to next keyword
00493                     return 0
00494                 else:
00495                     return -1
00496         else:
00497             if( 2 == len(keywordlist[amount_of_keywords])):
00498                 if (-1 == teststring.rfind(keywordlist[amount_of_keywords][0])):
00499                     # not in string go to next keyword
00500                     return self._test_keywords(keywordlist, amount_of_keywords-1, teststring)
00501                 else:
00502                     if( -1 == keywordlist[amount_of_keywords][1].rfind("!")):
00503                         # check for NO keyword
00504                         temp = keywordlist[amount_of_keywords][1].strip("!")
00505                         if ( -1 == teststring.rfind(temp)):
00506                             # go on to next keyword
00507                             return self._test_keywords(keywordlist, amount_of_keywords-1, teststring)
00508                         else:
00509                             return -1
00510                     else:
00511                         # there is no "!"
00512                         if ( -1 != teststring.rfind(keywordlist[amount_of_keywords][1])):
00513                             # string is there, go on to next keyword
00514                             return self._test_keywords(keywordlist, amount_of_keywords-1, teststring)
00515                         else:
00516                             return -1
00517             else:
00518                 if (-1 == teststring.rfind(keywordlist[amount_of_keywords][0])):
00519                     # not in string go to next keyword
00520                     return self._test_keywords(keywordlist, amount_of_keywords-1, teststring)
00521                 else:
00522                     return -1
00523 
00524     def update_keywords(self, keys):
00525         '''
00526         This function updates the member variable keywords.
00527         
00528         keys = new keyword list
00529         '''
00530         self._keywords = keys
00531         
00532     def update_ignore_error(self, error):
00533         '''
00534         This function updates the ignore_error list.
00535         
00536         error = new ignore list
00537         '''
00538         self._ignore_error = error
00539         
00540     def _extract_time(self, time_string, year):
00541         '''
00542         This function takes a line from the logfile and extracts the time from there.
00543         '''
00544         if ( None == re.search('^NOTICE: *[A-Z][a-z]{2} +[0-9]{1,2} +[0-9]{2}:[0-9]{2}:[0-9]{2}:', time_string)):
00545             return -1
00546         else:
00547             listus = time_string.split(":")
00548             zeit = year+listus[1]+":"+listus[2]+":"+listus[3]
00549             time_tupel = time.strptime(zeit, "%Y %b %d %X")
00550             date_time = range(2)
00551             date_time[0] = time.strftime("%Y-%m-%d", time_tupel)
00552             date_time[1] = time.strftime("%H:%M:%S", time_tupel)
00553             return date_time
00554 
00555     def _extract_error_number(self, text):
00556         '''
00557         Thhis function takes a line from the logfile and extract the error number.
00558         '''
00559         match = re.search('(status|errno) *= *-*\d{1,10}', text)
00560         if(None != match):
00561             #if match then give me number
00562             listus = match.string[match.start():match.end()]
00563             listus = re.findall('-*\d{1,10}', listus)
00564             if(1 == len(listus)):
00565                 return listus[0]
00566             else:
00567                 return None
00568         else:
00569             return None
00570 
00571 ################## CLASS  MyParserThread  ##################
00572 
00573 class MyParserThread(threading.Thread):
00574     '''
00575     This class is used to create a thread, which is doing all the necessery work in the background.
00576     '''
00577     def __init__(self, shared, timus, gz_path, logfilepath, logfilename, keyword, ignore_error, xml_file_path, xml_file_name, configfile, keywordfile, verbose):
00578         '''
00579         Constructor
00580         '''
00581         self._keywordfile = keywordfile
00582         temp = os.stat(self._keywordfile)
00583         self._keywordfile_time = temp[8]
00584         self._configfile = configfile
00585         temp = os.stat(self._configfile)
00586         self._configfile_time = temp[8]
00587         self._verbose = verbose
00588         self._id = thread.get_ident()
00589         self._shared_obj = shared
00590         self._interval = timus
00591         self._stop = 0
00592         threading.Thread.__init__(self)
00593         self._parser = LogFileParser (logfilepath, keyword, ignore_error, xml_file_path, xml_file_name, self._verbose)
00594         self._logfilepath = logfilepath
00595         self._log_file_name = logfilepath+"/"+logfilename
00596         self._gz_diect = gz_path
00597         self._list = [] #save *.gz files
00598         
00599     def run(self):
00600         '''
00601         This function overwrites the standard run method.
00602         '''
00603         block_counter = 0
00604         while(1):
00605             # acquire lock
00606             if self._stop == 1:
00607                 print "%s -> working thread stopped !!!" % time.ctime()
00608                 os._exit(0)
00609             # check if config file has changed
00610             if (self._configfile != ""):
00611                 temp = os.stat(self._configfile)
00612                 if self._configfile_time != temp[8]:
00613                     #if time has changed save new time
00614                     if self._verbose == 1:
00615                         print "%s -> config file has changed, reading new values ..." % time.ctime()
00616                     self._configfile_time = temp[8]
00617                     self._refresh_configuration()
00618                 
00619             # check if keyword file has changed
00620             if (self._keywordfile != ''):
00621                 temp = os.stat(self._keywordfile)
00622                 if self._keywordfile_time != temp[8]:
00623                     #if time has changed
00624                     if self._verbose == 1:
00625                         print "%s -> keyword file has changed, reading new values ..." % time.ctime()
00626                     self._keywordfile_time = temp[8]
00627                     self._refresh_keywords()
00628 
00629             if(-1 == self._shared_obj.set_variable_parsing(1, self)):
00630                 # client is busy
00631                 block_counter += 1
00632                 time.sleep(5*block_counter)
00633                 if self._verbose == 1:
00634                     print "%s -> client busy" % time.ctime()
00635                 if block_counter > 5:
00636                     if self._verbose == 1:
00637                         print "%s -> client needs a long time, miss this parsing period" % time.ctime()
00638                     block_counter = 0
00639                     time.sleep(self._interval*60)
00640             else:
00641                 # no client busy
00642                 try:
00643                     
00644                     # check if the first lines the same
00645                     if(0 == self._parser.test_first_lines(self._log_file_name)):
00646                         #if the first 15 lines still the same
00647                         if self._verbose == 1:
00648                             print "%s -> no log file rotation" % time.ctime()
00649                         self._parser.analyse_log_file(self._log_file_name)
00650                     else:
00651                         if self._verbose == 1:
00652                             print "%s -> new log file" % time.ctime()
00653                         # create gz file list
00654                         # empty list for the gz_files
00655                         self._list = []
00656 
00657                         os.path.walk(self._gz_diect, self.parse_directory, self._list)
00658                         if (0 < len(self._list)):
00659                             self.gunzip(self._list[0][1])
00660                             d = os.getcwd()
00661                             try:
00662                                 os.chdir(self._gz_diect)
00663                                 self.gunzip(self._list[0][1])    
00664                                 self._parser.analyse_log_file(self._gz_diect+"/temp_srbLog")
00665                                 delete_file("temp_srbLog", self._verbose)
00666                                 os.chdir(d)
00667                             except:
00668                                 if self._verbose == 1:
00669                                     print "%s -> could not find directory \"%s\"" % (time.ctime(), self._gz_diect)
00670                         else:
00671                             pass
00672                         
00673                         self._parser.reset()
00674                         self._parser.set_first_lines(self._log_file_name)
00675                         self._parser.analyse_log_file(self._log_file_name)
00676                     
00677                 finally:
00678                     
00679                     # release lock
00680                     self._shared_obj.set_variable_parsing(0, self)
00681                         
00682             time.sleep(self._interval*60)#
00683                 
00684             if self._verbose == 1:
00685                 print "\n%s -> -------- parse ------" % time.ctime() #--- debug ---
00686 
00687     def _refresh_keywords(self):
00688         '''
00689         This function gets keywords from the keyword file !
00690         '''
00691         if(1 == os.path.exists(self._keywordfile)):
00692             keyword = get_keywords(self._keywordfile)
00693             self._parser.update_keywords(keyword)
00694             return 0
00695         else:
00696             # if file NOT use old configuration
00697             if self._verbose == 1:
00698                 print "%s -> Sorry, the keyword file does NOT exist !\nUsing old configuration!\n\n" % time.ctime()
00699             return -1
00700                 
00701     def _refresh_configuration(self):
00702         '''
00703         This function gets the needed information from the configfile!
00704         '''
00705         if(1 == os.path.exists(self._configfile)):
00706             config = LoadConfig(self._configfile)
00707             error = config.get("misc.ignore_error")
00708             error = error.strip()
00709             error = error.strip(",")
00710             ignore_error = error.split(",")
00711             for i in range(len(ignore_error)):
00712                 # check if there is an entry at all
00713                 if ignore_error[i] != '':
00714                     try:
00715                         ignore_error[i] = int(ignore_error[i].strip())
00716                     except ValueError:
00717                         if self._verbose == 1:
00718                             print "%s -> \"ignore_error\" in the config file has NO valid values, use old configuration" % time.ctime()
00719                             return -1
00720                 else:
00721                     del ignore_error[i]
00722             
00723             self._parser.update_ignore_error(ignore_error)        
00724             
00725             return 0
00726         else:
00727             # if file NOT exists terminate program
00728             if self._verbose == 1:
00729                 print "%s -> Sorry, the given config file does NOT exist !\nUsing old configuration!\n\n" % time.ctime()
00730             return -1
00731 
00732     def stop_thread(self):
00733         """
00734         Stop the thread
00735         """
00736         self._stop = 1
00737 
00738     def parse_directory(self, arg, dirname, fnames):
00739         '''
00740         This function "walks" through a given directory and considers all srbLOG*.gz files. The name and last modified time are saved in a list (2 dimensional array). The function should be used with os.path.walk(path, function_name, arg)!
00741         '''
00742         d = os.getcwd()
00743         # change into log file directory
00744         try:
00745             os.chdir(dirname)
00746         except:
00747             if self._verbose == 1:
00748                 print "%s -> could not find directory \"%s\"" % (time.ctime(), dirname)
00749             return -1
00750         # for each file   
00751         for f in fnames:
00752             # check if file and if file is a log file e.g. srbLog.20051003.gz
00753             if (not os.path.isfile(f)) or (None == re.search('^srbLog[_0-9.-]*.gz', f)):
00754                 continue
00755             # get last modified time
00756             date = os.stat(f)[stat.ST_MTIME]
00757             # create tupel
00758             tupel = (date, f)
00759             # save last modified time and filename into am arrray (list)
00760             self._list.append(tupel)
00761         # change back into the working directory
00762         os.chdir(d)
00763         # sort list ascending (aufsteigend)
00764         self._list.sort() 
00765         # reverse list order, sorted descending (absteigend), the greater the time number the younger the file
00766         self._list.reverse()
00767         return 0
00768 
00769     def gunzip(self, filus, name_temp_file="temp_srbLog"):
00770         '''
00771         This function unzips a *.gz file using the system tool gunzip. Make sure when calling the function the file exists in this directory. The function creates a temporary file and leave the orignal *.gz file untouched!
00772         '''
00773         if (not os.path.isfile(filus)):
00774             return -1
00775         else:
00776             command = "gunzip -c %s > %s" % (filus, name_temp_file)
00777             try:
00778                 os.system(command)
00779                 return 0
00780             except:
00781                 return -1
00782 
00783 ##########################  C L A S S   M U T E X    #######################
00784 
00785 class Mutex:
00786     '''
00787     This class makes sure that server and client are not accessing the same file at the same time.
00788     '''
00789     # lock
00790     _locked = threading.Lock()
00791     
00792     
00793     def __init__(self):
00794         '''
00795         Constructor
00796         '''
00797         self.parsing = 0
00798         self._parsing_thread_id = 0
00799         self.client = 0
00800         
00801         
00802     def set_variable_parsing(self, value, the_thread):
00803         '''
00804         set variable parsing
00805         '''
00806         Mutex._locked.acquire()  # lock
00807         self._parsing_thread_id = the_thread
00808 
00809         if self.client == 0:
00810             #set variable
00811             self.parsing = value
00812             Mutex._locked.release()
00813             time.sleep(1)
00814             if value == 0:
00815                 # reset parsing thread identity
00816                 self._parsing_thread_id = 0
00817             return 0
00818         else:
00819             Mutex._locked.release()   # release lock
00820             time.sleep(1)
00821             return -1
00822            
00823     def set_variable_client(self, value):
00824         '''
00825         set variable client
00826         '''
00827         Mutex._locked.acquire()  # lock
00828         # if client is not fetching the file
00829         if self.parsing == 0:
00830             #set variable
00831             self.client = value
00832             #print "client variable gesetzt"
00833             Mutex._locked.release()   # release lock
00834             time.sleep(1)
00835             return 0
00836         else:
00837             if (0 != self._parsing_thread_id):
00838                 if (1 != self._parsing_thread_id.isAlive()):
00839                     # if parsing thread dead, reset semphore
00840                     self.parsing = 0
00841                     self._parsing_thread_id = 0
00842             Mutex._locked.release()   # release lock
00843             time.sleep(1)
00844             return -1
00845  
00846 ########################   C L A S S     R P C     #######################
00847  
00848 class RPC:
00849     '''
00850     This class contains the RPC functions.
00851     '''
00852     def __init__(self, verbose, semaphore, config_file, interval, keyword_path, keyword_name, xml_file_path):
00853         '''
00854         constructor
00855         '''
00856         self._verbose = verbose
00857         self._client = True
00858         self._share = semaphore
00859         self._config_file = config_file
00860         self._interval = interval
00861         self._keyword_path = keyword_path
00862         self._keyword_name = keyword_name
00863         self._xml_file_path = xml_file_path
00864         
00865         self._list = [] #for walking through the directory
00866         
00867     def rpc_stop_server(self):
00868         '''
00869         This function stops the server!
00870         '''
00871         command = "./stop_server"
00872         
00873         answer = os.system(command)
00874         print answer
00875         return answer
00876             
00877     def rpc_disable_rpc_calls(self):
00878         '''
00879         This function disables rpc.
00880         '''
00881         self._client = False
00882         if self._verbose == 1:
00883             print "%s -> RPC through \"admin tool\" disabled!" % time.ctime()
00884         return "RPC disabled"
00885             
00886     def rpc_enable_rpc_calls(self):
00887         '''
00888         This function enables rpc.
00889         '''
00890         self._client = True
00891         if self._verbose == 1:
00892             print "%s -> RPC through \"admin tool\" enabled!" % time.ctime()
00893         return "RPC enabled"
00894     
00895     def rpc_status(self):
00896         '''
00897         This function return the current status of the self._client variable.
00898         '''
00899         if self._client == True:
00900             return "RPC enabled"
00901         else:
00902             return "RPC disabled"
00903         
00904     def rpc_interval_status(self):
00905         '''
00906         This function returns the current parsing interval time.
00907         '''
00908         if self._client == True:
00909             return self._interval
00910         else:
00911             return -2
00912     
00913     def rpc_get_my_xml_file(self, filename):
00914         '''
00915         This functions gets the xml file from the server !
00916         '''
00917         if (self._client == True):
00918             if (0 == self._share.set_variable_client(1)):
00919                 # check if file is available
00920                 try:
00921                     filus = self._xml_file_path+"/"+filename
00922                     client_xml_fd = open(filus, 'r')
00923                     file_content = client_xml_fd.read()
00924                     client_xml_fd.close()
00925                 except  IOError:
00926                     self._share.set_variable_client(0)
00927                     return "no file"
00928                 #delete xml file
00929                 if (0 == delete_file((self._xml_file_path+"/"+filename), self._verbose)):
00930                     self._share.set_variable_client(0) # reset variable
00931                     return file_content
00932                 else:
00933                     if self._verbose == 1:
00934                         print "problems deleting file"
00935                     self._share.set_variable_client(0)
00936                     return -1
00937             else:
00938                 return -3   # server is busy parsing
00939         else:
00940             return -2 # rpc disalbed
00941 
00942     def rpc_check_availabitity(self):
00943         '''
00944         This function check if the server is still in the parsing process.
00945         '''
00946         if (self._client == True):
00947             if (0 == self._share.set_variable_client(1)):
00948                 return 0
00949             else:
00950                 return -3
00951         else:
00952             return -2
00953 
00954     def rpc_get_file_list(self):
00955         '''
00956         This function walks through the *.xml directory and finds all files, which need to be fetched form the client.
00957         '''
00958         if (self._client == True):
00959             self._list = [] # empty list
00960             try:
00961                 os.path.walk(self._xml_file_path, self._parse_directory, self._list)
00962                 if (0 < len(self._list)):
00963                     return self._list
00964                 else:
00965                     return 0
00966             except:
00967                 return -1
00968         else:
00969             return -2 # rpc disalbed
00970         
00971     def rpc_update_configuration(self, section, key, value, action):
00972         '''
00973         This functions adds or deletes values in the config.ini.
00974         action:
00975         0 = delete
00976         1 = add
00977         2 = exchange
00978         4 = info
00979         '''
00980         print "section: ", section
00981         print "key: ", key
00982         print "value: ", value
00983         print "action: ", action
00984         if(self._client == True):
00985             try:
00986                 config_fd = file(self._config_file, 'r+')
00987             except IOError, e:
00988                 return "Problem -> %s" % e
00989             
00990             byte_count = 0
00991             
00992             while(1):
00993                 byte_count = config_fd.tell()
00994                 line = config_fd.readline()
00995                 if line == '':
00996                     if self._verbose == 1:
00997                         print "%s -> Section: \"%s\" and key: \"%s\" do not exist in config file!" % (time.ctime(), section, key)
00998                     config_fd.close()
00999                     return "Section: \"%s\" and key: \"%s\" do not exist in config file!" % (section, key)
01000                 if (-1 != line.find(section)):
01001                     while(1):
01002                         byte_count = config_fd.tell()
01003                         line = config_fd.readline()
01004                         if line == '':
01005                             if self._verbose == 1:
01006                                 print "%s -> Key: \"%s\" do not exist under section \"%s\" in config file!" % (time.ctime(), key, section)
01007                             config_fd.close()
01008                             return "Key \"%s\" do not exist under section \"%s\" in config file!" % (key, section)
01009                         if (-1 != line.find(key) and -1 != line.find("=") and -1 == line.find("#", 0, 1)):
01010                             if action == 0:
01011                                 if self._verbose == 1:
01012                                     print "%s -> Delete \"%s:%s\" value \"%s\"" % (time.ctime(), section, key, value)
01013                                 listus = line.split("=")
01014                                 listus[1] = listus[1].strip()
01015                                 listus[1] = listus[1].strip(",")
01016                                 listus = listus[1].split(",")
01017                                 for i in range(len(listus)):
01018                                     listus[i] = listus[i].strip()
01019                                 new_content = ''
01020                                 for i in range(len(listus)):
01021                                     if int(listus[i]) != value:                      
01022                                         new_content = new_content+"%s, " % listus[i]
01023                                         
01024                                 new_content = new_content.strip()
01025                                 new_content = new_content.strip(",")
01026                                 new_content = "%s = %s\n" % (key, new_content)
01027                                 rest = config_fd.read()
01028                                 config_fd.truncate(byte_count)
01029                                 config_fd.seek(byte_count)                            
01030                                 config_fd.writelines(new_content)
01031                                 config_fd.write(rest)
01032                                 config_fd.close()                            
01033                                 return "Changes applied: %s" % new_content                            
01034                             elif action == 1:
01035                                 if self._verbose == 1:
01036                                     print "%s -> Add \"%s:%s\" value \"%s\"" % (time.ctime(), section, key, value)
01037                                 listus = line.split("=")
01038                                 listus[1] = listus[1].strip()
01039                                 listus[1] = listus[1].strip(",")
01040                                 listus = listus[1].split(",")
01041                                 finish = 0
01042                                 while (finish == 0):
01043                                     if len(listus) == 0:
01044                                         finish = 1
01045                                     for i in range(len(listus)):
01046                                         finish = 1 # break the while loop
01047                                         print listus[i]
01048                                         listus[i] = listus[i].strip()
01049                                         
01050                                         try:
01051                                             print "test auf int"
01052                                             temp_value = int(listus[i])
01053                                             if value == temp_value:
01054                                                 config_fd.close()
01055                                                 return "Value %d already exists!" % value
01056                                         except ValueError, e:
01057                                             finish = 0 # activate while loop
01058                                             print e, listus[i]
01059                                             del listus[i]
01060                                             break
01061                                     
01062                                     
01063                                 new_content = ''
01064                                 for i in range(len(listus)):
01065                                     new_content = new_content+"%s, " % listus[i]
01066                                 new_content = "%s = %s%s\n" % (key, new_content, value)
01067                                 rest = config_fd.read()
01068                                 config_fd.truncate(byte_count)
01069                                 config_fd.seek(byte_count)                            
01070                                 config_fd.writelines(new_content)
01071                                 config_fd.write(rest)
01072                                 config_fd.close()
01073                                 return "Changes applied: %s" % new_content
01074                             elif action == 2:
01075                                 # exchange
01076                                 if self._verbose == 1:
01077                                     print "%s -> Change \"%s:%s\" to value \"%s\"" % (time.ctime(), section, key, value)
01078                                 rest = config_fd.read()
01079                                 config_fd.truncate(byte_count)
01080                                 config_fd.seek(byte_count)
01081                                 new_content = "%s = %s\n" % (key, value)
01082                                 config_fd.writelines(new_content)
01083                                 config_fd.write(rest)
01084                                 config_fd.close()
01085                                 if section == 'misc' and key == 'minute':
01086                                     self._interval = int(value)
01087                                 return 0
01088                             elif action == 4:
01089                                 config_fd.close()
01090                                 return line
01091                             else:
01092                                 config_fd.close()
01093                                 return -1
01094         else:
01095             return "RPC disabled" # rpc disalbed
01096         
01097     def rpc_update_keyword_file(self, keyword, action):
01098         '''
01099         This function updates the keyword file.
01100         
01101         action:
01102         0 = delete
01103         1 = add
01104         2 = info
01105         '''
01106         if(self._client == True):
01107             byte_count = 0
01108             file_size = 0
01109             comments = ''
01110     
01111             filus = self._keyword_path+"/"+self._keyword_name
01112     
01113             try:
01114                 key_fd = file(filus, 'r+')
01115             except IOError, e:
01116                 if self._verbose == 1:
01117                     print "%s -> Problem open keyword file -> %s !" % (time.ctime(), e)
01118                 return "Problem -> %s" % e
01119     
01120             key_fd.seek(0, 2) # set cursor to end of file
01121             file_size = key_fd.tell()
01122             key_fd.seek(0) # set cursor to begining of file
01123     
01124             while(1):
01125                 # get comments 
01126                 byte_count = key_fd.tell()
01127                 line = key_fd.readline()
01128                 if byte_count >= file_size:
01129                     break
01130                 if (-1 != line.find("#", 0, 1)):
01131                     comments += line
01132             
01133             key_fd.close()
01134             keyword_list = get_keywords(filus)
01135             keyword = keyword.split(":")
01136             
01137             if action == 0:
01138                 # test if keyword is already there
01139                 for i in range(len(keyword_list)):
01140                     if 1 == len(keyword) and 1 == len(keyword_list[i]):
01141                         if keyword[0] == keyword_list[i][0]:
01142                             del keyword_list[i]
01143                             try:
01144                                 key_fd = file(filus, 'w+')
01145                                 print key_fd
01146                                 key_string = list_to_string(keyword_list)
01147                                 key_fd.write(comments)
01148                                 key_fd.write(key_string)
01149                                 key_fd.close()                            
01150                             except IOError, e:
01151                                 if self._verbose == 1:
01152                                     print "%s -> Problem open keyword file -> %s !" % (time.ctime(), e)
01153                                 return "Problem -> %s" % e                        
01154                             return "keyword %s from keyword list deleted" % keyword
01155                     elif 2 == len(keyword) and 2 == len(keyword_list[i]):
01156                         if keyword[0] == keyword_list[i][0] and keyword[1] == keyword_list[i][1]:
01157                             del keyword_list[i]
01158                             try:
01159                                 key_fd = file(filus, 'w+')
01160                                 key_string = list_to_string(keyword_list)
01161                                 key_fd.write(comments)
01162                                 key_fd.write(key_string)
01163                                 key_fd.close()                                                        
01164                             except IOError, e:
01165                                 if self._verbose == 1:
01166                                     print "%s -> Problem open keyword file -> %s !" % (time.ctime(), e)
01167                                 return "Problem -> %s" % e                        
01168                             return "keyword %s from keyword list deleted" % keyword
01169                         
01170                 return "keyword %s was not part of keyword list" % keyword
01171                 
01172             if action == 1:
01173                 # test if keyword is already there
01174                 for i in range(len(keyword_list)):
01175                     if 1 == len(keyword) and 1 == len(keyword_list[i]):
01176                         if keyword[0] == keyword_list[i][0]:
01177                             return "keyword %s already in keyword list" % keyword
01178                     elif 2 == len(keyword) and 2 == len(keyword_list[i]):
01179                         if keyword[0] == keyword_list[i][0] and keyword[1] == keyword_list[i][1]:
01180                             return "keywords %s already in keyword list" % keyword
01181     
01182                 keyword_list.append(keyword)
01183                 try:
01184                     key_fd = file(filus, 'w+')
01185                 except IOError, e:
01186                     if self._verbose == 1:
01187                         print "%s -> Problem open keyword file -> %s !" % (time.ctime(), e)
01188                     return "Problem -> %s" % e
01189                 
01190                 key_string = list_to_string(keyword_list)
01191                 key_fd.write(comments)
01192                 key_fd.write(key_string)
01193                 key_fd.close()
01194                     
01195                 return key_string
01196             
01197             if action == 2:
01198                 return list_to_string(keyword_list)
01199         
01200         else:
01201             return "RPC disabled"
01202 
01203     def _parse_directory(self, arg, dirname, fnames):
01204         '''
01205         This function "walks" through a given directory and looks for the client_log.xml file. The name and last modified time are saved in a list (2 dimensional array). The function should be used with os.path.walk(path, function_name, arg)!
01206         
01207         dirname = directory which need to be pared
01208         fnames = files within dirname
01209         '''
01210         d = os.getcwd()
01211         # change into log file directory
01212         try:
01213             os.chdir(dirname)
01214         except:
01215             if self._verbose == 1:
01216                 print "could not find directory \"%s\"" % dirname
01217             return -1
01218         # for each file   
01219         for f in fnames:
01220             # check if file and if file is a log file e.g. client_log.xml
01221             if (not os.path.isfile(f)) or (None == re.search('client_log.xml', f)):
01222                 continue
01223             else:
01224             # save filename into an arrray (list)
01225                 self._list.append(f)
01226         # change back into the working directory
01227         os.chdir(d)
01228 
01229 
01230 
01231 
01232 
01233 
01234 
01235 

Generated on Sun Mar 5 18:06:11 2006 for Server by  doxygen 1.4.6-NO