F:/thesis_austausch/dissertation/code_docu_doxygen/GZ_Parser/gz_parser.py

Go to the documentation of this file.

#!/usr/bin/env python

'''
This is the gz_parser.py module, which uses an external config file (e.g. config_gz_parser.ini) to parse through a directory with *.gz files. The server_classes.py is also needed.

Reading University
MSc in Network Centered Computing
a.weise - a.weise@reading.ac.uk - December 2005
'''

import os, sys, string, re, stat
from server_classes import LogFileParser
import ConfigParser, getopt

gz_list = [] #save *.gz files
  

def LoadConfig(file, config={}):
    """
    This functions returns a dictionary with key's of the form
    <section>.<option> and the values .
    
    source: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65334
    """
    config = config.copy()
    cp = ConfigParser.ConfigParser()
    cp.read(file)
    for sec in cp.sections():
        name = string.lower(sec)
        for opt in cp.options(sec):
            config[name + "." + string.lower(opt)] = string.strip(cp.get(sec, opt))
    return config

def parse_directory(arg, dirname, fnames):
    '''
    This function "walks" through a given directory and considers all srbLOG*.gz files. The name and last modified time are saved in a list (2 dimensional array). The function should be used with os.path.walk(path, function_name, arg)!
    '''
    d = os.getcwd()
    # change into log file directory
    try:
        os.chdir(dirname)
    except:
        print "could not find directory \"%s\"" % dirname
        return -1
    # for each file   
    for f in fnames:
        # check if file and if file is a log file e.g. srbLog.20051003.gz
        if (not os.path.isfile(f)) or (None == re.search('^srbLog[_0-9.-]*.gz', f)):
            continue
        # get last modified time
        date = os.stat(f)[stat.ST_MTIME]
        # create tupel
        tupel = (date, f)
        # save last modified time and filename into am arrray (list)
        gz_list.append(tupel)
    # change back into the working directory
    os.chdir(d)

def get_keywords(filus):
    '''
    This function extracts keyword from a give file!
    '''
    keys = []
    
    try:
        file_fd = file(filus, 'r')
    except IOError, e:
        print "Problem with keyword file -> ",  e
        return -1
    
    content = file_fd.readlines()# save file content as list (1 line == 1 entry)

    file_fd.close()

    content = remove_item(content, "#") # remove comments
    content = remove_item(content, "\n")# remove linebreaks

    for i in range(len(content)):
        content[i] = content[i].strip()
        content[i] = content[i].rstrip(",")
        content[i] = content[i].split(",")
        for a in range(len(content[i])):
            keys.append(content[i][a])
    
    for i in range(len(keys)):
        keys[i] = keys[i].strip() # remove whitespace
        keys[i] = keys[i].split(":")
        
    return keys
        
def remove_item(listus, item):
    '''
    This function removes "items" form a list object rekursiv.
    '''
    
    while(1):
        
        for i in range(len(listus)):
            if -1 != listus[i].find(item, 0, 1):
                del listus[i]
                remove_item(listus, item)
                break
        else:
            break
        
    return listus
     
def gunzip(filus, name_temp_file="temp_srbLog"):
    '''
    This function unzips a *.gz file using the system tool gunzip. Make sure when calling the function the file exists in this directory. The function creates a temporary file and leave the orignal *.gz file untouched!
    '''
    if (not os.path.isfile(filus)):
        return -1
    else:
        command = "gunzip -c %s > %s" % (filus, name_temp_file)
        os.system(command)
        return 0
    
def delete_file(filus):
    ''' 
    This functions deletes a given file.
    '''
    try:
        os.remove(filus)
        return 0
    except:
        print "could not delete -> ", filus
        return -1

def usage_exit(progname, msg=None):
    '''
    This function displays the usage of the program and terminated the script.
    '''
    if msg:
        print msg
        print
    print "usage: %s -h|--help -c|--config -v|--verbose " % progname
    os._exit(-1)

########################################################################

def start():
    '''
    This function starts the application.
    '''
    global gz_list
    gz_list = [] #save *.gz files

    configfile = ""
    verbose = 0

    # evaluate parameters    
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'c:vh', ['config=', 'verbose', 'help'])
        for opt, value in opts:
            if opt in ('-h','--help'):
                msg = "Help:\n-c or --config\t->\tdefines config file, if no config file given, default values are used\n-v or --verbose\t->\tactivates printing of messages [debug option]\n-h or --help\t->\tprints this help"
                usage_exit(sys.argv[0], msg)
            elif opt in ('-c','--config'):
                value = value.replace("=", "")
                configfile = os.getcwd()+"/"+value
            elif opt in ('-v','--verbose'):
                verbose = 1
            else:
                usage_exit(sys.argv[0], "Wrong use of parameter")
    except getopt.error, e:
        usage_exit(sys.argv[0], e)
    
    # load config file or default values
    if (configfile != ""):
        # check if file exists
        if(1 == os.path.exists(configfile)):
            config = LoadConfig(configfile)
        else:
            # if file NOT exists terminate program
            print "Sorry, a given file does NOT exist !\nPlease try again!\n\n"
            os._exit(-1)
    else:
        msg = "\nNo config file spezified !\n"
        usage_exit(sys.argv[0], msg)

    print "\n\n------ GZ SRB LOG FILE PARSER ------"
    
    workingpath = os.getcwd()
    
    path_srb_gz = config.get("path.path_srb_gz")
    path_srb_gz = path_srb_gz.rstrip("/")
    path_xml_file = config.get("path.path_xml_file")
    path_xml_file = path_xml_file.rstrip("/")
    xml_file_name = "gz_client_log.xml"
    
    # check if the configuration is correct
    if(0 == os.path.exists(path_srb_gz)):
        print "Could not locate log file archive path under %s !\nMaybe change configuration file and try again!\n\n" % path_srb_gz
        os._exit(-1)
    
    if(0 == os.path.exists(path_xml_file)):
        print "Could not locate xml path under %s !\nMaybe change configuration file and try again!\n\n" % path_xml_file
        os._exit(-1)
        
    keyword = config.get("file.keyword")
    keyword_path = config.get("path.path_keyword")
    if keyword != None:
        keyword = keyword.strip()
    if keyword_path != None:
        keyword_path = keyword_path.rstrip("/")
    if(keyword_path == '' or keyword_path == None):
        keyword_path = workingpath
    else:
        if (-1 != keyword_path.find("/", 0, 1)):
            # first character "/"
            pass
        else:
            keyword_path = workingpath+"/"+keyword_path

    keyword_list = get_keywords(keyword_path+"/"+keyword)
        
    ignore_error = config.get("misc.ignore_error")
    if ("" != ignore_error):
        ignore_error = ignore_error.split(",")
        for i in range(len(ignore_error)):
            ignore_error[i] = int(ignore_error[i].strip())
    
    parserus = LogFileParser(path_srb_gz, keyword_list, ignore_error, os.getcwd(), "temp_client_log.xml", verbose)

    os.path.walk(path_srb_gz, parse_directory, gz_list)
    d = os.getcwd()
    os.chdir(path_srb_gz)
    if (0 < len(gz_list)):
        try:
            for x in range(len(gz_list)):
                print "\n"
                print x+1,
                print ". parsing -> \"%s\"\n" % gz_list[x][1]
                gunzip(gz_list[x][1])
                status = os.stat(gz_list[x][1])
                parserus.analyse_log_file("temp_srbLog", file_time=status[8])
                delete_file("temp_srbLog")
        except:
            os.remove("temp_srbLog")
            os.chdir(d)
            os.remove("temp_client_log.xml")
            print "Problem parsing log files -> terminating !"
            os._exit(0)
       
    else:
        print "Could not find any srbLog*.gz files!"
        os._exit(0)
    
    os.chdir(d)
    
    
    test_file = "%s/%s" % (path_xml_file, xml_file_name)
    
    # check if a gz_client_log.xml already there, if yes change name
    c = 1
    while(1):
        if(0 == os.path.exists(test_file)):
            break
        test_file = "%s/%d_%s" % (path_xml_file, c, xml_file_name)
        c += 1
        
    print "\ncopy xml file ..."
    command = "cp temp_client_log.xml %s" % test_file
    os.system(command)
    
    # delete temporary xml file
    delete_file("temp_client_log.xml")
    
    print "\n\ndone ... \n\n"

if __name__ == '__main__':
    start()

---