日后在做整理

#!/usr/bin/env python# -*- coding: utf-8 -*-# Created by weilai on 2016/12/21# 自动分析火车票发送和接受日志工具，wl 201611221# 用于分析火车票发送和收通知的内容 和时差# v0.1import osimport datetimeimport timeimport jsonimport re# 定义基础目录path = 'X:\\AccDetail'send_log_base_path = 'X:\\InteractionDetail'receive_log_base_path = 'X:\\InteractionDetail'now = datetime.datetime.now()today_dir = '20161221' #now.strftime('%Y%m%d')# 定义基础目录和时间目录组合work_dir = path + '\\' + today_dirsend_log_path = send_log_base_path + '\\' + today_dirreceive_log_path = receive_log_base_path + '\\' + today_dir# 定义发送，接收，结果集合book_list = []receive_list = []ticket_list = []pattern_log_body = r'\[BEGIN\]([\s\S.]*?)\[END\]'pattern_log_time = r'\[L_B\]时间戳:(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\[L_E\]'pattern_log_run_time = r'\[L_B\]时间间隔:(\d{7}\.\d{3})\[L_E\]'pattern_log_func_name = r'\[L_B\](自定义消息:book)\[L_E\]'pattern_log_send_content = r'\[L_B\]发送内容:j=([\s\S.]*?)\[L_E\]'pattern_log_receive_content = r'\[L_B\]接收内容:([\s\S.]*?)\[L_E\]'pattern_log_notice_func_name = r'\[L_B\](自定义消息:预定结果通知)\[L_E\]'pattern_log_notice_send_content = r'\[L_B\]发送内容:([\s\S.]*?)\[L_E\]'pattern_log_notice_receive_content = r'\[L_B\]接收内容:j=([\s\S.]*?)\[L_E\]'class InteractiveInfo:    """初级分析结果类"""    log_time = ''    log_name = ''    log_run_time = ''    log_param = ''    log_result = ''    flow_id = ''    order_id = ''    out_order_id = ''    msg = ''    log_file = ''    valid_data = False    train = ''    passager = ''    ticket = ''class TicketInfo:    """最后分析结果类"""    ticket_order_id = ''    ticket_out_order_id = ''    ticket_flow_id = ''    ticket_send_time = ''    ticket_receive_time = ''    time_diff = 0    receive_times = 0    info_from_file = ''    msg = ''    source_send_data = None    source_receive_data = None# ------------------------------------------------def regex_text(pattern_str, txt):    pattern = re.compile(pattern_str, re.M)    match = pattern.findall(txt)    if match:        return match    return Nonedef regex_log_body(txt):    return regex_text(pattern_log_body, txt)def regex_log_time(txt):    return regex_text(pattern_log_time, txt)def regex_log_run_time(txt):    return regex_text(pattern_log_run_time, txt)def regex_log_func_name(txt):    return regex_text(pattern_log_func_name, txt)def regex_log_send_content(txt):    return regex_text(pattern_log_send_content, txt)def regex_log_receive_content(txt):    return regex_text(pattern_log_receive_content, txt)def regex_log_notice_func_name(txt):    return regex_text(pattern_log_notice_func_name, txt)def regex_log_notice_receive_content(txt):    return regex_text(pattern_log_notice_receive_content, txt)# ----------------------------------------------def analyse_send_file(log_file_path):    with open(log_file_path, 'r', encoding='utf-8') as f:        print(log_file_path)        log_file_str = f.read()        logs = regex_log_body(log_file_str)        for log in logs:            log_item = log            log_name = regex_log_func_name(log_item)            if log_name is not None:                send_log = InteractiveInfo()                send_log.log_name = log_name[0]                send_log.log_file = log_file_path                # print(log_name[0])                log_time = regex_log_time(log_item)                send_log.log_time = log_time[0]                # print(log_time[0])                log_run_time = regex_log_run_time(log_item)                send_log.log_run_time = log_run_time[0]                # print(log_run_time[0])                send_content = regex_log_send_content(log_item)                get_send_json(send_content[0], send_log)                # print(send_content[0])                receive_content = regex_log_receive_content(log_item)                get_send_result_json(receive_content[0], send_log)                # print(receive_content[0])                book_list.append(send_log)# ---------------------------------------------------def get_send_result_json(json_str, send_obj):    json_val = json.loads(json_str)    send_obj.flow_id = str(json_val['i'])    send_obj.msg += json_val['m'] + "|"def get_send_json(json_str, send_obj):    json_val = json.loads(json_str)    send_obj.order_id = json_val['d']['n']    x1 = lambda x: x['pn']    send_obj.passager = ','.join([x1(y) for y in json_val['d']['p']])    send_obj.train = '{0}|{1}|{2}|{3}'.format(json_val['d']['c'], json_val['d']['d'], json_val['d']['fn'], json_val['d']['tn'])    # print(send_obj.train)# -----------------------------------------------def analyse_receive_file(log_file_path):    with open(log_file_path, 'r', encoding='utf-8') as f:        print(log_file_path)        log_file_str = f.read()        logs = regex_log_body(log_file_str)        for log in logs:            log_item = log            log_name = regex_log_notice_func_name(log_item)            if log_name is not None:                receive_log = InteractiveInfo()                receive_log.log_name = log_name[0]                # print(receive_log.log_name)                receive_log.log_file = log_file_path                # print(log_name[0])                log_time = regex_log_time(log_item)                receive_log.log_time = log_time[0]                # print(log_time[0])                log_run_time = regex_log_run_time(log_item)                receive_log.log_run_time = log_run_time[0]                # print(log_run_time[0])                # 无需这两句 不分解发送内容                # send_content = regex_log_send_content(log_item)                # get_receive_json(send_content[0], receive_log)                # print(send_content[0])                receive_content = regex_log_notice_receive_content(log_item)                get_receive_result_json(receive_content[0], receive_log)                # print(receive_content[0])                receive_list.append(receive_log)def get_receive_result_json(json_str, receive_obj):    json_val = json.loads(json_str)    receive_obj.flow_id = str(json_val['i'])    receive_obj.out_order_id = str(json_val['d']['i'])    receive_obj.order_id = str(json_val['d']['n'])    receive_obj.msg += json_val['m'] + "|"    x1 = lambda x: x['tn']+'|'+x['si']+'|'+x['pr']    if 't' in json_val['d']:        receive_obj.ticket = json_val['d']['t'] + '|'    if 'p' in json_val['d']:        receive_obj.ticket = ','.join([x1(y) for y in json_val['d']['p']])    receive_obj.ticket.replace('\n', '')    # print(receive_obj.ticket)def get_receive_json(json_str, send_obj):    pass    # -----------------------------------------------# -----------------------------------------------def analyse_file(analyse_dir, dir_str, analyse_func):    """分析日志"""    file_list = os.listdir(analyse_dir)    for dir_name in file_list:        if dir_name.find(dir_str) >= 0:            full_path = analyse_dir + '\\' + dir_name            # print(full_path)            files = os.listdir(full_path)            for file in files:                file_path = full_path + '\\' + file                analyse_func(file_path)# ----------------------------------------------def analyse_send_log(analyse_dir):    analyse_file(analyse_dir, '火车票交互服务', analyse_send_file)def analyse_receive_log(analyse_dir):    analyse_file(analyse_dir, '火车票业务收通知网站基线', analyse_receive_file)def show_send_list(source_list):    """显示分析的发送日志"""    global book_list    book_list = sorted(source_list, key=lambda x: x.log_time)    i = 0    p_template = '{0})=={1}=={2}=={3}=={4}=={5};'    for item in book_list:        i += 1        p_str = p_template.format(str(i), item.log_time, item.log_name, item.flow_id, item.order_id, item.msg)        print(p_str)def show_receive_list(source_list):    """显示分析的接收日志"""    global receive_list    receive_list = sorted(source_list, key=lambda x: x.log_time)    i = 0    p_template = '{0})=={1}=={2}=={3}=={4}=={5};'    for item in receive_list:        i += 1        m = item.msg        if item.msg == '|':            m += item.ticket        p_str = p_template.format(str(i), item.log_time, item.log_name, item.flow_id, item.order_id, m)        print(p_str)# ----------------------------------------------def analyse_diff(send_list, result_list):    """对比发送数据和接收数据，组合对照数据，并计算运行时差"""    for b_item in send_list:        for r_item in result_list:            if b_item.flow_id == r_item.flow_id and b_item.flow_id != '': # 不能为空字符串                has_it = [x for x in ticket_list if x.ticket_flow_id == b_item.flow_id]                if len(has_it) <= 0:                    t = TicketInfo()                    t.ticket_send_time = b_item.log_time                    t.ticket_receive_time = r_item.log_time                    t.ticket_flow_id = b_item.flow_id                    t.ticket_order_id = b_item.order_id                    t.ticket_out_order_id = r_item.out_order_id                    t.msg = b_item.msg + '|' + r_item.msg                    t.receive_times = 1                    t.time_diff = 0                    t.info_from_file = '{f1}|{f2}'.format(f1=b_item.log_file, f2=r_item.log_file)                    time_a1 = time.strptime(t.ticket_receive_time, "%Y-%m-%d %H:%M:%S.%f")                    time_stamp1 = datetime.datetime(*time_a1[:6])                    time_a2 = time.strptime(t.ticket_send_time, "%Y-%m-%d %H:%M:%S.%f")                    time_stamp2 = datetime.datetime(*time_a2[:6])                    t.time_diff = (time_stamp1 - time_stamp2).total_seconds()                    # print(t.time_diff)                    ticket_list.append(t)                else:                    has_it[0].receive_times += 1# -------------------------------------------------def show_result_list(source_list):    """显示分析组合结果"""    global ticket_list    ticket_list = sorted(source_list, key=lambda x: x.ticket_send_time)    i = 0    p_template = '{0})={1}=={2}=={3}-{4}[{5}秒]{6}次'    for item in ticket_list:        i += 1        p_str = p_template.format(str(i), item.ticket_flow_id, item.ticket_order_id, item.ticket_send_time,                                  item.ticket_receive_time, str(item.time_diff), str(item.receive_times), item.info_from_file)        print(p_str)# ---------------------------------------------------print('/***********火车票发送交互服务日志[{0}]***********/'.format(today_dir))analyse_send_log(send_log_path)show_send_list(book_list)print('/***********火车票接收通知日志[{0}]***********/'.format(today_dir))analyse_receive_log(receive_log_path)show_receive_list(receive_list)print('/***********火车票占座时差计算[{0}]***********/'.format(today_dir))analyse_diff(book_list, receive_list)show_result_list(ticket_list)# 样品xx = """[BEGIN][L_B]日志ID:bb1f4d67-162f-47d0-90b0-c27bfbe218cb[L_E][L_B]时间点:636179106143493242[L_E][L_B]跟踪ID:00517WCF16122109492672990[L_E][L_B]时间戳:2016-12-21 09:50:14.349[L_E][L_B]时间间隔:0047620.052[L_E][L_B]应用程序名:火车票交互服务v0029[L_E][L_B]模块名:火车票交互[L_E][L_B]方法名:[L_E][L_B]方法说明:[L_E][L_B]进程ID:25968[L_E][L_B]线程ID:27[L_E][L_B]发送地址:http://121.41.85.136/a.ashx[L_E][L_B]异常ID:[L_E][L_B]异常级别:[L_E][L_B]自定义消息:book[L_E][L_B]交互类型:[L_E][L_B]关键信息1:1612210950140910f0500236688[L_E][L_B]关键信息2:[L_E][L_B]机器名:WIN-VDHFP5GHJ89[L_E][L_B]本地IP:172.17.1.242[L_E][L_B]发送内容:j={"a": true,"b":[{"PassagerID":1,"TicketKeyID":"1688"}],"c": "s55666","d": {  "c":"D2244","d":"2016-12-31","n":"161221095016770236688","w":false,"p":[{"pi":"555555555555555554","pn":"某","po":1,"ic":"1","xc":"O","pc":"1"}],"fc":"ICW","fn":"成都东","tc":"NIW","tn":"遂宁"    },     "f": "book",     "r":"http://9999.com:1401/Notify",   "s": "c722f24f1ec88271a3c0aed7e70f6835",            "t": "20161221095014",        "w": 30000 }[L_E][L_B]接收内容:{  "c": 0,  "d": null,  "m": "请等待异步",  "s": true,  "w": true,  "i": 2016122100000000298}[L_E][L_B]路由:r+zYFqAUxzhIEINyz9RyMppo9SMJ1oVwbqlQwyzn0Wol+lxGUB1MwjcWz88Pa5ah7StDIUP/0XBirsjsZMiW0BfC8lHJzAILcpwLva4Q8/zrzra0mjlKFg@@[L_E][END]"""y = re.findall(r'\[BEGIN\]([\s\S.]*?)\[END\]', xx, re.M)print(y)