Парсинг логов с использованием регулярных выражений на Python и их упаковка в структуру данных словарь словарей

import string
import re
 
allLogs = r"""2018-07-24 00:01:33,711 WARN  [r3223r2r3] [] [] [ewqrwerwer] [ewfwefwef] - Failed;
2018-07-24 00:01:33,712 DEBUG [r3223r2r3] [] [] [ewqrwerwer] [ewfwefwef] - Sample log
2018-07-24 00:02:33,712 DEBUG [r3223r2r3] [] [] [ewqrwerwer] [ewfwefwef] - Sample log
2018-07-24 00:20:33,830 DEBUG [r3223r2r3] [] [] [ewqrwerwer] [ewfwefwef] - Sample log
2018-07-24 00:20:37,731 DEBUG [r3223r2r3] [] [] [3r2r32r] [c.i.s.h.r.i.RssNewsServiceImpl] - 3r232r
2018-07-24 10:33:45,852 ERROR [r3223r2r3] [fgd] [gr] [reger] [regre] - Sample log
javax.servlet.ServletException: Something bad happened
   at com.example.myproject.OpenSessionInViewFilter.doFilter(OpenSessionInViewFilter.java:60)
   at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
   at com.example.myproject.ExceptionHandlerFilter.doFilter(ExceptionHandlerFilter.java:28)
   at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
   at com.example.myproject.OutputBufferFilter.doFilter(OutputBufferFilter.java:33)
   at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
   at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388)
   at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
   at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
   at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
   at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:418)
   at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
   at org.mortbay.jetty.Server.handle(Server.java:326)
   at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
   at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:943)
   at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
   at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
   at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
   at org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)
   at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Caused by: com.example.myproject.MyProjectServletException
   at com.example.myproject.MyServlet.doPost(MyServlet.java:169)
   at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
   at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
   at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)"""
 
keyPattern = re.compile(r"(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})")
valuePatternWithErrorMessage = re.compile(r"(?P<level>INFO|ERROR|WARN|TRACE|DEBUG|FATAL)\s+\[(?P<firstSquareBrackets>[^\]]*)]\s+\[(?P<secondSquareBrackets>[^\]]*)]\s+\[(?P<thirdSquareBrackets>[^\]]*)]\s+\[(?P<fourthSquareBrackets>[^\]]*)]\s+\[(?P<fifthSquareBrackets>[^\]]*)] - (?P<textMessage>.*?)(?=\Z|;\s+|\n)|(?P<exception>^.+Exception: .+)|(?P<stacktrace>^\s+at .+)|(^\s+... \d+ more)|(^\s*Caused by:.+)")
valuePatternWithoutErrorMessage = re.compile(r"(?P<level>INFO|ERROR|WARN|TRACE|DEBUG|FATAL)\s+\[(?P<firstSquareBrackets>[^\]]*)]\s+\[(?P<secondSquareBrackets>[^\]]*)]\s+\[(?P<thirdSquareBrackets>[^\]]*)]\s+\[(?P<fourthSquareBrackets>[^\]]*)]\s+\[(?P<fifthSquareBrackets>[^\]]*)] - (?P<textMessage>.*?)(?=\Z|;\s+|\n)")
 
keyLogslist = keyPattern.findall(allLogs)
 
#из лога вырезается timestamp и перед level оставляется пробел,
#разделение идет по \n вместе с этим пробелом
valueLogs = keyPattern.sub("",allLogs)
valueLogslist = valueLogs.split("\n ")
 
logDict = dict(zip(keyLogslist, valueLogslist))
 
for keysTimestamp in keyLogslist:
    match = valuePatternWithErrorMessage.search(logDict[keysTimestamp])
    if match:
        logDict[keysTimestamp] = match.groupdict()
    else:            
        match = valuePatternWithoutErrorMessage.search(logDict[keysTimestamp])
        if match:
            logDict[keysTimestamp] = match.groupdict()
        else:
            logDict[keysTimestamp] = "None"
 
print(logDict)
print(len(logDict))
print(type(logDict))

pattern = r'''(?xm)
^(\d{4}-\d{2}-\d{2}\ \d{2}:\d{2}:\d{2},\d{3})\ +
(?P<level>INFO|ERROR|WARN|TRACE|DEBUG|FATAL)\ +
\[ (?P<firstSquareBrackets> [^\]]* ) \] \ +
\[ (?P<secondSquareBrackets>[^\]]* ) \] \ +
\[ (?P<thirdSquareBrackets> [^\]]* ) \] \ +
\[ (?P<fourthSquareBrackets>[^\]]* ) \] \ +
\[ (?P<fifthSquareBrackets> [^\]]* ) \] \ +
- \ + (?P<textMessage>.*?)(?:\Z|;\s+|\n)
(?:
    (?P<exception> .+ Exception: .+)\n
    (?P<stacktrace> .+ (?:\n(?!\d{4}-\d{2}-\d{2}\ \d{2}:\d{2}:\d{2},\d{3}).+)* )
)?
'''
 
logDict = {}
for x in re.finditer(pattern, allLogs):
    logDict[x.group(1)] = x.groupdict()
 
 
print(logDict)