handle invalid json in log

Author: oxycoder

src/opnsense/scripts/syslog/logformats/grafana.py

@@ -10,10 +10,16 @@ def __init__(self, filename):
     def match(self, line):
         # need to check file name here otherwise it will fallback to another LogFormater
         # weird thing happen in FormatContainer.get_format that take first line match base on piority.
-        if self._filename.find('grafana') == -1:
+        if self._filename.find("grafana") == -1:
             return None
-        if bool(line and not line.isspace()):
+        try:
             return json.loads(line)
+        except:
+            if bool(line and not line.isspace()):
+                self._parts["msg"] = line
+                return True
+            else:
+                return None
 
     def get_key(self, key):
         try:
@@ -23,17 +29,20 @@ def get_key(self, key):
 
     def set_line(self, line):
         super().set_line(line)
-        self._parts = json.loads(self._line)
+        try:
+            self._parts = json.loads(self._line)
+        except:
+            self._parts["msg"] = line
 
     @property
     def timestamp(self):
-        return self.get_key('t')
+        return self.get_key("t")
 
     @property
     def line(self):
-        msg = self.get_key('msg')
-        err = self.get_key('error')
-        reason = self.get_key('reason')
+        msg = self.get_key("msg")
+        err = self.get_key("error")
+        reason = self.get_key("reason")
         if err is not None:
             msg += ". Error: " + err
         if reason is not None:
@@ -42,11 +51,11 @@ def line(self):
 
     @property
     def process_name(self):
-        return self.get_key('logger')
+        return self.get_key("logger")
 
     @property
     def severity(self):
-        level = self.get_key('level')
+        level = self.get_key("level")
         if level == "info":
             return 6
         if level == "debug":

src/opnsense/scripts/syslog/logformats/promtail.py

@@ -10,10 +10,17 @@ def __init__(self, filename):
     def match(self, line):
         # need to check file name here otherwise it will fallback to another LogFormater
         # weird thing happen in FormatContainer.get_format that take first line match base on piority.
-        if self._filename.find('promtail') == -1:
+        if self._filename.find("promtail") == -1:
             return None
-        if bool(line and not line.isspace()):
+        try:
             return json.loads(line)
+        except:
+            if bool(line and not line.isspace()):
+                self._parts["msg"] = line
+                return True
+            else:
+                return None
+            
 
     def get_key(self, key):
         try:
@@ -23,17 +30,20 @@ def get_key(self, key):
 
     def set_line(self, line):
         super().set_line(line)
-        self._parts = json.loads(self._line)
+        try:
+            self._parts = json.loads(self._line)
+        except:
+            self._parts["msg"] = line
 
     @property
     def timestamp(self):
-        return self.get_key('ts')
+        return self.get_key("ts")
 
     @property
     def line(self):
-        msg = self.get_key('msg')
-        err = self.get_key('err')
-        reason = self.get_key('reason')
+        msg = self.get_key("msg")
+        err = self.get_key("err")
+        reason = self.get_key("reason")
         if err is not None:
             msg += ". Error: " + err
         if reason is not None:
@@ -42,11 +52,11 @@ def line(self):
 
     @property
     def process_name(self):
-        self.get_key('caller')
+        self.get_key("caller")
 
     @property
     def severity(self):
-        level = self.get_key('level')
+        level = self.get_key("level")
         if level == "info":
             return 6
         if level == "debug":