Iprep compatibility

suricata/update/category.py

@@ -0,0 +1,157 @@
+# Copyright (C) 2017-2019 Open Information Security Foundation
+# Copyright (c) 2020 Michael Schem
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+""" Module for parsing categories.txt files.
+
+Parsing is done using regular expressions and the job of the module is
+to do its best at parsing out fields of interest from the categories file
+rather than perform a sanity check.
+
+"""
+
+from __future__ import print_function
+
+from suricata.update import fileparser
+
+import io
+import re
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Compile a re pattern for basic iprep directive matching
+category_pattern = re.compile(r"^(?P<enabled>#)*[\s#]*"
+                              r"(?P<id>\d+),"
+                              r"(?P<short_name>\w+),"
+                              r"(?P<description>.*$)")
+
+class Category(dict):
+    """ Class representing an iprep category
+
+    The category class also acts like a dictionary.
+
+    Dictionary fields:
+
+    - **enabled**: True if the category is enabled (uncommented), false is
+      disabled (commented)
+    - **id**: The maximum value for the category id is hard coded at 60
+      currently (Suricata 5.0.3).
+    - **short_name**: The shortname that refers to the category.
+    - **description**: A description of the category.
+
+    :param enabled: Optional parameter to set the enabled state of the category
+
+    """
+
+    def __init__(self, enabled=None):
+        dict.__init__(self)
+        self["enabled"] = enabled
+        self["id"] = None
+        self["short_name"] = None
+        self["description"] = None
+
+    def __getattr__(self, name):
+        return self[name]
+
+    @property
+    def id(self):
+        """ The ID of the category.
+
+        :returns: An int ID of the category
+        :rtype: int
+        """
+        return int(self["id"])
+
+    @property
+    def idstr(self):
+        """Return the gid and sid of the rule as a string formatted like:
+        '[id]'"""
+        return "[%s]" % str(self.id)
+
+    def __str__(self):
+        """ The string representation of the category.
+
+        If the category is disabled it will be returned as commented out.
+        """
+        return self.format()
+
+    def format(self):
+        return "{0}{1},{2},{3}".format(u"" if self["enabled"] else u"# ",
+                                       self['id'],
+                                       self['short_name'],
+                                       self['description'])
+
+
+def parse(buf, group=None):
+    """ Parse a single Iprep category from a string buffer.
+
+    :param buf: A string buffer containing a single Iprep category.
+
+    :returns: An instance of a :py:class:`.Category` representing the parsed Iprep category
+    """
+
+    if type(buf) == type(b""):
+        buf = buf.decode("utf-8")
+    buf = buf.strip()
+
+    m = category_pattern.match(buf)
+    if not m:
+        return None
+
+    if m.group("enabled") == "#":
+        enabled = False
+    else:
+        enabled = True
+
+    # header = m.group("header").strip()
+
+    category = Category(enabled=enabled)
+
+    category["id"] = int(m.group("id").strip())
+
+    if not 0 < category["id"] < 60:
+        logging.error("Category id of {0}, not valid. Id is required to be between 0 and 60.".format(category["id"]))
+        return None
+
+    category["short_name"] = m.group("short_name").strip()
+
+    category["description"] = m.group("description").strip()
+
+    return category
+
+
+def parse_fileobj(fileobj, group=None):
+    """ Parse multiple ipreps from a file like object.
+
+    Note: At this time ipreps must exist on one line.
+
+    :param fileobj: A file like object to parse rules from.
+
+    :returns: A list of :py:class:`.Iprep` instances, one for each rule parsed
+    """
+    return fileparser.parse_fileobj(fileobj, parse, group)
+
+def parse_file(filename, group=None):
+    """ Parse multiple ipreps from the provided filename.
+
+    :param filename: Name of file to parse ipreps from
+
+    :returns: A list of :py:class:`.Iprep` instances, one for each iprep parsed
+    """
+    with io.open(filename, encoding="utf-8") as fileobj:
+        return parse_fileobj(fileobj, group)
+

suricata/update/commands/addsource.py

@@ -37,6 +37,8 @@ def register(parser):
                         help="Additional HTTP header to add to requests")
     parser.add_argument("--no-checksum", action="store_false",
                         help="Skips downloading the checksum URL")
+    parser.add_argument("--iprep", action="store_true",
+                        help="Identifies source as an IPRep Source")
     parser.set_defaults(func=add_source)
 
 
@@ -67,6 +69,8 @@ def add_source():
 
     header = args.http_header if args.http_header else None
 
+    is_iprep = args.iprep
+
     source_config = sources.SourceConfiguration(
-        name, header=header, url=url, checksum=checksum)
+        name, header=header, url=url, checksum=checksum, is_iprep=is_iprep)
     sources.save_source_config(source_config)

suricata/update/config.py

@@ -49,6 +49,7 @@
 DROP_CONF_KEY = "drop-conf"
 LOCAL_CONF_KEY = "local"
 OUTPUT_KEY = "output"
+IPREP_OUTPUT_KEY = "iprep"
 DIST_RULE_DIRECTORY_KEY = "dist-rule-directory"
 
 if has_defaults:
@@ -83,6 +84,7 @@
     # The default file patterns to ignore.
     "ignore": [
         "*deleted.rules",
+        ".*"
     ],
 }
 
@@ -136,6 +138,12 @@ def get_output_dir():
         return _config[OUTPUT_KEY]
     return os.path.join(get_state_dir(), "rules")
 
+def get_iprep_dir():
+    """Get the rule output directory."""
+    if IPREP_OUTPUT_KEY in _config:
+        return _config[IPREP_OUTPUT_KEY]
+    return os.path.join(get_state_dir(), "iprep")
+
 def args():
     """Return sthe parsed argument object."""
     return _args

suricata/update/fileparser.py

@@ -0,0 +1,77 @@
+# Copyright (C) 2017-2019 Open Information Security Foundation
+# Copyright (c) 2020 Michael Schem
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+""" Module for files with either rules, ipreps, or category files.
+
+Parse funcitons
+
+"""
+
+from __future__ import print_function
+
+import sys
+import re
+import logging
+import io
+
+
+logger = logging.getLogger(__name__)
+
+
+def parse_fileobj(fileobj, parse, group=None):
+    """ Parse multiple line based items from a file like object.
+
+    Note: At this point items must exist on one line
+
+    :param fileobj: A file like object to parse items from.
+    :param parse: A function used to parse items.
+
+    :returns: A list of :py:class:`.Rule`, :py:class:`.Iprep`, or :py:class:`.Category`
+    instances depending on what parsing function is passed in.
+    """
+    items = []
+    buf = ""
+    for line in fileobj:
+        try:
+            if type(line) == type(b""):
+                line = line.decode()
+        except:
+            pass
+        if line.rstrip().endswith("\\"):
+            buf = "%s%s " % (buf, line.rstrip()[0:-1])
+            continue
+        buf = buf + line
+        try:
+            item = parse(buf, group)
+            if item:
+                items.append(item)
+        except Exception as err:
+            logger.error("Failed to parse: %s: %s", buf.rstrip(), err)
+        buf = ""
+    return items
+
+
+def parse_file(filename, group=None):
+    """ Parse multiple rules from the provided filename.
+
+    :param filename: Name of file to parse ipreps from
+
+    :returns: A list of .rules files or :py:class:`.Iprep` instances
+    for .list files, one for each rule parsed
+    """
+    with io.open(filename, encoding="utf-8") as fileobj:
+        return parse_fileobj(fileobj, group)