Add generic Zino config file

.gitignore

@@ -19,3 +19,4 @@ old-events/
 # avoid accidental commits of config and state files
 polldevs.cf
 zino-state.json
+zino.toml

README.md

@@ -190,6 +190,18 @@ Zino will check `polldevs.cf` for changes on a scheduled interval while it's
 running, so any changes made while Zino is running should be picked up without
 requiring a restart of the process.
 
+### Configuring other settings
+
+Other settings can be also configured in a separate [TOML](https://toml.io/en/) file,
+which defaults to `zino.toml` in the current working directory, but a different file
+can be specified using the `--config-file` command line option.
+
+See the [zino.toml.example](./zino.toml.example) file for the settings that can be
+configured and their default values.
+
+Zino does not currently check `zino.toml` for changes on a scheduled interval while
+it's running, so Zino needs to be restarted for changes to take effect.
+
 ### Configuring API users
 
 Zino 2 reimplements the text-based (vaguely SMTP-esque) API protocol from Zino

changelog.d/224.added.md

@@ -0,0 +1 @@
+Add generic Zino config file

pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
     "pysnmplib",
     "pyasn1<0.5.0",
     "aiodns",
+    "tomli; python_version < '3.11'",
 ]
 dynamic = ["version"]
 

src/zino/api/legacy.py

@@ -16,7 +16,7 @@
 from zino import version
 from zino.api import auth
 from zino.api.notify import Zino1NotificationProtocol
-from zino.state import ZinoState
+from zino.state import ZinoState, config
 from zino.statemodels import ClosedEventError, Event, EventState
 
 if TYPE_CHECKING:
@@ -38,7 +38,7 @@ def __init__(
         self,
         server: Optional["ZinoServer"] = None,
         state: Optional[ZinoState] = None,
-        secrets_file: Optional[Union[Path, str]] = "secrets",
+        secrets_file: Optional[Union[Path, str]] = None,
     ):
         """Initializes a protocol instance.
 
@@ -58,7 +58,7 @@ def __init__(
         self._authentication_challenge: Optional[str] = None
 
         self._state = state if state is not None else ZinoState()
-        self._secrets_file = secrets_file
+        self._secrets_file = secrets_file or config.authentication.file
 
     @property
     def peer_name(self) -> Optional[str]:

src/zino/config/__init__.py

@@ -1,0 +1,41 @@
+from typing import Optional
+
+try:
+    from tomllib import TOMLDecodeError, load
+except ImportError:
+    from tomli import TOMLDecodeError, load
+
+from .models import Configuration
+
+
+class InvalidConfigurationError(Exception):
+    """The configuration file is invalid toml"""
+
+
+def read_configuration(config_file_name: str, poll_file_name: Optional[str] = None) -> Configuration:
+    """
+    Reads and validates config toml file
+
+    Returns configuration if file name is given and file exists
+
+    Raises InvalidConfigurationError if toml file is invalid,
+    OSError if the config toml file could not be found and
+    pydantic.ValidationError if values in it are invalid or the specified files
+    don't exist
+    """
+    with open(config_file_name, mode="rb") as cf:
+        try:
+            config_dict = load(cf)
+        except TOMLDecodeError:
+            raise InvalidConfigurationError
+
+    # Polldevs by command line argument will override config file entry
+    if poll_file_name:
+        if "polling" not in config_dict.keys():
+            config_dict["polling"] = {"file": poll_file_name}
+        else:
+            config_dict["polling"]["file"] = poll_file_name
+
+    config = Configuration.model_validate(obj=config_dict, strict=True)
+
+    return config

src/zino/config/models.py

@@ -1,15 +1,30 @@
 """Zino configuration models"""
 
 from ipaddress import IPv4Address, IPv6Address
+from os import R_OK, access
+from os.path import isfile
 from typing import Optional, Union
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
+from pydantic.functional_validators import AfterValidator
+from typing_extensions import Annotated
 
 DEFAULT_INTERVAL_MINUTES = 5
+STATE_FILENAME = "zino-state.json"
+EVENT_DUMP_DIR = "old-events"
+POLLFILE = "polldevs.cf"
 
 IPAddress = Union[IPv4Address, IPv6Address]
 
 
+def validate_file_can_be_opened(filename: str) -> str:
+    assert isfile(filename) and access(filename, R_OK), f"File {filename} doesn't exist or isn't readable"
+    return filename
+
+
+ExistingFileName = Annotated[str, AfterValidator(validate_file_can_be_opened)]
+
+
 # config fields and default values from
 # https://gitlab.sikt.no/verktoy/zino/blob/master/common/config.tcl#L18-44
 class PollDevice(BaseModel):
@@ -30,3 +45,41 @@ class PollDevice(BaseModel):
     hcounters: bool = False
     do_bgp: bool = True
     port: int = 161
+
+
+class Archiving(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    old_events_dir: str = EVENT_DUMP_DIR
+
+
+class Authentication(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    file: ExistingFileName = "secrets"
+
+
+class Persistence(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    file: str = STATE_FILENAME
+    period: int = DEFAULT_INTERVAL_MINUTES
+
+
+class Polling(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    file: ExistingFileName = POLLFILE
+    period: int = 1
+
+
+class Configuration(BaseModel):
+    """Class for keeping track of the configuration set by zino.toml"""
+
+    # throw ValidationError on extra keys
+    model_config = ConfigDict(extra="forbid")
+
+    archiving: Archiving = Archiving()
+    authentication: Authentication = Authentication()
+    persistence: Persistence = Persistence()
+    polling: Polling = Polling()

src/zino/events.py

@@ -20,7 +20,6 @@
 _log = logging.getLogger(__name__)
 
 EVENT_EXPIRY = timedelta(hours=8)
-EVENT_DUMP_DIR = "old-events"
 
 
 class EventIndex(NamedTuple):
@@ -166,10 +165,13 @@ def commit(self, event: Event, user: str = "monitor"):
 
     def _delete(self, event: Event):
         """Removes a closed event from the events dict and notifies all observers"""
+        from zino.state import config
+
         if event.state != EventState.CLOSED:
             return
 
-        event.dump_event_to_file(dir_name=f"{EVENT_DUMP_DIR}/{now().year}-{now().month}/{now().day}")
+        base_dir = config.archiving.old_events_dir
+        event.dump_event_to_file(dir_name=f"{base_dir}/{now().year}-{now().month}/{now().day}")
         index = EventIndex(event.router, event.subindex, type(event))
         if self._closed_events_by_index.get(index) and event.id == self._closed_events_by_index[index].id:
             del self._closed_events_by_index[index]

src/zino/getuptime.py

@@ -6,6 +6,7 @@
 
 from zino.config.polldevs import read_polldevs
 from zino.snmp import SNMP
+from zino.state import config
 
 _log = logging.getLogger(__name__)
 
@@ -19,7 +20,7 @@
 
 
 async def run(args: argparse.Namespace):
-    devices = {d.name: d for d in read_polldevs("polldevs.cf")}
+    devices = {d.name: d for d in read_polldevs(config.polling.file)}
     device = devices[args.router]
 
     snmp = SNMP(device)
@@ -28,8 +29,8 @@
 
 
 def parse_args():
-    devicenames = [d.name for d in read_polldevs("polldevs.cf")]
-    parser = argparse.ArgumentParser(description="Fetch sysUptime from a device in polldevs.cf")
+    devicenames = [d.name for d in read_polldevs(config.polling.file)]
+    parser = argparse.ArgumentParser(description="Fetch sysUptime from a device in the pollfile")
     parser.add_argument("router", type=str, help="Zino router name", choices=devicenames)
     return parser.parse_args()
 

src/zino/scheduler.py

@@ -39,7 +39,7 @@ def get_scheduler() -> AsyncIOScheduler:
 
 
 def load_polldevs(polldevs_conf: str) -> Tuple[Set, Set]:
-    """Loads polldevs.cf into process state.
+    """Loads pollfile into process state.
 
     :returns: A tuple of (new_devices, deleted_devices)
     """

src/zino/state.py

@@ -8,20 +8,21 @@
 
 from pydantic import BaseModel, Field
 
-from zino.config.models import IPAddress, PollDevice
+from zino.config.models import Configuration, IPAddress, PollDevice
 from zino.events import Events
 from zino.planned_maintenance import PlannedMaintenances
 from zino.statemodels import DeviceStates
 
 _log = logging.getLogger(__name__)
-STATE_FILENAME = "zino-state.json"
 
 # Dictionary of configured devices
 polldevs: Dict[str, PollDevice] = {}
 
 # Global (sic) state
 state: "ZinoState" = None
 
+config: Configuration = Configuration()
+
 
 class ZinoState(BaseModel):
     """Holds all state that Zino needs to persist between runtimes"""
@@ -31,14 +32,14 @@ class ZinoState(BaseModel):
     addresses: dict[IPAddress, str] = {}
     planned_maintenances: PlannedMaintenances = Field(default_factory=PlannedMaintenances)
 
-    def dump_state_to_file(self, filename: str = STATE_FILENAME):
+    def dump_state_to_file(self, filename: str):
         """Dumps the full state to a file in JSON format"""
         _log.debug("dumping state to %s", filename)
         with open(filename, "w") as statefile:
             statefile.write(self.model_dump_json(exclude_none=True, indent=2))
 
     @classmethod
-    def load_state_from_file(cls, filename: str = STATE_FILENAME) -> Optional["ZinoState"]:
+    def load_state_from_file(cls, filename: str) -> Optional["ZinoState"]:
         """Loads and returns a previously persisted ZinoState from a JSON file dump.
 
         :returns: A ZinoState object if the state file was found, None if it wasn't.  If the state file is invalid or

src/zino/trapd.py

@@ -93,7 +93,7 @@ class TrapReceiver:
 
     A major difference to Zino 1 is that this receiver must explicitly be configured with SNMP community strings that
     will be accepted.  Zino 1 accepts traps with any community string, as long as their origin is any one of the
-    devices configured in `polldevs.cf`.  However, PySNMP places heavy emphasis on being standards compliant,
+    devices configured in the pollfile.  However, PySNMP places heavy emphasis on being standards compliant,
     and will not even pass on traps to our callbacks unless they match the authorization config for the SNMP engine.
     """
 

src/zino/zino.py

@@ -12,10 +12,11 @@
 from typing import Optional
 
 import tzlocal
+from pydantic import ValidationError
 
 from zino import state
 from zino.api.server import ZinoServer
-from zino.config.models import DEFAULT_INTERVAL_MINUTES
+from zino.config import InvalidConfigurationError, read_configuration
 from zino.scheduler import get_scheduler, load_and_schedule_polldevs
 from zino.statemodels import Event
 from zino.trapd import TrapReceiver
@@ -26,6 +27,7 @@
 STATE_DUMP_JOB_ID = "zino.dump_state"
 # Never try to dump state more often than this:
 MINIMUM_STATE_DUMP_INTERVAL = timedelta(seconds=10)
+DEFAULT_CONFIG_FILE = "zino.toml"
 _log = logging.getLogger("zino")
 
 
@@ -35,7 +37,20 @@
         level=logging.INFO if not args.debug else logging.DEBUG,
         format="%(asctime)s - %(levelname)s - %(name)s (%(threadName)s) - %(message)s",
     )
-    state.state = state.ZinoState.load_state_from_file() or state.ZinoState()
+    try:
+        state.config = read_configuration(args.config_file or DEFAULT_CONFIG_FILE, args.polldevs)
+    except OSError:
+        if args.config_file:
+            _log.fatal(f"No config file with the name {args.config_file} found.")
+            sys.exit(1)
+    except InvalidConfigurationError:
+        _log.fatal(f"Configuration file with the name {args.config_file or DEFAULT_CONFIG_FILE} is invalid TOML.")
+        sys.exit(1)
+    except ValidationError as e:
+        _log.fatal(e)
+        sys.exit(1)
+
+    state.state = state.ZinoState.load_state_from_file(state.config.persistence.file) or state.ZinoState()
     init_event_loop(args)
 
 
@@ -66,13 +81,18 @@
     scheduler.add_job(
         func=load_and_schedule_polldevs,
         trigger="interval",
-        args=(args.polldevs.name,),
-        minutes=1,
+        args=(state.config.polling.file,),
+        minutes=state.config.polling.period,
         next_run_time=datetime.now(),
     )
-    # Schedule state dumping every DEFAULT_INTERVAL_MINUTES and reschedule whenever events are committed
+    # Schedule state dumping as often as configured in
+    # 'config.persistence.period' and reschedule whenever events are committed
     scheduler.add_job(
-        func=state.state.dump_state_to_file, trigger="interval", id=STATE_DUMP_JOB_ID, minutes=DEFAULT_INTERVAL_MINUTES
+        func=state.state.dump_state_to_file,
+        trigger="interval",
+        args=(state.config.persistence.file,),
+        id=STATE_DUMP_JOB_ID,
+        minutes=state.config.persistence.period,
     )
     # Schedule planned maintenance
     scheduler.add_job(
@@ -179,7 +199,16 @@
 def parse_args(arguments=None):
     parser = argparse.ArgumentParser(description="Zino is not OpenView")
     parser.add_argument(
-        "--polldevs", type=argparse.FileType("r"), metavar="PATH", default="polldevs.cf", help="Path to polldevs.cf"
+        "--polldevs",
+        type=str,
+        required=False,
+        help="Path to the pollfile",
+    )
+    parser.add_argument(
+        "--config-file",
+        type=str,
+        required=False,
+        help="Path to zino configuration file",
     )
     parser.add_argument(
         "--debug", action="store_true", default=False, help="Set global log level to DEBUG. Very chatty!"
@@ -197,8 +226,6 @@
         "--user", metavar="USER", help="Switch to this user immediately after binding to privileged ports"
     )
     args = parser.parse_args(args=arguments)
-    if args.polldevs:
-        args.polldevs.close()  # don't leave this temporary file descriptor open
     return args