Add rotating file log handler and simplify streamed log messages (dat…

…alab-org#545)

pydatalab/pydatalab/config.py

@@ -141,6 +141,11 @@ class ServerConfig(BaseSettings):
         description="The path under which to place stored files uploaded to the server.",
     )
 
+    LOG_FILE: Union[str, Path] = Field(
+        Path(__file__).parent.joinpath("../logs/datalab.log").resolve(),
+        description="The path to the log file to use for the server and all associated processes (e.g., invoke tasks)",
+    )
+
     DEBUG: bool = Field(True, description="Whether to enable debug-level logging in the server.")
 
     TESTING: bool = Field(
@@ -273,6 +278,17 @@ def deactivate_backup_strategies_during_testing(cls, values):
 
         return values
 
+    @validator("LOG_FILE")
+    def make_missing_log_directory(cls, v):
+        """Make sure that the log directory exists and is writable."""
+        try:
+            v = Path(v)
+            v.parent.mkdir(exist_ok=True, parents=True)
+            v.touch(exist_ok=True)
+        except Exception as exc:
+            raise RuntimeError(f"Unable to create log file at {v}") from exc
+        return v
+
     class Config:
         env_prefix = "pydatalab_"
         extra = "allow"

pydatalab/pydatalab/logger.py

@@ -1,8 +1,11 @@
 import logging
+import logging.handlers
 import time
 from functools import wraps
 from typing import Callable, Optional
 
+LOG_FORMAT_STRING = "%(asctime)s | %(levelname)-8s: %(message)s (PID: %(process)d - %(name)s: %(pathname)s:%(funcName)s:%(lineno)d)"
+
 
 class AnsiColorHandler(logging.StreamHandler):
     """Colourful and truncated log handler, exfiltrated from/inspired
@@ -21,24 +24,12 @@ class AnsiColorHandler(logging.StreamHandler):
 
     max_width = 2000
 
-    def __init__(self) -> None:
-        super().__init__()
-        self.formatter = logging.Formatter(
-            "%(asctime)s | %(levelname)-8s: %(message)s (PID: %(process)d - %(name)s: %(pathname)s:%(funcName)s:%(lineno)d)"
-        )
-
     def format(self, record: logging.LogRecord) -> str:
-        from flask_login import current_user
-
-        prefix = "🔓"
-        if current_user and current_user.is_authenticated:
-            prefix = "🔒"
         message: str = super().format(record)
         if len(message) > self.max_width:
             message = message[: self.max_width] + "[...]"
         color = self.LOGLEVEL_COLORS[record.levelno]
-        message = f"\x1b[{color} {prefix} {message}\x1b[0m"
-        return message
+        return f"\x1b[{color} {message}\x1b[0m"
 
 
 def setup_log(log_name: str = "pydatalab", log_level: Optional[int] = None) -> logging.Logger:
@@ -60,8 +51,14 @@ def setup_log(log_name: str = "pydatalab", log_level: Optional[int] = None) -> l
     logger = logging.getLogger(log_name)
     logger.handlers = []
     logger.propagate = False
-    handler = AnsiColorHandler()
-    logger.addHandler(handler)
+    stream_handler = AnsiColorHandler()
+    stream_handler.setFormatter(logging.Formatter(LOG_FORMAT_STRING))
+    rotating_file_handler = logging.handlers.RotatingFileHandler(
+        CONFIG.LOG_FILE, maxBytes=1000000, backupCount=100
+    )
+    rotating_file_handler.setFormatter(logging.Formatter(LOG_FORMAT_STRING))
+    logger.addHandler(stream_handler)
+    logger.addHandler(rotating_file_handler)
     if log_level is None:
         log_level = logging.INFO