Metaflow/community

Hey all, I have a question about logging. In our project, we are using python standard logging. (https://docs.python.org/3/howto/logging.html) When we send a warning, debug vs. logs with it, Metaflow overrides these logs and sends it's to info.

Here is a code example;

import logging.config

from metaflow import FlowSpec, step

LOGGING_CONFIG = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format': '[%(levelname)s] %(name)s: %(message)s'
        },
    },
    'handlers': {
        'default': {
            'level': 'INFO',
            'formatter': 'standard',
            'class': 'logging.StreamHandler',
            'stream': 'ext://sys.stdout',
        },
    },
    'loggers': {
        '': {  # root logger
            'handlers': ['default'],
            'level': 'INFO',
            'propagate': False
        },
    }
}

class DebugFlow(FlowSpec):

    @step
    def start(self):
        self.next(self.a, self.b)

    @step
    def a(self):
        logger.debug("Hello Debug log")
        self.x = 1
        self.next(self.join)

    @step
    def b(self):
        self.x = int('2')
        logger.warning("Hello warning log")
        self.next(self.join)

    @step
    def join(self, inputs):
        logger.info('a is %s', inputs.a.x)
        logger.info('b is %s', inputs.b.x)
        logger.info('total is %d', sum(input.x for input in inputs))
        logger.error("Hello error log")
        self.next(self.end)

    @step
    def end(self):
        pass

if __name__ == '__main__':
    logger = logging.getLogger('DebugFlow')
    DebugFlow()

When I took a look at how Metaflow handles logging, I realized that Metaflow uses different logging systems. I also tested logging configuration with --event-logger. it looks like it doesn't work.

import logging.config

from metaflow.plugins import LOGGING_SIDECAR, SIDECAR

from metaflow import FlowSpec, step


LOGGING_CONFIG = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format': '[%(levelname)s] %(name)s: %(message)s'
        },
    },
    'handlers': {
        'default': {
            'level': 'INFO',
            'formatter': 'standard',
            'class': 'logging.StreamHandler',
            'stream': 'ext://sys.stdout',
        },
    },
    'loggers': {
        '': {  # root logger
            'handlers': ['default'],
            'level': 'INFO',
            'propagate': False
        },
    }
}


class DebugFlow(FlowSpec):

    @step
    def start(self):
        self.next(self.a, self.b)

    @step
    def a(self):
        logger.debug("Hello Debug log")
        self.x = 1
        self.next(self.join)

    @step
    def b(self):
        self.x = int('2')
        logger.warning("Hello warning log")
        self.next(self.join)

    @step
    def join(self, inputs):
        logger.info('a is %s', inputs.a.x)
        logger.info('b is %s', inputs.b.x)
        logger.info('total is %d', sum(input.x for input in inputs))
        logger.error("Hello error log")
        self.next(self.end)

    @step
    def end(self):
        pass


class CustomEventLogger(object):
    TYPE = 'customEventLogger'

    def __init__(self):
        self.logger = logging.getLogger('DebugFlow')

    def log(self, msg):
        self.logger.info('event_logger: %s', str(msg))

    def process_message(self, msg):
        # type: (Message) -> None
        self.log(msg.payload)

    def shutdown(self):
        pass


def setup_logger():
    logger_config = {
        'customEventLogger': CustomEventLogger
    }

    LOGGING_SIDECAR.update(logger_config)
    SIDECAR.update(logger_config)
    logging.config.dictConfig(LOGGING_CONFIG)


if __name__ == '__main__':
    setup_logger()
    logger = logging.getLogger('DebugFlow')
    DebugFlow()

python debug_flow.py --event-logger=customEventLogger run

How can I configure the Metaflow logger? if it is not possible, how can I send debug, warning logs with Metaflow logger? Thanks.