Python metaclasses & descriptors


Athens Python users meetup

Metaclasses

type

class MyClass(object):
    pass

MyClass            # __main__.MyClass
MyClass.__class__  # type
type(MyClass())    # __main__.MyClass
type(MyClass)      # type
type(object)       # type
type(type)         # type

>>> print(type.__doc__)
type(object_or_name, bases, dict)
type(object) -> the object's type
type(name, bases, dict) -> a new type

#      class     parent     class
#      name      classes    __dict__
type( 'MyClass',     (),        {}     )  # returns __main__.MyClass

Klass = type('Klass', (), {})

class Klass(object):
    pass

Klass2 = type('Klass2', (Klass, ), {})

class Klass2(Klass):
    pass

Klass3 = type('Klass3', (Klass, Klass2), {'a': 42})

class Klass3(Klass, Klass2):
    a = 42

Klass4 = type(
  'Klass4',
  (),
  {'a': 42, 'method': lambda self, num: self.a + num}
)

class Klass4(object):
    a = 42
    def method(self, x):
        return self.a + x

Metaclass API

Metaclass.__prepare__(mcls, name, bases)  # classmethod
Metaclass.__new__(mcls, name, bases, attrs, **kwargs)
Metaclass.__init__(cls, name, bases, attrs, **kwargs)
Metaclass.__call__(cls, *args, **kwargs)

__prepare__

  • returns dict-like object (empty or not)
  • if not dict, must be converted to dict before __new__ returns

Example: Allow only uppercase attribute names

class OnlyUppercase(dict):
    def __setitem__(self, key, value):
        if isinstance(key, str) and key.isupper():
            super().__setitem__(key, value)

class MyMeta(type):
    @classmethod
    def __prepare__(mcls, name, bases):
        return OnlyUppercase()

    def __new__(mcls, name, bases, attrs):
        return super().__new__(mcls, name, bases, dict(attrs))

class MyClass(metaclass=MyMeta):
    lowercase = 1
    UPPERCASE = 2

MyClass.__dict__  # {'UPPERCASE': 2}

__new__

  • class constructor (class MyClass ... -> __new__ runs)
  • most useful

__init__

  • class initializer (after __new__)
  • not generally useful

__call__

  • object instantiation (before Class.__new__ & object.__init__)
class MyMeta(type):
    def __call__(cls, *args, **kwargs):
        print('In metaclass', args, kwargs)
        return super().__call__(*args, **kwargs)

class MyClass(metaclass=MyMeta):
    def __init__(cls, *args, **kwargs):
        print('In class', args, kwargs)

>>> obj = MyClass(1, 2, foo=42)
In metaclass (1, 2), {'foo': 42}
In class (1, 2) {'foo': 42}

Example: Call order

class MyMeta(type):
    @classmethod
    def __prepare__(mcls, name, bases):
        print('Meta __prepare__')
        return super().__prepare__(mcls, name, bases)

    def __new__(mcls, name, bases, attrs):
        print('Meta __new__')
        return super().__new__(mcls, name, bases, attrs)

    def __init__(cls, name, bases, attrs):
        print('Meta __init__')
        return super().__init__(name, bases, attrs)

    def __call__(cls, *args, **kwargs):
        print('Meta __call__')
        return super().__call__(*args, **kwargs)

>>> class MyClass(metaclass=MyMeta):
...     def __new__(cls, *args, **kwargs):
...         print('Class __new__')
...         return super().__new__(cls)
...     def __init__(self, *args, **kwargs):
...         print('Class __init__')
...
Meta __prepare__
Meta __new__
Meta __init__
>>>
>>> obj = MyClass()
Meta __call__
Class __new__
Class __init__

Example: singleton

class SingletonMeta(type):

    def __call__(cls, *args, **kwargs):
        if not hasattr(cls, '_inst'):
            obj = super(SingletonMeta, cls).__call__(*args, **kwargs)
            cls._inst = obj
        return cls._inst

class MyClass(metaclass=SingletonMeta):
    pass

>>> a = MyClass()
>>> b = MyClass()
>>> a is b
True

Example: metaclass is a callable

>>> class MyClass(metaclass=print):
...     a = 1
...
MyClass () {'__qualname__': 'MyClass', '__module__': '__main__', 'a': 1}
>>> MyClass is None
True

Attribute lookup

Object-level (instance.attr)

  • attr in Class.__dict__ and attr is data descriptor -> Class.__dict__['attr'].__get__(instance, Class)
  • attr in instance.__dict__ -> instance.__dict__['attr']
  • attr in Class.__dict__ and attr is not a data descriptor -> Class.__dict__['attr'].__get__(instance, Class)
  • attr in Class.__dict__ -> Class.__dict__['attr']
  • Class.__getattr__ exists -> Class.__getattr__('attr')

Class-level (Class.attr)

  • attr in Metaclass.__dict__ and attr is data desciptor -> Metaclass.__dict__['attr'].__get__(Class, Metaclass)
  • attr in Class.__dict__ and attr is descriptor -> Class.__dict__['attr'].__get__(None, Class)
  • attr in Class.__dict__ -> Class.__dict__['attr']

Class-level (cont.)

  • attr in Metaclass.__dict__ and attr is not a data descriptor -> Metaclass.__dict__['attr'].__get__(Class, Metaclass)
  • attr in Metaclass.__dict__ -> Metaclass.__dict__['attr']
  • Metaclass.__getattr__ exists -> Metaclass.__getattr__('attr')

Descriptors

  • only defined in class-level (not in __init__ etc.)
  • objects with __get__, __set__ & __delete__ methods
  • __get__ & __set__ = data descriptors
  • only __get__ = non-data descriptors
  • e.g. property decorator (getter & setter)

Descriptor API

descr.__get__(self, obj, cls)  # -> value
descr.__set__(self, obj, value)  # -> None
descr.__delete__(self, obj)  # -> None

Example

class Descriptor(object):
    def __init__(self, initval=None, name='var'):
        self.val = initval
        self.name = name

    def __get__(self, obj, cls):
        print('get', self.name)
        return self.val

    def __set__(self, obj, val):
        print('set', self.name)
        self.val = val

class MyClass(object):
    attr = Descriptor(initval=10, name='attr')

>>> MyClass.attr
get attr
10
>>> MyClass.attr = 11
>>> MyClass.attr
11
>>> # oops

>>> a = MyClass()
>>> a.attr
get attr
10
>>> a.attr = 11
set attr
>>> a.attr
get attr
11

>>> b = MyClass()
>>> b.attr
get attr
11
>>> # wat

Example

  • use descriptors to indirectly set values on instance.__dict__
  • if called from class, just return the descriptor class
class Descriptor(object):
    def __init__(self, name):
        self.name = name

    def __get__(self, obj, cls):
        if obj is None:
            return self
        try:
            print('get', self.name)
            return obj.__dict__[self.name]
        except KeyError:
            raise AttributeError()

    def __set__(self, obj, val):
        print('set', self.name)
        obj.__dict__[self.name] = val

class MyClass(object):
    attr  = Descriptor('attr')

>>> MyClass.attr
<__main__.Descriptor at 0x312....>
>>>
>>> a = MyClass()
>>> a.attr
get attr
Traceback (most recent call last)
....
AttributeError: ...
>>> a.attr = 1
set attr
>>> a.attr
get attr
1
>>>
>>> b = MyClass()
>>> b.attr = 2
set attr
>>> b.attr
get attr
2
>>> a.attr
get attr
1

Metaclasses & descriptors

  • attr = Descriptor('attr') -> attr = Descriptor()

Example

class Descriptor(object):
    def __init__(self):
        self.name = None

    def __get__(self, obj, cls):
        if obj is None:
            return self
        try:
            print('get', self.name)
            return obj.__dict__[self.name]
        except KeyError:
            raise AttributeError()

    def __set__(self, obj, val):
        print('set', self.name)
        obj.__dict__[self.name] = val

class MyMeta(type):
    def __new__(mcls, name, bases, attrs):
        for k, v in attrs.items():
            if isinstance(v, Descriptor):
                v.name = k
        return super().__new__(mcls, name, bases, attrs)

class MyClass(metaclass=MyMeta):
    attr  = Descriptor()

>>> MyClass.attr
<__main__.Descriptor at 0x312....>
>>>
>>> a = MyClass()
>>> a.attr = 1
set attr
>>> a.attr
get attr
1
>>> a.__dict__
{'attr': 1}

Example: using annotations for typing (Python 3.6)

>>> class MyClass:
...     a: int
...     b: str
...
>>> MyClass.__annotations__
{'a': int, 'b': str}

>>> class MyClass(Typed):
...     a: int
...     b: str
...
>>> obj = MyClass()
>>> obj.a = 1
>>> obj.a = 'foo'
...
...
TypeError: 'foo' is not of type 'int'

class TypedDescriptor:
    def __init__(self, name, tp):
        self.name = name
        self.tp = tp

    def __get__(self, obj, cls):
        if obj is None:
            return self
        try:
            return obj.__dict__[self.name]
        except KeyError:
            raise AttributeError()

    def __set__(self, obj, val):
        if not isinstance(val, self.tp):
            raise TypeError()
        obj.__dict__[self.name] = val

class TypedMeta(type):
    def __new__(mcls, name, bases, attrs):
        ann = attrs.get('__annotations__', {})
        for k, v in ann.items():
            attrs[k] = TypedDescriptor(name=k, tp=v)
        return super().__new__(mcls, name, bases, attrs)

class Typed(metaclass=TypedMeta):
    pass

Links