Source code for tiled.structures.array

import enum
import os
import sys
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union

import numpy

[docs] class Endianness(str, enum.Enum): """ An enum of endian values: big, little, not_applicable. """ big = "big" little = "little" not_applicable = "not_applicable"
class ObjectArrayTypeDisabled(ValueError): pass
[docs] class Kind(str, enum.Enum): """ See The term "kind" comes from the numpy API as well. Note: At import time, the environment variable ``TILED_ALLOW_OBJECT_ARRAYS`` is checked. If it is set to anything other than ``"0"``, then this Enum gets an additional member:: object = "O" to support numpy 'object'-type arrays which hold generic Python objects. Numpy 'object'-type arrays are not enabled by default because their binary representation is not interpretable by clients other than Python. It is recommended to convert your data to a non-object type if possible so that it can be read by non-Python clients. """ bit_field = "t" boolean = "b" integer = "i" unsigned_integer = "u" floating_point = "f" complex_floating_point = "c" timedelta = "m" datetime = "M" string = "S" # fixed-length sequence of char unicode = "U" # fixed-length sequence of Py_UNICODE other = "V" # "V" is for "void" -- generic fixed-size chunk of memory # By default, do not tolerate numpy objectg arrays if os.getenv("TILED_ALLOW_OBJECT_ARRAYS", "0") != "0": object = "O" # Object (i.e. the memory contains a pointer to PyObject) @classmethod def _missing_(cls, key): if key == "O": raise ObjectArrayTypeDisabled( "Numpy 'object'-type arrays are not enabled by default " "because their binary representation is not interpretable " "by clients other than Python. " "It is recommended to convert your data to a non-object type " "if possible so that it can be read by non-Python clients. " "If this is not possible, you may enable 'object'-type arrays " "by setting the environment variable TILED_ALLOW_OBJECT_ARRAYS=1 " "in the server." )
[docs] @dataclass class BuiltinDtype: endianness: Endianness kind: Kind itemsize: int __endianness_map = { ">": "big", "<": "little", "=": sys.byteorder, "|": "not_applicable", } __endianness_reverse_map = {"big": ">", "little": "<", "not_applicable": "|"} @classmethod def from_numpy_dtype(cls, dtype) -> "BuiltinDtype": return cls( endianness=cls.__endianness_map[dtype.byteorder], kind=Kind(dtype.kind), itemsize=dtype.itemsize, ) def to_numpy_dtype(self) -> numpy.dtype: import numpy return numpy.dtype(self.to_numpy_str()) def to_numpy_str(self): endianness = self.__endianness_reverse_map[self.endianness] # dtype.itemsize always reports bytes. The format string from the # numeric types the string format is: {type_code}{byte_count} so we can # directly use the item size. # # for unicode the pattern is 'U{char_count}', however # which numpy always represents as 4 byte UCS4 encoding # (because variable width encodings do not fit with fixed-stride arrays) # so the reported itemsize is 4x the char count. To get back to the string # we need to divide by 4. size = self.itemsize if self.kind != Kind.unicode else self.itemsize // 4 return f"{endianness}{self.kind.value}{size}" @classmethod def from_json(cls, structure): return cls( kind=Kind(structure["kind"]), itemsize=structure["itemsize"], endianness=Endianness(structure["endianness"]), )
@dataclass class Field: name: str dtype: Union[BuiltinDtype, "StructDtype"] shape: Optional[Tuple[int, ...]] @classmethod def from_numpy_descr(cls, field): import numpy name, *rest = field if name == "": raise ValueError( f"You seem to have gotten descr of a base or subdtype: {field}" ) if len(rest) == 1: (f_type,) = rest shape = None else: f_type, shape = rest if isinstance(f_type, str): FType = BuiltinDtype.from_numpy_dtype(numpy.dtype(f_type)) else: FType = StructDtype.from_numpy_dtype(numpy.dtype(f_type)) return cls(name=name, dtype=FType, shape=shape) def to_numpy_descr(self): if isinstance(self.dtype, BuiltinDtype): base = [, self.dtype.to_numpy_str()] else: base = [, self.dtype.to_numpy_descr()] if self.shape is None: return tuple(base) else: return tuple(base + [self.shape]) @classmethod def from_json(cls, structure): name = structure["name"] if "fields" in structure["dtype"]: ftype = StructDtype.from_json(structure["dtype"]) else: ftype = BuiltinDtype.from_json(structure["dtype"]) return cls(name=name, dtype=ftype, shape=structure["shape"]) @dataclass class StructDtype: itemsize: int fields: List[Field] @classmethod def from_numpy_dtype(cls, dtype): # subdtypes push extra dimensions into arrays, we should handle these # a layer up and report an array with bigger dimensions. if dtype.subdtype is not None: raise ValueError(f"We do not know how to encode subdtypes: {dtype}") # If this is a builtin type, require the use of BuiltinDtype (nee .array.BuiltinDtype) if dtype.fields is None: raise ValueError(f"You have a base type: {dtype}") return cls( itemsize=dtype.itemsize, fields=[Field.from_numpy_descr(f) for f in dtype.descr], ) def to_numpy_dtype(self): import numpy return numpy.dtype(self.to_numpy_descr()) def to_numpy_descr(self): return [f.to_numpy_descr() for f in self.fields] def max_depth(self): return max( 1 if isinstance(f.dtype, BuiltinDtype) else 1 + f.dtype.max_depth() for f in self.fields ) @classmethod def from_json(cls, structure): return cls( itemsize=structure["itemsize"], fields=[Field.from_json(f) for f in structure["fields"]], )
[docs] @dataclass class ArrayStructure: data_type: Union[BuiltinDtype, StructDtype] chunks: Tuple[Tuple[int, ...], ...] # tuple-of-tuples-of-ints like ((3,), (3,)) shape: Tuple[int, ...] # tuple of ints like (3, 3) dims: Optional[Tuple[str, ...]] = None # None or tuple of names like ("x", "y") resizable: Union[bool, Tuple[bool, ...]] = False @classmethod def from_json(cls, structure): if "fields" in structure["data_type"]: data_type = StructDtype.from_json(structure["data_type"]) else: data_type = BuiltinDtype.from_json(structure["data_type"]) dims = structure["dims"] if dims is not None: dims = tuple(dims) return cls( data_type=data_type, chunks=tuple(map(tuple, structure["chunks"])), shape=tuple(structure["shape"]), dims=dims, resizable=structure.get("resizable", False), ) @classmethod def from_array(cls, array, shape=None, chunks=None, dims=None) -> "ArrayStructure": from dask.array.core import normalize_chunks if not hasattr(array, "__array__"): # may be a list of something; convert to array import numpy array = numpy.asanyarray(array) # Why would shape ever be different from array.shape, you ask? # Some formats (notably Zarr) force shape to be a multiple of # a chunk size, such that array.shape may include a margin beyond the # actual data. if shape is None: shape = array.shape if chunks is None: if hasattr(array, "chunks"): chunks = array.chunks # might be None else: chunks = None if chunks is None: chunks = ("auto",) * len(shape) normalized_chunks = normalize_chunks( chunks, shape=shape, dtype=array.dtype, ) if array.dtype.fields is not None: data_type = StructDtype.from_numpy_dtype(array.dtype) else: data_type = BuiltinDtype.from_numpy_dtype(array.dtype) return ArrayStructure( data_type=data_type, shape=shape, chunks=normalized_chunks, dims=dims )