Source code for metatensor.io._block

# from __future__ import annotations

import ctypes
import io
import pathlib
import warnings
from typing import BinaryIO, Optional, Union

import numpy as np

from .._block import TensorBlock
from .._c_api import mts_create_array_callback_t
from .._c_lib import _get_library
from .._data import ArrayBackend
from .._data._array import is_numpy_array, is_torch_array
from .._status import check_pointer
from ._labels import labels_from_mts, labels_to_mts
from ._utils import (
    CreateArrayCallback,
    array_backend_for_create_array,
    create_array_for_array_backend,
    save_buffer_raw,
)


[docs] def load_block( file: Union[str, pathlib.Path, BinaryIO], use_numpy=False, *, array_backend: ArrayBackend = "numpy", ) -> TensorBlock: """ Load a previously saved :py:class:`TensorBlock` from the given file. :param file: file to load: this can be a string, a :py:class:`pathlib.Path` containing the path to the file to load, or a file-like object that should be opened in binary mode. :param use_numpy: should we use numpy or the native implementation? Numpy should be able to process more dtypes than the native implementation, which is limited to float64, but the native implementation is usually faster than going through numpy. :param array_backend: array backend to use for all data in the :py:class:`TensorBlock`. See :ref:`python-api-array-backend` for more information. """ if use_numpy: if array_backend != "numpy": raise ValueError("`array_backend` must be 'numpy' when `use_numpy` is True") return _block_from_mts(file) else: create_array = create_array_for_array_backend(array_backend) if isinstance(file, (str, pathlib.Path)): return load_block_custom_array(file, create_array) else: # assume we have a file-like object buffer = file.read() assert isinstance(buffer, bytes) return load_block_buffer_custom_array(buffer, create_array)
[docs] def load_block_buffer( buffer: Union[bytes, bytearray, memoryview], use_numpy=False, *, array_backend: ArrayBackend = "numpy", ) -> TensorBlock: """ Load a previously saved :py:class:`TensorBlock` from an in-memory buffer. :param buffer: In-memory buffer containing a serialized ``TensorBlock`` :param use_numpy: should we use numpy or the native implementation? :param array_backend: array backend to use for all data in the :py:class:`TensorBlock`. See :ref:`python-api-array-backend` for more information. """ if use_numpy: if array_backend != "numpy": raise ValueError("`array_backend` must be 'numpy' when `use_numpy` is True") return _block_from_mts(io.BytesIO(buffer)) else: create_array = create_array_for_array_backend(array_backend) return load_block_buffer_custom_array(buffer, create_array)
[docs] def load_block_custom_array( path: Union[str, pathlib.Path], create_array: "CreateArrayCallback", *, array_backend: Optional[ArrayBackend] = None, ) -> TensorBlock: """ Load a previously saved :py:class:`TensorBlock` from the given path using a custom array creation callback. This advanced functionality should not be needed by most users. This function allows to specify the kind of array to use when loading the data through the ``create_array`` callback. This callback should take three arguments: a pointer to the shape, the number of elements in the shape, and a pointer to the ``mts_array_t`` to be filled. :py:func:`metatensor.io.create_numpy_array` and :py:func:`metatensor.io.create_torch_array` can be used to load data into numpy and torch arrays respectively. :param path: path of the file to load :param create_array: callback used to create arrays as needed :param array_backend: array backend to use for all data in the :py:class:`TensorBlock`. This is required for custom ``create_array`` callbacks, and guessed if the callback is one of :py:func:`metatensor.io.create_numpy_array` or :py:func:`metatensor.io.create_torch_array`. See :ref:`python-api-array-backend` for more information. """ lib = _get_library() if isinstance(path, str): path = path.encode("utf8") elif isinstance(path, pathlib.Path): path = bytes(path) ptr = lib.mts_block_load(path, mts_create_array_callback_t(create_array)) check_pointer(ptr) return TensorBlock.unsafe_from_ptr( ptr, array_backend=array_backend_for_create_array(create_array, array_backend) )
[docs] def load_block_buffer_custom_array( buffer: Union[bytes, bytearray, memoryview], create_array: "CreateArrayCallback", *, array_backend: Optional[ArrayBackend] = None, ) -> TensorBlock: """ Load a previously saved :py:class:`TensorBlock` from the given buffer using a custom array creation callback. This advanced functionality should not be needed by most users. This function allows to specify the kind of array to use when loading the data through the ``create_array`` callback. This callback should take three arguments: a pointer to the shape, the number of elements in the shape, and a pointer to the ``mts_array_t`` to be filled. :py:func:`metatensor.io.create_numpy_array` and :py:func:`metatensor.io.create_torch_array` can be used to load data into numpy and torch arrays respectively. :param buffer: in-memory buffer containing a saved :py:class:`TensorBlock` :param create_array: callback used to create arrays as needed :param array_backend: array backend to use for all data in the :py:class:`TensorBlock`. This is required for custom ``create_array`` callbacks, and guessed if the callback is one of :py:func:`metatensor.io.create_numpy_array` or :py:func:`metatensor.io.create_torch_array`. See :ref:`python-api-array-backend` for more information. """ lib = _get_library() array = np.frombuffer(buffer, dtype=np.uint8) ptr = lib.mts_block_load_buffer( array.ctypes.data_as(ctypes.c_char_p), array.nbytes, mts_create_array_callback_t(create_array), ) check_pointer(ptr) return TensorBlock.unsafe_from_ptr( ptr, array_backend=array_backend_for_create_array(create_array, array_backend) )
def save_block( file: Union[str, pathlib.Path, BinaryIO], block: TensorBlock, use_numpy=False, ): assert isinstance(block, TensorBlock) if use_numpy: all_entries = block_to_dict(block, prefix="", is_gradient=False) if not hasattr(file, "write"): # prevent numpy from adding a .npz extension by opening the file ourself with open(file, "wb") as fd: np.savez(fd, **all_entries) else: np.savez(file, **all_entries) else: lib = _get_library() if isinstance(file, str): if not file.endswith(".mts"): file += ".mts" warnings.warn( message="adding '.mts' extension," f" the file will be saved at '{file}'", stacklevel=1, ) path = file.encode("utf8") lib.mts_block_save(path, block._ptr) elif isinstance(file, pathlib.Path): if not file.name.endswith(".mts"): file = file.with_name(file.name + ".mts") warnings.warn( message="adding '.mts' extension," f" the file will be saved at '{file.name}'", stacklevel=1, ) path = bytes(file) lib.mts_block_save(path, block._ptr) else: # assume we have a file-like object buffer = save_block_buffer_raw(block) file.write(buffer.raw) def save_block_buffer_raw(block: TensorBlock) -> ctypes.Array: """ Save a TensorBlock to an in-memory buffer, returning the data as a ctypes array of ``ctypes.c_char``. """ lib = _get_library() return save_buffer_raw(lib.mts_block_save_buffer, block._ptr) def block_to_dict(block, prefix, is_gradient): result = {} result[f"{prefix}values"] = _array_to_numpy(block.values) result[f"{prefix}samples"] = labels_to_mts(block.samples) for i, component in enumerate(block.components): result[f"{prefix}components/{i}"] = labels_to_mts(component) if not is_gradient: result[f"{prefix}properties"] = labels_to_mts(block.properties) # sort the gradients to make sure the same file is generated every time for parameter in sorted(block.gradients_list()): result.update( block_to_dict( block.gradient(parameter), f"{prefix}gradients/{parameter}/", is_gradient=True, ) ) return result def _array_to_numpy(array): if is_numpy_array(array): return array elif is_torch_array(array): return array.cpu().numpy() else: raise ValueError("unknown array type passed to `metatensor.save`") def single_block_from_mts(prefix, dictionary, properties): values = dictionary[f"{prefix}values"] samples = labels_from_mts(dictionary[f"{prefix}samples"]) components = [] for i in range(len(values.shape) - 2): components.append(labels_from_mts(dictionary[f"{prefix}components/{i}"])) block = TensorBlock(values, samples, components, properties) parameters = set() gradient_prefix = f"{prefix}gradients/" for name in dictionary.keys(): if name.startswith(gradient_prefix) and name.endswith("/values"): parameter = name[len(gradient_prefix) :] parameter = parameter.split("/")[0] parameters.add(parameter) for parameter in parameters: gradient = single_block_from_mts( f"{prefix}gradients/{parameter}/", dictionary, properties, ) block.add_gradient(parameter, gradient) return block def _block_from_mts(file): dictionary = np.load(file) properties = labels_from_mts(dictionary["properties"]) return single_block_from_mts("", dictionary, properties)