Important

You can run this notebook in a live session Binder or view it on nbviewer or GitHub.

Higher Dimensional Data

[1]:
import pathlib
from collections import defaultdict

import h5py
import pandas as pd
import numpy as np
import xarray as xr

import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm,Normalize
import matplotlib.cm as cm

from bluesky_tutorial_utils import nexus, fetch
/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/pims/image_reader.py:26: RuntimeWarning: PIMS image_reader.py could not find scikit-image. Falling back to matplotlib's imread(), which uses floats instead of integers. This may break your scripts.
(To ignore this warning, include the line "warnings.simplefilter("ignore", RuntimeWarning)" in your script.)
  warnings.warn(RuntimeWarning(ski_preferred))
[2]:
# Un-comment this for better-looking plots on high-resolution screens.
# %config InlineBackend.figure_format = 'retina'
[3]:
fetch.rsoxs_simulation_data();

Working with Multiple xArrays: Gathering Data

Build Index Table (Table of Contents)

[4]:
def build_pandas_index(nxs_path):
    nxs_path = pathlib.Path(nxs_path)
    nxs_files = list(nxs_path.glob('*nxs'))

    #progress = ipywidgets.IntProgress(0,0,len(nxs_files))
    #display(progress)

    index_table = []
    for i,nxs_file in enumerate(nxs_files):
        #progress.value = i
        with h5py.File(nxs_file,'r') as nxs:
            notes = nxs[u'entry/instrument/simulation_engine/notes']
            config =  {k:v[()] for k,v in notes.items()}
            config['nxs'] = nxs_file
            index_table.append(config)
    return pd.DataFrame(index_table)
[5]:
toc = build_pandas_index('./rsoxs_simulation_data//')
[6]:
toc
[6]:
EndAngle EndEnergy Energy IncrementAngle IncrementEnergy NumThreads NumX NumY NumZ PhysSize Radius StartAngle StartEnergy nxs
0 360.0 2005.0 2000.0 2.0 1.0 4 512 512 128 5.0 10.0 0.0 2000.0 rsoxs_simulation_data/512-512-128-5.0-10.0-020...
1 360.0 290.0 288.0 2.0 1.0 4 512 512 128 5.0 5.0 0.0 285.0 rsoxs_simulation_data/512-512-128-5.0-5.0-0028...
2 360.0 2005.0 2003.0 2.0 1.0 4 512 512 128 5.0 5.0 0.0 2000.0 rsoxs_simulation_data/512-512-128-5.0-5.0-0200...
3 360.0 2005.0 2005.0 2.0 1.0 4 512 512 128 5.0 40.0 0.0 2000.0 rsoxs_simulation_data/512-512-128-5.0-40.0-020...
4 360.0 2005.0 2003.0 2.0 1.0 4 512 512 128 5.0 20.0 0.0 2000.0 rsoxs_simulation_data/512-512-128-5.0-20.0-020...
... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
85 360.0 2005.0 2005.0 2.0 1.0 4 512 512 128 5.0 80.0 0.0 2000.0 rsoxs_simulation_data/512-512-128-5.0-80.0-020...
86 360.0 290.0 286.0 2.0 1.0 4 512 512 128 5.0 5.0 0.0 285.0 rsoxs_simulation_data/512-512-128-5.0-5.0-0028...
87 360.0 20005.0 20001.0 2.0 1.0 4 512 512 128 5.0 40.0 0.0 20000.0 rsoxs_simulation_data/512-512-128-5.0-40.0-200...
88 360.0 290.0 289.0 2.0 1.0 4 512 512 128 5.0 10.0 0.0 285.0 rsoxs_simulation_data/512-512-128-5.0-10.0-002...
89 360.0 20005.0 20001.0 2.0 1.0 4 512 512 128 5.0 80.0 0.0 20000.0 rsoxs_simulation_data/512-512-128-5.0-80.0-200...

90 rows × 14 columns

[7]:
toc.describe().loc[['count','min','max']]
[7]:
EndAngle EndEnergy Energy IncrementAngle IncrementEnergy NumThreads NumX NumY NumZ PhysSize Radius StartAngle StartEnergy
count 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0
min 360.0 290.0 285.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 5.0 0.0 285.0
max 360.0 20005.0 20005.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 80.0 0.0 20000.0

Select subset of data From Index

[8]:
sdf = toc.query('Radius==40.0 & EndAngle==360.0 & PhysSize==5 & NumX==512')
sdf = sdf.sort_values('Energy')
sdf.describe().loc[['count','min','max']]
[8]:
EndAngle EndEnergy Energy IncrementAngle IncrementEnergy NumThreads NumX NumY NumZ PhysSize Radius StartAngle StartEnergy
count 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0
min 360.0 290.0 285.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 40.0 0.0 285.0
max 360.0 20005.0 20005.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 40.0 0.0 20000.0

Gather Data

[9]:
def gather(df):
    coords = defaultdict(list)
    data_arrays = []
    for row_index,row in df.iterrows():
        da_img = nexus.read_singleimg_nxs(row['nxs'])
        data_arrays.append(da_img)

        for col_index,value in row.iteritems():
            if col_index=='nxs':
                continue
            coords[col_index].append(value)
    return data_arrays,coords
[10]:
data_arrays,coords = gather(sdf)
[11]:
data_arrays
[11]:
[<xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False]
[12]:
coords
[12]:
defaultdict(list,
            {'EndAngle': [360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0],
             'EndEnergy': [290.0,
              290.0,
              290.0,
              290.0,
              290.0,
              290.0,
              2005.0,
              2005.0,
              2005.0,
              2005.0,
              2005.0,
              2005.0,
              20005.0,
              20005.0,
              20005.0,
              20005.0,
              20005.0,
              20005.0],
             'Energy': [285.0,
              286.0,
              287.0,
              288.0,
              289.0,
              290.0,
              2000.0,
              2001.0,
              2002.0,
              2003.0,
              2004.0,
              2005.0,
              20000.0,
              20001.0,
              20002.0,
              20003.0,
              20004.0,
              20005.0],
             'IncrementAngle': [2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0],
             'IncrementEnergy': [1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0],
             'NumThreads': [4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4],
             'NumX': [512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512],
             'NumY': [512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512],
             'NumZ': [128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128],
             'PhysSize': [5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0],
             'Radius': [40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0],
             'StartAngle': [0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0],
             'StartEnergy': [285.0,
              285.0,
              285.0,
              285.0,
              285.0,
              285.0,
              2000.0,
              2000.0,
              2000.0,
              2000.0,
              2000.0,
              2000.0,
              20000.0,
              20000.0,
              20000.0,
              20000.0,
              20000.0,
              20000.0]})
[13]:
data_arrays[1].plot(norm=LogNorm(1e-9,1),aspect=1.2,size=5)
[13]:
<matplotlib.collections.QuadMesh at 0x7fbc7db8ef60>
_images/Higher_Dimensional_Data_17_1.png

Multiple xArrays: simple xr.concat

[14]:
sdf = toc.query('Radius==40.0 & EndAngle==360.0 & PhysSize==5 & NumX==512')
sdf = sdf.sort_values('Energy')
display(sdf.describe().loc[['count','min','max']])

data_arrays,coords = gather(sdf)

da = xr.concat(data_arrays,dim='Energy')
da
EndAngle EndEnergy Energy IncrementAngle IncrementEnergy NumThreads NumX NumY NumZ PhysSize Radius StartAngle StartEnergy
count 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0
min 360.0 290.0 285.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 40.0 0.0 285.0
max 360.0 20005.0 20005.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 40.0 0.0 20000.0
[14]:
<xarray.DataArray 'I' (Energy: 18, Qx: 512, Qy: 512)>
array([[[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
...
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]]], dtype=float16)
Coordinates:
  * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
  * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
Dimensions without coordinates: Energy
Attributes:
    lazy:     False
[15]:
da = da.assign_coords(Energy=sdf.Energy.values)
da
[15]:
<xarray.DataArray 'I' (Energy: 18, Qx: 512, Qy: 512)>
array([[[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
...
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]]], dtype=float16)
Coordinates:
  * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
  * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
  * Energy   (Energy) float64 285.0 286.0 287.0 288.0 ... 2e+04 2e+04 2.000e+04
Attributes:
    lazy:     False
[16]:
da.sel(Qy=0,method='nearest').plot(norm=LogNorm(1e-9,1),yscale='log')
[16]:
<matplotlib.collections.QuadMesh at 0x7fbc7a0129e8>
_images/Higher_Dimensional_Data_21_1.png
[17]:
# da.plot(col='Energy',col_wrap=3,norm=LogNorm(1e-9,1))

Building xArrays: Multi-Index

[18]:
sdf = toc.query('EndAngle==360.0 & PhysSize==5 & NumX==512')
sdf = sdf.sort_values(['Energy','Radius'])
display(sdf.describe().loc[['count','min','max']])
EndAngle EndEnergy Energy IncrementAngle IncrementEnergy NumThreads NumX NumY NumZ PhysSize Radius StartAngle StartEnergy
count 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0
min 360.0 290.0 285.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 5.0 0.0 285.0
max 360.0 20005.0 20005.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 80.0 0.0 20000.0
[19]:
data_arrays,coords = gather(sdf)

da = xr.concat(data_arrays,dim=['Energy','Radius'])
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-19-d734cd483e55> in <module>
      1 data_arrays,coords = gather(sdf)
      2
----> 3 da = xr.concat(data_arrays,dim=['Energy','Radius'])

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/concat.py in concat(objs, dim, data_vars, coords, compat, positions, fill_value, join, combine_attrs)
    141         )
    142     return f(
--> 143         objs, dim, data_vars, coords, compat, positions, fill_value, join, combine_attrs
    144     )
    145

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/concat.py in _dataarray_concat(arrays, dim, data_vars, coords, compat, positions, fill_value, join, combine_attrs)
    465         fill_value=fill_value,
    466         join=join,
--> 467         combine_attrs="drop",
    468     )
    469

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/concat.py in _dataset_concat(datasets, dim, data_vars, coords, compat, positions, fill_value, join, combine_attrs)
    422     if coord is not None:
    423         # add concat dimension last to ensure that its in the final Dataset
--> 424         result[coord.name] = coord
    425
    426     return result

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/dataset.py in __setitem__(self, key, value)
   1286             )
   1287
-> 1288         self.update({key: value})
   1289
   1290     def __delitem__(self, key: Hashable) -> None:

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/dataset.py in update(self, other, inplace)
   3611         """
   3612         _check_inplace(inplace)
-> 3613         merge_result = dataset_update_method(self, other)
   3614         return self._replace(inplace=True, **merge_result._asdict())
   3615

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/merge.py in dataset_update_method(dataset, other)
    934         priority_arg=1,
    935         indexes=dataset.indexes,
--> 936         combine_attrs="override",
    937     )

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/merge.py in merge_core(objects, compat, join, combine_attrs, priority_arg, explicit_coords, indexes, fill_value)
    598     assert_unique_multiindex_level_names(variables)
    599
--> 600     dims = calculate_dimensions(variables)
    601
    602     coord_names, noncoord_names = determine_coords(coerced)

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/dataset.py in calculate_dimensions(variables)
    207                     "conflicting sizes for dimension %r: "
    208                     "length %s on %r and length %s on %r"
--> 209                     % (dim, size, k, dims[dim], last_used[dim])
    210                 )
    211     return dims

ValueError: conflicting sizes for dimension 'concat_dim': length 2 on 'concat_dim' and length 90 on <this-array>

hmmm…that didn’t work…

Let’s try a multi-index

[20]:
keys =  ['Energy','Radius']
tuples = [(i,j) for i,j in sdf[keys].values]
index = pd.MultiIndex.from_tuples(tuples,names=keys)
index.name = 'system'
da = xr.concat(data_arrays,dim=index)
da
[20]:
<xarray.DataArray 'I' (system: 90, Qx: 512, Qy: 512)>
array([[[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        ...,
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]],

       [[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        ...,
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]],

       [[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        ...,
...
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]],

       [[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        ...,
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]],

       [[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 6.e-08, 0.e+00, 0.e+00],
        ...,
        [0.e+00, 0.e+00, 6.e-08, ..., 1.e-07, 6.e-08, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 6.e-08, 0.e+00, 0.e+00],
        [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]]],
      dtype=float16)
Coordinates:
  * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
  * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
  * system   (system) MultiIndex
  - Energy   (system) float64 285.0 285.0 285.0 ... 2.000e+04 2.000e+04
  - Radius   (system) float64 5.0 10.0 20.0 40.0 80.0 ... 10.0 20.0 40.0 80.0
Attributes:
    lazy:     False
[21]:
da.sel(Energy=285.,method='nearest')
[21]:
<xarray.DataArray 'I' (Radius: 5, Qx: 512, Qy: 512)>
array([[[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]],

       [[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]]], dtype=float16)
Coordinates:
  * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
  * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
  * Radius   (Radius) float64 5.0 10.0 20.0 40.0 80.0
Attributes:
    lazy:     False
[22]:
da.sel(Energy=285.,Qy=0,method='nearest').plot.line(x='Qx',yscale='log',xscale='log')#(norm=LogNorm(1e-9,1))
[22]:
[<matplotlib.lines.Line2D at 0x7fbc785bb470>,
 <matplotlib.lines.Line2D at 0x7fbc785bb630>,
 <matplotlib.lines.Line2D at 0x7fbc785bb780>,
 <matplotlib.lines.Line2D at 0x7fbc785bb8d0>,
 <matplotlib.lines.Line2D at 0x7fbc785bba20>]
_images/Higher_Dimensional_Data_29_1.png
[23]:
da.sel(Energy=285.,method='nearest').plot(col='Radius',col_wrap=3,norm=LogNorm(1e-9,1))
[23]:
<xarray.plot.facetgrid.FacetGrid at 0x7fbc78331a90>
/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/IPython/core/pylabtools.py:132: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
  fig.canvas.print_figure(bytes_io, **kw)
_images/Higher_Dimensional_Data_30_2.png