Important

You can run this notebook in a live session Binder or view it on nbviewer or GitHub.

Higher Dimensional Data

[1]:
import pathlib
from collections import defaultdict

import h5py
import pandas as pd
import numpy as np
import xarray as xr

import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm,Normalize
import matplotlib.cm as cm

from bluesky_tutorial_utils import nexus, fetch
[2]:
# Un-comment this for better-looking plots on high-resolution screens.
# %config InlineBackend.figure_format = 'retina'
[3]:
fetch.rsoxs_simulation_data();

Working with Multiple xArrays: Gathering Data

Build Index Table (Table of Contents)

[4]:
def build_pandas_index(nxs_path):
    nxs_path = pathlib.Path(nxs_path)
    nxs_files = list(nxs_path.glob('*nxs'))

    #progress = ipywidgets.IntProgress(0,0,len(nxs_files))
    #display(progress)

    index_table = []
    for i,nxs_file in enumerate(nxs_files):
        #progress.value = i
        with h5py.File(nxs_file,'r') as nxs:
            notes = nxs[u'entry/instrument/simulation_engine/notes']
            config =  {k:v[()] for k,v in notes.items()}
            config['nxs'] = nxs_file
            index_table.append(config)
    return pd.DataFrame(index_table)
[5]:
toc = build_pandas_index('./rsoxs_simulation_data//')
[6]:
toc
[6]:
EndAngle EndEnergy Energy IncrementAngle IncrementEnergy NumThreads NumX NumY NumZ PhysSize Radius StartAngle StartEnergy nxs
0 360.0 2005.0 2000.0 2.0 1.0 4 512 512 128 5.0 10.0 0.0 2000.0 rsoxs_simulation_data/512-512-128-5.0-10.0-020...
1 360.0 290.0 288.0 2.0 1.0 4 512 512 128 5.0 5.0 0.0 285.0 rsoxs_simulation_data/512-512-128-5.0-5.0-0028...
2 360.0 2005.0 2003.0 2.0 1.0 4 512 512 128 5.0 5.0 0.0 2000.0 rsoxs_simulation_data/512-512-128-5.0-5.0-0200...
3 360.0 2005.0 2005.0 2.0 1.0 4 512 512 128 5.0 40.0 0.0 2000.0 rsoxs_simulation_data/512-512-128-5.0-40.0-020...
4 360.0 2005.0 2003.0 2.0 1.0 4 512 512 128 5.0 20.0 0.0 2000.0 rsoxs_simulation_data/512-512-128-5.0-20.0-020...
... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
85 360.0 2005.0 2005.0 2.0 1.0 4 512 512 128 5.0 80.0 0.0 2000.0 rsoxs_simulation_data/512-512-128-5.0-80.0-020...
86 360.0 290.0 286.0 2.0 1.0 4 512 512 128 5.0 5.0 0.0 285.0 rsoxs_simulation_data/512-512-128-5.0-5.0-0028...
87 360.0 20005.0 20001.0 2.0 1.0 4 512 512 128 5.0 40.0 0.0 20000.0 rsoxs_simulation_data/512-512-128-5.0-40.0-200...
88 360.0 290.0 289.0 2.0 1.0 4 512 512 128 5.0 10.0 0.0 285.0 rsoxs_simulation_data/512-512-128-5.0-10.0-002...
89 360.0 20005.0 20001.0 2.0 1.0 4 512 512 128 5.0 80.0 0.0 20000.0 rsoxs_simulation_data/512-512-128-5.0-80.0-200...

90 rows × 14 columns

[7]:
toc.describe().loc[['count','min','max']]
[7]:
EndAngle EndEnergy Energy IncrementAngle IncrementEnergy NumThreads NumX NumY NumZ PhysSize Radius StartAngle StartEnergy
count 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0
min 360.0 290.0 285.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 5.0 0.0 285.0
max 360.0 20005.0 20005.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 80.0 0.0 20000.0

Select subset of data From Index

[8]:
sdf = toc.query('Radius==40.0 & EndAngle==360.0 & PhysSize==5 & NumX==512')
sdf = sdf.sort_values('Energy')
sdf.describe().loc[['count','min','max']]
[8]:
EndAngle EndEnergy Energy IncrementAngle IncrementEnergy NumThreads NumX NumY NumZ PhysSize Radius StartAngle StartEnergy
count 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0
min 360.0 290.0 285.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 40.0 0.0 285.0
max 360.0 20005.0 20005.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 40.0 0.0 20000.0

Gather Data

[9]:
def gather(df):
    coords = defaultdict(list)
    data_arrays = []
    for row_index,row in df.iterrows():
        da_img = nexus.read_singleimg_nxs(row['nxs'])
        data_arrays.append(da_img)

        for col_index,value in row.iteritems():
            if col_index=='nxs':
                continue
            coords[col_index].append(value)
    return data_arrays,coords
[10]:
data_arrays,coords = gather(sdf)
[11]:
data_arrays
[11]:
[<xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False,
 <xarray.DataArray 'I' (Qx: 512, Qy: 512)>
 array([[0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        ...,
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.],
        [0., 0., 0., ..., 0., 0., 0.]], dtype=float16)
 Coordinates:
   * Qx       (Qx) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
   * Qy       (Qy) float16 -0.6284 -0.626 -0.6235 -0.621 ... 0.6235 0.626 0.6284
 Attributes:
     lazy:     False]
[12]:
coords
[12]:
defaultdict(list,
            {'EndAngle': [360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0,
              360.0],
             'EndEnergy': [290.0,
              290.0,
              290.0,
              290.0,
              290.0,
              290.0,
              2005.0,
              2005.0,
              2005.0,
              2005.0,
              2005.0,
              2005.0,
              20005.0,
              20005.0,
              20005.0,
              20005.0,
              20005.0,
              20005.0],
             'Energy': [285.0,
              286.0,
              287.0,
              288.0,
              289.0,
              290.0,
              2000.0,
              2001.0,
              2002.0,
              2003.0,
              2004.0,
              2005.0,
              20000.0,
              20001.0,
              20002.0,
              20003.0,
              20004.0,
              20005.0],
             'IncrementAngle': [2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0,
              2.0],
             'IncrementEnergy': [1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0,
              1.0],
             'NumThreads': [4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4,
              4],
             'NumX': [512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512],
             'NumY': [512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512,
              512],
             'NumZ': [128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128,
              128],
             'PhysSize': [5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0,
              5.0],
             'Radius': [40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0,
              40.0],
             'StartAngle': [0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0,
              0.0],
             'StartEnergy': [285.0,
              285.0,
              285.0,
              285.0,
              285.0,
              285.0,
              2000.0,
              2000.0,
              2000.0,
              2000.0,
              2000.0,
              2000.0,
              20000.0,
              20000.0,
              20000.0,
              20000.0,
              20000.0,
              20000.0]})
[13]:
data_arrays[1].plot(norm=LogNorm(1e-9,1),aspect=1.2,size=5)
[13]:
<matplotlib.collections.QuadMesh at 0x7f69ae145160>
_images/Higher_Dimensional_Data_17_1.png

Multiple xArrays: simple xr.concat

[14]:
sdf = toc.query('Radius==40.0 & EndAngle==360.0 & PhysSize==5 & NumX==512')
sdf = sdf.sort_values('Energy')
display(sdf.describe().loc[['count','min','max']])

data_arrays,coords = gather(sdf)

da = xr.concat(data_arrays,dim='Energy')
da
EndAngle EndEnergy Energy IncrementAngle IncrementEnergy NumThreads NumX NumY NumZ PhysSize Radius StartAngle StartEnergy
count 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0 18.0
min 360.0 290.0 285.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 40.0 0.0 285.0
max 360.0 20005.0 20005.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 40.0 0.0 20000.0
[14]:
Show/Hide data repr Show/Hide attributes
xarray.DataArray
'I'
  • Energy: 18
  • Qx: 512
  • Qy: 512
  • 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
    array([[[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           ...,
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]]], dtype=float16)
    • Qy
      (Qy)
      float16
      -0.6284 -0.626 ... 0.626 0.6284
      array([-0.6284, -0.626 , -0.6235, ...,  0.6235,  0.626 ,  0.6284],
            dtype=float16)
    • Qx
      (Qx)
      float16
      -0.6284 -0.626 ... 0.626 0.6284
      array([-0.6284, -0.626 , -0.6235, ...,  0.6235,  0.626 ,  0.6284],
            dtype=float16)
  • lazy :
    False
[15]:
da = da.assign_coords(Energy=sdf.Energy.values)
da
[15]:
Show/Hide data repr Show/Hide attributes
xarray.DataArray
'I'
  • Energy: 18
  • Qx: 512
  • Qy: 512
  • 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
    array([[[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           ...,
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]]], dtype=float16)
    • Qy
      (Qy)
      float16
      -0.6284 -0.626 ... 0.626 0.6284
      array([-0.6284, -0.626 , -0.6235, ...,  0.6235,  0.626 ,  0.6284],
            dtype=float16)
    • Qx
      (Qx)
      float16
      -0.6284 -0.626 ... 0.626 0.6284
      array([-0.6284, -0.626 , -0.6235, ...,  0.6235,  0.626 ,  0.6284],
            dtype=float16)
    • Energy
      (Energy)
      float64
      285.0 286.0 ... 2e+04 2.000e+04
      array([  285.,   286.,   287.,   288.,   289.,   290.,  2000.,  2001.,  2002.,
              2003.,  2004.,  2005., 20000., 20001., 20002., 20003., 20004., 20005.])
  • lazy :
    False
[16]:
da.sel(Qy=0,method='nearest').plot(norm=LogNorm(1e-9,1),yscale='log')
[16]:
<matplotlib.collections.QuadMesh at 0x7f69ac5a42b0>
_images/Higher_Dimensional_Data_21_1.png
[17]:
# da.plot(col='Energy',col_wrap=3,norm=LogNorm(1e-9,1))

Building xArrays: Multi-Index

[18]:
sdf = toc.query('EndAngle==360.0 & PhysSize==5 & NumX==512')
sdf = sdf.sort_values(['Energy','Radius'])
display(sdf.describe().loc[['count','min','max']])
EndAngle EndEnergy Energy IncrementAngle IncrementEnergy NumThreads NumX NumY NumZ PhysSize Radius StartAngle StartEnergy
count 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0 90.0
min 360.0 290.0 285.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 5.0 0.0 285.0
max 360.0 20005.0 20005.0 2.0 1.0 4.0 512.0 512.0 128.0 5.0 80.0 0.0 20000.0
[19]:
data_arrays,coords = gather(sdf)

da = xr.concat(data_arrays,dim=['Energy','Radius'])
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-19-d734cd483e55> in <module>
      1 data_arrays,coords = gather(sdf)
      2
----> 3 da = xr.concat(data_arrays,dim=['Energy','Radius'])

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/concat.py in concat(objs, dim, data_vars, coords, compat, positions, fill_value, join)
    133             "objects, got %s" % type(first_obj)
    134         )
--> 135     return f(objs, dim, data_vars, coords, compat, positions, fill_value, join)
    136
    137

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/concat.py in _dataarray_concat(arrays, dim, data_vars, coords, compat, positions, fill_value, join)
    453         positions,
    454         fill_value=fill_value,
--> 455         join=join,
    456     )
    457     return arrays[0]._from_temp_dataset(ds, name)

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/concat.py in _dataset_concat(datasets, dim, data_vars, coords, compat, positions, fill_value, join)
    412     if coord is not None:
    413         # add concat dimension last to ensure that its in the final Dataset
--> 414         result[coord.name] = coord
    415
    416     return result

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/dataset.py in __setitem__(self, key, value)
   1272             )
   1273
-> 1274         self.update({key: value})
   1275
   1276     def __delitem__(self, key: Hashable) -> None:

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/dataset.py in update(self, other, inplace)
   3573         """
   3574         _check_inplace(inplace)
-> 3575         merge_result = dataset_update_method(self, other)
   3576         return self._replace(inplace=True, **merge_result._asdict())
   3577

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/merge.py in dataset_update_method(dataset, other)
    862                     other[key] = value.drop_vars(coord_names)
    863
--> 864     return merge_core([dataset, other], priority_arg=1, indexes=dataset.indexes)

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/merge.py in merge_core(objects, compat, join, priority_arg, explicit_coords, indexes, fill_value)
    556     assert_unique_multiindex_level_names(variables)
    557
--> 558     dims = calculate_dimensions(variables)
    559
    560     coord_names, noncoord_names = determine_coords(coerced)

~/virtualenv/python3.7.1/lib/python3.7/site-packages/xarray/core/dataset.py in calculate_dimensions(variables)
    203                     "conflicting sizes for dimension %r: "
    204                     "length %s on %r and length %s on %r"
--> 205                     % (dim, size, k, dims[dim], last_used[dim])
    206                 )
    207     return dims

ValueError: conflicting sizes for dimension 'concat_dim': length 2 on 'concat_dim' and length 90 on <this-array>

hmmm…that didn’t work…

Let’s try a multi-index

[20]:
keys =  ['Energy','Radius']
tuples = [(i,j) for i,j in sdf[keys].values]
index = pd.MultiIndex.from_tuples(tuples,names=keys)
index.name = 'system'
da = xr.concat(data_arrays,dim=index)
da
[20]:
Show/Hide data repr Show/Hide attributes
xarray.DataArray
'I'
  • system: 90
  • Qx: 512
  • Qy: 512
  • 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 6e-08 6e-08 6e-08 0.0 0.0 0.0 0.0
    array([[[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            ...,
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]],
    
           [[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            ...,
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]],
    
           [[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            ...,
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]],
    
           ...,
    
           [[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            ...,
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]],
    
           [[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            ...,
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]],
    
           [[0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 6.e-08, 0.e+00, 0.e+00],
            ...,
            [0.e+00, 0.e+00, 6.e-08, ..., 1.e-07, 6.e-08, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 6.e-08, 0.e+00, 0.e+00],
            [0.e+00, 0.e+00, 0.e+00, ..., 0.e+00, 0.e+00, 0.e+00]]],
          dtype=float16)
    • Qy
      (Qy)
      float16
      -0.6284 -0.626 ... 0.626 0.6284
      array([-0.6284, -0.626 , -0.6235, ...,  0.6235,  0.626 ,  0.6284],
            dtype=float16)
    • Qx
      (Qx)
      float16
      -0.6284 -0.626 ... 0.626 0.6284
      array([-0.6284, -0.626 , -0.6235, ...,  0.6235,  0.626 ,  0.6284],
            dtype=float16)
    • system
      (system)
      MultiIndex
      (Energy, Radius)
      array([(285.0, 5.0), (285.0, 10.0), (285.0, 20.0), (285.0, 40.0),
             (285.0, 80.0), (286.0, 5.0), (286.0, 10.0), (286.0, 20.0),
             (286.0, 40.0), (286.0, 80.0), (287.0, 5.0), (287.0, 10.0),
             (287.0, 20.0), (287.0, 40.0), (287.0, 80.0), (288.0, 5.0),
             (288.0, 10.0), (288.0, 20.0), (288.0, 40.0), (288.0, 80.0),
             (289.0, 5.0), (289.0, 10.0), (289.0, 20.0), (289.0, 40.0),
             (289.0, 80.0), (290.0, 5.0), (290.0, 10.0), (290.0, 20.0),
             (290.0, 40.0), (290.0, 80.0), (2000.0, 5.0), (2000.0, 10.0),
             (2000.0, 20.0), (2000.0, 40.0), (2000.0, 80.0), (2001.0, 5.0),
             (2001.0, 10.0), (2001.0, 20.0), (2001.0, 40.0), (2001.0, 80.0),
             (2002.0, 5.0), (2002.0, 10.0), (2002.0, 20.0), (2002.0, 40.0),
             (2002.0, 80.0), (2003.0, 5.0), (2003.0, 10.0), (2003.0, 20.0),
             (2003.0, 40.0), (2003.0, 80.0), (2004.0, 5.0), (2004.0, 10.0),
             (2004.0, 20.0), (2004.0, 40.0), (2004.0, 80.0), (2005.0, 5.0),
             (2005.0, 10.0), (2005.0, 20.0), (2005.0, 40.0), (2005.0, 80.0),
             (20000.0, 5.0), (20000.0, 10.0), (20000.0, 20.0), (20000.0, 40.0),
             (20000.0, 80.0), (20001.0, 5.0), (20001.0, 10.0), (20001.0, 20.0),
             (20001.0, 40.0), (20001.0, 80.0), (20002.0, 5.0), (20002.0, 10.0),
             (20002.0, 20.0), (20002.0, 40.0), (20002.0, 80.0), (20003.0, 5.0),
             (20003.0, 10.0), (20003.0, 20.0), (20003.0, 40.0), (20003.0, 80.0),
             (20004.0, 5.0), (20004.0, 10.0), (20004.0, 20.0), (20004.0, 40.0),
             (20004.0, 80.0), (20005.0, 5.0), (20005.0, 10.0), (20005.0, 20.0),
             (20005.0, 40.0), (20005.0, 80.0)], dtype=object)
    • Energy
      (system)
      float64
      285.0 285.0 ... 2.000e+04 2.000e+04
      array([  285.,   285.,   285.,   285.,   285.,   286.,   286.,   286.,   286.,
               286.,   287.,   287.,   287.,   287.,   287.,   288.,   288.,   288.,
               288.,   288.,   289.,   289.,   289.,   289.,   289.,   290.,   290.,
               290.,   290.,   290.,  2000.,  2000.,  2000.,  2000.,  2000.,  2001.,
              2001.,  2001.,  2001.,  2001.,  2002.,  2002.,  2002.,  2002.,  2002.,
              2003.,  2003.,  2003.,  2003.,  2003.,  2004.,  2004.,  2004.,  2004.,
              2004.,  2005.,  2005.,  2005.,  2005.,  2005., 20000., 20000., 20000.,
             20000., 20000., 20001., 20001., 20001., 20001., 20001., 20002., 20002.,
             20002., 20002., 20002., 20003., 20003., 20003., 20003., 20003., 20004.,
             20004., 20004., 20004., 20004., 20005., 20005., 20005., 20005., 20005.])
    • Radius
      (system)
      float64
      5.0 10.0 20.0 ... 20.0 40.0 80.0
      array([ 5., 10., 20., 40., 80.,  5., 10., 20., 40., 80.,  5., 10., 20., 40.,
             80.,  5., 10., 20., 40., 80.,  5., 10., 20., 40., 80.,  5., 10., 20.,
             40., 80.,  5., 10., 20., 40., 80.,  5., 10., 20., 40., 80.,  5., 10.,
             20., 40., 80.,  5., 10., 20., 40., 80.,  5., 10., 20., 40., 80.,  5.,
             10., 20., 40., 80.,  5., 10., 20., 40., 80.,  5., 10., 20., 40., 80.,
              5., 10., 20., 40., 80.,  5., 10., 20., 40., 80.,  5., 10., 20., 40.,
             80.,  5., 10., 20., 40., 80.])
  • lazy :
    False
[21]:
da.sel(Energy=285.,method='nearest')
[21]:
Show/Hide data repr Show/Hide attributes
xarray.DataArray
'I'
  • Radius: 5
  • Qx: 512
  • Qy: 512
  • 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
    array([[[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]],
    
           [[0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            ...,
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.],
            [0., 0., 0., ..., 0., 0., 0.]]], dtype=float16)
    • Qy
      (Qy)
      float16
      -0.6284 -0.626 ... 0.626 0.6284
      array([-0.6284, -0.626 , -0.6235, ...,  0.6235,  0.626 ,  0.6284],
            dtype=float16)
    • Qx
      (Qx)
      float16
      -0.6284 -0.626 ... 0.626 0.6284
      array([-0.6284, -0.626 , -0.6235, ...,  0.6235,  0.626 ,  0.6284],
            dtype=float16)
    • Radius
      (Radius)
      float64
      5.0 10.0 20.0 40.0 80.0
      array([ 5., 10., 20., 40., 80.])
  • lazy :
    False
[22]:
da.sel(Energy=285.,Qy=0,method='nearest').plot.line(x='Qx',yscale='log',xscale='log')#(norm=LogNorm(1e-9,1))
[22]:
[<matplotlib.lines.Line2D at 0x7f69ad687dd8>,
 <matplotlib.lines.Line2D at 0x7f69ad687400>,
 <matplotlib.lines.Line2D at 0x7f69ad6877f0>,
 <matplotlib.lines.Line2D at 0x7f69ad687be0>,
 <matplotlib.lines.Line2D at 0x7f69ad687fd0>]
_images/Higher_Dimensional_Data_29_1.png
[23]:
da.sel(Energy=285.,method='nearest').plot(col='Radius',col_wrap=3,norm=LogNorm(1e-9,1))
[23]:
<xarray.plot.facetgrid.FacetGrid at 0x7f69ac1f4208>
/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/IPython/core/pylabtools.py:132: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
  fig.canvas.print_figure(bytes_io, **kw)
_images/Higher_Dimensional_Data_30_2.png