Loading viirs natural_color composite fails

[adybbroe]

Code Sample, a minimal, complete, and verifiable piece of code

from glob import glob
import os
from satpy import Scene
from satpy.utils import debug_on
debug_on()

BASEDIR = "/home/a000680/data/polar_in/direct_readout/npp/lvl1/"

if __name__ == '__main__':

    scn = Scene(filenames=glob(os.path.join(BASEDIR, "npp_20180102_1132_32037/*h5")), reader='viirs_sdr')

    areaid = 'euron1'
    scn.load(["natural_color"])

Problem description

Loading viirs natural_color composite fails

Expected Output

Data should load...

Actual Result, Traceback if applicable

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-2-eb43d0fe10d9> in <module>()
----> 1 scn.load(["natural_color"])

/home/a000680/usr/src/satpy/satpy/scene.pyc in load(self, wishlist, calibration, resolution, polarization, compute, unload, **kwargs)
    531         keepables = None
    532         if compute:
--> 533             keepables = self.compute()
    534         if self.missing_datasets:
    535             # copy the set of missing datasets because they won't be valid

/home/a000680/usr/src/satpy/satpy/scene.pyc in compute(self, nodes)
    478             nodes = set(self.dep_tree.trunk(nodes=required_nodes)) - \
    479                 set(self.datasets.keys())
--> 480         return self.read_composites(nodes)
    481 
    482     def _remove_failed_datasets(self, keepables):

/home/a000680/usr/src/satpy/satpy/scene.pyc in read_composites(self, compositor_nodes)
    452         keepables = set()
    453         for item in compositor_nodes:
--> 454             self._generate_composite(item, keepables)
    455         return keepables
    456 

/home/a000680/usr/src/satpy/satpy/scene.pyc in _generate_composite(self, comp_node, keepables)
    427             composite = compositor(prereq_datasets,
    428                                    optional_datasets=optional_datasets,
--> 429                                    **self.info)
    430             self.datasets[composite.id] = composite
    431             if comp_node.name in self.wishlist:

/home/a000680/usr/src/satpy/satpy/composites/__init__.pyc in __call__(self, projectables, optional_datasets, **info)
    372 
    373         try:
--> 374             refl_cor_band = corrector.get_reflectance(sunz, satz, ssadiff, vis.id.name, blue)
    375         except KeyError:
    376             LOG.warning("Could not get the reflectance correction using band name: %s", vis.id.name)

/home/a000680/usr/src/pyspectral/pyspectral/rayleigh.pyc in get_reflectance(self, sun_zenith, sat_zenith, azidiff, bandname, redband)
    217         if redband is not None:
    218             res = np.where(np.less(redband, 20.), res,
--> 219                            (1 - (redband - 20) / 80) * res)
    220 
    221         return np.clip(res, 0, 100)

/home/a000680/usr/src/satpy/satpy/dataset.pyc in wrapper(self, other, *args, **kwargs)
     97 
     98     def wrapper(self, other, *args, **kwargs):
---> 99         res = func(self, other, *args, **kwargs)
    100         res.info = combine_info(self, other)
    101         return res

/home/a000680/usr/src/satpy/satpy/dataset.pyc in __mul__(self, other)
    309     @copy_info
    310     def __mul__(self, other):
--> 311         return super(Dataset, self).__mul__(other)
    312 
    313     @copy_info

/home/a000680/.local/lib/python2.7/site-packages/numpy/ma/core.pyc in __mul__(self, other)
   4084         if self._delegate_binop(other):
   4085             return NotImplemented
-> 4086         return multiply(self, other)
   4087 
   4088     def __rmul__(self, other):

/home/a000680/.local/lib/python2.7/site-packages/numpy/ma/core.pyc in __call__(self, a, b, *args, **kwargs)
   1009         with np.errstate():
   1010             np.seterr(divide='ignore', invalid='ignore')
-> 1011             result = self.f(da, db, *args, **kwargs)
   1012         # Get the mask for the result
   1013         (ma, mb) = (getmask(a), getmask(b))

ValueError: operands could not be broadcast together with shapes (8448,3200) (16896,6400) 

Versions of Python, package at hand and relevant dependencies

Python version 2.7.5
satpy version 0.7.8
numpy version 1.13.1

Thank you for reporting an issue !

[adybbroe]

Oh, sorry, realize I forgot to update my master branch!
But, the above script still doesn't work. But now a different error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/home/a000680/laptop/Pytroll/demo/viirs/viirs_satpy.py in <module>()
     45     # scn = Scene(filenames=glob(os.path.join(BASEDIR, "noaa20_20180129_1358_01025/*h5")),
     46     scn = Scene(filenames=glob(os.path.join(BASEDIR, "npp_20180102_1132_32037/*h5")),
---> 47                 reader='viirs_sdr')
     48 
     49     # areaid = 'england_400m'

/home/a000680/usr/src/satpy/satpy/scene.pyc in __init__(self, filenames, reader, filter_parameters, reader_kwargs, ppp_config_dir, base_dir, sensor, start_time, end_time, area)
    141         self.readers = self.create_reader_instances(filenames=filenames,
    142                                                     reader=reader,
--> 143                                                     reader_kwargs=reader_kwargs)
    144         self.info.update(self._compute_metadata_from_readers())
    145         self.datasets = DatasetDict()

/home/a000680/usr/src/satpy/satpy/scene.pyc in create_reader_instances(self, filenames, reader, reader_kwargs)
    186                             reader=reader,
    187                             reader_kwargs=reader_kwargs,
--> 188                             ppp_config_dir=self.ppp_config_dir)
    189 
    190     @property

/home/a000680/usr/src/satpy/satpy/readers/__init__.pyc in load_readers(filenames, reader, reader_kwargs, ppp_config_dir)
    410             loadables = reader_instance.select_files_from_pathnames(readers_files)
    411         if loadables:
--> 412             reader_instance.create_filehandlers(loadables)
    413             reader_instances[reader_instance.name] = reader_instance
    414             remaining_filenames -= set(loadables)

/home/a000680/usr/src/satpy/satpy/readers/yaml_reader.pyc in create_filehandlers(self, filenames)
    597         for filetype, filetype_info in self.sorted_filetype_items():
    598             filehandlers = self.new_filehandlers_for_filetype(filetype_info,
--> 599                                                               filename_set)
    600 
    601             filename_set -= set([fhd.filename for fhd in filehandlers])

/home/a000680/usr/src/satpy/satpy/readers/yaml_reader.pyc in new_filehandlers_for_filetype(self, filetype_info, filenames)
    586                                                           filename_iter)
    587         filtered_iter = self.filter_fh_by_metadata(filehandler_iter)
--> 588         return list(filtered_iter)
    589 
    590     def create_filehandlers(self, filenames):

/home/a000680/usr/src/satpy/satpy/readers/yaml_reader.pyc in filter_fh_by_metadata(self, filehandlers)
    559     def filter_fh_by_metadata(self, filehandlers):
    560         """Filter out filehandlers using provide filter parameters."""
--> 561         for filehandler in filehandlers:
    562             filehandler.metadata['start_time'] = filehandler.start_time
    563             filehandler.metadata['end_time'] = filehandler.end_time

/home/a000680/usr/src/satpy/satpy/readers/yaml_reader.pyc in new_filehandler_instances(self, filetype_info, filename_items)
    503                 continue
    504 
--> 505             yield filetype_cls(filename, filename_info, filetype_info, *req_fh)
    506 
    507     def time_matches(self, fstart, fend):

/home/a000680/usr/src/satpy/satpy/readers/hdf5_utils.pyc in __init__(self, filename, filename_info, filetype_info)
     53             raise
     54 
---> 55         file_handle.visititems(self.collect_metadata)
     56         self._collect_attrs('', file_handle.attrs)
     57         file_handle.close()

/home/a000680/.local/lib/python2.7/site-packages/h5py/_hl/group.pyc in visititems(self, func)
    467                 name = self._d(name)
    468                 return func(name, self[name])
--> 469             return h5o.visit(self.id, proxy)
    470 
    471     @with_phil

h5py/_objects.pyx in h5py._objects.with_phil.wrapper()

h5py/_objects.pyx in h5py._objects.with_phil.wrapper()

h5py/h5o.pyx in h5py.h5o.visit()

h5py/defs.pyx in h5py.defs.H5Ovisit_by_name()

h5py/h5o.pyx in h5py.h5o.cb_obj_simple()

/home/a000680/.local/lib/python2.7/site-packages/h5py/_hl/group.pyc in proxy(name)
    466                 """ Use the text name of the object, not bytes """
    467                 name = self._d(name)
--> 468                 return func(name, self[name])
    469             return h5o.visit(self.id, proxy)
    470 

/home/a000680/usr/src/satpy/satpy/readers/hdf5_utils.pyc in collect_metadata(self, name, obj)
     67             self.file_content[name + "/dtype"] = obj.dtype
     68             self.file_content[name + "/shape"] = obj.shape
---> 69         self._collect_attrs(name, obj.attrs)
     70 
     71     def __getitem__(self, key):

/home/a000680/usr/src/satpy/satpy/readers/hdf5_utils.pyc in _collect_attrs(self, name, attrs)
     60         for key, value in six.iteritems(attrs):
     61             value = np.squeeze(value)
---> 62             self.file_content["{}/attr/{}".format(name, key)] = np2str(value)
     63 
     64     def collect_metadata(self, name, obj):

/home/a000680/usr/src/satpy/satpy/readers/helper_functions.pyc in np2str(value)
     36     """Convert an np.string_ to str."""
     37     if issubclass(value.dtype.type, np.string_):
---> 38         value = np.asscalar(value)
     39         if not isinstance(value, str):
     40             # python 3 - was scalar numpy array of bytes

/home/a000680/.local/lib/python2.7/site-packages/numpy/lib/type_check.pyc in asscalar(a)
    478 
    479     """
--> 480     return a.item()
    481 
    482 #-----------------------------------------------------------------------------

ValueError: can only convert an array of size 1 to a Python scalar

Same Python and Numpy versions. But satpy 0.8.1
And h5py version 2.7.1

[adybbroe]

I checked satpy version 0.8.0, that works. With the following code:

from glob import glob
import os
from datetime import datetime
from satpy import find_files_and_readers, Scene
from satpy.utils import debug_on
debug_on()


BASEDIR = "/home/a000680/data/polar_in/direct_readout/npp/lvl1/"

if __name__ == '__main__':

    my_files = find_files_and_readers(base_dir=BASEDIR + "npp_20180102_1132_32037",
                                      start_time=datetime(2018, 1, 2, 11, 34),
                                      end_time=datetime(2018, 1, 2, 11, 44),
                                      reader='viirs_sdr')
    scn = Scene(filenames=my_files)
    scn.load(['natural_color'])
    scn.show('natural_color')

[djhoese]

Looks like this was fixed with your PR. Since it hasn't gone in to the master branch yet this is still open. Closing manually.