diff --git a/.travis.yml b/.travis.yml index 94af6b8..2630226 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ +### XXX DOCS SHOULD INCLUDE LINKCHECK BUT IT'S BUSTED ON CURRENT SPHINX (3.1, 2020-Jun-17): +### https://github.com/sphinx-doc/sphinx/issues/7806 + language: c os: @@ -34,7 +37,7 @@ script: - python setup.py build_ext --inplace - py.test --cov toasty toasty # Different stdlib organization breaks API doc files on Python 2: - - if [[ $PYTHON_VERSION == 3.7 ]] ; then make -C docs html linkcheck ; fi + - if [[ $PYTHON_VERSION == 3.7 ]] ; then make -C docs html ; fi after_success: - coveralls diff --git a/MANIFEST.in b/MANIFEST.in index beea1f9..5774b98 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,7 +3,7 @@ include .coveragerc include *.md include *.yml -recursive-include toasty *.png +recursive-include toasty *.fits *.png *.wwtxml graft docs prune docs/_build diff --git a/docs/api/toasty.cli.indent_xml.rst b/docs/api/toasty.cli.indent_xml.rst deleted file mode 100644 index a9986f2..0000000 --- a/docs/api/toasty.cli.indent_xml.rst +++ /dev/null @@ -1,6 +0,0 @@ -indent_xml -========== - -.. currentmodule:: toasty.cli - -.. autofunction:: indent_xml diff --git a/docs/api/toasty.cli.multi_tan_make_wtml_getparser.rst b/docs/api/toasty.cli.multi_tan_make_wtml_getparser.rst deleted file mode 100644 index 0b279ad..0000000 --- a/docs/api/toasty.cli.multi_tan_make_wtml_getparser.rst +++ /dev/null @@ -1,6 +0,0 @@ -multi_tan_make_wtml_getparser -============================= - -.. currentmodule:: toasty.cli - -.. autofunction:: multi_tan_make_wtml_getparser diff --git a/docs/api/toasty.cli.multi_tan_make_wtml_impl.rst b/docs/api/toasty.cli.multi_tan_make_wtml_impl.rst deleted file mode 100644 index a2c567a..0000000 --- a/docs/api/toasty.cli.multi_tan_make_wtml_impl.rst +++ /dev/null @@ -1,6 +0,0 @@ -multi_tan_make_wtml_impl -======================== - -.. currentmodule:: toasty.cli - -.. autofunction:: multi_tan_make_wtml_impl diff --git a/docs/api/toasty.cli.wwtl_sample_image_tiles_getparser.rst b/docs/api/toasty.cli.wwtl_sample_image_tiles_getparser.rst new file mode 100644 index 0000000..09d1808 --- /dev/null +++ b/docs/api/toasty.cli.wwtl_sample_image_tiles_getparser.rst @@ -0,0 +1,6 @@ +wwtl_sample_image_tiles_getparser +================================= + +.. currentmodule:: toasty.cli + +.. autofunction:: wwtl_sample_image_tiles_getparser diff --git a/docs/api/toasty.cli.wwtl_sample_image_tiles_impl.rst b/docs/api/toasty.cli.wwtl_sample_image_tiles_impl.rst new file mode 100644 index 0000000..2f697a7 --- /dev/null +++ b/docs/api/toasty.cli.wwtl_sample_image_tiles_impl.rst @@ -0,0 +1,6 @@ +wwtl_sample_image_tiles_impl +============================ + +.. currentmodule:: toasty.cli + +.. autofunction:: wwtl_sample_image_tiles_impl diff --git a/docs/api/toasty.io.read_image_as_pil.rst b/docs/api/toasty.io.read_image_as_pil.rst new file mode 100644 index 0000000..71b13cd --- /dev/null +++ b/docs/api/toasty.io.read_image_as_pil.rst @@ -0,0 +1,6 @@ +read_image_as_pil +================= + +.. currentmodule:: toasty.io + +.. autofunction:: read_image_as_pil diff --git a/docs/api/toasty.pipeline.AzureBlobPipelineIo.rst b/docs/api/toasty.pipeline.AzureBlobPipelineIo.rst index 5b939c4..b934b5c 100644 --- a/docs/api/toasty.pipeline.AzureBlobPipelineIo.rst +++ b/docs/api/toasty.pipeline.AzureBlobPipelineIo.rst @@ -10,12 +10,14 @@ AzureBlobPipelineIo .. autosummary:: + ~AzureBlobPipelineIo.check_exists ~AzureBlobPipelineIo.get_item ~AzureBlobPipelineIo.list_items ~AzureBlobPipelineIo.put_item .. rubric:: Methods Documentation + .. automethod:: check_exists .. automethod:: get_item .. automethod:: list_items .. automethod:: put_item diff --git a/docs/api/toasty.pipeline.LocalPipelineIo.rst b/docs/api/toasty.pipeline.LocalPipelineIo.rst index 29af4db..762f98d 100644 --- a/docs/api/toasty.pipeline.LocalPipelineIo.rst +++ b/docs/api/toasty.pipeline.LocalPipelineIo.rst @@ -10,12 +10,14 @@ LocalPipelineIo .. autosummary:: + ~LocalPipelineIo.check_exists ~LocalPipelineIo.get_item ~LocalPipelineIo.list_items ~LocalPipelineIo.put_item .. rubric:: Methods Documentation + .. automethod:: check_exists .. automethod:: get_item .. automethod:: list_items .. automethod:: put_item diff --git a/docs/api/toasty.pipeline.PipelineIo.rst b/docs/api/toasty.pipeline.PipelineIo.rst index 2fb004d..e4beec6 100644 --- a/docs/api/toasty.pipeline.PipelineIo.rst +++ b/docs/api/toasty.pipeline.PipelineIo.rst @@ -10,12 +10,14 @@ PipelineIo .. autosummary:: + ~PipelineIo.check_exists ~PipelineIo.get_item ~PipelineIo.list_items ~PipelineIo.put_item .. rubric:: Methods Documentation + .. automethod:: check_exists .. automethod:: get_item .. automethod:: list_items .. automethod:: put_item diff --git a/setup.py b/setup.py index 41cc563..cf96096 100644 --- a/setup.py +++ b/setup.py @@ -69,10 +69,10 @@ setup_args = dict( }, install_requires = [ - 'cython', - 'numpy', - 'pillow', - 'wwt_data_formats', + 'cython>=0.20', + 'numpy>=1.7', + 'pillow>=7.0', + 'wwt_data_formats>=0.2.0', ], extras_require = { diff --git a/toasty/cli.py b/toasty/cli.py index f5e2f4f..b0d11a8 100644 --- a/toasty/cli.py +++ b/toasty/cli.py @@ -22,35 +22,9 @@ def warn(msg): print('warning:', msg, file=sys.stderr) -# TODO: This should be superseded by wwt_data_formats -def indent_xml(elem, level=0): - """A dumb XML indenter. - - We create XML files using xml.etree.ElementTree, which is careful about - spacing and so by default creates ugly files with no linewraps or - indentation. This function is copied from `ElementLib - `_ and implements - basic, sensible indentation using "tail" text. - - """ - i = "\n" + level * " " - - if len(elem): - if not elem.text or not elem.text.strip(): - elem.text = i + " " - if not elem.tail or not elem.tail.strip(): - elem.tail = i - for elem in elem: # intentionally updating "elem" here! - indent_xml(elem, level + 1) - if not elem.tail or not elem.tail.strip(): - elem.tail = i - else: - if level and (not elem.tail or not elem.tail.strip()): - elem.tail = i - - -def stub_wtml(imgset, wtml_path): - """Given an ImageSet object, save its information into a stub WTML file. +def stub_wtml(imgset, wtml_path, place=None): + """Given an ImageSet object and potential a Place, save its information into a + stub WTML file. """ from wwt_data_formats import write_xml_doc @@ -59,9 +33,15 @@ def stub_wtml(imgset, wtml_path): from wwt_data_formats.place import Place folder = Folder() - place = Place() - place.data_set_type = DataSetType.SKY - place.foreground_image_set = imgset + + if place is None: + place = Place() + place.data_set_type = DataSetType.SKY + place.foreground_image_set = imgset + place.name = 'Toasty' + place.thumbnail = imgset.thumbnail_url + place.zoom_level = 1.0 + folder.children = [place] with open(wtml_path, 'wt') as f: @@ -216,93 +196,7 @@ def multi_tan_make_data_tiles_impl(settings): for p in sorted(percentiles.keys()): print(' {} = {}'.format(p, percentiles[p])) - -# "multi_tan_make_wtml" subcommand - -def multi_tan_make_wtml_getparser(parser): - parser.add_argument( - '--hdu-index', - metavar = 'INDEX', - type = int, - default = 0, - help = 'Which HDU to load in each input FITS file', - ) - parser.add_argument( - '--name', - metavar = 'NAME', - default = 'MultiTan', - help = 'The dataset name to embed in the WTML file', - ) - parser.add_argument( - '--url-prefix', - metavar = 'PREFIX', - default = './', - help = 'The prefix to the tile URL that will be embedded in the WTML', - ) - parser.add_argument( - '--fov-factor', - metavar = 'NUMBER', - type = float, - default = 1.7, - help = 'How tall the FOV should be (ie the zoom level) when viewing this image, in units of the image height', - ) - parser.add_argument( - '--bandpass', - metavar = 'BANDPASS-NAME', - default = 'Visible', - help = 'The bandpass of the image data: "Gamma", "HydrogenAlpha", "IR", "Microwave", "Radio", "Ultraviolet", "Visible", "VisibleNight", "XRay"', - ) - parser.add_argument( - '--description', - metavar = 'TEXT', - default = '', - help = 'Free text describing what this image is', - ) - parser.add_argument( - '--credits-text', - metavar = 'TEXT', - default = 'Created by toasty, part of the AAS WorldWide Telescope.', - help = 'A brief credit of who created and processed the image data', - ) - parser.add_argument( - '--credits-url', - metavar = 'URL', - default = '', - help = 'A URL with additional credit information', - ) - parser.add_argument( - '--thumbnail-url', - metavar = 'URL', - default = '', - help = 'A URL of a thumbnail image (96x45 JPEG) representing this dataset', - ) - parser.add_argument( - 'paths', - metavar = 'PATHS', - nargs = '+', - help = 'The FITS files with image data', - ) - -def multi_tan_make_wtml_impl(settings): - from xml.etree import ElementTree as etree - from .multi_tan import MultiTanDataSource - - ds = MultiTanDataSource(settings.paths, hdu_index=settings.hdu_index) - ds.compute_global_pixelization() - - folder = ds.create_wtml( - name = settings.name, - url_prefix = settings.url_prefix, - fov_factor = settings.fov_factor, - bandpass = settings.bandpass, - description_text = settings.description, - credits_text = settings.credits_text, - credits_url = settings.credits_url, - thumbnail_url = settings.thumbnail_url, - ) - indent_xml(folder) - doc = etree.ElementTree(folder) - doc.write(sys.stdout, encoding='utf-8', xml_declaration=True) + # TODO: this should populate and emit a stub index_rel.wtml file. # "pipeline_fetch_inputs" subcommand @@ -449,16 +343,24 @@ def study_sample_image_tiles_getparser(parser): def study_sample_image_tiles_impl(settings): + import numpy as np + import PIL.Image from wwt_data_formats.imageset import ImageSet - from .io import read_image + from .io import read_image_as_pil from .pyramid import PyramidIO - from .study import tile_study_image + from .study import make_thumbnail_bitmap, tile_study_image - # Create the base tiles. + # Prevent max image size aborts: + PIL.Image.MAX_IMAGE_PIXELS = None + # Load image. pio = PyramidIO(settings.outdir) - img = read_image(settings.imgpath) - tiling = tile_study_image(img, pio) + img = read_image_as_pil(settings.imgpath) + tiling = tile_study_image(np.asarray(img), pio) + + # Thumbnail. + thumb = make_thumbnail_bitmap(img) + thumb.save(os.path.join(settings.outdir, 'thumb.jpg'), format='JPEG') # Write out a stub WTML file. The only information this will actually # contain is the number of tile levels. Other information can be filled @@ -466,8 +368,96 @@ def study_sample_image_tiles_impl(settings): imgset = ImageSet() tiling.apply_to_imageset(imgset) imgset.base_degrees_per_tile = 1.0 # random default to make it viewable + imgset.name = 'Toasty' + imgset.thumbnail_url = 'thumb.jpg' imgset.url = pio.get_path_scheme() + '.png' - stub_wtml(imgset, os.path.join(settings.outdir, 'toasty.wtml')) + stub_wtml(imgset, os.path.join(settings.outdir, 'index_rel.wtml')) + + +# "wwtl_sample_image_tiles" subcommand + +def wwtl_sample_image_tiles_getparser(parser): + parser.add_argument( + '--outdir', + metavar = 'PATH', + default = '.', + help = 'The root directory of the output tile pyramid', + ) + parser.add_argument( + 'wwtl_path', + metavar = 'WWTL-PATH', + help = 'The WWTL layer file to be processed', + ) + + +def wwtl_sample_image_tiles_impl(settings): + from io import BytesIO + import numpy as np + import PIL.Image + + from wwt_data_formats.enums import DataSetType, ProjectionType + from wwt_data_formats.layers import ImageSetLayer, LayerContainerReader + from wwt_data_formats.place import Place + + from .io import read_image_as_pil + from .pyramid import PyramidIO + from .study import make_thumbnail_bitmap, tile_study_image + + # Prevent max image size aborts: + PIL.Image.MAX_IMAGE_PIXELS = None + + # Load WWTL and see if it matches expectations + lc = LayerContainerReader.from_file(settings.wwtl_path) + + if len(lc.layers) != 1: + die('WWTL file must contain exactly one layer') + + layer = lc.layers[0] + if not isinstance(layer, ImageSetLayer): + die('WWTL file must contain an imageset layer') + + imgset = layer.image_set + if imgset.projection != ProjectionType.SKY_IMAGE: + die('WWTL imageset layer must have "SkyImage" projection type') + + # Looks OK. Read and parse the image. + img_data = lc.read_layer_file(layer, layer.extension) + img = PIL.Image.open(BytesIO(img_data)) + + # Tile it! + pio = PyramidIO(settings.outdir) + tiling = tile_study_image(np.asarray(img), pio) + + # Thumbnail. + thumb = make_thumbnail_bitmap(img) + thumb.save(os.path.join(settings.outdir, 'thumb.jpg'), format='JPEG') + + # Write a WTML file. We reuse the existing imageset as much as possible, + # but update the parameters that change in the tiling process. + + place = Place() + wcs_keywords = imgset.wcs_headers_from_position() + tiling.apply_to_imageset(imgset) + imgset.set_position_from_wcs(wcs_keywords, img.width, img.height, place=place) + + if not imgset.name: + imgset.name = 'Toasty' + imgset.thumbnail_url = 'thumb.jpg' + imgset.url = pio.get_path_scheme() + '.png' + + place.data_set_type = DataSetType.SKY + place.foreground_image_set = imgset + place.name = imgset.name + place.thumbnail = imgset.thumbnail_url + + stub_wtml(imgset, os.path.join(settings.outdir, 'index_rel.wtml'), place=place) + + # Helpful hint: + + print(f'Successfully tiled input "{settings.wwtl_path}" at level {imgset.tile_levels}.') + print('To create parent tiles, consider running:') + print() + print(f' toasty cascade --start {imgset.tile_levels} {settings.outdir}') # The CLI driver: diff --git a/toasty/io.py b/toasty/io.py index 800927c..fb5a590 100644 --- a/toasty/io.py +++ b/toasty/io.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function __all__ = ''' +read_image_as_pil read_image save_png '''.split() @@ -20,24 +21,61 @@ def save_png(pth, array): Parameters ---------- pth : str - Path to write to + Path to write to array : array-like - Image to save + Image to save """ Image.fromarray(array).save(pth) +def read_image_as_pil(path): + """Load a bitmap image into a PIL Image. + + The loading supports whatever image formats PIL does. As a special-case + hack, if the input path has extension ``.psd`` or ``.psb``, the + ``psd_tools`` module will be used if available. + + Parameters + ---------- + path : str + The path of the image to read + + Returns + ------- + img : :class:`PIL.Image.Image` + The image data. + """ + if path.endswith('.psd') or path.endswith('.psb'): + try: + from psd_tools import PSDImage + except ImportError: + pass + else: + psd = PSDImage.open(path) + return psd.composite() + + return Image.open(path) + + def read_image(path): """Load a bitmap image into a Numpy array. - The loading is done using PIL (the Python Imaging Library, usually the + The loading is generally done using PIL (the Python Imaging Library, usually the "pillow" implementation these days) so it supports whatever image formats - PIL does. + PIL does. As a special-case hack, if the input path has extension ``.psd`` or ``.psb``, + the ``psd_tools`` module will be used if available. Parameters ---------- path : str - The path of the image to read + The path of the image to read + + Returns + ------- + data : :class:`numpy.ndarray` + The image data. The array will have shape ``(height, width, planes)``, where + the first two axes are the image shape and the third is the number of color planes: + 3 for RGB or potentially 4 for RGBA. The data type will be ``uint8``. """ - return np.asarray(Image.open(path)) + return np.asarray(read_image_as_pil(path)) diff --git a/toasty/pipeline.py b/toasty/pipeline.py index 7feb2c7..d56a98c 100644 --- a/toasty/pipeline.py +++ b/toasty/pipeline.py @@ -544,8 +544,8 @@ class InputImage(ABC): ------- None. - Remarks - ------- + Notes + ----- This function should also take care of creating the thumbnail. """ @@ -644,8 +644,8 @@ class BitmapInputImage(InputImage): ------- A :class:`PIL.Image` of the image data. - Remarks - ------- + Notes + ----- This function will only be called once. It can assume that :meth:`InputImage.ensure_input_cached` has already been called. @@ -989,6 +989,8 @@ class PipelineManager(object): for stem, is_folder in self._pipeio.list_items(): if not is_folder: continue + if not self._pipeio.check_exists(stem, 'index.wtml'): + continue wtml_data = BytesIO() self._pipeio.get_item(stem, 'index.wtml', dest=wtml_data) diff --git a/toasty/study.py b/toasty/study.py index ef935d5..7d0ab83 100644 --- a/toasty/study.py +++ b/toasty/study.py @@ -95,13 +95,21 @@ class StudyTiling(object): imgset : ``wwt_data_formats.imageset.ImageSet`` The object to modify - Remarks - ------- - The only setting currently transferred is the number of tile levels. + Notes + ----- + The settings currently transferred are the number of tile levels and + the projection type. """ + from wwt_data_formats.enums import ProjectionType + imgset.tile_levels = self._tile_levels + if self._tile_levels == 0: + imgset.projection = ProjectionType.SKY_IMAGE + else: + imgset.projection = ProjectionType.TAN + def image_to_tile(self, im_ix, im_iy): """Convert an image pixel position to a tiled pixel position. @@ -113,8 +121,8 @@ class StudyTiling(object): im_iy : integer A 0-based vertical pixel position in the image coordinate system. - Remarks - ------- + Notes + ----- ``(0, 0)`` is the top-left corner of the image. The input values need not lie on the image. (I.e., they may be negative.) diff --git a/toasty/tests/layercontainer.wwtxml b/toasty/tests/layercontainer.wwtxml new file mode 100644 index 0000000..4fcb45a --- /dev/null +++ b/toasty/tests/layercontainer.wwtxml @@ -0,0 +1,32 @@ + + + + + + + + + + + diff --git a/toasty/tests/test_multi_tan.py b/toasty/tests/test_multi_tan.py index c68af0c..eb810fe 100644 --- a/toasty/tests/test_multi_tan.py +++ b/toasty/tests/test_multi_tan.py @@ -131,35 +131,3 @@ class TestMultiTan(object): test_path('wcs512.fits.gz') ] cli.entrypoint(args) - - - def test_basic_wtml_cli(self): - from xml.etree import ElementTree as etree - expected = etree.fromstring(self.WTML) - - prev_stdout = sys.stdout - from io import BytesIO - output = BytesIO() - - try: - sys.stdout = output - args = [ - 'multi-tan-make-wtml', - '--hdu-index', '0', - '--name', 'TestName', - '--url-prefix', 'UP', - '--fov-factor', '1.0', - '--bandpass', 'Gamma', - '--description', 'DT', - '--credits-text', 'CT', - '--credits-url', 'CU', - '--thumbnail-url', 'TU', - test_path('wcs512.fits.gz') - ] - cli.entrypoint(args) - finally: - sys.stdout = prev_stdout - - observed = etree.fromstring(output.getvalue()) - output.close() - assert_xml_elements_equal(observed, expected) diff --git a/toasty/tests/test_study.py b/toasty/tests/test_study.py index 222f2f0..cd64fac 100644 --- a/toasty/tests/test_study.py +++ b/toasty/tests/test_study.py @@ -17,10 +17,11 @@ from .. import study class TestStudy(object): WTML = """ - - + + - + + thumb.jpg @@ -80,7 +81,7 @@ class TestStudy(object): ] cli.entrypoint(args) - with open(self.work_path('toasty.wtml')) as f: + with open(self.work_path('index_rel.wtml')) as f: observed = etree.fromstring(f.read()) assert_xml_elements_equal(observed, expected) diff --git a/toasty/tests/test_wwtl.py b/toasty/tests/test_wwtl.py new file mode 100644 index 0000000..05574ff --- /dev/null +++ b/toasty/tests/test_wwtl.py @@ -0,0 +1,58 @@ +# -*- mode: python; coding: utf-8 -*- +# Copyright 2020 the AAS WorldWide Telescope project +# Licensed under the MIT License. + +from __future__ import absolute_import, division, print_function + +import numpy as np +from numpy import testing as nt +import os.path +import pytest +import sys + +from wwt_data_formats.filecabinet import FileCabinetWriter + +from . import assert_xml_elements_equal, test_path +from .. import cli +from .. import study + + +class TestStudy(object): + def setup_method(self, method): + from tempfile import mkdtemp + self.work_dir = mkdtemp() + + def teardown_method(self, method): + from shutil import rmtree + rmtree(self.work_dir) + + def work_path(self, *pieces): + return os.path.join(self.work_dir, *pieces) + + def test_basic_cli(self): + # First, create a WWTL. NB, filenames should match the ID's in the XML + # file. + + fw = FileCabinetWriter() + + with open(test_path('layercontainer.wwtxml'), 'rb') as f: + b = f.read() + + fw.add_file_with_data('55cb0cce-c44a-4a44-a509-ea66fce643a5.wwtxml', b) + + with open(test_path('NGC253ALMA.jpg'), 'rb') as f: + b = f.read() + + fw.add_file_with_data('55cb0cce-c44a-4a44-a509-ea66fce643a5\\7ecb6411-e4ee-4dfa-90ef-77d6f486c7d2.jpg', b) + + with open(self.work_path('image.wwtl'), 'wb') as f: + fw.emit(f) + + # Now run it through the CLI. + + args = [ + 'wwtl-sample-image-tiles', + '--outdir', self.work_path('tiles'), + self.work_path('image.wwtl') + ] + cli.entrypoint(args)