From 9a6180cbc0f3300f506286cbebc1994e1fc2140a Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Tue, 31 Mar 2020 14:11:57 -0400 Subject: [PATCH 01/11] toasty/pipeline.py: when reindexing, skip folders without index.wtml --- toasty/pipeline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/toasty/pipeline.py b/toasty/pipeline.py index 7feb2c7..2a86701 100644 --- a/toasty/pipeline.py +++ b/toasty/pipeline.py @@ -989,6 +989,8 @@ class PipelineManager(object): for stem, is_folder in self._pipeio.list_items(): if not is_folder: continue + if not self._pipeio.check_exists(stem, 'index.wtml'): + continue wtml_data = BytesIO() self._pipeio.get_item(stem, 'index.wtml', dest=wtml_data) From c2414918e50dfeb5766bf54f95b2de434605ac92 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Tue, 16 Jun 2020 16:44:24 -0400 Subject: [PATCH 02/11] toasty/io.py: document the data returned by read_image() --- toasty/io.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/toasty/io.py b/toasty/io.py index 800927c..39fd9fe 100644 --- a/toasty/io.py +++ b/toasty/io.py @@ -20,9 +20,9 @@ def save_png(pth, array): Parameters ---------- pth : str - Path to write to + Path to write to array : array-like - Image to save + Image to save """ Image.fromarray(array).save(pth) @@ -37,7 +37,14 @@ def read_image(path): Parameters ---------- path : str - The path of the image to read + The path of the image to read + + Returns + ------- + data : :class:`numpy.ndarray` + The image data. The array will have shape ``(height, width, planes)``, where + the first two axes are the image shape and the third is the number of color planes: + 3 for RGB or potentially 4 for ARGB. The data type will be ``uint8``. """ return np.asarray(Image.open(path)) From ddfaed29dae95ee29067780802f456d6b3f87145 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Wed, 17 Jun 2020 13:36:32 -0400 Subject: [PATCH 03/11] toasty/io.py: little hack, special-case loading of PSD/PSB files This lets us tile the PHAT image of Andromeda/M31, which is 5 GB and stored as a Photoshop file. --- toasty/io.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/toasty/io.py b/toasty/io.py index 39fd9fe..5ba41e3 100644 --- a/toasty/io.py +++ b/toasty/io.py @@ -30,9 +30,10 @@ def save_png(pth, array): def read_image(path): """Load a bitmap image into a Numpy array. - The loading is done using PIL (the Python Imaging Library, usually the + The loading is generally done using PIL (the Python Imaging Library, usually the "pillow" implementation these days) so it supports whatever image formats - PIL does. + PIL does. As a special-case hack, if the input path has extension ``.psd`` or ``.psb``, + the ``psd_tools`` module will be used if available. Parameters ---------- @@ -47,4 +48,13 @@ def read_image(path): 3 for RGB or potentially 4 for ARGB. The data type will be ``uint8``. """ + if path.endswith('.psd') or path.endswith('.psb'): + try: + from psd_tools import PSDImage + except ImportError: + pass + else: + psd = PSDImage.open(path) + return np.asarray(psd.composite()) + return np.asarray(Image.open(path)) From 1a9bb6c97745a6b8ae6a1ded87a0ef69ce01ada8 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Wed, 17 Jun 2020 14:14:24 -0400 Subject: [PATCH 04/11] Improve the output from `toasty study-sample-image-tiles` Output a thumbnail image now that we're equipped to do so. Rename the output WTML file to our maybe-new-standard `index_rel.wtml` and populate it with relative URLs. Correct some parameters in the WTML to make it so that the imageset can be viewed in WWT with minimal edits. --- toasty/cli.py | 25 +++++++++++++++++++------ toasty/io.py | 43 ++++++++++++++++++++++++++++++++----------- toasty/study.py | 10 +++++++++- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/toasty/cli.py b/toasty/cli.py index f5e2f4f..178acf6 100644 --- a/toasty/cli.py +++ b/toasty/cli.py @@ -62,6 +62,9 @@ def stub_wtml(imgset, wtml_path): place = Place() place.data_set_type = DataSetType.SKY place.foreground_image_set = imgset + place.name = 'Toasty' + place.thumbnail = imgset.thumbnail_url + place.zoom_level = 1.0 folder.children = [place] with open(wtml_path, 'wt') as f: @@ -449,16 +452,24 @@ def study_sample_image_tiles_getparser(parser): def study_sample_image_tiles_impl(settings): + import numpy as np + import PIL.Image from wwt_data_formats.imageset import ImageSet - from .io import read_image + from .io import read_image_as_pil from .pyramid import PyramidIO - from .study import tile_study_image + from .study import make_thumbnail_bitmap, tile_study_image - # Create the base tiles. + # Prevent max image size aborts: + PIL.Image.MAX_IMAGE_PIXELS = None + # Load image. pio = PyramidIO(settings.outdir) - img = read_image(settings.imgpath) - tiling = tile_study_image(img, pio) + img = read_image_as_pil(settings.imgpath) + tiling = tile_study_image(np.asarray(img), pio) + + # Thumbnail. + thumb = make_thumbnail_bitmap(img) + thumb.save(os.path.join(settings.outdir, 'thumb.jpg'), format='JPEG') # Write out a stub WTML file. The only information this will actually # contain is the number of tile levels. Other information can be filled @@ -466,8 +477,10 @@ def study_sample_image_tiles_impl(settings): imgset = ImageSet() tiling.apply_to_imageset(imgset) imgset.base_degrees_per_tile = 1.0 # random default to make it viewable + imgset.name = 'Toasty' + imgset.thumbnail_url = 'thumb.jpg' imgset.url = pio.get_path_scheme() + '.png' - stub_wtml(imgset, os.path.join(settings.outdir, 'toasty.wtml')) + stub_wtml(imgset, os.path.join(settings.outdir, 'index_rel.wtml')) # The CLI driver: diff --git a/toasty/io.py b/toasty/io.py index 5ba41e3..57be19d 100644 --- a/toasty/io.py +++ b/toasty/io.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function __all__ = ''' +read_image_as_pil read_image save_png '''.split() @@ -27,6 +28,35 @@ def save_png(pth, array): Image.fromarray(array).save(pth) +def read_image_as_pil(path): + """Load a bitmap image into a PIL Image. + + The loading supports whatever image formats PIL does. As a special-case + hack, if the input path has extension ``.psd`` or ``.psb``, the + ``psd_tools`` module will be used if available. + + Parameters + ---------- + path : str + The path of the image to read + + Returns + ------- + img : :class:`PIL.Image` + The image data. + """ + if path.endswith('.psd') or path.endswith('.psb'): + try: + from psd_tools import PSDImage + except ImportError: + pass + else: + psd = PSDImage.open(path) + return psd.composite() + + return Image.open(path) + + def read_image(path): """Load a bitmap image into a Numpy array. @@ -45,16 +75,7 @@ def read_image(path): data : :class:`numpy.ndarray` The image data. The array will have shape ``(height, width, planes)``, where the first two axes are the image shape and the third is the number of color planes: - 3 for RGB or potentially 4 for ARGB. The data type will be ``uint8``. + 3 for RGB or potentially 4 for RGBA. The data type will be ``uint8``. """ - if path.endswith('.psd') or path.endswith('.psb'): - try: - from psd_tools import PSDImage - except ImportError: - pass - else: - psd = PSDImage.open(path) - return np.asarray(psd.composite()) - - return np.asarray(Image.open(path)) + return np.asarray(read_image_as_pil(path)) diff --git a/toasty/study.py b/toasty/study.py index ef935d5..f1fc448 100644 --- a/toasty/study.py +++ b/toasty/study.py @@ -97,11 +97,19 @@ class StudyTiling(object): Remarks ------- - The only setting currently transferred is the number of tile levels. + The settings currently transferred are the number of tile levels and + the projection type. """ + from wwt_data_formats.enums import ProjectionType + imgset.tile_levels = self._tile_levels + if self._tile_levels == 0: + imgset.projection = ProjectionType.SKY_IMAGE + else: + imgset.projection = ProjectionType.TAN + def image_to_tile(self, im_ix, im_iy): """Convert an image pixel position to a tiled pixel position. From 4d0d4b7f3a5b5300484acd0e21c025a8168eb7e6 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Wed, 17 Jun 2020 16:19:07 -0400 Subject: [PATCH 05/11] toasty/cli.py: add wwtl-sample-image-tiles command This adds a command that implements a workflow motivated by David Weigel's usage. If you have a very large sky image without astrometry, you can open it up in WWT and manually position it. But how do you export it for viewing on the web? Our solution here is to ask the user to save their image layer in a WWTL layer export file containing just the image set layer. This tool then reads in that file, extracts the underlying (large) image, tiles it, and also translates the astrometric information appropriately into a WTML file expressing the tiled output. The output WTML file still needs editing to absolute-ize the URLs, but it's pretty close to a one-stop-shop for the data processing. --- toasty/cli.py | 106 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 8 deletions(-) diff --git a/toasty/cli.py b/toasty/cli.py index 178acf6..39cca8a 100644 --- a/toasty/cli.py +++ b/toasty/cli.py @@ -49,8 +49,9 @@ def indent_xml(elem, level=0): elem.tail = i -def stub_wtml(imgset, wtml_path): - """Given an ImageSet object, save its information into a stub WTML file. +def stub_wtml(imgset, wtml_path, place=None): + """Given an ImageSet object and potential a Place, save its information into a + stub WTML file. """ from wwt_data_formats import write_xml_doc @@ -59,12 +60,15 @@ def stub_wtml(imgset, wtml_path): from wwt_data_formats.place import Place folder = Folder() - place = Place() - place.data_set_type = DataSetType.SKY - place.foreground_image_set = imgset - place.name = 'Toasty' - place.thumbnail = imgset.thumbnail_url - place.zoom_level = 1.0 + + if place is None: + place = Place() + place.data_set_type = DataSetType.SKY + place.foreground_image_set = imgset + place.name = 'Toasty' + place.thumbnail = imgset.thumbnail_url + place.zoom_level = 1.0 + folder.children = [place] with open(wtml_path, 'wt') as f: @@ -483,6 +487,92 @@ def study_sample_image_tiles_impl(settings): stub_wtml(imgset, os.path.join(settings.outdir, 'index_rel.wtml')) +# "wwtl_sample_image_tiles" subcommand + +def wwtl_sample_image_tiles_getparser(parser): + parser.add_argument( + '--outdir', + metavar = 'PATH', + default = '.', + help = 'The root directory of the output tile pyramid', + ) + parser.add_argument( + 'wwtl_path', + metavar = 'WWTL-PATH', + help = 'The WWTL layer file to be processed', + ) + + +def wwtl_sample_image_tiles_impl(settings): + from io import BytesIO + import numpy as np + import PIL.Image + + from wwt_data_formats.enums import DataSetType, ProjectionType + from wwt_data_formats.layers import ImageSetLayer, LayerContainerReader + from wwt_data_formats.place import Place + + from .io import read_image_as_pil + from .pyramid import PyramidIO + from .study import make_thumbnail_bitmap, tile_study_image + + # Prevent max image size aborts: + PIL.Image.MAX_IMAGE_PIXELS = None + + # Load WWTL and see if it matches expectations + lc = LayerContainerReader.from_file(settings.wwtl_path) + + if len(lc.layers) != 1: + die('WWTL file must contain exactly one layer') + + layer = lc.layers[0] + if not isinstance(layer, ImageSetLayer): + die('WWTL file must contain an imageset layer') + + imgset = layer.image_set + if imgset.projection != ProjectionType.SKY_IMAGE: + die('WWTL imageset layer must have "SkyImage" projection type') + + # Looks OK. Read and parse the image. + img_data = lc.read_layer_file(layer, layer.extension) + img = PIL.Image.open(BytesIO(img_data)) + + # Tile it! + pio = PyramidIO(settings.outdir) + tiling = tile_study_image(np.asarray(img), pio) + + # Thumbnail. + thumb = make_thumbnail_bitmap(img) + thumb.save(os.path.join(settings.outdir, 'thumb.jpg'), format='JPEG') + + # Write a WTML file. We reuse the existing imageset as much as possible, + # but update the parameters that change in the tiling process. + + place = Place() + wcs_keywords = imgset.wcs_headers_from_position() + tiling.apply_to_imageset(imgset) + imgset.set_position_from_wcs(wcs_keywords, img.width, img.height, place=place) + + if not imgset.name: + imgset.name = 'Toasty' + imgset.thumbnail_url = 'thumb.jpg' + imgset.url = pio.get_path_scheme() + '.png' + + place.data_set_type = DataSetType.SKY + place.foreground_image_set = imgset + place.name = imgset.name + place.thumbnail = imgset.thumbnail_url + + stub_wtml(imgset, os.path.join(settings.outdir, 'index_rel.wtml'), place=place) + + # Helpful hint: + + print(f'Successfully tiled input "{settings.wwtl_path}" at level {imgset.tile_levels}.') + print('To create parent tiles, consider running:') + print() + print(f' toasty cascade --start {imgset.tile_levels} {settings.outdir}') + + # The CLI driver: def entrypoint(args=None): From 8f82ed2890f87599038b5b18af5cdd2b24f03b35 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Wed, 17 Jun 2020 16:27:47 -0400 Subject: [PATCH 06/11] Remove multi-tan-make-wtml and update docs The `multi-tan-make-wtml` subcommand was aimed at a workflow that is being superseded, and it didn't use wwt_data_formats. Also update the docs files to account for the deleted API functions. --- docs/api/toasty.cli.indent_xml.rst | 6 - ...asty.cli.multi_tan_make_wtml_getparser.rst | 6 - .../toasty.cli.multi_tan_make_wtml_impl.rst | 6 - ....cli.wwtl_sample_image_tiles_getparser.rst | 6 + ...oasty.cli.wwtl_sample_image_tiles_impl.rst | 6 + docs/api/toasty.io.read_image_as_pil.rst | 6 + .../toasty.pipeline.AzureBlobPipelineIo.rst | 2 + docs/api/toasty.pipeline.LocalPipelineIo.rst | 2 + docs/api/toasty.pipeline.PipelineIo.rst | 2 + toasty/cli.py | 115 +----------------- toasty/io.py | 2 +- 11 files changed, 26 insertions(+), 133 deletions(-) delete mode 100644 docs/api/toasty.cli.indent_xml.rst delete mode 100644 docs/api/toasty.cli.multi_tan_make_wtml_getparser.rst delete mode 100644 docs/api/toasty.cli.multi_tan_make_wtml_impl.rst create mode 100644 docs/api/toasty.cli.wwtl_sample_image_tiles_getparser.rst create mode 100644 docs/api/toasty.cli.wwtl_sample_image_tiles_impl.rst create mode 100644 docs/api/toasty.io.read_image_as_pil.rst diff --git a/docs/api/toasty.cli.indent_xml.rst b/docs/api/toasty.cli.indent_xml.rst deleted file mode 100644 index a9986f2..0000000 --- a/docs/api/toasty.cli.indent_xml.rst +++ /dev/null @@ -1,6 +0,0 @@ -indent_xml -========== - -.. currentmodule:: toasty.cli - -.. autofunction:: indent_xml diff --git a/docs/api/toasty.cli.multi_tan_make_wtml_getparser.rst b/docs/api/toasty.cli.multi_tan_make_wtml_getparser.rst deleted file mode 100644 index 0b279ad..0000000 --- a/docs/api/toasty.cli.multi_tan_make_wtml_getparser.rst +++ /dev/null @@ -1,6 +0,0 @@ -multi_tan_make_wtml_getparser -============================= - -.. currentmodule:: toasty.cli - -.. autofunction:: multi_tan_make_wtml_getparser diff --git a/docs/api/toasty.cli.multi_tan_make_wtml_impl.rst b/docs/api/toasty.cli.multi_tan_make_wtml_impl.rst deleted file mode 100644 index a2c567a..0000000 --- a/docs/api/toasty.cli.multi_tan_make_wtml_impl.rst +++ /dev/null @@ -1,6 +0,0 @@ -multi_tan_make_wtml_impl -======================== - -.. currentmodule:: toasty.cli - -.. autofunction:: multi_tan_make_wtml_impl diff --git a/docs/api/toasty.cli.wwtl_sample_image_tiles_getparser.rst b/docs/api/toasty.cli.wwtl_sample_image_tiles_getparser.rst new file mode 100644 index 0000000..09d1808 --- /dev/null +++ b/docs/api/toasty.cli.wwtl_sample_image_tiles_getparser.rst @@ -0,0 +1,6 @@ +wwtl_sample_image_tiles_getparser +================================= + +.. currentmodule:: toasty.cli + +.. autofunction:: wwtl_sample_image_tiles_getparser diff --git a/docs/api/toasty.cli.wwtl_sample_image_tiles_impl.rst b/docs/api/toasty.cli.wwtl_sample_image_tiles_impl.rst new file mode 100644 index 0000000..2f697a7 --- /dev/null +++ b/docs/api/toasty.cli.wwtl_sample_image_tiles_impl.rst @@ -0,0 +1,6 @@ +wwtl_sample_image_tiles_impl +============================ + +.. currentmodule:: toasty.cli + +.. autofunction:: wwtl_sample_image_tiles_impl diff --git a/docs/api/toasty.io.read_image_as_pil.rst b/docs/api/toasty.io.read_image_as_pil.rst new file mode 100644 index 0000000..71b13cd --- /dev/null +++ b/docs/api/toasty.io.read_image_as_pil.rst @@ -0,0 +1,6 @@ +read_image_as_pil +================= + +.. currentmodule:: toasty.io + +.. autofunction:: read_image_as_pil diff --git a/docs/api/toasty.pipeline.AzureBlobPipelineIo.rst b/docs/api/toasty.pipeline.AzureBlobPipelineIo.rst index 5b939c4..b934b5c 100644 --- a/docs/api/toasty.pipeline.AzureBlobPipelineIo.rst +++ b/docs/api/toasty.pipeline.AzureBlobPipelineIo.rst @@ -10,12 +10,14 @@ AzureBlobPipelineIo .. autosummary:: + ~AzureBlobPipelineIo.check_exists ~AzureBlobPipelineIo.get_item ~AzureBlobPipelineIo.list_items ~AzureBlobPipelineIo.put_item .. rubric:: Methods Documentation + .. automethod:: check_exists .. automethod:: get_item .. automethod:: list_items .. automethod:: put_item diff --git a/docs/api/toasty.pipeline.LocalPipelineIo.rst b/docs/api/toasty.pipeline.LocalPipelineIo.rst index 29af4db..762f98d 100644 --- a/docs/api/toasty.pipeline.LocalPipelineIo.rst +++ b/docs/api/toasty.pipeline.LocalPipelineIo.rst @@ -10,12 +10,14 @@ LocalPipelineIo .. autosummary:: + ~LocalPipelineIo.check_exists ~LocalPipelineIo.get_item ~LocalPipelineIo.list_items ~LocalPipelineIo.put_item .. rubric:: Methods Documentation + .. automethod:: check_exists .. automethod:: get_item .. automethod:: list_items .. automethod:: put_item diff --git a/docs/api/toasty.pipeline.PipelineIo.rst b/docs/api/toasty.pipeline.PipelineIo.rst index 2fb004d..e4beec6 100644 --- a/docs/api/toasty.pipeline.PipelineIo.rst +++ b/docs/api/toasty.pipeline.PipelineIo.rst @@ -10,12 +10,14 @@ PipelineIo .. autosummary:: + ~PipelineIo.check_exists ~PipelineIo.get_item ~PipelineIo.list_items ~PipelineIo.put_item .. rubric:: Methods Documentation + .. automethod:: check_exists .. automethod:: get_item .. automethod:: list_items .. automethod:: put_item diff --git a/toasty/cli.py b/toasty/cli.py index 39cca8a..b0d11a8 100644 --- a/toasty/cli.py +++ b/toasty/cli.py @@ -22,33 +22,6 @@ def warn(msg): print('warning:', msg, file=sys.stderr) -# TODO: This should be superseded by wwt_data_formats -def indent_xml(elem, level=0): - """A dumb XML indenter. - - We create XML files using xml.etree.ElementTree, which is careful about - spacing and so by default creates ugly files with no linewraps or - indentation. This function is copied from `ElementLib - `_ and implements - basic, sensible indentation using "tail" text. - - """ - i = "\n" + level * " " - - if len(elem): - if not elem.text or not elem.text.strip(): - elem.text = i + " " - if not elem.tail or not elem.tail.strip(): - elem.tail = i - for elem in elem: # intentionally updating "elem" here! - indent_xml(elem, level + 1) - if not elem.tail or not elem.tail.strip(): - elem.tail = i - else: - if level and (not elem.tail or not elem.tail.strip()): - elem.tail = i - - def stub_wtml(imgset, wtml_path, place=None): """Given an ImageSet object and potential a Place, save its information into a stub WTML file. @@ -223,93 +196,7 @@ def multi_tan_make_data_tiles_impl(settings): for p in sorted(percentiles.keys()): print(' {} = {}'.format(p, percentiles[p])) - -# "multi_tan_make_wtml" subcommand - -def multi_tan_make_wtml_getparser(parser): - parser.add_argument( - '--hdu-index', - metavar = 'INDEX', - type = int, - default = 0, - help = 'Which HDU to load in each input FITS file', - ) - parser.add_argument( - '--name', - metavar = 'NAME', - default = 'MultiTan', - help = 'The dataset name to embed in the WTML file', - ) - parser.add_argument( - '--url-prefix', - metavar = 'PREFIX', - default = './', - help = 'The prefix to the tile URL that will be embedded in the WTML', - ) - parser.add_argument( - '--fov-factor', - metavar = 'NUMBER', - type = float, - default = 1.7, - help = 'How tall the FOV should be (ie the zoom level) when viewing this image, in units of the image height', - ) - parser.add_argument( - '--bandpass', - metavar = 'BANDPASS-NAME', - default = 'Visible', - help = 'The bandpass of the image data: "Gamma", "HydrogenAlpha", "IR", "Microwave", "Radio", "Ultraviolet", "Visible", "VisibleNight", "XRay"', - ) - parser.add_argument( - '--description', - metavar = 'TEXT', - default = '', - help = 'Free text describing what this image is', - ) - parser.add_argument( - '--credits-text', - metavar = 'TEXT', - default = 'Created by toasty, part of the AAS WorldWide Telescope.', - help = 'A brief credit of who created and processed the image data', - ) - parser.add_argument( - '--credits-url', - metavar = 'URL', - default = '', - help = 'A URL with additional credit information', - ) - parser.add_argument( - '--thumbnail-url', - metavar = 'URL', - default = '', - help = 'A URL of a thumbnail image (96x45 JPEG) representing this dataset', - ) - parser.add_argument( - 'paths', - metavar = 'PATHS', - nargs = '+', - help = 'The FITS files with image data', - ) - -def multi_tan_make_wtml_impl(settings): - from xml.etree import ElementTree as etree - from .multi_tan import MultiTanDataSource - - ds = MultiTanDataSource(settings.paths, hdu_index=settings.hdu_index) - ds.compute_global_pixelization() - - folder = ds.create_wtml( - name = settings.name, - url_prefix = settings.url_prefix, - fov_factor = settings.fov_factor, - bandpass = settings.bandpass, - description_text = settings.description, - credits_text = settings.credits_text, - credits_url = settings.credits_url, - thumbnail_url = settings.thumbnail_url, - ) - indent_xml(folder) - doc = etree.ElementTree(folder) - doc.write(sys.stdout, encoding='utf-8', xml_declaration=True) + # TODO: this should populate and emit a stub index_rel.wtml file. # "pipeline_fetch_inputs" subcommand diff --git a/toasty/io.py b/toasty/io.py index 57be19d..fb5a590 100644 --- a/toasty/io.py +++ b/toasty/io.py @@ -42,7 +42,7 @@ def read_image_as_pil(path): Returns ------- - img : :class:`PIL.Image` + img : :class:`PIL.Image.Image` The image data. """ if path.endswith('.psd') or path.endswith('.psb'): From 8f4fa1b76e1dea366a7dc15f0bb1fd74e4666092 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Wed, 17 Jun 2020 16:35:47 -0400 Subject: [PATCH 07/11] toasty/tests: update these to pass again Combination of remove CLI function, new fields from newer wwt_data_formats, and a few output changes in the study tiling CLI. --- toasty/tests/test_multi_tan.py | 32 -------------------------------- toasty/tests/test_study.py | 9 +++++---- 2 files changed, 5 insertions(+), 36 deletions(-) diff --git a/toasty/tests/test_multi_tan.py b/toasty/tests/test_multi_tan.py index c68af0c..eb810fe 100644 --- a/toasty/tests/test_multi_tan.py +++ b/toasty/tests/test_multi_tan.py @@ -131,35 +131,3 @@ class TestMultiTan(object): test_path('wcs512.fits.gz') ] cli.entrypoint(args) - - - def test_basic_wtml_cli(self): - from xml.etree import ElementTree as etree - expected = etree.fromstring(self.WTML) - - prev_stdout = sys.stdout - from io import BytesIO - output = BytesIO() - - try: - sys.stdout = output - args = [ - 'multi-tan-make-wtml', - '--hdu-index', '0', - '--name', 'TestName', - '--url-prefix', 'UP', - '--fov-factor', '1.0', - '--bandpass', 'Gamma', - '--description', 'DT', - '--credits-text', 'CT', - '--credits-url', 'CU', - '--thumbnail-url', 'TU', - test_path('wcs512.fits.gz') - ] - cli.entrypoint(args) - finally: - sys.stdout = prev_stdout - - observed = etree.fromstring(output.getvalue()) - output.close() - assert_xml_elements_equal(observed, expected) diff --git a/toasty/tests/test_study.py b/toasty/tests/test_study.py index 222f2f0..cd64fac 100644 --- a/toasty/tests/test_study.py +++ b/toasty/tests/test_study.py @@ -17,10 +17,11 @@ from .. import study class TestStudy(object): WTML = """ - - + + - + + thumb.jpg @@ -80,7 +81,7 @@ class TestStudy(object): ] cli.entrypoint(args) - with open(self.work_path('toasty.wtml')) as f: + with open(self.work_path('index_rel.wtml')) as f: observed = etree.fromstring(f.read()) assert_xml_elements_equal(observed, expected) From 1390e218e0e505a0eab2af806c774d68a44bc527 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Wed, 17 Jun 2020 18:58:23 -0400 Subject: [PATCH 08/11] .travis.yml: work around temporary Sphinx linkcheck problem --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 94af6b8..2630226 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ +### XXX DOCS SHOULD INCLUDE LINKCHECK BUT IT'S BUSTED ON CURRENT SPHINX (3.1, 2020-Jun-17): +### https://github.com/sphinx-doc/sphinx/issues/7806 + language: c os: @@ -34,7 +37,7 @@ script: - python setup.py build_ext --inplace - py.test --cov toasty toasty # Different stdlib organization breaks API doc files on Python 2: - - if [[ $PYTHON_VERSION == 3.7 ]] ; then make -C docs html linkcheck ; fi + - if [[ $PYTHON_VERSION == 3.7 ]] ; then make -C docs html ; fi after_success: - coveralls From 21bceb98bc280d592788dd4a50eaa1b9c0985599 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Wed, 17 Jun 2020 18:59:05 -0400 Subject: [PATCH 09/11] setup.py: set some minimum required versions Although most of them are made-up. We do now require wwt_data_formats >= 0.2.0 though. --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 41cc563..cf96096 100644 --- a/setup.py +++ b/setup.py @@ -69,10 +69,10 @@ setup_args = dict( }, install_requires = [ - 'cython', - 'numpy', - 'pillow', - 'wwt_data_formats', + 'cython>=0.20', + 'numpy>=1.7', + 'pillow>=7.0', + 'wwt_data_formats>=0.2.0', ], extras_require = { From 9cf21f23c6364466a1186bc74de5a984d033fde7 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Wed, 17 Jun 2020 18:59:21 -0400 Subject: [PATCH 10/11] In docs, it's Notes not Remarks, which I can never get right. --- toasty/pipeline.py | 8 ++++---- toasty/study.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/toasty/pipeline.py b/toasty/pipeline.py index 2a86701..d56a98c 100644 --- a/toasty/pipeline.py +++ b/toasty/pipeline.py @@ -544,8 +544,8 @@ class InputImage(ABC): ------- None. - Remarks - ------- + Notes + ----- This function should also take care of creating the thumbnail. """ @@ -644,8 +644,8 @@ class BitmapInputImage(InputImage): ------- A :class:`PIL.Image` of the image data. - Remarks - ------- + Notes + ----- This function will only be called once. It can assume that :meth:`InputImage.ensure_input_cached` has already been called. diff --git a/toasty/study.py b/toasty/study.py index f1fc448..7d0ab83 100644 --- a/toasty/study.py +++ b/toasty/study.py @@ -95,8 +95,8 @@ class StudyTiling(object): imgset : ``wwt_data_formats.imageset.ImageSet`` The object to modify - Remarks - ------- + Notes + ----- The settings currently transferred are the number of tile levels and the projection type. @@ -121,8 +121,8 @@ class StudyTiling(object): im_iy : integer A 0-based vertical pixel position in the image coordinate system. - Remarks - ------- + Notes + ----- ``(0, 0)`` is the top-left corner of the image. The input values need not lie on the image. (I.e., they may be negative.) From 0c7ccfbc67deeb13a72bfaaadd981149f524f061 Mon Sep 17 00:00:00 2001 From: Peter Williams Date: Thu, 18 Jun 2020 15:17:10 -0400 Subject: [PATCH 11/11] toasty/tests/test_wwtl.py: smoketest --- MANIFEST.in | 2 +- toasty/tests/layercontainer.wwtxml | 32 +++++++++++++++++ toasty/tests/test_wwtl.py | 58 ++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 toasty/tests/layercontainer.wwtxml create mode 100644 toasty/tests/test_wwtl.py diff --git a/MANIFEST.in b/MANIFEST.in index beea1f9..5774b98 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,7 +3,7 @@ include .coveragerc include *.md include *.yml -recursive-include toasty *.png +recursive-include toasty *.fits *.png *.wwtxml graft docs prune docs/_build diff --git a/toasty/tests/layercontainer.wwtxml b/toasty/tests/layercontainer.wwtxml new file mode 100644 index 0000000..4fcb45a --- /dev/null +++ b/toasty/tests/layercontainer.wwtxml @@ -0,0 +1,32 @@ + + + + + + + + + + + diff --git a/toasty/tests/test_wwtl.py b/toasty/tests/test_wwtl.py new file mode 100644 index 0000000..05574ff --- /dev/null +++ b/toasty/tests/test_wwtl.py @@ -0,0 +1,58 @@ +# -*- mode: python; coding: utf-8 -*- +# Copyright 2020 the AAS WorldWide Telescope project +# Licensed under the MIT License. + +from __future__ import absolute_import, division, print_function + +import numpy as np +from numpy import testing as nt +import os.path +import pytest +import sys + +from wwt_data_formats.filecabinet import FileCabinetWriter + +from . import assert_xml_elements_equal, test_path +from .. import cli +from .. import study + + +class TestStudy(object): + def setup_method(self, method): + from tempfile import mkdtemp + self.work_dir = mkdtemp() + + def teardown_method(self, method): + from shutil import rmtree + rmtree(self.work_dir) + + def work_path(self, *pieces): + return os.path.join(self.work_dir, *pieces) + + def test_basic_cli(self): + # First, create a WWTL. NB, filenames should match the ID's in the XML + # file. + + fw = FileCabinetWriter() + + with open(test_path('layercontainer.wwtxml'), 'rb') as f: + b = f.read() + + fw.add_file_with_data('55cb0cce-c44a-4a44-a509-ea66fce643a5.wwtxml', b) + + with open(test_path('NGC253ALMA.jpg'), 'rb') as f: + b = f.read() + + fw.add_file_with_data('55cb0cce-c44a-4a44-a509-ea66fce643a5\\7ecb6411-e4ee-4dfa-90ef-77d6f486c7d2.jpg', b) + + with open(self.work_path('image.wwtl'), 'wb') as f: + fw.emit(f) + + # Now run it through the CLI. + + args = [ + 'wwtl-sample-image-tiles', + '--outdir', self.work_path('tiles'), + self.work_path('image.wwtl') + ] + cli.entrypoint(args)