Merge pull request #13 from pkgw/next

Improve study tiling and add WWTL tiling
This commit is contained in:
Peter Williams 2020-06-18 15:33:48 -04:00 коммит произвёл GitHub
Родитель b1dd50543f 0c7ccfbc67
Коммит 629a91119a
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
20 изменённых файлов: 306 добавлений и 200 удалений

Просмотреть файл

@ -1,3 +1,6 @@
### XXX DOCS SHOULD INCLUDE LINKCHECK BUT IT'S BUSTED ON CURRENT SPHINX (3.1, 2020-Jun-17):
### https://github.com/sphinx-doc/sphinx/issues/7806
language: c
os:
@ -34,7 +37,7 @@ script:
- python setup.py build_ext --inplace
- py.test --cov toasty toasty
# Different stdlib organization breaks API doc files on Python 2:
- if [[ $PYTHON_VERSION == 3.7 ]] ; then make -C docs html linkcheck ; fi
- if [[ $PYTHON_VERSION == 3.7 ]] ; then make -C docs html ; fi
after_success:
- coveralls

Просмотреть файл

@ -3,7 +3,7 @@ include .coveragerc
include *.md
include *.yml
recursive-include toasty *.png
recursive-include toasty *.fits *.png *.wwtxml
graft docs
prune docs/_build

Просмотреть файл

@ -1,6 +0,0 @@
indent_xml
==========
.. currentmodule:: toasty.cli
.. autofunction:: indent_xml

Просмотреть файл

@ -1,6 +0,0 @@
multi_tan_make_wtml_getparser
=============================
.. currentmodule:: toasty.cli
.. autofunction:: multi_tan_make_wtml_getparser

Просмотреть файл

@ -1,6 +0,0 @@
multi_tan_make_wtml_impl
========================
.. currentmodule:: toasty.cli
.. autofunction:: multi_tan_make_wtml_impl

Просмотреть файл

@ -0,0 +1,6 @@
wwtl_sample_image_tiles_getparser
=================================
.. currentmodule:: toasty.cli
.. autofunction:: wwtl_sample_image_tiles_getparser

Просмотреть файл

@ -0,0 +1,6 @@
wwtl_sample_image_tiles_impl
============================
.. currentmodule:: toasty.cli
.. autofunction:: wwtl_sample_image_tiles_impl

Просмотреть файл

@ -0,0 +1,6 @@
read_image_as_pil
=================
.. currentmodule:: toasty.io
.. autofunction:: read_image_as_pil

Просмотреть файл

@ -10,12 +10,14 @@ AzureBlobPipelineIo
.. autosummary::
~AzureBlobPipelineIo.check_exists
~AzureBlobPipelineIo.get_item
~AzureBlobPipelineIo.list_items
~AzureBlobPipelineIo.put_item
.. rubric:: Methods Documentation
.. automethod:: check_exists
.. automethod:: get_item
.. automethod:: list_items
.. automethod:: put_item

Просмотреть файл

@ -10,12 +10,14 @@ LocalPipelineIo
.. autosummary::
~LocalPipelineIo.check_exists
~LocalPipelineIo.get_item
~LocalPipelineIo.list_items
~LocalPipelineIo.put_item
.. rubric:: Methods Documentation
.. automethod:: check_exists
.. automethod:: get_item
.. automethod:: list_items
.. automethod:: put_item

Просмотреть файл

@ -10,12 +10,14 @@ PipelineIo
.. autosummary::
~PipelineIo.check_exists
~PipelineIo.get_item
~PipelineIo.list_items
~PipelineIo.put_item
.. rubric:: Methods Documentation
.. automethod:: check_exists
.. automethod:: get_item
.. automethod:: list_items
.. automethod:: put_item

Просмотреть файл

@ -69,10 +69,10 @@ setup_args = dict(
},
install_requires = [
'cython',
'numpy',
'pillow',
'wwt_data_formats',
'cython>=0.20',
'numpy>=1.7',
'pillow>=7.0',
'wwt_data_formats>=0.2.0',
],
extras_require = {

Просмотреть файл

@ -22,35 +22,9 @@ def warn(msg):
print('warning:', msg, file=sys.stderr)
# TODO: This should be superseded by wwt_data_formats
def indent_xml(elem, level=0):
"""A dumb XML indenter.
We create XML files using xml.etree.ElementTree, which is careful about
spacing and so by default creates ugly files with no linewraps or
indentation. This function is copied from `ElementLib
<http://effbot.org/zone/element-lib.htm#prettyprint>`_ and implements
basic, sensible indentation using "tail" text.
"""
i = "\n" + level * " "
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem: # intentionally updating "elem" here!
indent_xml(elem, level + 1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
def stub_wtml(imgset, wtml_path):
"""Given an ImageSet object, save its information into a stub WTML file.
def stub_wtml(imgset, wtml_path, place=None):
"""Given an ImageSet object and potential a Place, save its information into a
stub WTML file.
"""
from wwt_data_formats import write_xml_doc
@ -59,9 +33,15 @@ def stub_wtml(imgset, wtml_path):
from wwt_data_formats.place import Place
folder = Folder()
place = Place()
place.data_set_type = DataSetType.SKY
place.foreground_image_set = imgset
if place is None:
place = Place()
place.data_set_type = DataSetType.SKY
place.foreground_image_set = imgset
place.name = 'Toasty'
place.thumbnail = imgset.thumbnail_url
place.zoom_level = 1.0
folder.children = [place]
with open(wtml_path, 'wt') as f:
@ -216,93 +196,7 @@ def multi_tan_make_data_tiles_impl(settings):
for p in sorted(percentiles.keys()):
print(' {} = {}'.format(p, percentiles[p]))
# "multi_tan_make_wtml" subcommand
def multi_tan_make_wtml_getparser(parser):
parser.add_argument(
'--hdu-index',
metavar = 'INDEX',
type = int,
default = 0,
help = 'Which HDU to load in each input FITS file',
)
parser.add_argument(
'--name',
metavar = 'NAME',
default = 'MultiTan',
help = 'The dataset name to embed in the WTML file',
)
parser.add_argument(
'--url-prefix',
metavar = 'PREFIX',
default = './',
help = 'The prefix to the tile URL that will be embedded in the WTML',
)
parser.add_argument(
'--fov-factor',
metavar = 'NUMBER',
type = float,
default = 1.7,
help = 'How tall the FOV should be (ie the zoom level) when viewing this image, in units of the image height',
)
parser.add_argument(
'--bandpass',
metavar = 'BANDPASS-NAME',
default = 'Visible',
help = 'The bandpass of the image data: "Gamma", "HydrogenAlpha", "IR", "Microwave", "Radio", "Ultraviolet", "Visible", "VisibleNight", "XRay"',
)
parser.add_argument(
'--description',
metavar = 'TEXT',
default = '',
help = 'Free text describing what this image is',
)
parser.add_argument(
'--credits-text',
metavar = 'TEXT',
default = 'Created by toasty, part of the AAS WorldWide Telescope.',
help = 'A brief credit of who created and processed the image data',
)
parser.add_argument(
'--credits-url',
metavar = 'URL',
default = '',
help = 'A URL with additional credit information',
)
parser.add_argument(
'--thumbnail-url',
metavar = 'URL',
default = '',
help = 'A URL of a thumbnail image (96x45 JPEG) representing this dataset',
)
parser.add_argument(
'paths',
metavar = 'PATHS',
nargs = '+',
help = 'The FITS files with image data',
)
def multi_tan_make_wtml_impl(settings):
from xml.etree import ElementTree as etree
from .multi_tan import MultiTanDataSource
ds = MultiTanDataSource(settings.paths, hdu_index=settings.hdu_index)
ds.compute_global_pixelization()
folder = ds.create_wtml(
name = settings.name,
url_prefix = settings.url_prefix,
fov_factor = settings.fov_factor,
bandpass = settings.bandpass,
description_text = settings.description,
credits_text = settings.credits_text,
credits_url = settings.credits_url,
thumbnail_url = settings.thumbnail_url,
)
indent_xml(folder)
doc = etree.ElementTree(folder)
doc.write(sys.stdout, encoding='utf-8', xml_declaration=True)
# TODO: this should populate and emit a stub index_rel.wtml file.
# "pipeline_fetch_inputs" subcommand
@ -449,16 +343,24 @@ def study_sample_image_tiles_getparser(parser):
def study_sample_image_tiles_impl(settings):
import numpy as np
import PIL.Image
from wwt_data_formats.imageset import ImageSet
from .io import read_image
from .io import read_image_as_pil
from .pyramid import PyramidIO
from .study import tile_study_image
from .study import make_thumbnail_bitmap, tile_study_image
# Create the base tiles.
# Prevent max image size aborts:
PIL.Image.MAX_IMAGE_PIXELS = None
# Load image.
pio = PyramidIO(settings.outdir)
img = read_image(settings.imgpath)
tiling = tile_study_image(img, pio)
img = read_image_as_pil(settings.imgpath)
tiling = tile_study_image(np.asarray(img), pio)
# Thumbnail.
thumb = make_thumbnail_bitmap(img)
thumb.save(os.path.join(settings.outdir, 'thumb.jpg'), format='JPEG')
# Write out a stub WTML file. The only information this will actually
# contain is the number of tile levels. Other information can be filled
@ -466,8 +368,96 @@ def study_sample_image_tiles_impl(settings):
imgset = ImageSet()
tiling.apply_to_imageset(imgset)
imgset.base_degrees_per_tile = 1.0 # random default to make it viewable
imgset.name = 'Toasty'
imgset.thumbnail_url = 'thumb.jpg'
imgset.url = pio.get_path_scheme() + '.png'
stub_wtml(imgset, os.path.join(settings.outdir, 'toasty.wtml'))
stub_wtml(imgset, os.path.join(settings.outdir, 'index_rel.wtml'))
# "wwtl_sample_image_tiles" subcommand
def wwtl_sample_image_tiles_getparser(parser):
parser.add_argument(
'--outdir',
metavar = 'PATH',
default = '.',
help = 'The root directory of the output tile pyramid',
)
parser.add_argument(
'wwtl_path',
metavar = 'WWTL-PATH',
help = 'The WWTL layer file to be processed',
)
def wwtl_sample_image_tiles_impl(settings):
from io import BytesIO
import numpy as np
import PIL.Image
from wwt_data_formats.enums import DataSetType, ProjectionType
from wwt_data_formats.layers import ImageSetLayer, LayerContainerReader
from wwt_data_formats.place import Place
from .io import read_image_as_pil
from .pyramid import PyramidIO
from .study import make_thumbnail_bitmap, tile_study_image
# Prevent max image size aborts:
PIL.Image.MAX_IMAGE_PIXELS = None
# Load WWTL and see if it matches expectations
lc = LayerContainerReader.from_file(settings.wwtl_path)
if len(lc.layers) != 1:
die('WWTL file must contain exactly one layer')
layer = lc.layers[0]
if not isinstance(layer, ImageSetLayer):
die('WWTL file must contain an imageset layer')
imgset = layer.image_set
if imgset.projection != ProjectionType.SKY_IMAGE:
die('WWTL imageset layer must have "SkyImage" projection type')
# Looks OK. Read and parse the image.
img_data = lc.read_layer_file(layer, layer.extension)
img = PIL.Image.open(BytesIO(img_data))
# Tile it!
pio = PyramidIO(settings.outdir)
tiling = tile_study_image(np.asarray(img), pio)
# Thumbnail.
thumb = make_thumbnail_bitmap(img)
thumb.save(os.path.join(settings.outdir, 'thumb.jpg'), format='JPEG')
# Write a WTML file. We reuse the existing imageset as much as possible,
# but update the parameters that change in the tiling process.
place = Place()
wcs_keywords = imgset.wcs_headers_from_position()
tiling.apply_to_imageset(imgset)
imgset.set_position_from_wcs(wcs_keywords, img.width, img.height, place=place)
if not imgset.name:
imgset.name = 'Toasty'
imgset.thumbnail_url = 'thumb.jpg'
imgset.url = pio.get_path_scheme() + '.png'
place.data_set_type = DataSetType.SKY
place.foreground_image_set = imgset
place.name = imgset.name
place.thumbnail = imgset.thumbnail_url
stub_wtml(imgset, os.path.join(settings.outdir, 'index_rel.wtml'), place=place)
# Helpful hint:
print(f'Successfully tiled input "{settings.wwtl_path}" at level {imgset.tile_levels}.')
print('To create parent tiles, consider running:')
print()
print(f' toasty cascade --start {imgset.tile_levels} {settings.outdir}')
# The CLI driver:

Просмотреть файл

@ -5,6 +5,7 @@
from __future__ import absolute_import, division, print_function
__all__ = '''
read_image_as_pil
read_image
save_png
'''.split()
@ -20,24 +21,61 @@ def save_png(pth, array):
Parameters
----------
pth : str
Path to write to
Path to write to
array : array-like
Image to save
Image to save
"""
Image.fromarray(array).save(pth)
def read_image_as_pil(path):
"""Load a bitmap image into a PIL Image.
The loading supports whatever image formats PIL does. As a special-case
hack, if the input path has extension ``.psd`` or ``.psb``, the
``psd_tools`` module will be used if available.
Parameters
----------
path : str
The path of the image to read
Returns
-------
img : :class:`PIL.Image.Image`
The image data.
"""
if path.endswith('.psd') or path.endswith('.psb'):
try:
from psd_tools import PSDImage
except ImportError:
pass
else:
psd = PSDImage.open(path)
return psd.composite()
return Image.open(path)
def read_image(path):
"""Load a bitmap image into a Numpy array.
The loading is done using PIL (the Python Imaging Library, usually the
The loading is generally done using PIL (the Python Imaging Library, usually the
"pillow" implementation these days) so it supports whatever image formats
PIL does.
PIL does. As a special-case hack, if the input path has extension ``.psd`` or ``.psb``,
the ``psd_tools`` module will be used if available.
Parameters
----------
path : str
The path of the image to read
The path of the image to read
Returns
-------
data : :class:`numpy.ndarray`
The image data. The array will have shape ``(height, width, planes)``, where
the first two axes are the image shape and the third is the number of color planes:
3 for RGB or potentially 4 for RGBA. The data type will be ``uint8``.
"""
return np.asarray(Image.open(path))
return np.asarray(read_image_as_pil(path))

Просмотреть файл

@ -544,8 +544,8 @@ class InputImage(ABC):
-------
None.
Remarks
-------
Notes
-----
This function should also take care of creating the thumbnail.
"""
@ -644,8 +644,8 @@ class BitmapInputImage(InputImage):
-------
A :class:`PIL.Image` of the image data.
Remarks
-------
Notes
-----
This function will only be called once. It can assume that
:meth:`InputImage.ensure_input_cached` has already been called.
@ -989,6 +989,8 @@ class PipelineManager(object):
for stem, is_folder in self._pipeio.list_items():
if not is_folder:
continue
if not self._pipeio.check_exists(stem, 'index.wtml'):
continue
wtml_data = BytesIO()
self._pipeio.get_item(stem, 'index.wtml', dest=wtml_data)

Просмотреть файл

@ -95,13 +95,21 @@ class StudyTiling(object):
imgset : ``wwt_data_formats.imageset.ImageSet``
The object to modify
Remarks
-------
The only setting currently transferred is the number of tile levels.
Notes
-----
The settings currently transferred are the number of tile levels and
the projection type.
"""
from wwt_data_formats.enums import ProjectionType
imgset.tile_levels = self._tile_levels
if self._tile_levels == 0:
imgset.projection = ProjectionType.SKY_IMAGE
else:
imgset.projection = ProjectionType.TAN
def image_to_tile(self, im_ix, im_iy):
"""Convert an image pixel position to a tiled pixel position.
@ -113,8 +121,8 @@ class StudyTiling(object):
im_iy : integer
A 0-based vertical pixel position in the image coordinate system.
Remarks
-------
Notes
-----
``(0, 0)`` is the top-left corner of the image. The input values need
not lie on the image. (I.e., they may be negative.)

Просмотреть файл

@ -0,0 +1,32 @@
<?xml version='1.0' encoding='UTF-8'?>
<!-- Note these values are not correct for the image we use! -->
<LayerContainer ID="55cb0cce-c44a-4a44-a509-ea66fce643a5">
<Layers>
<Layer Id="7ecb6411-e4ee-4dfa-90ef-77d6f486c7d2"
Type="TerraViewer.ImageSetLayer"
Name="Toasty Test"
ReferenceFrame="Sky"
Color="NamedColor:White" Opacity="1"
StartTime="1/1/0001 12:00:00 AM" EndTime="12/31/9999 11:59:59 PM"
FadeSpan="00:00:00" FadeType="None"
Extension=".jpg"
OverrideDefault="False">
<ImageSet DataSetType="Sky" BandPass="Visible" Name="Toasty Test"
Projection="SkyImage" ReferenceFrame=""
CenterX="85.5" CenterY="-2.5"
OffsetX="22.2" OffsetY="15.5"
Rotation="-89.99"
BaseDegreesPerTile="0.002"
QuadTreeMap=""
Url="X:\InternalPath.jpg" DemUrl=""
FileType=".jpg"
BaseTileLevel="0"
TileLevels="0"
WidthFactor="1"
MeanRadius="0"
BottomsUp="False" Sparse="False" ElevationModel="False" StockSet="False" Generic="False">
<ThumbnailUrl />
</ImageSet>
</Layer>
</Layers>
</LayerContainer>

Просмотреть файл

@ -131,35 +131,3 @@ class TestMultiTan(object):
test_path('wcs512.fits.gz')
]
cli.entrypoint(args)
def test_basic_wtml_cli(self):
from xml.etree import ElementTree as etree
expected = etree.fromstring(self.WTML)
prev_stdout = sys.stdout
from io import BytesIO
output = BytesIO()
try:
sys.stdout = output
args = [
'multi-tan-make-wtml',
'--hdu-index', '0',
'--name', 'TestName',
'--url-prefix', 'UP',
'--fov-factor', '1.0',
'--bandpass', 'Gamma',
'--description', 'DT',
'--credits-text', 'CT',
'--credits-url', 'CU',
'--thumbnail-url', 'TU',
test_path('wcs512.fits.gz')
]
cli.entrypoint(args)
finally:
sys.stdout = prev_stdout
observed = etree.fromstring(output.getvalue())
output.close()
assert_xml_elements_equal(observed, expected)

Просмотреть файл

@ -17,10 +17,11 @@ from .. import study
class TestStudy(object):
WTML = """<?xml version='1.0' encoding='UTF-8'?>
<Folder Browseable="True" Group="Explorer" Searchable="True" Type="Sky">
<Place Angle="0.0" AngularSize="0.0" DataSetType="Sky" Dec="0.0" Distance="0.0" DomeAlt="0.0" DomeAz="0.0" Lat="0.0" Lng="0.0" Magnitude="0.0" Opacity="100.0" RA="0.0" Rotation="0.0" ZoomLevel="0.0">
<Folder Browseable="True" Group="Explorer" MSRCommunityId="0" MSRComponentId="0" Permission="0" Searchable="True" Type="Sky">
<Place Angle="0.0" AngularSize="0.0" DataSetType="Sky" Dec="0.0" Distance="0.0" DomeAlt="0.0" DomeAz="0.0" Lat="0.0" Lng="0.0" Magnitude="0.0" MSRCommunityId="0" MSRComponentId="0" Name="Toasty" Opacity="100.0" Permission="0" RA="0.0" Rotation="0.0" Thumbnail="thumb.jpg" ZoomLevel="1.0">
<ForegroundImageSet>
<ImageSet BandPass="Visible" BaseDegreesPerTile="1.0" BaseTileLevel="0" BottomsUp="False" CenterX="0.0" CenterY="0.0" DataSetType="Sky" ElevationModel="False" FileType=".png" Generic="False" MeanRadius="0.0" OffsetX="0.0" OffsetY="0.0" Projection="SkyImage" Rotation="0.0" Sparse="True" StockSet="False" TileLevels="4" Url="{1}/{3}/{3}_{2}.png" WidthFactor="2">
<ImageSet BandPass="Visible" BaseDegreesPerTile="1.0" BaseTileLevel="0" BottomsUp="False" CenterX="0.0" CenterY="0.0" DataSetType="Sky" ElevationModel="False" FileType=".png" Generic="False" MeanRadius="0.0" MSRCommunityId="0" MSRComponentId="0" Name="Toasty" OffsetX="0.0" OffsetY="0.0" Permission="0" Projection="Tan" Rotation="0.0" Sparse="True" StockSet="False" TileLevels="4" Url="{1}/{3}/{3}_{2}.png" WidthFactor="2">
<ThumbnailUrl>thumb.jpg</ThumbnailUrl>
</ImageSet>
</ForegroundImageSet>
</Place>
@ -80,7 +81,7 @@ class TestStudy(object):
]
cli.entrypoint(args)
with open(self.work_path('toasty.wtml')) as f:
with open(self.work_path('index_rel.wtml')) as f:
observed = etree.fromstring(f.read())
assert_xml_elements_equal(observed, expected)

58
toasty/tests/test_wwtl.py Normal file
Просмотреть файл

@ -0,0 +1,58 @@
# -*- mode: python; coding: utf-8 -*-
# Copyright 2020 the AAS WorldWide Telescope project
# Licensed under the MIT License.
from __future__ import absolute_import, division, print_function
import numpy as np
from numpy import testing as nt
import os.path
import pytest
import sys
from wwt_data_formats.filecabinet import FileCabinetWriter
from . import assert_xml_elements_equal, test_path
from .. import cli
from .. import study
class TestStudy(object):
def setup_method(self, method):
from tempfile import mkdtemp
self.work_dir = mkdtemp()
def teardown_method(self, method):
from shutil import rmtree
rmtree(self.work_dir)
def work_path(self, *pieces):
return os.path.join(self.work_dir, *pieces)
def test_basic_cli(self):
# First, create a WWTL. NB, filenames should match the ID's in the XML
# file.
fw = FileCabinetWriter()
with open(test_path('layercontainer.wwtxml'), 'rb') as f:
b = f.read()
fw.add_file_with_data('55cb0cce-c44a-4a44-a509-ea66fce643a5.wwtxml', b)
with open(test_path('NGC253ALMA.jpg'), 'rb') as f:
b = f.read()
fw.add_file_with_data('55cb0cce-c44a-4a44-a509-ea66fce643a5\\7ecb6411-e4ee-4dfa-90ef-77d6f486c7d2.jpg', b)
with open(self.work_path('image.wwtl'), 'wb') as f:
fw.emit(f)
# Now run it through the CLI.
args = [
'wwtl-sample-image-tiles',
'--outdir', self.work_path('tiles'),
self.work_path('image.wwtl')
]
cli.entrypoint(args)