Merge pull request #51 from pkgw/pipeline-ignore-rejects

pipeline: add ignore-rejects command

docs/cli.rst

@@ -9,6 +9,7 @@ CLI Reference
    cli/make-thumbnail
    cli/pipeline-approve
    cli/pipeline-fetch
+   cli/pipeline-ignore-rejects
    cli/pipeline-init
    cli/pipeline-process-todos
    cli/pipeline-publish

docs/cli/pipeline-ignore-rejects.rst

@@ -0,0 +1,50 @@
+.. _cli-pipeline-ignore-rejects:
+
+==================================
+``toasty pipeline ignore-rejects``
+==================================
+
+The ``ignore-rejects`` :ref:`pipeline command <pipeline>` marks all of the
+rejected images in a workspace to be ignored by future processing runs.
+
+
+Usage
+=====
+
+.. code-block:: shell
+
+   toasty pipeline ignore-rejects [--workdir=WORKDIR]
+
+The ``WORKDIR`` argument optionally specifies the location of the pipeline
+workspace directory. The default is the current directory.
+
+
+Example
+=======
+
+Ignore every image in the ``rejects`` directory:
+
+.. code-block:: shell
+
+   toasty pipeline ignore-rejects
+
+Subsequent invocations of :ref:`cli-pipeline-refresh` will ignore these images
+when searching for new processing candidates.
+
+
+Notes
+=====
+
+This command will ignore every image in the “reject” state.  That is, each file
+inside the ``rejects`` subfolder of the pipeline workspace will be taken to be
+an image ID that should be ignored. If you have rejects that should *not* be
+permanently ignored (maybe you can't solve their coordinates right now, but will
+be able to later), delete their files from the ``rejects`` subfolder before
+running this command.
+
+
+See Also
+========
+
+- :ref:`The toasty pipeline processing overview <pipeline>`
+- :ref:`cli-pipeline-refresh`

docs/cli/pipeline-refresh.rst

@@ -31,9 +31,14 @@ image, a file will be created in the ``candidates`` subdirectory. The next step
 is to select candidates for processing and download them using
 :ref:`cli-pipeline-fetch`.
 
+If there are candidates that should *never* be processed, they can be marked to
+be ignored by subsequent refreshes. Currently you can only do this with images
+that are rejected during processing, using :ref:`cli-pipeline-ignore-rejects`.
+
 
 See Also
 ========
 
 - :ref:`The toasty pipeline processing overview <pipeline>`
 - :ref:`cli-pipeline-fetch`
+- :ref:`cli-pipeline-ignore-rejects`

toasty/pipeline/__init__.py

@@ -462,3 +462,21 @@ class PipelineManager(object):
                     self._pipeio.put_item(*sub_components[1:], source=f)
 
             os.rename(os.path.join(todo_dir, uniq_id), os.path.join(done_dir, uniq_id))
+
+    def ignore_rejects(self):
+        from io import BytesIO
+
+        rejects_dir = self._path('rejects')
+        n = 0
+
+        # maybe one day this will be JSON with data?
+        flag_content = BytesIO(b'{}')
+
+        for uniq_id in os.listdir(rejects_dir):
+            print(f'ignoring {uniq_id} ...')
+            self._pipeio.put_item(uniq_id, 'skip.flag', source=flag_content)
+            n += 1
+
+        if n > 1:
+            print()
+            print(f'marked a total of {n} images to be permanently ignored')

toasty/pipeline/cli.py

@@ -294,28 +294,24 @@ def refresh_impl(settings):
 
 def pipeline_getparser(parser):
     subparsers = parser.add_subparsers(dest='pipeline_command')
+
+    def add_manager_command(name):
+        subp = subparsers.add_parser(name)
+        subp.add_argument(
+            '--workdir',
+            nargs = '?',
+            metavar = 'WORKDIR',
+            default = '.',
+            help = 'The local working directory',
+        )
+        return subp
+
     approve_setup_parser(subparsers.add_parser('approve'))
     fetch_setup_parser(subparsers.add_parser('fetch'))
+    add_manager_command('ignore-rejects')
     init_setup_parser(subparsers.add_parser('init'))
-
-    parser = subparsers.add_parser('process-todos')
-    parser.add_argument(
-        '--workdir',
-        nargs = '?',
-        metavar = 'WORKDIR',
-        default = '.',
-        help = 'The local working directory',
-    )
-
-    parser = subparsers.add_parser('publish')
-    parser.add_argument(
-        '--workdir',
-        nargs = '?',
-        metavar = 'WORKDIR',
-        default = '.',
-        help = 'The local working directory',
-    )
-
+    add_manager_command('process-todos')
+    add_manager_command('publish')
     refresh_setup_parser(subparsers.add_parser('refresh'))
 
 
@@ -330,6 +326,9 @@ def pipeline_impl(settings):
         approve_impl(settings)
     elif settings.pipeline_command == 'fetch':
         fetch_impl(settings)
+    elif settings.pipeline_command == 'ignore-rejects':
+        mgr = PipelineManager(settings.workdir)
+        mgr.ignore_rejects()
     elif settings.pipeline_command == 'init':
         init_impl(settings)
     elif settings.pipeline_command == 'process-todos':

toasty/tests/test_pipeline.py

@@ -112,6 +112,12 @@ class TestPipeline(object):
         ]
         cli.entrypoint(args)
 
+        args = [
+            'pipeline', 'ignore-rejects',
+            '--workdir', self.work_path('work'),
+        ]
+        cli.entrypoint(args)
+
     def test_args(self):
         with pytest.raises(SystemExit):
             args = [