Deprecate experimental API (#9888)

airflow/utils/docs.py

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Optional
+
+from airflow import version
+
+
+def get_docs_url(page: Optional[str] = None) -> str:
+    """Prepare link to Airflow documentation."""
+    if "dev" in version.version:
+        result = "https://airflow.readthedocs.io/en/latest/"
+    else:
+        result = 'https://airflow.apache.org/docs/{}/'.format(version.version)
+    if page:
+        result = result + page
+    return result

airflow/www/api/experimental/endpoints.py

@@ -19,7 +19,7 @@ import logging
 from functools import wraps
 from typing import Callable, TypeVar, cast
 
-from flask import Blueprint, current_app, g, jsonify, request, url_for
+from flask import Blueprint, Response, current_app, g, jsonify, request, url_for
 
 from airflow import models
 from airflow.api.common.experimental import delete_dag as delete, pool as pool_api, trigger_dag as trigger
@@ -31,6 +31,7 @@ from airflow.api.common.experimental.get_task import get_task
 from airflow.api.common.experimental.get_task_instance import get_task_instance
 from airflow.exceptions import AirflowException
 from airflow.utils import timezone
+from airflow.utils.docs import get_docs_url
 from airflow.utils.strings import to_boolean
 from airflow.version import version
 
@@ -51,6 +52,25 @@ def requires_authentication(function: T):
 api_experimental = Blueprint('api_experimental', __name__)
 
 
+def add_deprecation_headers(response: Response):
+    """
+    Add `Deprecation HTTP Header Field
+    <https://tools.ietf.org/id/draft-dalal-deprecation-header-03.html>`__.
+    """
+    response.headers['Deprecation'] = 'true'
+    doc_url = get_docs_url("stable-rest-api/migration.html")
+    deprecation_link = f'<{doc_url}>; rel="deprecation"; type="text/html"'
+    if 'link' in response.headers:
+        response.headers['Link'] += f', {deprecation_link}'
+    else:
+        response.headers['Link'] = f'{deprecation_link}'
+
+    return response
+
+
+api_experimental.after_request(add_deprecation_headers)
+
+
 @api_experimental.route('/dags/<string:dag_id>/dag_runs', methods=['POST'])
 @requires_authentication
 def trigger_dag(dag_id):

airflow/www/extensions/init_appbuilder_links.py

@@ -15,21 +15,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from airflow import version
+from airflow.utils.docs import get_docs_url
 
 
 def init_appbuilder_links(app):
     """Add links to Docs menu in navbar"""
     appbuilder = app.appbuilder
-    if "dev" in version.version:
-        doc_site = "https://airflow.readthedocs.io/en/latest"
-    else:
-        doc_site = 'https://airflow.apache.org/docs/{}'.format(version.version)
 
     appbuilder.add_link(
         "Website", href='https://airflow.apache.org', category="Docs", category_icon="fa-globe"
     )
-    appbuilder.add_link("Documentation", href=doc_site, category="Docs", category_icon="fa-cube")
+    appbuilder.add_link("Documentation", href=get_docs_url(), category="Docs", category_icon="fa-cube")
     appbuilder.add_link("GitHub", href='https://github.com/apache/airflow', category="Docs")
     appbuilder.add_link(
         "REST API Reference (Swagger UI)", href='/api/v1./api/v1_swagger_ui_index', category="Docs"

docs/index.rst

@@ -95,8 +95,8 @@ Content
     kubernetes
     lineage
     dag-serialization
-    Using the REST API <stable-rest-api/index.rst>
-    REST API Migration Guide <stable-rest-api/migration.rst>
+    Using the REST API <stable-rest-api/index>
+    REST API Migration Guide <stable-rest-api/migration>
     changelog
     best-practices
     faq

docs/rest-api-ref.rst

@@ -21,9 +21,10 @@ Experimental REST API Reference
 Airflow exposes an REST API. It is available through the webserver. Endpoints are
 available at ``/api/experimental/``.
 
-.. warning::
+.. deprecated:: 2.0
 
-  The API structure is not stable. We expect the endpoint definitions to change.
+  This REST API is deprecated. Please consider using :doc:`the stable REST API <stable-rest-api/redoc>`.
+  For more information on migration, see: :doc:`stable-rest-api/migration`.
 
 Endpoints
 ---------

tests/utils/test_docs.py

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from unittest import mock
+
+from parameterized import parameterized
+
+from airflow.utils.docs import get_docs_url
+
+
+class TestGetDocsUrl(unittest.TestCase):
+    @parameterized.expand([
+        ('2.0.0.dev0', None, 'https://airflow.readthedocs.io/en/latest/'),
+        ('2.0.0.dev0', 'migration.html', 'https://airflow.readthedocs.io/en/latest/migration.html'),
+        ('1.10.0', None, 'https://airflow.apache.org/docs/1.10.0/'),
+        ('1.10.0', 'migration.html', 'https://airflow.apache.org/docs/1.10.0/migration.html'),
+    ])
+    def test_should_return_link(self, version, page, expected_urk):
+        with mock.patch('airflow.version.version', version):
+            self.assertEqual(expected_urk, get_docs_url(page))

tests/www/api/experimental/test_endpoints.py

@@ -52,6 +52,14 @@ class TestBase(unittest.TestCase):
         settings.configure_orm()
         self.session = Session
 
+    def assert_deprecated(self, resp):
+        self.assertEqual('true', resp.headers['Deprecation'])
+        self.assertRegex(
+            resp.headers['Link'],
+            r'\<.+/stable-rest-api/migration.html\>; '
+            'rel="deprecation"; type="text/html"',
+        )
+
 
 @parameterized_class([
     {"dag_serialization": "False"},
@@ -89,6 +97,7 @@ class TestApiExperimental(TestBase):
         resp = json.loads(resp_raw.data.decode('utf-8'))
 
         self.assertEqual(version, resp['version'])
+        self.assert_deprecated(resp_raw)
 
     def test_task_info(self):
         with conf_vars(
@@ -99,6 +108,8 @@ class TestApiExperimental(TestBase):
             response = self.client.get(
                 url_template.format('example_bash_operator', 'runme_0')
             )
+            self.assert_deprecated(response)
+
             self.assertIn('"email"', response.data.decode('utf-8'))
             self.assertNotIn('error', response.data.decode('utf-8'))
             self.assertEqual(200, response.status_code)
@@ -124,6 +135,7 @@ class TestApiExperimental(TestBase):
             response = self.client.get(
                 url_template.format('example_bash_operator')
             )
+            self.assert_deprecated(response)
             self.assertIn('BashOperator(', response.data.decode('utf-8'))
             self.assertEqual(200, response.status_code)
 
@@ -143,6 +155,7 @@ class TestApiExperimental(TestBase):
             response = self.client.get(
                 pause_url_template.format('example_bash_operator', 'true')
             )
+            self.assert_deprecated(response)
             self.assertIn('ok', response.data.decode('utf-8'))
             self.assertEqual(200, response.status_code)
 
@@ -173,6 +186,7 @@ class TestApiExperimental(TestBase):
                 data=json.dumps({'run_id': run_id}),
                 content_type="application/json"
             )
+            self.assert_deprecated(response)
 
             self.assertEqual(200, response.status_code)
             response_execution_date = parse_datetime(
@@ -211,6 +225,7 @@ class TestApiExperimental(TestBase):
                 data=json.dumps({'execution_date': datetime_string}),
                 content_type="application/json"
             )
+            self.assert_deprecated(response)
             self.assertEqual(200, response.status_code)
             self.assertEqual(datetime_string, json.loads(response.data.decode('utf-8'))['execution_date'])
 
@@ -277,6 +292,7 @@ class TestApiExperimental(TestBase):
             response = self.client.get(
                 url_template.format(dag_id, datetime_string, task_id)
             )
+            self.assert_deprecated(response)
             self.assertEqual(200, response.status_code)
             self.assertIn('state', response.data.decode('utf-8'))
             self.assertNotIn('error', response.data.decode('utf-8'))
@@ -331,6 +347,7 @@ class TestApiExperimental(TestBase):
             response = self.client.get(
                 url_template.format(dag_id, datetime_string)
             )
+            self.assert_deprecated(response)
             self.assertEqual(200, response.status_code)
             self.assertIn('state', response.data.decode('utf-8'))
             self.assertNotIn('error', response.data.decode('utf-8'))
@@ -401,6 +418,7 @@ class TestLineageApiExperimental(TestBase):
             response = self.client.get(
                 url_template.format(dag_id, datetime_string)
             )
+            self.assert_deprecated(response)
             self.assertEqual(200, response.status_code)
             self.assertIn('task_ids', response.data.decode('utf-8'))
             self.assertNotIn('error', response.data.decode('utf-8'))
@@ -461,6 +479,7 @@ class TestPoolApiExperimental(TestBase):
         response = self.client.get(
             '/api/experimental/pools/{}'.format(self.pool.pool),
         )
+        self.assert_deprecated(response)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(json.loads(response.data.decode('utf-8')),
                          self.pool.to_json())
@@ -473,6 +492,7 @@ class TestPoolApiExperimental(TestBase):
 
     def test_get_pools(self):
         response = self.client.get('/api/experimental/pools')
+        self.assert_deprecated(response)
         self.assertEqual(response.status_code, 200)
         pools = json.loads(response.data.decode('utf-8'))
         self.assertEqual(len(pools), self.TOTAL_POOL_COUNT)
@@ -489,6 +509,7 @@ class TestPoolApiExperimental(TestBase):
             }),
             content_type='application/json',
         )
+        self.assert_deprecated(response)
         self.assertEqual(response.status_code, 200)
         pool = json.loads(response.data.decode('utf-8'))
         self.assertEqual(pool['pool'], 'foo')
@@ -518,6 +539,7 @@ class TestPoolApiExperimental(TestBase):
         response = self.client.delete(
             '/api/experimental/pools/{}'.format(self.pool.pool),
         )
+        self.assert_deprecated(response)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(json.loads(response.data.decode('utf-8')),
                          self.pool.to_json())