[AIRFLOW-5614] Enable Fernet by default (#6282)

UPDATING.md

@@ -41,6 +41,15 @@ assists users migrating to a new version.
 
 ## Airflow Master
 
+### Fernet is enabled by default
+
+The fernet mechanism is enabled by default to increase the security of the default installation.  In order to
+restore the previous behavior, the user must consciously set an empty key in the ``fernet_key`` option of
+section ``[core]`` in the ``airflow.cfg`` file.
+
+At the same time, this means that the `apache-airflow[crypto]` extra-packages are always installed.
+However, this requires that your operating system has ``libffi-dev`` installed.
+
 ### Changes to Google PubSub Operators, Hook and Sensor
 In the `PubSubPublishOperator` and `PubSubHook.publsh` method the data field in a message should be bytestring (utf-8 encoded) rather than base64 encoded string.
 

airflow/configuration.py

@@ -29,6 +29,7 @@ from collections import OrderedDict
 # Ignored Mypy on configparser because it thinks the configparser module has no _UNSET attribute
 from configparser import _UNSET, ConfigParser, NoOptionError, NoSectionError  # type: ignore
 
+from cryptography.fernet import Fernet
 from zope.deprecation import deprecated
 
 from airflow.exceptions import AirflowConfigException
@@ -43,15 +44,6 @@ warnings.filterwarnings(
     action='default', category=PendingDeprecationWarning, module='airflow')
 
 
-def generate_fernet_key():
-    try:
-        from cryptography.fernet import Fernet
-    except ImportError:
-        return ''
-    else:
-        return Fernet.generate_key().decode()
-
-
 def expand_env_var(env_var):
     """
     Expands (potentially nested) env vars by repeatedly applying
@@ -517,7 +509,7 @@ TEST_CONFIG_FILE = get_airflow_test_config(AIRFLOW_HOME)
 
 # only generate a Fernet key if we need to create a new config file
 if not os.path.isfile(TEST_CONFIG_FILE) or not os.path.isfile(AIRFLOW_CONFIG):
-    FERNET_KEY = generate_fernet_key()
+    FERNET_KEY = Fernet.generate_key().decode()
 else:
     FERNET_KEY = ''
 

airflow/models/crypto.py

@@ -19,6 +19,8 @@
 
 from typing import Optional
 
+from cryptography.fernet import Fernet, MultiFernet
+
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException
 from airflow.typing import Protocol
@@ -33,12 +35,6 @@ class FernetProtocol(Protocol):
         ...
 
 
-class InvalidFernetToken(Exception):
-    # If Fernet isn't loaded we need a valid exception class to catch. If it is
-    # loaded this will get reset to the actual class once get_fernet() is called
-    pass
-
-
 class NullFernet:
     """
     A "Null" encryptor class that doesn't encrypt or decrypt but that presents
@@ -75,17 +71,6 @@ def get_fernet():
 
     if _fernet:
         return _fernet
-    try:
-        from cryptography.fernet import Fernet, MultiFernet, InvalidToken
-        global InvalidFernetToken
-        InvalidFernetToken = InvalidToken
-
-    except ImportError:
-        log.warning(
-            "cryptography not found - values will not be stored encrypted."
-        )
-        _fernet = NullFernet()
-        return _fernet
 
     try:
         fernet_key = conf.get('core', 'FERNET_KEY')

airflow/models/variable.py

@@ -20,12 +20,13 @@
 import json
 from typing import Any
 
+from cryptography.fernet import InvalidToken as InvalidFernetToken
 from sqlalchemy import Boolean, Column, Integer, String, Text
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import synonym
 
 from airflow.models.base import ID_LEN, Base
-from airflow.models.crypto import InvalidFernetToken, get_fernet
+from airflow.models.crypto import get_fernet
 from airflow.utils.db import provide_session
 from airflow.utils.log.logging_mixin import LoggingMixin
 
@@ -50,12 +51,10 @@ class Variable(Base, LoggingMixin):
                 fernet = get_fernet()
                 return fernet.decrypt(bytes(self._val, 'utf-8')).decode()
             except InvalidFernetToken:
-                log.error("Can't decrypt _val for key={}, invalid token "
-                          "or value".format(self.key))
+                log.error("Can't decrypt _val for key=%s, invalid token or value", self.key)
                 return None
             except Exception:
-                log.error("Can't decrypt _val for key={}, FERNET_KEY "
-                          "configuration missing".format(self.key))
+                log.error("Can't decrypt _val for key=%s, FERNET_KEY configuration missing", self.key)
                 return None
         else:
             return self._val

docs/faq.rst

@@ -86,11 +86,6 @@ How do I trigger tasks based on another task's failure?
 
 Check out the :ref:`concepts/trigger_rule`.
 
-Why are connection passwords still not encrypted in the metadata db after I installed airflow[crypto]?
-------------------------------------------------------------------------------------------------------
-
-Check out the :doc:`howto/secure-connections`.
-
 What's the deal with ``start_date``?
 ------------------------------------
 

docs/howto/connection/gcp.rst

@@ -60,7 +60,7 @@ Keyfile Path
 Keyfile JSON
     Contents of a `service account
     <https://cloud.google.com/docs/authentication/#service_accounts>`_ key
-    file (JSON format) on disk. It is recommended to :doc:`Secure your connections <../secure-connections>` if using this method to authenticate.
+    file (JSON format) on disk.
 
     Not required if using application default credentials.
 

docs/howto/index.rst

@@ -34,7 +34,6 @@ configuring an Airflow environment.
     initialize-database
     operator/index
     connection/index
-    secure-connections
     write-logs
     run-behind-proxy
     run-with-systemd

docs/howto/secure-connections.rst

@@ -1,63 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-
- ..   http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
-
-
-
-Securing Connections
-====================
-
-By default, Airflow will save the passwords for the connection in plain text
-within the metadata database. The ``crypto`` package is highly recommended
-during installation. The ``crypto`` package does require that your operating
-system has ``libffi-dev`` installed.
-
-If ``crypto`` package was not installed initially, it means that your Fernet key in ``airflow.cfg`` is empty.
-
-You can still enable encryption for passwords within connections by following below steps:
-
-#. Install crypto package ``pip install 'apache-airflow[crypto]'``
-#. Generate fernet_key, using this code snippet below. ``fernet_key`` must be a base64-encoded 32-byte key:
-
-    .. code:: python
-
-      from cryptography.fernet import Fernet
-      fernet_key= Fernet.generate_key()
-      print(fernet_key.decode()) # your fernet_key, keep it in secured place!
-
-#. Replace ``airflow.cfg`` fernet_key value with the one from ``Step 2``. *Alternatively,* you can store your ``fernet_key`` in OS environment variable - You do not need to change ``airflow.cfg`` in this case as Airflow will use environment variable over the value in ``airflow.cfg``:
-
-    .. code-block:: bash
-
-      # Note the double underscores
-      export AIRFLOW__CORE__FERNET_KEY=your_fernet_key
-
-#. Restart the webserver
-#. For existing connections (the ones that you had defined before installing ``airflow[crypto]`` and creating a Fernet key), you need to open each connection in the connection admin UI, re-type the password, and save the change
-
-Rotating encryption keys
-========================
-
-Once connection credentials and variables have been encrypted using a fernet
-key, changing the key will cause decryption of existing credentials to fail. To
-rotate the fernet key without invalidating existing encrypted values, prepend
-the new key to the ``fernet_key`` setting, run
-``airflow rotate_fernet_key``, and then drop the original key from
-``fernet_keys``:
-
-#. Set ``fernet_key`` to ``new_fernet_key,old_fernet_key``
-#. Run ``airflow rotate_fernet_key`` to re-encrypt existing credentials with the new fernet key
-#. Set ``fernet_key`` to ``new_fernet_key``

docs/installation.rst

@@ -35,6 +35,8 @@ You can also install Airflow with support for extra features like ``gcp`` or ``p
 
     pip install 'apache-airflow[postgres,gcp]'
 
+Airflow require that your operating system has ``libffi-dev`` installed.
+
 Extra Packages
 ''''''''''''''
 

docs/security.rst

@@ -516,3 +516,50 @@ DAG Level Role
 ``Admin`` can create a set of roles which are only allowed to view a certain set of dags. This is called DAG level access. Each dag defined in the dag model table
 is treated as a ``View`` which has two permissions associated with it (``can_dag_read`` and ``can_dag_edit``). There is a special view called ``all_dags`` which
 allows the role to access all the dags. The default ``Admin``, ``Viewer``, ``User``, ``Op`` roles can all access ``all_dags`` view.
+
+
+.. _security/fernet:
+
+Securing Connections
+--------------------
+
+Airflow uses `Fernet <https://github.com/fernet/spec/>`__ to encrypt passwords in the connection
+configuration. It guarantees that a password encrypted using it cannot be manipulated or read without the key.
+Fernet is an implementation of symmetric (also known as “secret key”) authenticated cryptography.
+
+The first time Airflow is started, the ``airflow.cfg`` file is generated with the default configuration and the unique Fernet
+key. The key is saved to option ``fernet_key`` of section ``[core]``.
+
+You can also configure a fernet key using environment variables. This will overwrite the value from the
+`airflow.cfg` file
+
+    .. code-block:: bash
+
+      # Note the double underscores
+      export AIRFLOW__CORE__FERNET_KEY=your_fernet_key
+
+Generating fernet key
+'''''''''''''''''''''
+
+If you need to generate a new fernet key you can use the following code snippet.
+
+    .. code-block:: python
+
+      from cryptography.fernet import Fernet
+      fernet_key= Fernet.generate_key()
+      print(fernet_key.decode()) # your fernet_key, keep it in secured place!
+
+
+Rotating encryption keys
+''''''''''''''''''''''''
+
+Once connection credentials and variables have been encrypted using a fernet
+key, changing the key will cause decryption of existing credentials to fail. To
+rotate the fernet key without invalidating existing encrypted values, prepend
+the new key to the ``fernet_key`` setting, run
+``airflow rotate_fernet_key``, and then drop the original key from
+``fernet_keys``:
+
+#. Set ``fernet_key`` to ``new_fernet_key,old_fernet_key``
+#. Run ``airflow rotate_fernet_key`` to re-encrypt existing credentials with the new fernet key
+#. Set ``fernet_key`` to ``new_fernet_key``

setup.py

@@ -169,7 +169,6 @@ cgroups = [
     'cgroupspy>=0.1.4',
 ]
 cloudant = ['cloudant>=2.0']
-crypto = ['cryptography>=0.9.3']
 dask = [
     'distributed>=1.17.1, <2'
 ]
@@ -316,7 +315,7 @@ else:
 
 devel_minreq = devel + kubernetes + mysql + doc + password + cgroups
 devel_hadoop = devel_minreq + hive + hdfs + webhdfs + kerberos
-devel_all = (sendgrid + devel + all_dbs + doc + samba + slack + crypto + oracle +
+devel_all = (sendgrid + devel + all_dbs + doc + samba + slack + oracle +
              docker + ssh + kubernetes + celery + redis + gcp + grpc +
              datadog + zendesk + jdbc + ldap + kerberos + password + webhdfs + jenkins +
              druid + pinot + segment + snowflake + elasticsearch + sentry +
@@ -356,6 +355,7 @@ def do_setup():
             'cached_property~=1.5',
             'colorlog==4.0.2',
             'croniter>=0.3.17, <0.4',
+            'cryptography>=0.9.3',
             'dill>=0.2.2, <0.3',
             'flask>=1.1.0, <2.0',
             'flask-appbuilder>=1.12.5, <2.0.0',
@@ -413,7 +413,6 @@ def do_setup():
             'celery': celery,
             'cgroups': cgroups,
             'cloudant': cloudant,
-            'crypto': crypto,
             'dask': dask,
             'databricks': databricks,
             'datadog': datadog,