[AIRFLOW-3771] Minor refactor securityManager (#4594)
This commit is contained in:
Tao Feng
GitHub
Родитель
40f4370324
Коммит
2f70347bdc
18
UPDATING.md
18
UPDATING.md
|
|
@ -107,14 +107,6 @@ so you might need to update your config.
|
|||
|
||||
`task_runner = StandardTaskRunner`
|
||||
|
||||
### DAG level Access Control for new RBAC UI
|
||||
|
||||
Extend and enhance new Airflow RBAC UI to support DAG level ACL. Each dag now has two permissions(one for write, one for read) associated('can_dag_edit', 'can_dag_read').
|
||||
The admin will create new role, associate the dag permission with the target dag and assign that role to users. That user can only access / view the certain dags on the UI
|
||||
that he has permissions on. If a new role wants to access all the dags, the admin could associate dag permissions on an artificial view(``all_dags``) with that role.
|
||||
|
||||
We also provide a new cli command(``sync_perm``) to allow admin to auto sync permissions.
|
||||
|
||||
|
||||
### min_file_parsing_loop_time config option temporarily disabled
|
||||
|
||||
|
|
@ -154,8 +146,16 @@ airflow users --remove-role --username jondoe --role Public
|
|||
|
||||
## Airflow 1.10.2
|
||||
|
||||
### DAG level Access Control for new RBAC UI
|
||||
|
||||
Extend and enhance new Airflow RBAC UI to support DAG level ACL. Each dag now has two permissions(one for write, one for read) associated('can_dag_edit', 'can_dag_read').
|
||||
The admin will create new role, associate the dag permission with the target dag and assign that role to users. That user can only access / view the certain dags on the UI
|
||||
that he has permissions on. If a new role wants to access all the dags, the admin could associate dag permissions on an artificial view(``all_dags``) with that role.
|
||||
|
||||
We also provide a new cli command(``sync_perm``) to allow admin to auto sync permissions.
|
||||
|
||||
### Modification to `ts_nodash` macro
|
||||
`ts_nodash` previously contained TimeZone information alongwith execution date. For Example: `20150101T000000+0000`. This is not user-friendly for file or folder names which was a popular use case for `ts_nodash`. Hence this behavior has been changed and using `ts_nodash` will no longer contain TimeZone information, restoring the pre-1.10 behavior of this macro. And a new macro `ts_nodash_with_tz` has been added which can be used to get a string with execution date and timezone info without dashes.
|
||||
`ts_nodash` previously contained TimeZone information alongwith execution date. For Example: `20150101T000000+0000`. This is not user-friendly for file or folder names which was a popular use case for `ts_nodash`. Hence this behavior has been changed and using `ts_nodash` will no longer contain TimeZone information, restoring the pre-1.10 behavior of this macro. And a new macro `ts_nodash_with_tz` has been added which can be used to get a string with execution date and timezone info without dashes.
|
||||
|
||||
Examples:
|
||||
* `ts_nodash`: `20150101T000000`
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
# under the License.
|
||||
#
|
||||
|
||||
import logging
|
||||
from flask import g
|
||||
from flask_appbuilder.security.sqla import models as sqla_models
|
||||
from flask_appbuilder.security.sqla.manager import SecurityManager
|
||||
|
|
@ -27,11 +26,12 @@ from sqlalchemy import or_
|
|||
from airflow import models
|
||||
from airflow.www.app import appbuilder
|
||||
from airflow.utils.db import provide_session
|
||||
from airflow.utils.log.logging_mixin import LoggingMixin
|
||||
|
||||
###########################################################################
|
||||
# VIEW MENUS
|
||||
###########################################################################
|
||||
viewer_vms = {
|
||||
VIEWER_VMS = {
|
||||
'Airflow',
|
||||
'DagModelView',
|
||||
'Browse',
|
||||
|
|
@ -53,9 +53,9 @@ viewer_vms = {
|
|||
'VersionView',
|
||||
}
|
||||
|
||||
user_vms = viewer_vms
|
||||
USER_VMS = VIEWER_VMS
|
||||
|
||||
op_vms = {
|
||||
OP_VMS = {
|
||||
'Admin',
|
||||
'Configurations',
|
||||
'ConfigurationView',
|
||||
|
|
@ -73,7 +73,7 @@ op_vms = {
|
|||
# PERMISSIONS
|
||||
###########################################################################
|
||||
|
||||
viewer_perms = {
|
||||
VIEWER_PERMS = {
|
||||
'menu_access',
|
||||
'can_index',
|
||||
'can_list',
|
||||
|
|
@ -100,7 +100,7 @@ viewer_perms = {
|
|||
'can_version',
|
||||
}
|
||||
|
||||
user_perms = {
|
||||
USER_PERMS = {
|
||||
'can_dagrun_clear',
|
||||
'can_run',
|
||||
'can_trigger',
|
||||
|
|
@ -118,17 +118,17 @@ user_perms = {
|
|||
'can_clear',
|
||||
}
|
||||
|
||||
op_perms = {
|
||||
OP_PERMS = {
|
||||
'can_conf',
|
||||
'can_varimport',
|
||||
}
|
||||
|
||||
# global view-menu for dag-level access
|
||||
dag_vms = {
|
||||
DAG_VMS = {
|
||||
'all_dags'
|
||||
}
|
||||
|
||||
dag_perms = {
|
||||
DAG_PERMS = {
|
||||
'can_dag_read',
|
||||
'can_dag_edit',
|
||||
}
|
||||
|
|
@ -140,18 +140,18 @@ dag_perms = {
|
|||
ROLE_CONFIGS = [
|
||||
{
|
||||
'role': 'Viewer',
|
||||
'perms': viewer_perms | dag_perms,
|
||||
'vms': viewer_vms | dag_vms
|
||||
'perms': VIEWER_PERMS | DAG_PERMS,
|
||||
'vms': VIEWER_VMS | DAG_VMS
|
||||
},
|
||||
{
|
||||
'role': 'User',
|
||||
'perms': viewer_perms | user_perms | dag_perms,
|
||||
'vms': viewer_vms | dag_vms | user_vms,
|
||||
'perms': VIEWER_PERMS | USER_PERMS | DAG_PERMS,
|
||||
'vms': VIEWER_VMS | DAG_VMS | USER_VMS,
|
||||
},
|
||||
{
|
||||
'role': 'Op',
|
||||
'perms': viewer_perms | user_perms | op_perms | dag_perms,
|
||||
'vms': viewer_vms | dag_vms | user_vms | op_vms,
|
||||
'perms': VIEWER_PERMS | USER_PERMS | OP_PERMS | DAG_PERMS,
|
||||
'vms': VIEWER_VMS | DAG_VMS | USER_VMS | OP_VMS,
|
||||
},
|
||||
]
|
||||
|
||||
|
|
@ -164,7 +164,7 @@ EXISTING_ROLES = {
|
|||
}
|
||||
|
||||
|
||||
class AirflowSecurityManager(SecurityManager):
|
||||
class AirflowSecurityManager(SecurityManager, LoggingMixin):
|
||||
|
||||
def init_role(self, role_name, role_vms, role_perms):
|
||||
"""
|
||||
|
|
@ -183,7 +183,7 @@ class AirflowSecurityManager(SecurityManager):
|
|||
role = self.add_role(role_name)
|
||||
|
||||
if len(role.permissions) == 0:
|
||||
logging.info('Initializing permissions for role:%s in the database.', role_name)
|
||||
self.log.info('Initializing permissions for role:%s in the database.', role_name)
|
||||
role_pvms = []
|
||||
for pvm in pvms:
|
||||
if pvm.view_menu.name in role_vms and pvm.permission.name in role_perms:
|
||||
|
|
@ -192,11 +192,14 @@ class AirflowSecurityManager(SecurityManager):
|
|||
self.get_session.merge(role)
|
||||
self.get_session.commit()
|
||||
else:
|
||||
logging.info('Existing permissions for the role:%s within the database will persist.', role_name)
|
||||
self.log.info('Existing permissions for the role:%s within the database will persist.', role_name)
|
||||
|
||||
def get_user_roles(self, user=None):
|
||||
"""
|
||||
Get all the roles associated with the user.
|
||||
|
||||
:param user: the ab_user in FAB model.
|
||||
:return: a list of roles associated with the user.
|
||||
"""
|
||||
if user is None:
|
||||
user = g.user
|
||||
|
|
@ -227,16 +230,16 @@ class AirflowSecurityManager(SecurityManager):
|
|||
username = g.user
|
||||
|
||||
if username.is_anonymous or 'Public' in username.roles:
|
||||
# return an empty list if the role is public
|
||||
# return an empty set if the role is public
|
||||
return set()
|
||||
|
||||
roles = {role.name for role in username.roles}
|
||||
if {'Admin', 'Viewer', 'User', 'Op'} & roles:
|
||||
return dag_vms
|
||||
return DAG_VMS
|
||||
|
||||
user_perms_views = self.get_all_permissions_views()
|
||||
# return all dags that the user could access
|
||||
return set([view for perm, view in user_perms_views if perm in dag_perms])
|
||||
# return a set of all dags that the user could access
|
||||
return set([view for perm, view in user_perms_views if perm in DAG_PERMS])
|
||||
|
||||
def has_access(self, permission, view_name, user=None):
|
||||
"""
|
||||
|
|
@ -296,7 +299,7 @@ class AirflowSecurityManager(SecurityManager):
|
|||
"""
|
||||
FAB leaves faulty permissions that need to be cleaned up
|
||||
"""
|
||||
logging.info('Cleaning faulty perms')
|
||||
self.log.info('Cleaning faulty perms')
|
||||
sesh = self.get_session
|
||||
pvms = (
|
||||
sesh.query(sqla_models.PermissionView)
|
||||
|
|
@ -308,7 +311,7 @@ class AirflowSecurityManager(SecurityManager):
|
|||
deleted_count = pvms.delete()
|
||||
sesh.commit()
|
||||
if deleted_count:
|
||||
logging.info('Deleted {} faulty permissions'.format(deleted_count))
|
||||
self.log.info('Deleted {} faulty permissions'.format(deleted_count))
|
||||
|
||||
def _merge_perm(self, permission_name, view_menu_name):
|
||||
"""
|
||||
|
|
@ -334,16 +337,18 @@ class AirflowSecurityManager(SecurityManager):
|
|||
def create_custom_dag_permission_view(self, session=None):
|
||||
"""
|
||||
Workflow:
|
||||
1. when scheduler found a new dag, we will create an entry in ab_view_menu
|
||||
2. we fetch all the roles associated with dag users.
|
||||
3. we join and create all the entries for ab_permission_view_menu
|
||||
(predefined permissions * dag-view_menus)
|
||||
4. Create all the missing role-permission-views for the ab_role_permission_views
|
||||
1. Fetch all the existing (permissions, view-menu) from Airflow DB.
|
||||
2. Fetch all the existing dag models that are either active or paused. Exclude the subdags.
|
||||
3. Create both read and write permission view-menus relation for every dags from step 2
|
||||
4. Find out all the dag specific roles(excluded pubic, admin, viewer, op, user)
|
||||
5. Get all the permission-vm owned by the user role.
|
||||
6. Grant all the user role's permission-vm except the all-dag view-menus to the dag roles.
|
||||
7. Commit the updated permission-vm-role into db
|
||||
|
||||
:return: None.
|
||||
"""
|
||||
# todo(Tao): should we put this function here or in scheduler loop?
|
||||
logging.info('Fetching a set of all permission, view_menu from FAB meta-table')
|
||||
self.log.info('Fetching a set of all permission, view_menu from FAB meta-table')
|
||||
|
||||
def merge_pv(perm, view_menu):
|
||||
"""Create permission view menu only if it doesn't exist"""
|
||||
|
|
@ -360,8 +365,9 @@ class AirflowSecurityManager(SecurityManager):
|
|||
.filter(or_(models.DagModel.is_active, models.DagModel.is_paused))\
|
||||
.filter(~models.DagModel.is_subdag).all()
|
||||
|
||||
# create can_dag_edit and can_dag_read permissions for every dag(vm)
|
||||
for dag in all_dags_models:
|
||||
for perm in dag_perms:
|
||||
for perm in DAG_PERMS:
|
||||
merge_pv(perm, dag.dag_id)
|
||||
|
||||
# for all the dag-level role, add the permission of viewer
|
||||
|
|
@ -372,13 +378,12 @@ class AirflowSecurityManager(SecurityManager):
|
|||
dag_role = [role for role in all_roles if role.name not in EXISTING_ROLES]
|
||||
update_perm_views = []
|
||||
|
||||
# todo(tao) need to remove all_dag vm
|
||||
# need to remove all_dag vm from all the existing view-menus
|
||||
dag_vm = self.find_view_menu('all_dags')
|
||||
ab_perm_view_role = sqla_models.assoc_permissionview_role
|
||||
perm_view = self.permissionview_model
|
||||
view_menu = self.viewmenu_model
|
||||
|
||||
# todo(tao) comment on the query
|
||||
all_perm_view_by_user = session.query(ab_perm_view_role)\
|
||||
.join(perm_view, perm_view.id == ab_perm_view_role
|
||||
.columns.permission_view_id)\
|
||||
|
|
@ -430,7 +435,7 @@ class AirflowSecurityManager(SecurityManager):
|
|||
|
||||
:return: None.
|
||||
"""
|
||||
logging.info('Start syncing user roles.')
|
||||
self.log.info('Start syncing user roles.')
|
||||
# Create global all-dag VM
|
||||
self.create_perm_vm_for_all_dag()
|
||||
|
||||
|
|
@ -449,12 +454,12 @@ class AirflowSecurityManager(SecurityManager):
|
|||
def sync_perm_for_dag(self, dag_id):
|
||||
"""
|
||||
Sync permissions for given dag id. The dag id surely exists in our dag bag
|
||||
as only /refresh button will call this function
|
||||
as only / refresh button will call this function
|
||||
|
||||
:param dag_id:
|
||||
:return:
|
||||
"""
|
||||
for dag_perm in dag_perms:
|
||||
for dag_perm in DAG_PERMS:
|
||||
perm_on_dag = self.find_permission_view_menu(dag_perm, dag_id)
|
||||
if perm_on_dag is None:
|
||||
self.add_permission_view_menu(dag_perm, dag_id)
|
||||
|
|
@ -464,7 +469,7 @@ class AirflowSecurityManager(SecurityManager):
|
|||
Create perm-vm if not exist and insert into FAB security model for all-dags.
|
||||
"""
|
||||
# create perm for global logical dag
|
||||
for dag_vm in dag_vms:
|
||||
for perm in dag_perms:
|
||||
for dag_vm in DAG_VMS:
|
||||
for perm in DAG_PERMS:
|
||||
self._merge_perm(permission_name=perm,
|
||||
view_menu_name=dag_vm)
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from flask_appbuilder.views import ModelView, BaseView
|
|||
|
||||
from sqlalchemy import Column, Integer, String, Date, Float
|
||||
|
||||
from airflow.www.security import AirflowSecurityManager, dag_perms
|
||||
from airflow.www.security import AirflowSecurityManager, DAG_PERMS
|
||||
|
||||
|
||||
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(name)s:%(message)s')
|
||||
|
|
@ -176,7 +176,7 @@ class TestSecurity(unittest.TestCase):
|
|||
def test_sync_perm_for_dag(self):
|
||||
test_dag_id = 'TEST_DAG'
|
||||
self.security_manager.sync_perm_for_dag(test_dag_id)
|
||||
for dag_perm in dag_perms:
|
||||
for dag_perm in DAG_PERMS:
|
||||
self.assertIsNotNone(self.security_manager.
|
||||
find_permission_view_menu(dag_perm, test_dag_id))
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче