[AIRFLOW-1942] Update Sphinx docs to remove deprecated import structure

Update Sphinx docs to use correct import
structure. Fixes improperly
mocked modules that resulted in hooks not
displaying. Fixes executors
and operators section, which weren't displaying
anything.

Closes #2894 from andyxhadji/AIRFLOW-1942

docs/code.rst

@@ -44,58 +44,54 @@ Operator API
 ''''''''''''
 
 .. automodule:: airflow.operators
-    :show-inheritance:
+    :no-members:
-    :members:
+.. deprecated:: 1.8
-        BashOperator,
+ Use :code:`from airflow.operators.bash_operator import BashOperator` instead.
-        BranchPythonOperator,
-        TriggerDagRunOperator,
-        DummyOperator,
-        EmailOperator,
-        ExternalTaskSensor,
-        GenericTransfer,
-        HdfsSensor,
-        Hive2SambaOperator,
-        HiveOperator,
-        HivePartitionSensor,
-        HiveToDruidTransfer,
-        HiveToMySqlTransfer,
-        SimpleHttpOperator,
-        HttpSensor,
-        MetastorePartitionSensor,
-        MsSqlOperator,
-        MsSqlToHiveTransfer,
-        MySqlOperator,
-        MySqlToHiveTransfer,
-        NamedHivePartitionSensor,
-        PostgresOperator,
-        PrestoCheckOperator,
-        PrestoIntervalCheckOperator,
-        PrestoValueCheckOperator,
-        PythonOperator,
-        PythonVirtualenvOperator,
-        S3KeySensor,
-        S3ToHiveTransfer,
-        ShortCircuitOperator,
-        SlackAPIOperator,
-        SlackAPIPostOperator,
-        SqlSensor,
-        SubDagOperator,
-        TimeSensor,
-        WebHdfsSensor
 
+.. autoclass:: airflow.operators.bash_operator.BashOperator
+.. autoclass:: airflow.operators.python_operator.BranchPythonOperator
+.. autoclass:: airflow.operators.dagrun_operator.TriggerDagRunOperator
 .. autoclass:: airflow.operators.docker_operator.DockerOperator
-
+.. autoclass:: airflow.operators.dummy_operator.DummyOperator
+.. autoclass:: airflow.operators.email_operator.EmailOperator
+.. autoclass:: airflow.operators.sensors.ExternalTaskSensor
+.. autoclass:: airflow.operators.generic_transfer.GenericTransfer
+.. autoclass:: airflow.operators.sensors.HdfsSensor
+.. autoclass:: airflow.operators.hive_to_samba_operator.Hive2SambaOperator
+.. autoclass:: airflow.operators.hive_operator.HiveOperator
+.. autoclass:: airflow.operators.sensors.HivePartitionSensor
+.. autoclass:: airflow.operators.hive_to_druid.HiveToDruidTransfer
+.. autoclass:: airflow.operators.hive_to_mysql.HiveToMySqlTransfer
+.. autoclass:: airflow.operators.http_operator.SimpleHttpOperator
+.. autoclass:: airflow.operators.sensors.HttpSensor
+.. autoclass:: airflow.operators.sensors.MetastorePartitionSensor
+.. autoclass:: airflow.operators.mssql_operator.MsSqlOperator
+.. autoclass:: airflow.operators.mssql_to_hive.MsSqlToHiveTransfer
+.. autoclass:: airflow.operators.sensors.NamedHivePartitionSensor
+.. autoclass:: airflow.operators.postgres_operator.PostgresOperator
+.. autoclass:: airflow.operators.presto_check_operator.PrestoCheckOperator
+.. autoclass:: airflow.operators.presto_check_operator.PrestoIntervalCheckOperator
+.. autoclass:: airflow.operators.presto_check_operator.PrestoValueCheckOperator
+.. autoclass:: airflow.operators.python_operator.PythonOperator
+.. autoclass:: airflow.operators.python_operator.PythonVirtualenvOperator
+.. autoclass:: airflow.operators.sensors.S3KeySensor
+.. autoclass:: airflow.operators.s3_to_hive_operator.S3ToHiveTransfer
+.. autoclass:: airflow.operators.ShortCircuitOperator
+.. autoclass:: airflow.operators.slack_operator.SlackAPIOperator
+.. autoclass:: airflow.operators.sensors.SqlSensor
+.. autoclass:: airflow.operators.subdag_operator.SubDagOperator
+.. autoclass:: airflow.operators.sensors.TimeSensor
+.. autoclass:: airflow.operators.sensors.HdfsSensor
 
 Community-contributed Operators
 '''''''''''''''''''''''''''''''
 
 .. automodule:: airflow.contrib.operators
-    :show-inheritance:
+    :no-members:
-    :members:
+.. deprecated:: 1.8
-        SSHOperator,
+ Use :code:`from airflow.operators.bash_operator import BashOperator` instead.
-        VerticaOperator,
-        VerticaToHiveTransfer
 
+.. autoclass:: airflow.contrib.sensors.aws_redshift_cluster_sensor.AwsRedshiftClusterSensor
 .. autoclass:: airflow.contrib.operators.bigquery_operator.BigQueryOperator
 .. autoclass:: airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator
 .. autoclass:: airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator
@@ -108,9 +104,12 @@ Community-contributed Operators
 .. autoclass:: airflow.contrib.operators.pubsub_operator.PubSubSubscriptionDeleteOperator
 .. autoclass:: airflow.contrib.operators.pubsub_operator.PubSubPublishOperator
 .. autoclass:: airflow.contrib.sensors.pubsub_sensor.PubSubPullSensor
-.. autoclass:: airflow.contrib.operators.QuboleOperator
 .. autoclass:: airflow.contrib.operators.hipchat_operator.HipChatAPIOperator
 .. autoclass:: airflow.contrib.operators.hipchat_operator.HipChatAPISendRoomNotificationOperator
+.. autoclass:: airflow.contrib.operators.qubole_operator.QuboleOperator
+.. autoclass:: airflow.contrib.operators.ssh_operator.SSHOperator
+.. autoclass:: airflow.contrib.operators.vertica_operator.VerticaOperator
+.. autoclass:: airflow.contrib.operators.vertica_to_hive.VerticaToHiveTransfer
 
 .. _macros:
 
@@ -199,9 +198,8 @@ Some airflow specific macros are also defined:
     :show-inheritance:
     :members:
 
-.. automodule:: airflow.macros.hive
+.. autofunction:: airflow.macros.hive.closest_ds_partition
-    :show-inheritance:
+.. autofunction:: airflow.macros.hive.max_partition
-    :members:
 
 .. _models_ref:
 
@@ -219,46 +217,51 @@ persisted in the database.
 Hooks
 -----
 .. automodule:: airflow.hooks
-    :show-inheritance:
+    :no-members:
+.. deprecated:: 1.8
+ Use :code:`from airflow.operators.bash_operator import BashOperator` instead.
+
+.. autoclass:: airflow.hooks.dbapi_hook.DbApiHook
+.. autoclass:: airflow.hooks.docker_hook.DockerHook
+.. automodule:: airflow.hooks.hive_hooks
     :members:
-        DbApiHook,
+      HiveCliHook,
-        DockerHook,
+      HiveMetastoreHook,
-        HiveCliHook,
+      HiveServer2Hook
-        HiveMetastoreHook,
+.. autoclass:: airflow.hooks.http_hook.HttpHook
-        HiveServer2Hook,
+.. autoclass:: airflow.hooks.druid_hook.DruidHook
-        HttpHook,
+.. autoclass:: airflow.hooks.mssql_hook.MsSqlHook
-        DruidHook,
+.. autoclass:: airflow.hooks.mysql_hook.MySqlHook
-        MsSqlHook,
+.. autoclass:: airflow.hooks.postgres_hook.PostgresHook
-        MySqlHook,
+.. autoclass:: airflow.hooks.presto_hook.PrestoHook
-        PostgresHook,
+.. autoclass:: airflow.hooks.S3_hook.S3Hook
-        PrestoHook,
+.. autoclass:: airflow.hooks.sqlite_hook.SqliteHook
-        S3Hook,
+.. autoclass:: airflow.hooks.webhdfs_hook.WebHDFSHook
-        SqliteHook,
-        WebHDFSHook
 
 Community contributed hooks
 '''''''''''''''''''''''''''
 
 .. automodule:: airflow.contrib.hooks
-    :show-inheritance:
+    :no-members:
-    :members:
+.. deprecated:: 1.8
-        BigQueryHook,
+ Use :code:`from airflow.operators.bash_operator import BashOperator` instead.
-        GoogleCloudStorageHook,
-        VerticaHook,
-        FTPHook,
-        SSHHook,
-        CloudantHook,
-        PubSubHook
 
+.. autoclass:: airflow.contrib.hooks.redshift_hook.RedshiftHook
+.. autoclass:: airflow.contrib.hooks.bigquery_hook.BigQueryHook
+.. autoclass:: airflow.contrib.hooks.vertica_hook.VerticaHook
+.. autoclass:: airflow.contrib.hooks.ftp_hook.FTPHook
+.. autoclass:: airflow.contrib.hooks.ssh_hook.SSHHook
+.. autoclass:: airflow.contrib.hooks.cloudant_hook.CloudantHook
 .. autoclass:: airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook
+.. autoclass:: airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook
 
 Executors
 ---------
 Executors are the mechanism by which task instances get run.
 
-.. automodule:: airflow.executors
+.. autoclass:: airflow.executors.local_executor.LocalExecutor
-    :show-inheritance:
+.. autoclass:: airflow.executors.celery_executor.CeleryExecutor
-    :members: LocalExecutor, CeleryExecutor, SequentialExecutor
+.. autoclass:: airflow.executors.sequential_executor.SequentialExecutor
 
 Community-contributed executors
 '''''''''''''''''''''''''''''''

docs/conf.py

@@ -24,6 +24,8 @@ MOCK_MODULES = [
     'mesos.native',
     'oauth2client.service_account',
     'pandas.io.gbq',
+    'vertica_python',
+    'pymssql'
 ]
 for mod_name in MOCK_MODULES:
     sys.modules[mod_name] = mock.Mock()
@@ -53,6 +55,9 @@ extensions = [
     'sphinx.ext.viewcode',
     'sphinxarg.ext',
 ]
+
+autodoc_default_flags = ['show-inheritance', 'members']
+
 viewcode_import = True
 
 # Add any paths that contain templates here, relative to this directory.

docs/integration.rst

@@ -161,8 +161,24 @@ AWSBatchOperator
 AWS RedShift
 '''''''''''''
 
+- :ref:`AwsRedshiftClusterSensor` : Waits for a Redshift cluster to reach a specific status.
+- :ref:`RedshiftHook` : Interact with AWS Redshift, using the boto3 library.
 - :ref:`RedshiftToS3Transfer` : Executes an unload command to S3 as a CSV with headers.
 
+.. _AwsRedshiftClusterSensor:
+
+AwsRedshiftClusterSensor
+"""""""""""""""""""""""""
+
+.. autoclass:: airflow.contrib.sensors.aws_redshift_cluster_sensor.AwsRedshiftClusterSensor
+
+.. _RedshiftHook:
+
+RedshiftHook
+"""""""""""""
+
+.. autoclass:: airflow.contrib.hooks.redshift_hook.RedshiftHook
+
 .. _RedshiftToS3Transfer:
 
 RedshiftToS3Transfer
@@ -171,7 +187,6 @@ RedshiftToS3Transfer
 .. autoclass:: airflow.operators.redshift_to_s3_operator.RedshiftToS3Transfer
 
 
-
 .. _Databricks:
 
 Databricks