Refactor Google operators guides (#9766)

* Refactor Google guides

* fixup! Refactor Google guides

* fixup! fixup! Refactor Google guides
This commit is contained in:
Kamil Breguła 2020-07-13 18:35:00 +02:00 коммит произвёл GitHub
Родитель 514eb6d1e3
Коммит 1ea7316ffa
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
54 изменённых файлов: 763 добавлений и 518 удалений

Просмотреть файл

@ -347,7 +347,8 @@ def check_enforce_code_block():
)
MISSING_GCP_DOC_GUIDES = {
MISSING_GOOGLLE_DOC_GUIDES = {
"ads_to_gcs",
'adls_to_gcs',
'bigquery_to_bigquery',
'bigquery_to_gcs',
@ -367,19 +368,22 @@ MISSING_GCP_DOC_GUIDES = {
}
def check_gcp_guides():
doc_files = glob(f"{ROOT_PROJECT_DIR}/docs/howto/operator/gcp/*.rst")
def check_google_guides():
doc_files = glob(f"{ROOT_PROJECT_DIR}/docs/howto/operator/google/**/*.rst", recursive=True)
doc_names = {f.split("/")[-1].rsplit(".")[0] for f in doc_files}
operators_files = glob(f"{ROOT_PROJECT_DIR}/airflow/providers/google/*/operators/*.py")
operators_files = chain(*[
glob(f"{ROOT_PROJECT_DIR}/airflow/providers/google/*/{resource_type}/*.py")
for resource_type in ["operators", "sensors", "transfers"]
])
operators_files = (f for f in operators_files if not f.endswith("__init__.py"))
operator_names = {f.split("/")[-1].rsplit(".")[0] for f in operators_files}
# Detect missing docs:
missing_guide = operator_names - doc_names
missing_guide -= MISSING_GCP_DOC_GUIDES
missing_guide -= MISSING_GOOGLLE_DOC_GUIDES
if missing_guide:
missing_guide_text = " * " + "\n* ".join(missing_guide)
missing_guide_text = " * " + "\n * ".join(missing_guide)
message = (
"You've added a new operators, but it looks like you haven't added the guide.\n"
f"{missing_guide_text}"
@ -389,9 +393,9 @@ def check_gcp_guides():
build_errors.append(DocBuildError(file_path=None, line_no=None, message=message))
# Keep update missing missing guide list
new_guides = set(doc_names).intersection(set(MISSING_GCP_DOC_GUIDES))
new_guides = set(doc_names).intersection(set(MISSING_GOOGLLE_DOC_GUIDES))
if new_guides:
new_guides_text = " * " + "\n* ".join(new_guides)
new_guides_text = " * " + "\n * ".join(new_guides)
message = (
"You've added a guide currently listed as missing:\n"
f"{new_guides_text}"
@ -504,7 +508,7 @@ check_class_links_in_operators_and_hooks_ref()
check_guide_links_in_operators_and_hooks_ref()
check_enforce_code_block()
check_exampleinclude_for_example_dags()
check_gcp_guides()
check_google_guides()
CHANNEL_INVITATION = """\
If you need help, write to #documentation channel on Airflow's Slack.

Просмотреть файл

@ -135,6 +135,7 @@ extensions = [
'docroles',
'removemarktransform',
'sphinx_copybutton',
'redirects',
]
autodoc_default_options = {
@ -210,7 +211,7 @@ exclude_patterns: List[str] = [
'_api/airflow/providers/google/cloud/_internal_client',
# Templates or partials
'autoapi_templates',
'howto/operator/gcp/_partials',
'howto/operator/google/_partials',
]
ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
@ -515,6 +516,9 @@ autoapi_root = '_api'
# -- Options for example include ------------------------------------------
exampleinclude_sourceroot = os.path.abspath('..')
# -- Options for sphinxcontrib-redirects ----------------------------------
redirects_file = 'redirects.txt'
# -- Additional HTML Context variable
html_context = {
# Google Analytics ID.

79
docs/exts/redirects.py Normal file
Просмотреть файл

@ -0,0 +1,79 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""
Based on: https://github.com/sphinx-contrib/redirects
"""
import os
from sphinx.builders import html as builders
from sphinx.errors import ExtensionError
from sphinx.util import logging
TEMPLATE = '<html><head><meta http-equiv="refresh" content="0; url={}"/></head></html>'
log = logging.getLogger(__name__)
def generate_redirects(app):
"""Generaate redirects files."""
redirect_file_path = os.path.join(app.srcdir, app.config.redirects_file)
if not os.path.exists(redirect_file_path):
raise ExtensionError(f"Could not find redirects file at '{redirect_file_path}'")
in_suffix = next(iter(app.config.source_suffix.keys()))
if not isinstance(app.builder, builders.StandaloneHTMLBuilder):
log.warning(
"The plugin is support only 'html' builder, but you are using '{type(app.builder)}'. Skipping..."
)
return
with open(redirect_file_path) as redirects:
for line in redirects.readlines():
# Skip empty line
if not line.strip():
continue
# Skip comments
if line.startswith("#"):
continue
from_path, _, to_path = line.rstrip().partition(" ")
log.debug("Redirecting '%s' to '%s'", from_path, to_path)
from_path = from_path.replace(in_suffix, '.html')
to_path = to_path.replace(in_suffix, ".html")
to_path_prefix = "..%s" % os.path.sep * (len(from_path.split(os.path.sep)) - 1)
to_path = to_path_prefix + to_path
log.debug("Resolved redirect '%s' to '%s'", from_path, to_path)
redirected_filename = os.path.join(app.builder.outdir, from_path)
redirected_directory = os.path.dirname(redirected_filename)
os.makedirs(redirected_directory, exist_ok=True)
with open(redirected_filename, "w") as f:
f.write(TEMPLATE.format(to_path))
def setup(app):
"""Setup plugin"""
app.add_config_value("redirects_file", "redirects", "env")
app.connect("builder-inited", generate_redirects)

Просмотреть файл

@ -27,7 +27,7 @@ businesses to advertise on Google Search, YouTube and other sites across the web
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GoogleAdsToGcsOperator:

Просмотреть файл

@ -31,7 +31,7 @@ and then integrate those models into your applications and web sites.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudAutoMLDocuments:
@ -48,7 +48,7 @@ To create a Google AutoML dataset you can use
:class:`~airflow.providers.google.cloud.operators.automl.AutoMLCreateDatasetOperator`.
The operator returns dataset id in :ref:`XCom <concepts:xcom>` under ``dataset_id`` key.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_automl_create_dataset]
@ -57,7 +57,7 @@ The operator returns dataset id in :ref:`XCom <concepts:xcom>` under ``dataset_i
After creating a dataset you can use it to import some data using
:class:`~airflow.providers.google.cloud.operators.automl.AutoMLImportDataOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_automl_import_data]
@ -66,7 +66,7 @@ After creating a dataset you can use it to import some data using
To update dataset you can use
:class:`~airflow.providers.google.cloud.operators.automl.AutoMLTablesUpdateDatasetOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_automl_update_dataset]
@ -81,7 +81,7 @@ Listing Table And Columns Specs
To list table specs you can use
:class:`~airflow.providers.google.cloud.operators.automl.AutoMLTablesListTableSpecsOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_automl_specs]
@ -90,7 +90,7 @@ To list table specs you can use
To list column specs you can use
:class:`~airflow.providers.google.cloud.operators.automl.AutoMLTablesListColumnSpecsOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_automl_column_specs]
@ -109,7 +109,7 @@ To create a Google AutoML model you can use
The operator will wait for the operation to complete. Additionally the operator
returns the id of model in :ref:`XCom <concepts:xcom>` under ``model_id`` key.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_automl_create_model]
@ -118,7 +118,7 @@ returns the id of model in :ref:`XCom <concepts:xcom>` under ``model_id`` key.
To get existing model one can use
:class:`~airflow.providers.google.cloud.operators.automl.AutoMLGetModelOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_get_model]
@ -127,7 +127,7 @@ To get existing model one can use
Once a model is created it could be deployed using
:class:`~airflow.providers.google.cloud.operators.automl.AutoMLDeployModelOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_deploy_model]
@ -136,7 +136,7 @@ Once a model is created it could be deployed using
If you wish to delete a model you can use
:class:`~airflow.providers.google.cloud.operators.automl.AutoMLDeleteModelOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_automl_delete_model]
@ -153,13 +153,13 @@ To obtain predictions from Google Cloud AutoML model you can use
:class:`~airflow.providers.google.cloud.operators.automl.AutoMLBatchPredictOperator`. In the first case
the model must be deployed.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_prediction]
:end-before: [END howto_operator_prediction]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_batch_prediction]
@ -175,7 +175,7 @@ You can get a list of AutoML models using
:class:`~airflow.providers.google.cloud.operators.automl.AutoMLListDatasetOperator`. The operator returns list
of datasets ids in :ref:`XCom <concepts:xcom>` under ``dataset_id_list`` key.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_list_dataset]
@ -184,7 +184,7 @@ of datasets ids in :ref:`XCom <concepts:xcom>` under ``dataset_id_list`` key.
To delete a model you can use :class:`~airflow.providers.google.cloud.operators.automl.AutoMLDeleteDatasetOperator`.
The delete operator allows also to pass list or coma separated string of datasets ids to be deleted.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_automl_tables.py
:language: python
:dedent: 4
:start-after: [START howto_operator_delete_dataset]

Просмотреть файл

@ -33,7 +33,7 @@ data.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
Manage datasets
^^^^^^^^^^^^^^^
@ -46,7 +46,7 @@ Create dataset
To create an empty dataset in a BigQuery database you can use
:class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyDatasetOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_create_dataset]
@ -62,7 +62,7 @@ To get the details of an existing dataset you can use
This operator returns a `Dataset Resource <https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource>`__.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_get_dataset]
@ -76,7 +76,7 @@ List tables in dataset
To retrieve the list of tables in a given dataset use
:class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryGetDatasetTablesOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_get_dataset_tables]
@ -93,7 +93,7 @@ To patch a dataset in BigQuery you can use
Note, this operator only replaces fields that are provided in the submitted dataset
resource.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_patch_dataset]
@ -110,7 +110,7 @@ To update a dataset in BigQuery you can use
The update method replaces the entire dataset resource, whereas the patch
method only replaces fields that are provided in the submitted dataset resource.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_update_dataset]
@ -124,7 +124,7 @@ Delete dataset
To delete an existing dataset from a BigQuery database you can use
:class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteDatasetOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_delete_dataset]
@ -147,7 +147,7 @@ ways. You may either directly pass the schema fields in, or you may point the
operator to a Google Cloud Storage object name. The object in Google Cloud
Storage must be a JSON file with the schema fields in it.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_create_table]
@ -155,7 +155,7 @@ Storage must be a JSON file with the schema fields in it.
You can use this operator to create a view on top of an existing table.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_create_view]
@ -175,7 +175,7 @@ Similarly to
you may either directly pass the schema fields in, or you may point the operator
to a Google Cloud Storage object name.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_create_external_table]
@ -196,7 +196,7 @@ returned list will be equal to the number of rows fetched. Each element in the
list will again be a list where elements would represent the column values for
that row.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
:language: python
:dedent: 8
:start-after: [START howto_operator_bigquery_get_data]
@ -213,7 +213,7 @@ To upsert a table you can use
This operator either updates the existing table or creates a new, empty table
in the given dataset.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_upsert_table]
@ -227,7 +227,7 @@ Delete table
To delete an existing table you can use
:class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteTableOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_delete_table]
@ -235,7 +235,7 @@ To delete an existing table you can use
You can also use this operator to delete a view.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
:language: python
:dedent: 4
:start-after: [START howto_operator_bigquery_delete_view]
@ -248,7 +248,7 @@ Execute BigQuery jobs
Let's say you would like to execute the following query.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
:language: python
:dedent: 0
:start-after: [START howto_operator_bigquery_query]
@ -258,7 +258,7 @@ To execute the SQL query in a specific BigQuery database you can use
:class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryInsertJobOperator` with
proper query job configuration that can be Jinja templated.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
:language: python
:dedent: 8
:start-after: [START howto_operator_bigquery_insert_job]
@ -270,7 +270,7 @@ For more information on types of BigQuery job please check
If you want to include some files in your configuration you can use ``include`` clause of Jinja template
language as follow:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
:language: python
:dedent: 8
:start-after: [START howto_operator_bigquery_select_job]
@ -299,7 +299,7 @@ This operator expects a sql query that will return a single row. Each value on
that first row is evaluated using python ``bool`` casting. If any of the values
return ``False`` the check is failed and errors out.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
:language: python
:dedent: 8
:start-after: [START howto_operator_bigquery_check]
@ -317,7 +317,7 @@ This operator expects a sql query that will return a single row. Each value on
that first row is evaluated against ``pass_value`` which can be either a string
or numeric value. If numeric, you can also specify ``tolerance``.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
:language: python
:dedent: 8
:start-after: [START howto_operator_bigquery_value_check]
@ -332,7 +332,7 @@ To check that the values of metrics given as SQL expressions are within a certai
tolerance of the ones from ``days_back`` before you can use
:class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryIntervalCheckOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
:language: python
:dedent: 8
:start-after: [START howto_operator_bigquery_interval_check]

Просмотреть файл

@ -33,7 +33,7 @@ gain access to data connectors that allow you to easily transfer data from Terad
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:BigQueryDTSDocuments:
@ -53,7 +53,7 @@ for example :class:`~airflow.providers.google.cloud.operators.bigquery_dts.BigQu
scheduling option is present in passed configuration. If present then nothing is done, otherwise it's value is
set to ``True``.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_dts.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_dts.py
:language: python
:start-after: [START howto_bigquery_dts_create_args]
:end-before: [END howto_bigquery_dts_create_args]
@ -61,7 +61,7 @@ set to ``True``.
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Basic usage of the operator:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_dts.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_dts.py
:language: python
:dedent: 4
:start-after: [START howto_bigquery_create_data_transfer]
@ -84,7 +84,7 @@ To delete DTS transfer configuration you can use
Basic usage of the operator:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_dts.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_dts.py
:language: python
:dedent: 4
:start-after: [START howto_bigquery_delete_data_transfer]
@ -105,7 +105,7 @@ Start manual transfer runs to be executed now with schedule_time equal to curren
Basic usage of the operator:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_dts.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_dts.py
:language: python
:dedent: 4
:start-after: [START howto_bigquery_start_transfer]
@ -118,7 +118,7 @@ parameters which allows you to dynamically determine values.
To check if operation succeeded you can use
:class:`~airflow.providers.google.cloud.sensors.bigquery_dts.BigQueryDataTransferServiceTransferRunSensor`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigquery_dts.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigquery_dts.py
:language: python
:dedent: 4
:start-after: [START howto_bigquery_dts_sensor]

Просмотреть файл

@ -27,7 +27,7 @@ Google Cloud Bigtable Operators
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:BigtableCreateInstanceOperator:
@ -47,7 +47,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_bigtable_instance_create]
@ -67,7 +67,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_bigtable_instance_delete]
@ -87,7 +87,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_bigtable_cluster_update]
@ -111,7 +111,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_bigtable_table_create]
@ -139,7 +139,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_bigtable_table_delete]
@ -164,7 +164,7 @@ timeout hits and does not raise any exception.
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_bigtable.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_bigtable_table_wait_for_replication]

Просмотреть файл

@ -33,7 +33,7 @@ artifacts such as Docker containers or Java archives.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudBuildBuild:
@ -43,7 +43,7 @@ Build configuration overview
In order to trigger a build, it is necessary to pass the build configuration.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_build.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_build.py
:language: python
:dedent: 0
:start-after: [START howto_operator_gcp_create_build_from_storage_body]
@ -51,7 +51,7 @@ In order to trigger a build, it is necessary to pass the build configuration.
The source code for the build can come from `Google Cloud Build Storage <https://cloud.google.com/storage/>`__:
.. literalinclude:: ../../../../tests/providers/google/cloud/operators/test_cloud_build.py
.. literalinclude:: ../../../../../tests/providers/google/cloud/operators/test_cloud_build.py
:language: python
:dedent: 12
:start-after: [START howto_operator_gcp_cloud_build_source_gcs_dict]
@ -59,7 +59,7 @@ The source code for the build can come from `Google Cloud Build Storage <https:/
It is also possible to specify it using the URL:
.. literalinclude:: ../../../../tests/providers/google/cloud/operators/test_cloud_build.py
.. literalinclude:: ../../../../../tests/providers/google/cloud/operators/test_cloud_build.py
:language: python
:dedent: 12
:start-after: [START howto_operator_gcp_cloud_build_source_gcs_url]
@ -67,7 +67,7 @@ It is also possible to specify it using the URL:
In addition, a build can refer to source stored in `Google Cloud Source Repositories <https://cloud.google.com/source-repositories/docs/>`__.
.. literalinclude:: ../../../../tests/providers/google/cloud/operators/test_cloud_build.py
.. literalinclude:: ../../../../../tests/providers/google/cloud/operators/test_cloud_build.py
:language: python
:dedent: 12
:start-after: [START howto_operator_gcp_cloud_build_source_repo_dict]
@ -75,7 +75,7 @@ In addition, a build can refer to source stored in `Google Cloud Source Reposito
It is also possible to specify it using the URL:
.. literalinclude:: ../../../../tests/providers/google/cloud/operators/test_cloud_build.py
.. literalinclude:: ../../../../../tests/providers/google/cloud/operators/test_cloud_build.py
:language: python
:dedent: 12
:start-after: [START howto_operator_gcp_cloud_build_source_repo_url]
@ -83,7 +83,7 @@ It is also possible to specify it using the URL:
It is also possible to specify it using a YAML or JSON format.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_build.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_build.py
:language: python
:dedent: 0
:start-after: [START howto_operator_gcp_create_build_from_yaml_body]
@ -100,7 +100,7 @@ Trigger a build
Trigger a build is performed with the
:class:`~airflow.providers.google.cloud.operators.cloud_build.CloudBuildCreateBuildOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_build.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_build.py
:language: python
:dedent: 4
:start-after: [START howto_operator_create_build_from_storage]
@ -111,7 +111,7 @@ You can use :ref:`Jinja templating <jinja-templating>` with
parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_build.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_build.py
:language: python
:dedent: 4
:start-after: [START howto_operator_create_build_from_storage_result]

Просмотреть файл

@ -32,7 +32,7 @@ of managing complex Redis deployments.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudMemorystoreInstance:
@ -45,7 +45,7 @@ presented as a compatible dictionary also.
Here is an example of instance
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:start-after: [START howto_operator_instance]
:end-before: [END howto_operator_instance]
@ -63,7 +63,7 @@ make a use of the service account listed under ``persistenceIamIdentity``.
You can use :class:`~airflow.providers.google.cloud.operators.gcs.GCSBucketCreateAclEntryOperator`
operator to set permissions.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_set_acl_permission]
@ -80,7 +80,7 @@ Create instance
Create a instance is performed with the
:class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreCreateInstanceOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_create_instance]
@ -91,7 +91,7 @@ You can use :ref:`Jinja templating <jinja-templating>` with
parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_create_instance_result]
@ -106,7 +106,7 @@ Delete instance
Delete a instance is performed with the
:class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreDeleteInstanceOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_delete_instance]
@ -124,7 +124,7 @@ Export instance
Delete a instance is performed with the
:class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreExportInstanceOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_export_instance]
@ -142,7 +142,7 @@ Failover instance
Delete a instance is performed with the
:class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreFailoverInstanceOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_failover_instance]
@ -160,7 +160,7 @@ Get instance
Delete a instance is performed with the
:class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreGetInstanceOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_get_instance]
@ -178,7 +178,7 @@ Import instance
Delete a instance is performed with the
:class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreImportOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_import_instance]
@ -196,7 +196,7 @@ List instances
List a instances is performed with the
:class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreListInstancesOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_list_instances]
@ -207,7 +207,7 @@ You can use :ref:`Jinja templating <jinja-templating>` with
parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_list_instances_result]
@ -221,7 +221,7 @@ Update instance
Update a instance is performed with the
:class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreUpdateInstanceOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_update_instance]
@ -240,7 +240,7 @@ Scale instance
Scale a instance is performed with the
:class:`~airflow.providers.google.cloud.operators.cloud_memorystore.CloudMemorystoreScaleInstanceOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_scale_instance]

Просмотреть файл

@ -27,7 +27,7 @@ Google Cloud SQL Operators
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudSQLCreateInstanceDatabaseOperator:
@ -46,7 +46,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_db_create]
@ -54,7 +54,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Example request body:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:start-after: [START howto_operator_cloudsql_db_create_body]
:end-before: [END howto_operator_cloudsql_db_create_body]
@ -62,7 +62,7 @@ Example request body:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_sql.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_sql.py
:language: python
:dedent: 4
:start-after: [START gcp_sql_db_create_template_fields]
@ -91,7 +91,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_db_delete]
@ -100,7 +100,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_sql.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_sql.py
:language: python
:dedent: 4
:start-after: [START gcp_sql_db_delete_template_fields]
@ -131,7 +131,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_db_patch]
@ -139,7 +139,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Example request body:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:start-after: [START howto_operator_cloudsql_db_patch_body]
:end-before: [END howto_operator_cloudsql_db_patch_body]
@ -147,7 +147,7 @@ Example request body:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_sql.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_sql.py
:language: python
:dedent: 4
:start-after: [START gcp_sql_db_patch_template_fields]
@ -178,7 +178,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_delete]
@ -187,7 +187,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Note: If the instance has read or failover replicas you need to delete them before you delete the primary instance.
Replicas are deleted the same way as primary instances:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_replicas_delete]
@ -196,7 +196,7 @@ Replicas are deleted the same way as primary instances:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_sql.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_sql.py
:language: python
:dedent: 4
:start-after: [START gcp_sql_delete_template_fields]
@ -228,7 +228,7 @@ Arguments
Example body defining the export operation:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:start-after: [START howto_operator_cloudsql_export_body]
:end-before: [END howto_operator_cloudsql_export_body]
@ -239,7 +239,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_export]
@ -248,7 +248,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_sql.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_sql.py
:language: python
:dedent: 4
:start-after: [START gcp_sql_export_template_fields]
@ -273,7 +273,7 @@ To grant the service account with the appropriate WRITE permissions for the GCS
you can use the :class:`~airflow.providers.google.cloud.operators.gcs.GCSBucketCreateAclEntryOperator`,
as shown in the example:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_export_gcs_permissions]
@ -313,7 +313,7 @@ Arguments
Example body defining the import operation:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:start-after: [START howto_operator_cloudsql_import_body]
:end-before: [END howto_operator_cloudsql_import_body]
@ -324,7 +324,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_import]
@ -333,7 +333,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_sql.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_sql.py
:language: python
:dedent: 4
:start-after: [START gcp_sql_import_template_fields]
@ -358,7 +358,7 @@ To grant the service account with the appropriate READ permissions for the GCS o
you can use the :class:`~airflow.providers.google.cloud.operators.gcs.GCSBucketCreateAclEntryOperator`,
as shown in the example:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_import_gcs_permissions]
@ -384,14 +384,14 @@ Arguments
Example body defining the instance with failover replica:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:start-after: [START howto_operator_cloudsql_create_body]
:end-before: [END howto_operator_cloudsql_create_body]
Example body defining read replica for the instance above:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:start-after: [START howto_operator_cloudsql_create_replica]
:end-before: [END howto_operator_cloudsql_create_replica]
@ -405,7 +405,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_create]
@ -414,7 +414,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_sql.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_sql.py
:language: python
:dedent: 4
:start-after: [START gcp_sql_create_template_fields]
@ -445,7 +445,7 @@ Arguments
Example body defining the instance:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:start-after: [START howto_operator_cloudsql_patch_body]
:end-before: [END howto_operator_cloudsql_patch_body]
@ -456,7 +456,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql.py
:language: python
:dedent: 4
:start-after: [START howto_operator_cloudsql_patch]
@ -465,7 +465,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_sql.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_sql.py
:language: python
:dedent: 4
:start-after: [START gcp_sql_patch_template_fields]
@ -530,7 +530,7 @@ NFS-like volumes in the same path for all the workers.
Example connection definitions for all connectivity cases. Note that all the components
of the connection URI should be URL-encoded:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
:language: python
:start-after: [START howto_operator_cloudsql_query_connections]
:end-before: [END howto_operator_cloudsql_query_connections]
@ -542,7 +542,7 @@ Example operators below are using all connectivity options. Note connection id
from the operator matches the :envvar:`AIRFLOW_CONN_{CONN_ID}` postfix uppercase. This is
standard AIRFLOW notation for defining connection via environment variables):
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
:language: python
:start-after: [START howto_operator_cloudsql_query_operators]
:end-before: [END howto_operator_cloudsql_query_operators]
@ -550,7 +550,7 @@ standard AIRFLOW notation for defining connection via environment variables):
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_sql.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_sql.py
:language: python
:dedent: 4
:start-after: [START gcp_sql_query_template_fields]

Просмотреть файл

@ -27,7 +27,7 @@ Google Cloud Transfer Service Operators
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudDataTransferServiceCreateJobOperator:
@ -52,7 +52,7 @@ The function accepts time in two formats:
- as an :class:`~datetime.time` object
If you want to create a job transfer that copies data from AWS S3 then you must have a connection configured. Information about configuration for AWS is available: :doc:`../../connection/aws`
If you want to create a job transfer that copies data from AWS S3 then you must have a connection configured. Information about configuration for AWS is available: :doc:`/howto/connection/aws`
The selected connection for AWS can be indicated by the parameter ``aws_conn_id``.
For parameter definition, take a look at
@ -62,17 +62,17 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_gcp.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_gcp.py
:language: python
:start-after: [START howto_operator_gcp_transfer_create_job_body_gcp]
:end-before: [END howto_operator_gcp_transfer_create_job_body_gcp]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
:language: python
:start-after: [START howto_operator_gcp_transfer_create_job_body_aws]
:end-before: [END howto_operator_gcp_transfer_create_job_body_aws]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_transfer_create_job]
@ -81,7 +81,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
:language: python
:dedent: 4
:start-after: [START gcp_transfer_job_create_template_fields]
@ -107,7 +107,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_transfer_delete_job]
@ -116,7 +116,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
:language: python
:dedent: 4
:start-after: [START gcp_transfer_job_delete_template_fields]
@ -142,12 +142,12 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_gcp.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_gcp.py
:language: python
:start-after: [START howto_operator_gcp_transfer_update_job_body]
:end-before: [END howto_operator_gcp_transfer_update_job_body]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_gcp.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_gcp.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_transfer_update_job]
@ -156,7 +156,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
:language: python
:dedent: 4
:start-after: [START gcp_transfer_job_update_template_fields]
@ -181,7 +181,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_transfer_cancel_operation]
@ -190,7 +190,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
:language: python
:dedent: 4
:start-after: [START gcp_transfer_operation_cancel_template_fields]
@ -217,7 +217,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_transfer_get_operation]
@ -226,7 +226,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
:language: python
:dedent: 4
:start-after: [START gcp_transfer_operation_get_template_fields]
@ -252,7 +252,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_transfer_list_operations]
@ -261,7 +261,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
:language: python
:dedent: 4
:start-after: [START gcp_transfer_operations_list_template_fields]
@ -286,7 +286,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_transfer_pause_operation]
@ -295,7 +295,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
:language: python
:dedent: 4
:start-after: [START gcp_transfer_operation_pause_template_fields]
@ -320,7 +320,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_transfer_resume_operation]
@ -329,7 +329,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
:language: python
:dedent: 4
:start-after: [START gcp_transfer_operation_resume_template_fields]
@ -355,7 +355,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_transfer_wait_operation]
@ -364,7 +364,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py
:language: python
:dedent: 4
:start-after: [START gcp_transfer_job_sensor_template_fields]

Просмотреть файл

@ -27,7 +27,7 @@ Google Compute Engine Operators
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:ComputeEngineStartInstanceOperator:
@ -43,7 +43,7 @@ Using the operator
The code to create the operator:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gce_start]
@ -52,7 +52,7 @@ The code to create the operator:
You can also create the operator without project id - project id will be retrieved
from the GCP connection id used:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gce_start_no_project_id]
@ -62,7 +62,7 @@ from the GCP connection id used:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/compute.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/compute.py
:language: python
:dedent: 4
:start-after: [START gce_instance_start_template_fields]
@ -89,7 +89,7 @@ Using the operator
The code to create the operator:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gce_stop]
@ -98,7 +98,7 @@ The code to create the operator:
You can also create the operator without project id - project id will be retrieved
from the GCP connection used:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gce_stop_no_project_id]
@ -107,7 +107,7 @@ from the GCP connection used:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/compute.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/compute.py
:language: python
:dedent: 4
:start-after: [START gce_instance_stop_template_fields]
@ -139,7 +139,7 @@ Using the operator
The code to create the operator:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gce_set_machine_type]
@ -148,7 +148,7 @@ The code to create the operator:
You can also create the operator without project id - project id will be retrieved
from the GCP connection used:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gce_set_machine_type_no_project_id]
@ -157,7 +157,7 @@ from the GCP connection used:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/compute.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/compute.py
:language: python
:dedent: 4
:start-after: [START gce_instance_set_machine_type_template_fields]
@ -185,12 +185,12 @@ Using the operator
The code to create the operator:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
:language: python
:start-after: [START howto_operator_compute_template_copy_args]
:end-before: [END howto_operator_compute_template_copy_args]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gce_igm_copy_template]
@ -199,7 +199,7 @@ The code to create the operator:
You can also create the operator without project id - project id will be retrieved
from the GCP connection used:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gce_igm_copy_template_no_project_id]
@ -208,7 +208,7 @@ from the GCP connection used:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/compute.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/compute.py
:language: python
:dedent: 4
:start-after: [START gce_instance_template_copy_operator_template_fields]
@ -239,12 +239,12 @@ Using the operator
The code to create the operator:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
:language: python
:start-after: [START howto_operator_compute_igm_update_template_args]
:end-before: [END howto_operator_compute_igm_update_template_args]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gce_igm_update_template]
@ -253,7 +253,7 @@ The code to create the operator:
You can also create the operator without project id - project id will be retrieved
from the GCP connection used:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_compute_igm.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gce_igm_update_template_no_project_id]
@ -263,7 +263,7 @@ from the GCP connection used:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/compute.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/compute.py
:language: python
:dedent: 4
:start-after: [START gce_igm_update_template_operator_template_fields]

Просмотреть файл

@ -36,7 +36,7 @@ Google Cloud. It offers:
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudDataCatalogEntryOperators:
@ -63,7 +63,7 @@ operators.
The ``CloudDataCatalogGetEntryOperator`` use Project ID, Entry Group ID, Entry ID to get the entry.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_get_entry]
@ -75,7 +75,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_get_entry_result]
@ -83,7 +83,7 @@ The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used b
The ``CloudDataCatalogLookupEntryOperator`` use the resource name to get the entry.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_lookup_entry_linked_resource]
@ -95,7 +95,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_lookup_entry_result]
@ -109,7 +109,7 @@ Creating an entry
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogCreateEntryOperator`
operator create the entry.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_entry_gcs]
@ -121,7 +121,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_entry_gcs_result2]
@ -129,7 +129,7 @@ The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used b
The newly created entry ID can be read with the ``entry_id`` key.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_entry_gcs_result]
@ -143,7 +143,7 @@ Updating an entry
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogUpdateEntryOperator`
operator update the entry.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_update_entry]
@ -161,7 +161,7 @@ Deleting a entry
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteEntryOperator`
operator delete the entry.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_delete_entry]
@ -190,7 +190,7 @@ Creating an entry group
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogCreateEntryGroupOperator`
operator create the entry group.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_entry_group]
@ -202,7 +202,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_entry_group_result2]
@ -210,7 +210,7 @@ The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used b
The newly created entry group ID can be read with the ``entry_group_id`` key.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_entry_group_result2]
@ -224,7 +224,7 @@ Getting an entry group
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogGetEntryGroupOperator`
operator get the entry group.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_get_entry_group]
@ -236,7 +236,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_get_entry_group_result]
@ -250,7 +250,7 @@ Deleting an entry group
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteEntryGroupOperator`
operator delete the entry group.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_delete_entry_group]
@ -279,7 +279,7 @@ Creating a tag templates
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogCreateTagTemplateOperator`
operator get the tag template.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_tag_template]
@ -291,7 +291,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_tag_template_result2]
@ -299,7 +299,7 @@ The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used b
The newly created tag template ID can be read with the ``tag_template_id`` key.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_tag_template_result]
@ -313,7 +313,7 @@ Deleting a tag template
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteTagTemplateOperator`
operator delete the tag template.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_delete_tag_template]
@ -332,7 +332,7 @@ Getting a tag template
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogGetTagTemplateOperator`
operator get the tag template.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_get_tag_template]
@ -344,7 +344,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_get_tag_template_result]
@ -358,7 +358,7 @@ Updating a tag template
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogUpdateTagTemplateOperator`
operator update the tag template.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_update_tag_template]
@ -387,7 +387,7 @@ Creating a tag on an entry
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogCreateTagOperator`
operator get the tag template.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_tag]
@ -399,7 +399,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_tag_result2]
@ -407,7 +407,7 @@ The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used b
The newly created tag ID can be read with the ``tag_id`` key.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_entry_group_result2]
@ -421,7 +421,7 @@ Updating an tag
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogUpdateTagOperator`
operator update the tag template.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_update_tag_template]
@ -439,7 +439,7 @@ Deleting an tag
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteTagOperator`
operator delete the tag template.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_delete_tag_template]
@ -457,7 +457,7 @@ Listing an tags on an entry
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogListTagsOperator`
operator get list of the tags on the entry.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_list_tags]
@ -469,7 +469,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_list_tags_result]
@ -495,7 +495,7 @@ Creating a field
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogCreateTagTemplateFieldOperator`
operator get the tag template field.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_tag_template_field]
@ -507,7 +507,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_tag_template_field_result2]
@ -515,7 +515,7 @@ The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used b
The newly created field ID can be read with the ``tag_template_field_id`` key.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_create_entry_group_result2]
@ -529,7 +529,7 @@ Renaming a field
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogRenameTagTemplateFieldOperator`
operator rename the tag template field.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_rename_tag_template_field]
@ -547,7 +547,7 @@ Updating a field
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogUpdateTagTemplateFieldOperator`
operator get the tag template field.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_update_tag_template_field]
@ -566,7 +566,7 @@ Deleting a field
The :class:`~airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogDeleteTagTemplateFieldOperator`
operator delete the tag template field.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_delete_tag_template_field]
@ -587,7 +587,7 @@ operator searches Data Catalog for multiple resources like entries, tags that ma
The ``query`` parameters should defined using `search syntax <https://cloud.google.com/data-catalog/docs/how-to/search-reference>`__.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_search_catalog]
@ -599,7 +599,7 @@ parameters which allows you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datacatalog.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_datacatalog_search_catalog_result]

Просмотреть файл

@ -33,7 +33,7 @@ and action.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudDataFusionRestartInstanceOperator:
@ -44,7 +44,7 @@ Restart DataFusion Instance
To restart Data Fusion instance use:
:class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionRestartInstanceOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
:language: python
:dedent: 4
:start-after: [START howto_cloud_data_fusion_restart_instance_operator]
@ -63,7 +63,7 @@ Delete DataFusion Instance
To delete Data Fusion instance use:
:class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionDeleteInstanceOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
:language: python
:dedent: 4
:start-after: [START howto_cloud_data_fusion_delete_instance_operator]
@ -83,7 +83,7 @@ Create DataFusion Instance
To create Data Fusion instance use:
:class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionCreateInstanceOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
:language: python
:dedent: 4
:start-after: [START howto_cloud_data_fusion_create_instance_operator]
@ -103,7 +103,7 @@ Update DataFusion Instance
To update Data Fusion instance use:
:class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionUpdateInstanceOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
:language: python
:dedent: 4
:start-after: [START howto_cloud_data_fusion_update_instance_operator]
@ -122,7 +122,7 @@ Get DataFusion Instance
To retrieve Data Fusion instance use:
:class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionGetInstanceOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
:language: python
:dedent: 4
:start-after: [START howto_cloud_data_fusion_get_instance_operator]
@ -142,7 +142,7 @@ Create a DataFusion pipeline
To create Data Fusion pipeline use:
:class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionCreatePipelineOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
:language: python
:dedent: 4
:start-after: [START howto_cloud_data_fusion_create_pipeline]
@ -161,7 +161,7 @@ Start a DataFusion pipeline
To start Data Fusion pipeline use:
:class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionStartPipelineOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
:language: python
:dedent: 4
:start-after: [START howto_cloud_data_fusion_start_pipeline]
@ -180,7 +180,7 @@ Stop a DataFusion pipeline
To stop Data Fusion pipeline use:
:class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionStopPipelineOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
:language: python
:dedent: 4
:start-after: [START howto_cloud_data_fusion_stop_pipeline]
@ -199,7 +199,7 @@ Delete a DataFusion pipeline
To delete Data Fusion pipeline use:
:class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionDeletePipelineOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
:language: python
:dedent: 4
:start-after: [START howto_cloud_data_fusion_delete_pipeline]
@ -219,7 +219,7 @@ List DataFusion pipelines
To list Data Fusion pipelines use:
:class:`~airflow.providers.google.cloud.operators.datafusion.CloudDataFusionListPipelinesOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_datafusion.py
:language: python
:dedent: 4
:start-after: [START howto_cloud_data_fusion_list_pipelines]

Просмотреть файл

@ -32,7 +32,7 @@ For more information about the service visit `Dataproc production documentation
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:DataprocCreateClusterOperator:
@ -46,7 +46,7 @@ For more information about the available fields to pass when creating a cluster,
A cluster configuration can look as followed:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 0
:start-after: [START how_to_cloud_dataproc_create_cluster]
@ -55,7 +55,7 @@ A cluster configuration can look as followed:
With this configuration we can create the cluster:
:class:`~airflow.providers.google.cloud.operators.dataproc.DataprocCreateClusterOperator`
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 4
:start-after: [START how_to_cloud_dataproc_create_cluster_operator]
@ -69,7 +69,7 @@ For more information on updateMask and other parameters take a look at `Dataproc
An example of a new cluster config and the updateMask:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 0
:start-after: [START how_to_cloud_dataproc_updatemask_cluster_operator]
@ -78,7 +78,7 @@ An example of a new cluster config and the updateMask:
To update a cluster you can use:
:class:`~airflow.providers.google.cloud.operators.dataproc.DataprocUpdateClusterOperator`
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 4
:start-after: [START how_to_cloud_dataproc_update_cluster_operator]
@ -91,7 +91,7 @@ To delete a cluster you can use:
:class:`~airflow.providers.google.cloud.operators.dataproc.DataprocDeleteClusterOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 4
:start-after: [START how_to_cloud_dataproc_delete_cluster_operator]
@ -110,7 +110,7 @@ file system. You can specify a file:/// path to refer to a local file on a clust
The job configuration can be submitted by using:
:class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 4
:start-after: [START how_to_cloud_dataproc_submit_job_to_cluster_operator]
@ -125,7 +125,7 @@ There are more arguments to provide in the jobs than the examples show. For the
Example of the configuration for a PySpark Job:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 0
:start-after: [START how_to_cloud_dataproc_pyspark_config]
@ -133,7 +133,7 @@ Example of the configuration for a PySpark Job:
Example of the configuration for a SparkSQl Job:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 0
:start-after: [START how_to_cloud_dataproc_sparksql_config]
@ -141,7 +141,7 @@ Example of the configuration for a SparkSQl Job:
Example of the configuration for a Spark Job:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 0
:start-after: [START how_to_cloud_dataproc_spark_config]
@ -149,7 +149,7 @@ Example of the configuration for a Spark Job:
Example of the configuration for a Hive Job:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 0
:start-after: [START how_to_cloud_dataproc_hive_config]
@ -157,7 +157,7 @@ Example of the configuration for a Hive Job:
Example of the configuration for a Hadoop Job:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 0
:start-after: [START how_to_cloud_dataproc_hadoop_config]
@ -165,7 +165,7 @@ Example of the configuration for a Hadoop Job:
Example of the configuration for a Pig Job:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 0
:start-after: [START how_to_cloud_dataproc_pig_config]
@ -174,7 +174,7 @@ Example of the configuration for a Pig Job:
Example of the configuration for a SparkR:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_dataproc.py
:language: python
:dedent: 0
:start-after: [START how_to_cloud_dataproc_sparkr_config]

Просмотреть файл

@ -27,7 +27,7 @@ Google Cloud Functions Operators
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudFunctionDeleteFunctionOperator:
@ -42,7 +42,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_functions.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_functions.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcf_delete]
@ -51,7 +51,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/functions.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/functions.py
:language: python
:dedent: 4
:start-after: [START gcf_function_delete_template_fields]
@ -81,7 +81,7 @@ Arguments
When a DAG is created, the default_args dictionary can be used to pass
arguments common with other tasks:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_functions.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_functions.py
:language: python
:start-after: [START howto_operator_gcf_default_args]
:end-before: [END howto_operator_gcf_default_args]
@ -105,19 +105,19 @@ Using the operator
Depending on the combination of parameters, the Function's source code can be obtained
from different sources:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_functions.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_functions.py
:language: python
:start-after: [START howto_operator_gcf_deploy_body]
:end-before: [END howto_operator_gcf_deploy_body]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_functions.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_functions.py
:language: python
:start-after: [START howto_operator_gcf_deploy_variants]
:end-before: [END howto_operator_gcf_deploy_variants]
The code to create the operator:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_functions.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_functions.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcf_deploy]
@ -126,7 +126,7 @@ The code to create the operator:
You can also create the operator without project id - project id will be retrieved
from the GCP connection used:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_functions.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_functions.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcf_deploy_no_project_id]
@ -135,7 +135,7 @@ from the GCP connection used:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/functions.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/functions.py
:language: python
:dedent: 4
:start-after: [START gcf_function_deploy_template_fields]

Просмотреть файл

@ -27,7 +27,7 @@ Google Cloud Storage Operators
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GCSToBigQueryOperator:
@ -38,7 +38,7 @@ Use the
:class:`~airflow.providers.google.cloud.transfers.gcs_to_bigquery.GCSToBigQueryOperator`
to execute a BigQuery load job.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_bigquery.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_bigquery.py
:language: python
:start-after: [START howto_operator_gcs_to_bigquery]
:end-before: [END howto_operator_gcs_to_bigquery]
@ -56,7 +56,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_bucket_create_acl_entry_task]
@ -65,7 +65,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/gcs.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/gcs.py
:language: python
:dedent: 4
:start-after: [START gcs_bucket_create_acl_template_fields]
@ -90,7 +90,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_object_create_acl_entry_task]
@ -99,7 +99,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/gcs.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/gcs.py
:language: python
:dedent: 4
:start-after: [START gcs_object_create_acl_template_fields]
@ -128,7 +128,7 @@ Deleting Bucket allows you to remove bucket object from the Google Cloud Storage
It is performed through the
:class:`~airflow.providers.google.cloud.operators.gcs.GCSDeleteBucketOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_delete_bucket]

Просмотреть файл

@ -28,5 +28,5 @@ Google Cloud Operators
*
.. note::
You can learn how to use GCP integrations by analyzing the
You can learn how to use Google Cloud integrations by analyzing the
`source code <https://github.com/apache/airflow/tree/master/airflow/providers/google/cloud/example_dags/>`_ of the particular example DAGs.

Просмотреть файл

@ -31,7 +31,7 @@ consists of multiple machines (specifically, Compute Engine instances) grouped t
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
Manage GKE cluster
^^^^^^^^^^^^^^^^^^
@ -47,7 +47,7 @@ Create GKE cluster
Here is an example of a cluster definition:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_kubernetes_engine.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_kubernetes_engine.py
:language: python
:start-after: [START howto_operator_gcp_gke_create_cluster_definition]
:end-before: [END howto_operator_gcp_gke_create_cluster_definition]
@ -57,7 +57,7 @@ A dict object like this, or a
definition, is required when creating a cluster with
:class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKECreateClusterOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_kubernetes_engine.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_kubernetes_engine.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gke_create_cluster]
@ -72,7 +72,7 @@ To delete a cluster, use
:class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKEDeleteClusterOperator`.
This would also delete all the nodes allocated to the cluster.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_kubernetes_engine.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_kubernetes_engine.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gke_delete_cluster]
@ -121,7 +121,7 @@ is the path ``/airflow/xcom``. To provide values to the XCom, ensure your Pod wr
``return.json`` in the sidecar. The contents of this can then be used downstream in your DAG.
Here is an example of it being used:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_kubernetes_engine.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_kubernetes_engine.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gke_start_pod_xcom]
@ -129,7 +129,7 @@ Here is an example of it being used:
And then use it in other operators:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_kubernetes_engine.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_kubernetes_engine.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gke_xcom_result]

Просмотреть файл

@ -31,7 +31,7 @@ and biomedical data at scale.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
Pipeline Configuration
@ -39,7 +39,7 @@ Pipeline Configuration
In order to run the pipeline, it is necessary to configure the request body.
Here is an example of the pipeline configuration with a single action.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_life_sciences.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_life_sciences.py
:language: python
:dedent: 0
:start-after: [START howto_configure_simple_action_pipeline]
@ -47,7 +47,7 @@ Here is an example of the pipeline configuration with a single action.
The pipeline can also be configured with multiple action.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_life_sciences.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_life_sciences.py
:language: python
:dedent: 0
:start-after: [START howto_configure_multiple_action_pipeline]
@ -64,7 +64,7 @@ Use the
:class:`~airflow.providers.google.cloud.operators.life_sciences.LifeSciencesRunPipelineOperator`
to execute pipelines.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_life_sciences.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_life_sciences.py
:language: python
:dedent: 0
:start-after: [START howto_run_pipeline]

Просмотреть файл

@ -35,7 +35,7 @@ messaging app.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudNaturalLanguageDocuments:
@ -48,14 +48,14 @@ representing text.
Here is an example of document with text provided as a string:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
:language: python
:start-after: [START howto_operator_gcp_natural_language_document_text]
:end-before: [END howto_operator_gcp_natural_language_document_text]
In addition to supplying string, a document can refer to content stored in Google Cloud Storage.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
:language: python
:start-after: [START howto_operator_gcp_natural_language_document_gcs]
:end-before: [END howto_operator_gcp_natural_language_document_gcs]
@ -70,7 +70,7 @@ public figures, landmarks, etc.), and returns information about those entities.
Entity analysis is performed with the
:class:`~airflow.providers.google.cloud.operators.natural_language.CloudNaturalLanguageAnalyzeEntitiesOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_natural_language_analyze_entities]
@ -81,7 +81,7 @@ You can use :ref:`Jinja templating <jinja-templating>` with
parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_natural_language_analyze_entities_result]
@ -98,7 +98,7 @@ as positive, negative, or neutral. Sentiment analysis is performed through
the :class:`~airflow.providers.google.cloud.operators.natural_language.CloudNaturalLanguageAnalyzeEntitySentimentOperator`
operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_natural_language_analyze_entity_sentiment]
@ -109,7 +109,7 @@ You can use :ref:`Jinja templating <jinja-templating>` with
parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_natural_language_analyze_entity_sentiment_result]
@ -127,7 +127,7 @@ through the
:class:`~airflow.providers.google.cloud.operators.natural_language.CloudNaturalLanguageAnalyzeSentimentOperator`
operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_natural_language_analyze_sentiment]
@ -138,7 +138,7 @@ You can use :ref:`Jinja templating <jinja-templating>` with
parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_natural_language_analyze_sentiment_result]
@ -155,7 +155,7 @@ content in a document, use the
:class:`~airflow.providers.google.cloud.operators.natural_language.CloudNaturalLanguageClassifyTextOperator`
operator.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_natural_language_analyze_classify_text]
@ -166,7 +166,7 @@ You can use :ref:`Jinja templating <jinja-templating>` with
parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
to be used by other operators.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_natural_language_analyze_classify_text_result]

Просмотреть файл

@ -35,7 +35,7 @@ By decoupling senders and receivers Google Cloud PubSub allows developers to com
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:PubSubCreateTopicOperator:
@ -45,7 +45,7 @@ Creating a PubSub topic
The PubSub topic is a named resource to which messages are sent by publishers.
The :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubCreateTopicOperator` operator creates a topic.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
:language: python
:start-after: [START howto_operator_gcp_pubsub_create_topic]
:end-before: [END howto_operator_gcp_pubsub_create_topic]
@ -60,7 +60,7 @@ A ``Subscription`` is a named resource representing the stream of messages from
to be delivered to the subscribing application.
The :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubCreateSubscriptionOperator` operator creates the subscription.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
:language: python
:start-after: [START howto_operator_gcp_pubsub_create_subscription]
:end-before: [END howto_operator_gcp_pubsub_create_subscription]
@ -74,7 +74,7 @@ Publishing PubSub messages
A ``Message`` is a combination of data and (optional) attributes that a publisher sends to a topic and is eventually delivered to subscribers.
The :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubPublishMessageOperator` operator would publish messages.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
:language: python
:start-after: [START howto_operator_gcp_pubsub_publish]
:end-before: [END howto_operator_gcp_pubsub_publish]
@ -87,24 +87,24 @@ Pulling messages from a PubSub subscription
The :class:`~airflow.providers.google.cloud.sensors.pubsub.PubSubPullSensor` sensor pulls messages from a PubSub subscription
and pass them through XCom.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
:language: python
:start-after: [START howto_operator_gcp_pubsub_pull_message_with_sensor]
:end-before: [END howto_operator_gcp_pubsub_pull_message_with_sensor]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
:language: python
:start-after: [START howto_operator_gcp_pubsub_pull_message_with_operator]
:end-before: [END howto_operator_gcp_pubsub_pull_message_with_operator]
To pull messages from XCom use the :class:`~airflow.operators.bash.BashOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
:language: python
:start-after: [START howto_operator_gcp_pubsub_pull_messages_result_cmd]
:end-before: [END howto_operator_gcp_pubsub_pull_messages_result_cmd]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
:language: python
:start-after: [START howto_operator_gcp_pubsub_pull_messages_result]
:end-before: [END howto_operator_gcp_pubsub_pull_messages_result]
@ -117,7 +117,7 @@ Deleting a PubSub subscription
The :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubDeleteSubscriptionOperator` operator deletes the subscription.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
:language: python
:start-after: [START howto_operator_gcp_pubsub_unsubscribe]
:end-before: [END howto_operator_gcp_pubsub_unsubscribe]
@ -130,7 +130,7 @@ Deleting a PubSub topic
The :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubDeleteTopicOperator` operator deletes topic.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_pubsub.py
:language: python
:start-after: [START howto_operator_gcp_pubsub_delete_topic]
:end-before: [END howto_operator_gcp_pubsub_delete_topic]

Просмотреть файл

@ -27,7 +27,7 @@ Google Cloud Spanner Operators
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:SpannerDeployInstanceOperator:
@ -45,7 +45,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
:language: python
:dedent: 4
:start-after: [START howto_operator_spanner_deploy]
@ -54,7 +54,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/spanner.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/spanner.py
:language: python
:dedent: 4
:start-after: [START gcp_spanner_deploy_template_fields]
@ -84,7 +84,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
:language: python
:dedent: 4
:start-after: [START howto_operator_spanner_database_delete]
@ -93,7 +93,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/spanner.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/spanner.py
:language: python
:dedent: 4
:start-after: [START gcp_spanner_delete_template_fields]
@ -124,7 +124,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
:language: python
:dedent: 4
:start-after: [START howto_operator_spanner_database_deploy]
@ -133,7 +133,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/spanner.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/spanner.py
:language: python
:dedent: 4
:start-after: [START gcp_spanner_database_deploy_template_fields]
@ -168,13 +168,13 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
:language: python
:dedent: 4
:start-after: [START howto_operator_spanner_database_update]
:end-before: [END howto_operator_spanner_database_update]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
:language: python
:dedent: 4
:start-after: [START howto_operator_spanner_database_update_idempotent]
@ -183,7 +183,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/spanner.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/spanner.py
:language: python
:dedent: 4
:start-after: [START gcp_spanner_database_update_template_fields]
@ -211,7 +211,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
:language: python
:dedent: 4
:start-after: [START howto_operator_spanner_query]
@ -220,7 +220,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/spanner.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/spanner.py
:language: python
:dedent: 4
:start-after: [START gcp_spanner_query_template_fields]
@ -250,7 +250,7 @@ Using the operator
You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_spanner.py
:language: python
:dedent: 4
:start-after: [START howto_operator_spanner_delete]
@ -259,7 +259,7 @@ it will be retrieved from the GCP connection used. Both variants are shown:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/spanner.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/spanner.py
:language: python
:dedent: 4
:start-after: [START gcp_spanner_delete_template_fields]

Просмотреть файл

@ -22,7 +22,7 @@ Google Cloud Speech to Text Operators
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudSpeechToTextRecognizeSpeechOperator:
@ -42,14 +42,14 @@ google.cloud.speech_v1.types module
for more information, see: https://googleapis.github.io/google-cloud-python/latest/speech/gapic/v1/api.html#google.cloud.speech_v1.SpeechClient.recognize
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_speech_to_text.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_speech_to_text.py
:language: python
:start-after: [START howto_operator_text_to_speech_api_arguments]
:end-before: [END howto_operator_text_to_speech_api_arguments]
filename is a simple string argument:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_speech_to_text.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_speech_to_text.py
:language: python
:start-after: [START howto_operator_speech_to_text_api_arguments]
:end-before: [END howto_operator_speech_to_text_api_arguments]
@ -57,7 +57,7 @@ filename is a simple string argument:
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_speech_to_text.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_speech_to_text.py
:language: python
:dedent: 4
:start-after: [START howto_operator_speech_to_text_recognize]
@ -66,7 +66,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/speech_to_text.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/speech_to_text.py
:language: python
:dedent: 4
:start-after: [START gcp_speech_to_text_synthesize_template_fields]

Просмотреть файл

@ -27,7 +27,7 @@ Google Cloud Stackdriver Operators
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:StackdriverListAlertPoliciesOperator:
@ -44,7 +44,7 @@ Using the operator
You can use this operator with or without project id to fetch all the alert policies.
If project is is missing it wil be retrieved from GCP connection used.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_stackdriver_list_alert_policy]
@ -64,7 +64,7 @@ Using the operator
You can use this operator with or without project id to fetch all the alert policies.
If project is is missing it wil be retrieved from GCP connection used.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_stackdriver_enable_alert_policy]
@ -84,7 +84,7 @@ Using the operator
You can use this operator with or without project id to fetch all the alert policies.
If project is is missing it wil be retrieved from GCP connection used.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_stackdriver_disable_alert_policy]
@ -105,7 +105,7 @@ Using the operator
You can use this operator with or without project id to fetch all the alert policies.
If project is is missing it wil be retrieved from GCP connection used.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_stackdriver_upsert_alert_policy]
@ -124,7 +124,7 @@ Using the operator
The name of the alert to be deleted should be given in the format projects/<PROJECT_NAME>/alertPolicies/<ALERT_NAME>
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_stackdriver_delete_alert_policy]
@ -144,7 +144,7 @@ Using the operator
You can use this operator with or without project id to fetch all the alert policies.
If project is is missing it wil be retrieved from GCP connection used.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_stackdriver_list_notification_channel]
@ -164,7 +164,7 @@ Using the operator
You can use this operator with or without project id to fetch all the alert policies.
If project is is missing it wil be retrieved from GCP connection used.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_stackdriver_enable_notification_channel]
@ -184,7 +184,7 @@ Using the operator
You can use this operator with or without project id to fetch all the alert policies.
If project is is missing it wil be retrieved from GCP connection used.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_stackdriver_disable_notification_channel]
@ -205,7 +205,7 @@ Using the operator
You can use this operator with or without project id to fetch all the alert policies.
If project is is missing it wil be retrieved from GCP connection used.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_stackdriver_disable_notification_channel]
@ -224,7 +224,7 @@ Using the operator
You can use this operator with or without project id to fetch all the alert policies.
If project is is missing it wil be retrieved from GCP connection used.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_stackdriver.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcp_stackdriver_delete_notification_channel]

Просмотреть файл

@ -22,7 +22,7 @@ Google Cloud Text to Speech Operators
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudTextToSpeechSynthesizeOperator:
@ -42,14 +42,14 @@ The ``input``, ``voice`` and ``audio_config`` arguments need to be dicts or obje
for more information, see: https://googleapis.github.io/google-cloud-python/latest/texttospeech/gapic/v1/api.html#google.cloud.texttospeech_v1.TextToSpeechClient.synthesize_speech
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_text_to_speech.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_text_to_speech.py
:language: python
:start-after: [START howto_operator_text_to_speech_api_arguments]
:end-before: [END howto_operator_text_to_speech_api_arguments]
The ``filename`` argument is a simple string argument:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_text_to_speech.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_text_to_speech.py
:language: python
:start-after: [START howto_operator_text_to_speech_gcp_filename]
:end-before: [END howto_operator_text_to_speech_gcp_filename]
@ -57,7 +57,7 @@ The ``filename`` argument is a simple string argument:
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_text_to_speech.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_text_to_speech.py
:language: python
:dedent: 4
:start-after: [START howto_operator_text_to_speech_synthesize]
@ -66,7 +66,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/text_to_speech.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/text_to_speech.py
:language: python
:dedent: 4
:start-after: [START gcp_text_to_speech_synthesize_template_fields]

Просмотреть файл

@ -27,7 +27,7 @@ Google Cloud Translate Operators
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudTranslateTextOperator:
@ -44,7 +44,7 @@ Using the operator
Basic usage of the operator:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_translate.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_translate.py
:language: python
:dedent: 4
:start-after: [START howto_operator_translate_text]
@ -53,7 +53,7 @@ Basic usage of the operator:
The result of translation is available as dictionary or array of dictionaries accessible via the usual
XCom mechanisms of Airflow:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_translate.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_translate.py
:language: python
:dedent: 4
:start-after: [START howto_operator_translate_access]
@ -63,7 +63,7 @@ XCom mechanisms of Airflow:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/translate.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/translate.py
:language: python
:dedent: 4
:start-after: [START translate_template_fields]

Просмотреть файл

@ -25,7 +25,7 @@ Google Cloud Speech Translate Operators
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudTranslateSpeechOperator:
@ -47,7 +47,7 @@ for more information, see: https://googleapis.github.io/google-cloud-python/late
Arguments for translation need to be specified.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_translate_speech.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_translate_speech.py
:language: python
:start-after: [START howto_operator_translate_speech_arguments]
:end-before: [END howto_operator_translate_speech_arguments]
@ -56,7 +56,7 @@ Arguments for translation need to be specified.
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_translate_speech.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_translate_speech.py
:language: python
:dedent: 4
:start-after: [START howto_operator_translate_speech]
@ -65,7 +65,7 @@ Using the operator
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/translate_speech.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/translate_speech.py
:language: python
:dedent: 4
:start-after: [START translate_speech_template_fields]

Просмотреть файл

@ -42,7 +42,7 @@ Google Cloud Video Intelligence Operators
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudVideoIntelligenceDetectVideoLabelsOperator:
@ -59,12 +59,12 @@ Using the operator
Input uri is an uri to a file in Google Cloud Storage
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
:language: python
:start-after: [START howto_operator_video_intelligence_other_args]
:end-before: [END howto_operator_video_intelligence_other_args]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_labels]
@ -72,7 +72,7 @@ Input uri is an uri to a file in Google Cloud Storage
You can use the annotation output via Xcom:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_labels_result]
@ -81,7 +81,7 @@ You can use the annotation output via Xcom:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/video_intelligence.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/video_intelligence.py
:language: python
:dedent: 4
:start-after: [START gcp_video_intelligence_detect_labels_template_fields]
@ -108,7 +108,7 @@ Arguments
Input uri is an uri to a file in Google Cloud Storage
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
:language: python
:start-after: [START howto_operator_video_intelligence_other_args]
:end-before: [END howto_operator_video_intelligence_other_args]
@ -116,7 +116,7 @@ Input uri is an uri to a file in Google Cloud Storage
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_explicit_content]
@ -124,7 +124,7 @@ Using the operator
You can use the annotation output via Xcom:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_explicit_content_result]
@ -133,7 +133,7 @@ You can use the annotation output via Xcom:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/video_intelligence.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/video_intelligence.py
:language: python
:dedent: 4
:start-after: [START gcp_video_intelligence_detect_explicit_content_template_fields]
@ -160,7 +160,7 @@ Arguments
Input uri is an uri to a file in Google Cloud Storage
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
:language: python
:start-after: [START howto_operator_video_intelligence_other_args]
:end-before: [END howto_operator_video_intelligence_other_args]
@ -168,7 +168,7 @@ Input uri is an uri to a file in Google Cloud Storage
Using the operator
""""""""""""""""""
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_video_shots]
@ -176,7 +176,7 @@ Using the operator
You can use the annotation output via Xcom:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_video_shots_result]
@ -185,7 +185,7 @@ You can use the annotation output via Xcom:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/video_intelligence.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/video_intelligence.py
:language: python
:dedent: 4
:start-after: [START gcp_video_intelligence_detect_video_shots_template_fields]

Просмотреть файл

@ -27,7 +27,7 @@ Google Cloud Vision Operators
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudVisionAddProductToProductSetOperator:
@ -46,17 +46,17 @@ We are using the :class:`~google.cloud.vision_v1.types.Product`,
:class:`~google.cloud.vision_v1.types.ProductSet` and :class:`~google.api_core.retry.Retry` objects from
Google libraries:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_retry_import]
:end-before: [END howto_operator_vision_retry_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product_set_import]
:end-before: [END howto_operator_vision_product_set_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product_import]
:end-before: [END howto_operator_vision_product_import]
@ -64,7 +64,7 @@ Google libraries:
If ``product_set_id`` and ``product_id`` was generated by the API it can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_add_product_to_product_set]
@ -72,7 +72,7 @@ If ``product_set_id`` and ``product_id`` was generated by the API it can be extr
Otherwise it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_add_product_to_product_set_2]
@ -82,7 +82,7 @@ Otherwise it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_add_product_to_product_set_template_fields]
@ -111,18 +111,18 @@ Using the operator
We are using the :class:`~google.cloud.vision.enums` and :class:`~google.api_core.retry.Retry` objects from
Google libraries:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_retry_import]
:end-before: [END howto_operator_vision_retry_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_enums_import]
:end-before: [END howto_operator_vision_enums_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_annotate_image]
@ -130,7 +130,7 @@ Google libraries:
The result can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_annotate_image_result]
@ -140,7 +140,7 @@ The result can be extracted from XCOM:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_annotate_image_template_fields]
@ -173,24 +173,24 @@ Using the operator
We are using the ``Product`` and :class:`~google.api_core.retry.Retry` objects from Google libraries:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product_import]
:end-before: [END howto_operator_vision_product_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_retry_import]
:end-before: [END howto_operator_vision_retry_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product]
:end-before: [END howto_operator_vision_product]
The ``product_id`` argument can be omitted (it will be generated by the API):
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_create]
@ -198,7 +198,7 @@ The ``product_id`` argument can be omitted (it will be generated by the API):
Or it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_create_2]
@ -208,7 +208,7 @@ Or it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_product_create_template_fields]
@ -243,7 +243,7 @@ Using the operator
If ``product_id`` was generated by the API it can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_delete]
@ -251,7 +251,7 @@ If ``product_id`` was generated by the API it can be extracted from XCOM:
Otherwise it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_delete_2]
@ -260,7 +260,7 @@ Otherwise it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_product_delete_template_fields]
@ -291,7 +291,7 @@ Using the operator
If ``product_id`` was generated by the API it can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_get]
@ -299,7 +299,7 @@ If ``product_id`` was generated by the API it can be extracted from XCOM:
Otherwise it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_get_2]
@ -308,7 +308,7 @@ Otherwise it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_product_get_template_fields]
@ -335,24 +335,24 @@ Using the operator
We are using the ``ProductSet`` and :class:`~google.api_core.retry.Retry` objects from Google libraries:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product_set_import]
:end-before: [END howto_operator_vision_product_set_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_retry_import]
:end-before: [END howto_operator_vision_retry_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product_set]
:end-before: [END howto_operator_vision_product_set]
The ``product_set_id`` argument can be omitted (it will be generated by the API):
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_set_create]
@ -360,7 +360,7 @@ The ``product_set_id`` argument can be omitted (it will be generated by the API)
Or it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_set_create_2]
@ -370,7 +370,7 @@ Or it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_productset_create_template_fields]
@ -399,7 +399,7 @@ Using the operator
If ``product_set_id`` was generated by the API it can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_set_delete]
@ -407,7 +407,7 @@ If ``product_set_id`` was generated by the API it can be extracted from XCOM:
Otherwise it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_set_delete_2]
@ -416,7 +416,7 @@ Otherwise it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_productset_delete_template_fields]
@ -443,7 +443,7 @@ Using the operator
If ``product_set_id`` was generated by the API it can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_set_get]
@ -451,7 +451,7 @@ If ``product_set_id`` was generated by the API it can be extracted from XCOM:
Otherwise it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_set_get_2]
@ -460,7 +460,7 @@ Otherwise it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_productset_get_template_fields]
@ -499,12 +499,12 @@ Using the operator
We are using the ``ProductSet`` object from the Google Cloud Vision library:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product_set_import]
:end-before: [END howto_operator_vision_product_set_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product_set]
:end-before: [END howto_operator_vision_product_set]
@ -513,7 +513,7 @@ Initialization of the task:
If ``product_set_id`` was generated by the API it can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_set_update]
@ -521,7 +521,7 @@ If ``product_set_id`` was generated by the API it can be extracted from XCOM:
Otherwise it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_set_update_2]
@ -530,7 +530,7 @@ Otherwise it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_productset_update_template_fields]
@ -580,19 +580,19 @@ Using the operator
We are using the ``Product`` object from the Google Cloud Vision library:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product_import]
:end-before: [END howto_operator_vision_product_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product]
:end-before: [END howto_operator_vision_product]
If ``product_id`` was generated by the API it can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_update]
@ -600,7 +600,7 @@ If ``product_id`` was generated by the API it can be extracted from XCOM:
Otherwise it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_product_update_2]
@ -609,7 +609,7 @@ Otherwise it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_product_update_template_fields]
@ -636,24 +636,24 @@ Using the operator
We are using the :class:`~google.cloud.vision_v1.types.ReferenceImage` and :class:`~google.api_core.retry.Retry` objects from Google libraries:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_reference_image_import]
:end-before: [END howto_operator_vision_reference_image_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_retry_import]
:end-before: [END howto_operator_vision_retry_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_reference_image]
:end-before: [END howto_operator_vision_reference_image]
The ``product_set_id`` argument can be omitted (it will be generated by the API):
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_reference_image_create]
@ -661,7 +661,7 @@ The ``product_set_id`` argument can be omitted (it will be generated by the API)
Or it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_reference_image_create_2]
@ -671,7 +671,7 @@ Or it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_reference_image_create_template_fields]
@ -700,17 +700,17 @@ We are using the :class:`~google.cloud.vision_v1.types.Product`,
:class:`~google.cloud.vision_v1.types.ProductSet` and :class:`~google.api_core.retry.Retry` objects from
Google libraries:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_retry_import]
:end-before: [END howto_operator_vision_retry_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product_set_import]
:end-before: [END howto_operator_vision_product_set_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_product_import]
:end-before: [END howto_operator_vision_product_import]
@ -718,7 +718,7 @@ Google libraries:
If ``product_set_id`` and ``product_id`` was generated by the API it can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_remove_product_from_product_set]
@ -726,7 +726,7 @@ If ``product_set_id`` and ``product_id`` was generated by the API it can be extr
Otherwise it can be specified explicitly:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_remove_product_from_product_set_2]
@ -736,7 +736,7 @@ Otherwise it can be specified explicitly:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_remove_product_from_product_set_template_fields]
@ -766,12 +766,12 @@ Using the operator
We are using the :class:`~google.api_core.retry.Retry` objects from
Google libraries:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_retry_import]
:end-before: [END howto_operator_vision_retry_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_detect_text]
@ -779,7 +779,7 @@ Google libraries:
The result can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_detect_text_result]
@ -789,7 +789,7 @@ The result can be extracted from XCOM:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_detect_text_set_template_fields]
@ -818,12 +818,12 @@ Using the operator
We are using the :class:`~google.api_core.retry.Retry` objects from
Google libraries:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_retry_import]
:end-before: [END howto_operator_vision_retry_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_document_detect_text]
@ -831,7 +831,7 @@ Google libraries:
The result can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_document_detect_text_result]
@ -841,7 +841,7 @@ The result can be extracted from XCOM:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_document_detect_text_set_template_fields]
@ -871,12 +871,12 @@ Using the operator
We are using the :class:`~google.api_core.retry.Retry` objects from
Google libraries:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_retry_import]
:end-before: [END howto_operator_vision_retry_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_detect_labels]
@ -884,7 +884,7 @@ Google libraries:
The result can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_detect_labels_result]
@ -894,7 +894,7 @@ The result can be extracted from XCOM:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_detect_labels_template_fields]
@ -923,12 +923,12 @@ Using the operator
We are using the :class:`~google.api_core.retry.Retry` objects from
Google libraries:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:start-after: [START howto_operator_vision_retry_import]
:end-before: [END howto_operator_vision_retry_import]
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_detect_safe_search]
@ -936,7 +936,7 @@ Google libraries:
The result can be extracted from XCOM:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_vision.py
:language: python
:dedent: 4
:start-after: [START howto_operator_vision_detect_safe_search_result]
@ -946,7 +946,7 @@ The result can be extracted from XCOM:
Templating
""""""""""
.. literalinclude:: ../../../../airflow/providers/google/cloud/operators/vision.py
.. literalinclude:: ../../../../../airflow/providers/google/cloud/operators/vision.py
:language: python
:dedent: 4
:start-after: [START vision_detect_safe_search_template_fields]

Просмотреть файл

@ -34,7 +34,7 @@ Cloud Functions.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:CloudFirestoreExportDatabaseOperator:
@ -45,7 +45,7 @@ Export database
Exports a copy of all or a subset of documents from Google Cloud Firestore to Google Cloud Storage is performed with the
:class:`~airflow.providers.google.firebase.operators.firestore.CloudFirestoreExportDatabaseOperator` operator.
.. exampleinclude:: ../../../../airflow/providers/google/firebase/example_dags/example_firestore.py
.. exampleinclude:: ../../../../../airflow/providers/google/firebase/example_dags/example_firestore.py
:language: python
:dedent: 4
:start-after: [START howto_operator_export_database_to_gcs]

Просмотреть файл

@ -0,0 +1,31 @@
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
.. http://www.apache.org/licenses/LICENSE-2.0
.. Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
Google Operators
================
.. toctree::
:maxdepth: 1
cloud/index
firebase/firestore
marketing_platform/index
suite/sheets
ads
transfer/index

Просмотреть файл

@ -30,7 +30,7 @@ For more information about the Google Analytics 360 API check
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GoogleAnalyticsListAccountsOperator:
@ -40,7 +40,7 @@ List the Accounts
To list accounts from Analytics you can use the
:class:`~airflow.providers.google.marketing_platform.operators.analytics.GoogleAnalyticsListAccountsOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_analytics.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_analytics.py
:language: python
:dedent: 4
:start-after: [START howto_marketing_platform_list_accounts_operator]
@ -58,7 +58,7 @@ Returns a web property-Google Ads link to which the user has access.
To list web property-Google Ads link you can use the
:class:`~airflow.providers.google.marketing_platform.operators.analytics.GoogleAnalyticsGetAdsLinkOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_analytics.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_analytics.py
:language: python
:dedent: 4
:start-after: [START howto_marketing_platform_get_ads_link_operator]
@ -76,7 +76,7 @@ Operator returns a list of entity Google Ads links.
To list Google Ads links you can use the
:class:`~airflow.providers.google.marketing_platform.operators.analytics.GoogleAnalyticsRetrieveAdsLinksListOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_analytics.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_analytics.py
:language: python
:dedent: 4
:start-after: [START howto_marketing_platform_retrieve_ads_links_list_operator]

Просмотреть файл

@ -30,7 +30,7 @@ reports. For more information about the Campaign Manager API check
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GoogleCampaignManagerDeleteReportOperator:
@ -41,7 +41,7 @@ To delete Campaign Manager report you can use the
:class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerDeleteReportOperator`.
It deletes a report by its unique ID.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
:language: python
:dedent: 4
:start-after: [START howto_campaign_manager_delete_report_operator]
@ -59,7 +59,7 @@ Downloading a report
The :class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerDownloadReportOperator`.
allows you to download a Campaign Manager to Google Cloud Storage bucket.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
:language: python
:dedent: 4
:start-after: [START howto_campaign_manager_get_report_operator]
@ -77,7 +77,7 @@ Waiting for a report
Report are generated asynchronously. To wait for report to be ready for downloading
you can use :class:`~airflow.providers.google.marketing_platform.sensors.campaign_manager.GoogleCampaignManagerReportSensor`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
:language: python
:dedent: 4
:start-after: [START howto_campaign_manager_wait_for_operation]
@ -96,7 +96,7 @@ To insert a Campaign Manager report you can use the
:class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerInsertReportOperator`.
Running this operator creates a new report.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
:language: python
:dedent: 4
:start-after: [START howto_campaign_manager_insert_report_operator]
@ -116,7 +116,7 @@ Running a report
To run Campaign Manager report you can use the
:class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerRunReportOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
:language: python
:dedent: 4
:start-after: [START howto_campaign_manager_run_report_operator]
@ -135,7 +135,7 @@ Inserting a conversions
To insert Campaign Manager conversions you can use the
:class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerBatchInsertConversionsOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
:language: python
:dedent: 4
:start-after: [START howto_campaign_manager_insert_conversions]
@ -154,7 +154,7 @@ Updating a conversions
To update Campaign Manager conversions you can use the
:class:`~airflow.providers.google.marketing_platform.operators.campaign_manager.GoogleCampaignManagerBatchUpdateConversionsOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_campaign_manager.py
:language: python
:dedent: 4
:start-after: [START howto_campaign_manager_update_conversions]

Просмотреть файл

@ -27,7 +27,7 @@ campaign management features you need.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GoogleDisplayVideo360CreateReportOperator:
@ -37,7 +37,7 @@ Creating a report
To create Display&Video 360 report use
:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360CreateReportOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
:start-after: [START howto_google_display_video_createquery_report_operator]
@ -57,7 +57,7 @@ Deleting a report
To delete Display&Video 360 report use
:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DeleteReportOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
:start-after: [START howto_google_display_video_deletequery_report_operator]
@ -75,7 +75,7 @@ Waiting for report
To wait for the report use
:class:`~airflow.providers.google.marketing_platform.sensors.display_video.GoogleDisplayVideo360ReportSensor`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
:start-after: [START howto_google_display_video_wait_report_operator]
@ -93,7 +93,7 @@ Downloading a report
To download a report to GCS bucket use
:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DownloadReportOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
:start-after: [START howto_google_display_video_getquery_report_operator]
@ -112,7 +112,7 @@ Running a report
To run Display&Video 360 report use
:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360RunReportOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
:start-after: [START howto_google_display_video_runquery_report_operator]
@ -141,7 +141,7 @@ The operator accepts body request:
To download line items in CSV format report use
:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DownloadLineItemsOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
:start-after: [START howto_google_display_video_download_line_items_operator]
@ -160,7 +160,7 @@ Upload line items
To run Display&Video 360 uploading line items use
:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360UploadLineItemsOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
:start-after: [START howto_google_display_video_upload_line_items_operator]
@ -178,7 +178,7 @@ Create SDF download task
To create SDF download task use
:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360CreateSDFDownloadTaskOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
:start-after: [START howto_google_display_video_create_sdf_download_task_operator]
@ -197,7 +197,7 @@ Save SDF files in the Google Cloud Storage
To save SDF files and save them in the Google Cloud Storage use
:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360SDFtoGCSOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
:start-after: [START howto_google_display_video_save_sdf_in_gcs_operator]
@ -215,7 +215,7 @@ Waiting for SDF operation
Wait for SDF operation is executed by:
:class:`~airflow.providers.google.marketing_platform.sensors.display_video.GoogleDisplayVideo360GetSDFDownloadOperationSensor`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
:start-after: [START howto_google_display_video_wait_for_operation_sensor]

Просмотреть файл

@ -0,0 +1,32 @@
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
.. http://www.apache.org/licenses/LICENSE-2.0
.. Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
Google Marketing Platform Operators
===================================
.. toctree::
:maxdepth: 1
:glob:
*
.. note::
You can learn how to use Google Cloud integrations by analyzing the
`source code <https://github.com/apache/airflow/tree/master/airflow/providers/google/marketing_platform/example_dags/>`_ of the particular example DAGs.

Просмотреть файл

@ -28,7 +28,7 @@ For more information check `Google Search Ads <https://developers.google.com/sea
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GoogleSearchAdsInsertReportOperator:
@ -38,7 +38,7 @@ Inserting a report
To insert a Search Ads report use the
:class:`~airflow.providers.google.marketing_platform.operators.search_ads.GoogleSearchAdsInsertReportOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
:language: python
:dedent: 4
:start-after: [START howto_search_ads_generate_report_operator]
@ -50,7 +50,7 @@ parameters which allows you to dynamically determine values. You can provide rep
.json`` file as this operator supports this template extension.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows it to be used by other operators:
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
:language: python
:dedent: 4
:start-after: [START howto_search_ads_get_report_id]
@ -64,7 +64,7 @@ Awaiting for a report
To wait for a report to be ready for download use
:class:`~airflow.providers.google.marketing_platform.sensors.search_ads.GoogleSearchAdsReportSensor`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
:language: python
:dedent: 4
:start-after: [START howto_search_ads_get_report_operator]
@ -82,7 +82,7 @@ Downloading a report
To download a Search Ads report to Google Cloud Storage bucket use the
:class:`~airflow.providers.google.marketing_platform.operators.search_ads.GoogleSearchAdsDownloadReportOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
.. exampleinclude:: ../../../../../airflow/providers/google/marketing_platform/example_dags/example_search_ads.py
:language: python
:dedent: 4
:start-after: [START howto_search_ads_getfile_report_operator]

Просмотреть файл

@ -18,7 +18,7 @@
Google Sheets Operators
=======================
cThe latest version of the Sheets API lets developers programmatically:
The latest version of the Sheets API lets developers programmatically:
- Read and write data
- Format text and numbers
@ -39,7 +39,7 @@ For more information check `official documentation <https://developers.google.co
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GoogleSheetsCreateSpreadsheetOperator:
@ -49,7 +49,7 @@ Create spreadsheet
To create new spreadsheet you can use the
:class:`~airflow.providers.google.suite.operators.sheets.GoogleSheetsCreateSpreadsheetOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/suite/example_dags/example_sheets.py
.. exampleinclude:: ../../../../../airflow/providers/google/suite/example_dags/example_sheets.py
:language: python
:dedent: 4
:start-after: [START create_spreadsheet]
@ -60,7 +60,7 @@ You can use :ref:`Jinja templating <jinja-templating>` with
To get the URL of newly created spreadsheet use XCom value:
.. exampleinclude:: ../../../../airflow/providers/google/suite/example_dags/example_sheets.py
.. exampleinclude:: ../../../../../airflow/providers/google/suite/example_dags/example_sheets.py
:language: python
:dedent: 4
:start-after: [START print_spreadsheet_url]

Просмотреть файл

@ -27,7 +27,7 @@ Facebook Ads To GCS Operators
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:FacebookAdsReportToGcsOperator:
@ -38,7 +38,7 @@ Use the
:class:`~airflow.providers.google.cloud.transfers.facebook_ads_to_gcs.FacebookAdsReportToGcsOperator`
to execute a Facebook ads report fetch and load to GCS.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py
:language: python
:start-after: [START howto_operator_facebook_ads_to_gcs]
:end-before: [END howto_operator_facebook_ads_to_gcs]

Просмотреть файл

@ -49,7 +49,7 @@ from the source after they are transferred to the sink.
Source objects can be specified using include and exclusion prefixes, as well as based on the file
modification date.
If you need information on how to use it, look at the guide: :doc:`cloud_storage_transfer_service`
If you need information on how to use it, look at the guide: :doc:`/howto/operator/google/cloud/cloud_storage_transfer_service`
Local transfer
~~~~~~~~~~~~~~
@ -61,7 +61,7 @@ In the next section they will be described.
Prerequisite Tasks
------------------
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
Operators
@ -94,7 +94,7 @@ Copy single file
The following example would copy a single file, ``OBJECT_1`` from the ``BUCKET_1_SRC`` GCS bucket to the ``BUCKET_1_DST`` bucket.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_gcs_single_file]
@ -105,7 +105,7 @@ Copy multiple files
There are several ways to copy multiple files, various examples of which are presented following.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_gcs_wildcard]
@ -115,7 +115,7 @@ The ``source_object`` value may contain one wild card, denoted as "*". All files
be copied. In this example, all root level files ending with ``.txt`` in ``BUCKET_1_SRC`` will be copied to the ``data``
folder in ``BUCKET_1_DST``, with file names unchanged.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_gcs_delimiter]
@ -125,7 +125,7 @@ The delimiter filed may be specified to select any source files starting with ``
value supplied to ``delimiter``. This example uses the ``delimiter`` value to implement the same functionality as the
prior example.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_gcs_list]
@ -141,7 +141,7 @@ Move single file
Supplying ``True`` to the ``move`` argument causes the operator to delete ``source_object`` once the copy is complete.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_gcs_single_file_move]
@ -153,7 +153,7 @@ Move multiple files
Multiple files may be moved by supplying ``True`` to the ``move`` argument. The same rules concerning wild cards and
the ``delimiter`` argument apply to moves as well as copies.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_gcs_list_move]
@ -186,7 +186,7 @@ The following example will ensure all files in ``BUCKET_1_SRC``, including any i
``BUCKET_1_DST``. It will not overwrite identically named files in ``BUCKET_1_DST`` if they already exist. It will not
delete any files in ``BUCKET_1_DST`` not in ``BUCKET_1_SRC``.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_synch_bucket]
@ -199,7 +199,7 @@ This example will ensure all files in ``BUCKET_1_SRC``, including any in subdire
``BUCKET_1_DST``. It will overwrite identically named files in ``BUCKET_1_DST`` if they already exist. It will
delete any files in ``BUCKET_1_DST`` not in ``BUCKET_1_SRC``.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_synch_full_bucket]
@ -212,7 +212,7 @@ The following example will ensure all files in ``BUCKET_1_SRC``, including any i
``subdir`` folder in ``BUCKET_1_DST``. It will not overwrite identically named files in ``BUCKET_1_DST/subdir`` if they
already exist and it will not delete any files in ``BUCKET_1_DST/subdir`` not in ``BUCKET_1_SRC``.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_synch_to_subdir]
@ -225,7 +225,7 @@ This example will ensure all files in ``BUCKET_1_SRC/subdir``, including any in
in ``BUCKET_1_DST``. It will not overwrite identically named files in ``BUCKET_1_DST`` if they
already exist and it will not delete any files in ``BUCKET_1_DST`` not in ``BUCKET_1_SRC/subdir``.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_sync_from_subdir]

Просмотреть файл

@ -33,7 +33,7 @@ document editor, file sharing mechanisms.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GCSToGoogleDriveOperator:
@ -52,7 +52,7 @@ Copy single files
The following Operator would copy a single file.
.. exampleinclude:: ../../../../airflow/providers/google/suite/example_dags/example_gcs_to_gdrive.py
.. exampleinclude:: ../../../../../airflow/providers/google/suite/example_dags/example_gcs_to_gdrive.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_gdrive_copy_single_file]
@ -63,7 +63,7 @@ Copy multiple files
The following Operator would copy all the multiples files (i.e. using wildcard).
.. exampleinclude:: ../../../../airflow/providers/google/suite/example_dags/example_gcs_to_gdrive.py
.. exampleinclude:: ../../../../../airflow/providers/google/suite/example_dags/example_gcs_to_gdrive.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_gdrive_copy_files]
@ -75,7 +75,7 @@ Move files
Using the ``move_object`` parameter allows you to move the files. After copying the file to Google Drive,
the original file from the bucket is deleted.
.. exampleinclude:: ../../../../airflow/providers/google/suite/example_dags/example_gcs_to_gdrive.py
.. exampleinclude:: ../../../../../airflow/providers/google/suite/example_dags/example_gcs_to_gdrive.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_gdrive_move_files]

Просмотреть файл

@ -29,7 +29,7 @@ This page shows how to download data from GCS to local filesystem.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GCSToLocalFilesystemOperator:
@ -42,7 +42,7 @@ data from GCS to local filesystem.
Below is an example of using this operator to upload a file to GCS.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs.py
:language: python
:dedent: 0
:start-after: [START howto_operator_gcs_download_file_task]

Просмотреть файл

@ -32,7 +32,7 @@ It runs over the SSH protocol. It supports the full security and authentication
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GCSToSFTPOperator:
@ -53,7 +53,7 @@ Copying a single file
The following Operator copies a single file.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_sftp.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_sftp.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_sftp_copy_single_file]
@ -66,7 +66,7 @@ To move the file use the ``move_object`` parameter. Once the file is copied to S
the original file from the Google Storage is deleted. The ``destination_path`` parameter defines the
full path of the file on the SFTP server.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_sftp.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_sftp.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_sftp_move_single_file_destination]
@ -78,7 +78,7 @@ Copying a directory
Use the ``wildcard`` in ``source_path`` parameter to copy a directory.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_sftp.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_sftp.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_sftp_copy_directory]
@ -90,7 +90,7 @@ Moving specific files
Use the ``wildcard`` in ``source_path`` parameter to move the specific files.
The ``destination_path`` defines the path that is prefixed to all copied files.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_sftp.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_gcs_to_sftp.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_sftp_move_specific_files]

Просмотреть файл

@ -32,7 +32,7 @@ common spreadsheet tasks.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GCSToGoogleSheets:
@ -42,7 +42,7 @@ Upload data from GCS to Google Sheets
To upload data from Google Cloud Storage to Google Spreadsheet you can use the
:class:`~airflow.providers.google.suite.transfers.gcs_to_sheets.GCSToGoogleSheetsOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/suite/example_dags/example_gcs_to_sheets.py
.. exampleinclude:: ../../../../../airflow/providers/google/suite/example_dags/example_gcs_to_sheets.py
:language: python
:dedent: 4
:start-after: [START upload_gcs_to_sheets]

Просмотреть файл

@ -0,0 +1,28 @@
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
.. http://www.apache.org/licenses/LICENSE-2.0
.. Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
Google Transfer Operators
===================================
.. toctree::
:maxdepth: 1
:glob:
*

Просмотреть файл

@ -29,7 +29,7 @@ This page shows how to upload data from local filesystem to GCS.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:LocalFilesystemToGCSOperator:
@ -43,7 +43,7 @@ When you use this operator, you can optionally compress the data being uploaded.
Below is an example of using this operator to upload a file to GCS.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_local_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_local_to_gcs.py
:language: python
:dedent: 0
:start-after: [START howto_operator_local_filesystem_to_gcs]

Просмотреть файл

@ -49,7 +49,7 @@ All parameters are described in the reference documentation - :class:`~airflow.p
An example operator call might look like this:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_presto_to_gcs_basic]
@ -67,7 +67,7 @@ You can specify these options by the ``export_format`` parameter.
If you want a CSV file to be created, your operator call might look like this:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_presto_to_gcs_csv]
@ -81,7 +81,7 @@ will be dumped from the database and upload to the bucket.
If you want to create a schema file, then an example operator call might look like this:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_presto_to_gcs_multiple_types]
@ -102,7 +102,7 @@ maximum allowed file size for a single object.
If you want to create 10 MB files, your code might look like this:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_read_data_from_gcs_many_chunks]
@ -123,7 +123,7 @@ For example, if you want to create an external table that allows you to create q
read data directly from GCS, then you can use :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateExternalTableOperator`.
Using this operator looks like this:
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_create_external_table_multiple_types]

Просмотреть файл

@ -32,7 +32,7 @@ It runs over the SSH protocol. It supports the full security and authentication
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:SFTPToGCSOperator:
@ -51,7 +51,7 @@ Copying single files
The following Operator copies a single file.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_sftp_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_sftp_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_sftp_to_gcs_copy_single_file]
@ -64,7 +64,7 @@ To move the file use the ``move_object`` parameter. Once the file is copied to G
the original file from the SFTP is deleted.
The ``destination_path`` parameter defines the full path of the file in the bucket.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_sftp_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_sftp_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_sftp_to_gcs_move_single_file_destination]
@ -76,7 +76,7 @@ Copying directory
Use the ``wildcard`` in ``source_path`` parameter to copy the directory.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_sftp_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_sftp_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_sftp_to_gcs_copy_directory]
@ -92,7 +92,7 @@ e.g. ``tests_sftp_hook_dir/subdir/parent-1.bin`` is copied to ``specific_files/p
and ``tests_sftp_hook_dir/subdir/parent-2.bin`` is copied to ``specific_files/parent-2.bin`` .
``tests_sftp_hook_dir/subdir/parent-3.txt`` is skipped.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_sftp_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_sftp_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_sftp_to_gcs_move_specific_files]

Просмотреть файл

@ -32,7 +32,7 @@ common spreadsheet tasks.
Prerequisite Tasks
^^^^^^^^^^^^^^^^^^
.. include:: /howto/operator/gcp/_partials/prerequisite_tasks.rst
.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst
.. _howto/operator:GoogleSheetsToGCSOperator:
@ -42,7 +42,7 @@ Upload data from Google Sheets to GCS
To upload data from Google Spreadsheet to Google Cloud Storage you can use the
:class:`~airflow.providers.google.cloud.transfers.sheets_to_gcs.GoogleSheetsToGCSOperator`.
.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_sheets_to_gcs.py
.. exampleinclude:: ../../../../../airflow/providers/google/cloud/example_dags/example_sheets_to_gcs.py
:language: python
:dedent: 4
:start-after: [START upload_sheet_to_gcs]

Просмотреть файл

@ -35,7 +35,7 @@ determine what actually executes when your DAG runs.
amazon/aws/index
apache/index
dingding
gcp/index
google/index
http/index
kubernetes
papermill

Просмотреть файл

@ -555,7 +555,7 @@ These integrations allow you to copy data from/to Amazon Web Services.
* - `Amazon Simple Storage Service (S3) <https://aws.amazon.com/s3/>`__
- `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/cloud_storage_transfer_service>`
- :doc:`How to use <howto/operator/google/cloud/cloud_storage_transfer_service>`
- :mod:`airflow.providers.google.cloud.transfers.s3_to_gcs`,
:mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service`
@ -646,37 +646,37 @@ These integrations allow you to perform various operations within the Google Clo
* - `AutoML <https://cloud.google.com/automl/>`__
- :doc:`How to use <howto/operator/gcp/automl>`
- :doc:`How to use <howto/operator/google/cloud/automl>`
- :mod:`airflow.providers.google.cloud.hooks.automl`
- :mod:`airflow.providers.google.cloud.operators.automl`
-
* - `BigQuery <https://cloud.google.com/bigquery/>`__
- :doc:`How to use <howto/operator/gcp/bigquery>`
- :doc:`How to use <howto/operator/google/cloud/bigquery>`
- :mod:`airflow.providers.google.cloud.hooks.bigquery`
- :mod:`airflow.providers.google.cloud.operators.bigquery`
- :mod:`airflow.providers.google.cloud.sensors.bigquery`
* - `BigQuery Data Transfer Service <https://cloud.google.com/bigquery/transfer/>`__
- :doc:`How to use <howto/operator/gcp/bigquery_dts>`
- :doc:`How to use <howto/operator/google/cloud/bigquery_dts>`
- :mod:`airflow.providers.google.cloud.hooks.bigquery_dts`
- :mod:`airflow.providers.google.cloud.operators.bigquery_dts`
- :mod:`airflow.providers.google.cloud.sensors.bigquery_dts`
* - `Bigtable <https://cloud.google.com/bigtable/>`__
- :doc:`How to use <howto/operator/gcp/bigtable>`
- :doc:`How to use <howto/operator/google/cloud/bigtable>`
- :mod:`airflow.providers.google.cloud.hooks.bigtable`
- :mod:`airflow.providers.google.cloud.operators.bigtable`
- :mod:`airflow.providers.google.cloud.sensors.bigtable`
* - `Cloud Build <https://cloud.google.com/cloud-build/>`__
- :doc:`How to use <howto/operator/gcp/cloud_build>`
- :doc:`How to use <howto/operator/google/cloud/cloud_build>`
- :mod:`airflow.providers.google.cloud.hooks.cloud_build`
- :mod:`airflow.providers.google.cloud.operators.cloud_build`
-
* - `Compute Engine <https://cloud.google.com/compute/>`__
- :doc:`How to use <howto/operator/gcp/compute>`
- :doc:`How to use <howto/operator/google/cloud/compute>`
- :mod:`airflow.providers.google.cloud.hooks.compute`
- :mod:`airflow.providers.google.cloud.operators.compute`
-
@ -688,13 +688,13 @@ These integrations allow you to perform various operations within the Google Clo
-
* - `DataFusion <https://cloud.google.com/data-fusion/>`__
- :doc:`How to use <howto/operator/gcp/datafusion>`
- :doc:`How to use <howto/operator/google/cloud/datafusion>`
- :mod:`airflow.providers.google.cloud.hooks.datafusion`
- :mod:`airflow.providers.google.cloud.operators.datafusion`
-
* - `Datacatalog <https://cloud.google.com/data-catalog>`__
- :doc:`How to use <howto/operator/gcp/datacatalog>`
- :doc:`How to use <howto/operator/google/cloud/datacatalog>`
- :mod:`airflow.providers.google.cloud.hooks.datacatalog`
- :mod:`airflow.providers.google.cloud.operators.datacatalog`
-
@ -706,7 +706,7 @@ These integrations allow you to perform various operations within the Google Clo
-
* - `Dataproc <https://cloud.google.com/dataproc/>`__
- :doc:`How to use <howto/operator/gcp/dataproc>`
- :doc:`How to use <howto/operator/google/cloud/dataproc>`
- :mod:`airflow.providers.google.cloud.hooks.dataproc`
- :mod:`airflow.providers.google.cloud.operators.dataproc`
-
@ -718,13 +718,13 @@ These integrations allow you to perform various operations within the Google Clo
-
* - `Cloud Functions <https://cloud.google.com/functions/>`__
- :doc:`How to use <howto/operator/gcp/functions>`
- :doc:`How to use <howto/operator/google/cloud/functions>`
- :mod:`airflow.providers.google.cloud.hooks.functions`
- :mod:`airflow.providers.google.cloud.operators.functions`
-
* - `Cloud Firestore <https://firebase.google.com/docs/firestore>`__
- :doc:`How to use <howto/operator/gcp/firestore>`
- :doc:`How to use <howto/operator/google/firebase/firestore>`
- :mod:`airflow.providers.google.firebase.hooks.firestore`
- :mod:`airflow.providers.google.firebase.operators.firestore`
-
@ -735,13 +735,13 @@ These integrations allow you to perform various operations within the Google Clo
-
-
* - `Cloud Life Sciences <https://cloud.google.com/life-sciences/>`__
- :doc:`How to use <howto/operator/gcp/life_sciences>`
- :doc:`How to use <howto/operator/google/cloud/life_sciences>`
- :mod:`airflow.providers.google.cloud.hooks.life_sciences`
- :mod:`airflow.providers.google.cloud.operators.life_sciences`
-
* - `Kubernetes Engine <https://cloud.google.com/kubernetes_engine/>`__
- :doc:`How to use <howto/operator/gcp/kubernetes_engine>`
- :doc:`How to use <howto/operator/google/cloud/kubernetes_engine>`
- :mod:`airflow.providers.google.cloud.hooks.kubernetes_engine`
- :mod:`airflow.providers.google.cloud.operators.kubernetes_engine`
-
@ -753,19 +753,19 @@ These integrations allow you to perform various operations within the Google Clo
-
* - `Cloud Memorystore <https://cloud.google.com/memorystore/>`__
- :doc:`How to use <howto/operator/gcp/cloud_memorystore>`
- :doc:`How to use <howto/operator/google/cloud/cloud_memorystore>`
- :mod:`airflow.providers.google.cloud.hooks.cloud_memorystore`
- :mod:`airflow.providers.google.cloud.operators.cloud_memorystore`
-
* - `Natural Language <https://cloud.google.com/natural-language/>`__
- :doc:`How to use <howto/operator/gcp/natural_language>`
- :doc:`How to use <howto/operator/google/cloud/natural_language>`
- :mod:`airflow.providers.google.cloud.hooks.natural_language`
- :mod:`airflow.providers.google.cloud.operators.natural_language`
-
* - `Cloud Pub/Sub <https://cloud.google.com/pubsub/>`__
- :doc:`How to use <howto/operator/gcp/pubsub>`
- :doc:`How to use <howto/operator/google/cloud/pubsub>`
- :mod:`airflow.providers.google.cloud.hooks.pubsub`
- :mod:`airflow.providers.google.cloud.operators.pubsub`
- :mod:`airflow.providers.google.cloud.sensors.pubsub`
@ -777,37 +777,37 @@ These integrations allow you to perform various operations within the Google Clo
-
* - `Cloud Spanner <https://cloud.google.com/spanner/>`__
- :doc:`How to use <howto/operator/gcp/spanner>`
- :doc:`How to use <howto/operator/google/cloud/spanner>`
- :mod:`airflow.providers.google.cloud.hooks.spanner`
- :mod:`airflow.providers.google.cloud.operators.spanner`
-
* - `Cloud Speech-to-Text <https://cloud.google.com/speech-to-text/>`__
- :doc:`How to use <howto/operator/gcp/speech_to_text>`
- :doc:`How to use <howto/operator/google/cloud/speech_to_text>`
- :mod:`airflow.providers.google.cloud.hooks.speech_to_text`
- :mod:`airflow.providers.google.cloud.operators.speech_to_text`
-
* - `Cloud SQL <https://cloud.google.com/sql/>`__
- :doc:`How to use <howto/operator/gcp/cloud_sql>`
- :doc:`How to use <howto/operator/google/cloud/cloud_sql>`
- :mod:`airflow.providers.google.cloud.hooks.cloud_sql`
- :mod:`airflow.providers.google.cloud.operators.cloud_sql`
-
* - `Cloud Stackdriver <https://cloud.google.com/stackdriver>`__
- :doc:`How to use <howto/operator/gcp/stackdriver>`
- :doc:`How to use <howto/operator/google/cloud/stackdriver>`
- :mod:`airflow.providers.google.cloud.hooks.stackdriver`
- :mod:`airflow.providers.google.cloud.operators.stackdriver`
-
* - `Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/gcs>`
- :doc:`How to use <howto/operator/google/cloud/gcs>`
- :mod:`airflow.providers.google.cloud.hooks.gcs`
- :mod:`airflow.providers.google.cloud.operators.gcs`
- :mod:`airflow.providers.google.cloud.sensors.gcs`
* - `Storage Transfer Service <https://cloud.google.com/storage/transfer/>`__
- :doc:`How to use <howto/operator/gcp/cloud_storage_transfer_service>`
- :doc:`How to use <howto/operator/google/cloud/cloud_storage_transfer_service>`
- :mod:`airflow.providers.google.cloud.hooks.cloud_storage_transfer_service`
- :mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service`
- :mod:`airflow.providers.google.cloud.sensors.cloud_storage_transfer_service`
@ -819,25 +819,25 @@ These integrations allow you to perform various operations within the Google Clo
-
* - `Cloud Text-to-Speech <https://cloud.google.com/text-to-speech/>`__
- :doc:`How to use <howto/operator/gcp/text_to_speech>`
- :doc:`How to use <howto/operator/google/cloud/text_to_speech>`
- :mod:`airflow.providers.google.cloud.hooks.text_to_speech`
- :mod:`airflow.providers.google.cloud.operators.text_to_speech`
-
* - `Cloud Translation <https://cloud.google.com/translate/>`__
- :doc:`How to use <howto/operator/gcp/translate>`
- :doc:`How to use <howto/operator/google/cloud/translate>`
- :mod:`airflow.providers.google.cloud.hooks.translate`
- :mod:`airflow.providers.google.cloud.operators.translate`
-
* - `Cloud Video Intelligence <https://cloud.google.com/video_intelligence/>`__
- :doc:`How to use <howto/operator/gcp/video_intelligence>`
- :doc:`How to use <howto/operator/google/cloud/video_intelligence>`
- :mod:`airflow.providers.google.cloud.hooks.video_intelligence`
- :mod:`airflow.providers.google.cloud.operators.video_intelligence`
-
* - `Cloud Vision <https://cloud.google.com/vision/>`__
- :doc:`How to use <howto/operator/gcp/vision>`
- :doc:`How to use <howto/operator/google/cloud/vision>`
- :mod:`airflow.providers.google.cloud.hooks.vision`
- :mod:`airflow.providers.google.cloud.operators.vision`
-
@ -865,7 +865,7 @@ These integrations allow you to copy data from/to Google Cloud Platform.
* - `Amazon Simple Storage Service (S3) <https://aws.amazon.com/s3/>`__
- `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/cloud_storage_transfer_service>`
- :doc:`How to use <howto/operator/google/cloud/cloud_storage_transfer_service>`
- :mod:`airflow.providers.google.cloud.transfers.s3_to_gcs`,
:mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service`
@ -881,13 +881,13 @@ These integrations allow you to copy data from/to Google Cloud Platform.
* - `Facebook Ads <http://business.facebook.com>`__
- `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/facebook_ads_to_gcs>`
- :doc:`How to use <howto/operator/google/transfer/facebook_ads_to_gcs>`
- :mod:`airflow.providers.google.cloud.transfers.facebook_ads_to_gcs`
* - `Google Ads <https://ads.google.com/>`__
- `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/ads>`
- :doc:`How to use <howto/operator/google/ads>`
- :mod:`airflow.providers.google.ads.transfers.ads_to_gcs`
* - `Google BigQuery <https://cloud.google.com/bigquery/>`__
@ -907,7 +907,7 @@ These integrations allow you to copy data from/to Google Cloud Platform.
* - `Cloud Firestore <https://firebase.google.com/docs/firestore>`__
- `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/firestore>`
- :doc:`How to use <howto/operator/google/firebase/firestore>`
- :mod:`airflow.providers.google.firebase.operators.firestore`
* - `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
@ -922,14 +922,14 @@ These integrations allow you to copy data from/to Google Cloud Platform.
* - `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/gcs_to_gcs>`,
:doc:`How to use <howto/operator/gcp/cloud_storage_transfer_service>`
- :doc:`How to use <howto/operator/google/transfer/gcs_to_gcs>`,
:doc:`How to use <howto/operator/google/cloud/cloud_storage_transfer_service>`
- :mod:`airflow.providers.google.cloud.transfers.gcs_to_gcs`,
:mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service`
* - `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- Local
- :doc:`How to use <howto/operator/gcp/gcs_to_local>`
- :doc:`How to use <howto/operator/google/transfer/gcs_to_local>`
- :mod:`airflow.providers.google.cloud.transfers.gcs_to_local`
* - `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
@ -939,12 +939,12 @@ These integrations allow you to copy data from/to Google Cloud Platform.
* - `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- SFTP
- :doc:`How to use <howto/operator/gcp/gcs_to_sftp>`
- :doc:`How to use <howto/operator/google/transfer/gcs_to_sftp>`
- :mod:`airflow.providers.google.cloud.transfers.gcs_to_sftp`
* - Local
- `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/local_to_gcs>`
- :doc:`How to use <howto/operator/google/transfer/local_to_gcs>`
- :mod:`airflow.providers.google.cloud.transfers.local_to_gcs`
* - `Microsoft SQL Server (MSSQL) <https://www.microsoft.com/pl-pl/sql-server/sql-server-downloads>`__
@ -964,12 +964,12 @@ These integrations allow you to copy data from/to Google Cloud Platform.
* - `Presto <https://prestodb.io/>`__
- `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/presto_to_gcs>`
- :doc:`How to use <howto/operator/google/transfer/presto_to_gcs>`
- :mod:`airflow.providers.google.cloud.transfers.presto_to_gcs`
* - SFTP
- `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/sftp_to_gcs>`
- :doc:`How to use <howto/operator/google/transfer/sftp_to_gcs>`
- :mod:`airflow.providers.google.cloud.transfers.sftp_to_gcs`
* - SQL
@ -979,12 +979,12 @@ These integrations allow you to copy data from/to Google Cloud Platform.
* - `Google Spreadsheet <https://www.google.com/intl/en/sheets/about/>`__
- `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- :doc:`How to use <howto/operator/gcp/sheets_to_gcs>`
- :doc:`How to use <howto/operator/google/transfer/sheets_to_gcs>`
- :mod:`airflow.providers.google.cloud.transfers.sheets_to_gcs`
* - `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- `Google Spreadsheet <https://www.google.com/intl/en/sheets/about/>`__
- :doc:`How to use <howto/operator/gcp/gcs_to_sheets>`
- :doc:`How to use <howto/operator/google/transfer/gcs_to_sheets>`
- :mod:`airflow.providers.google.suite.transfers.gcs_to_sheets`
.. _integration:GCP-Discovery:
@ -1006,7 +1006,7 @@ Other operators and hooks
- Operator
- Hook
* - :doc:`How to use <howto/operator/gcp/translate_speech>`
* - :doc:`How to use <howto/operator/google/cloud/translate_speech>`
- :mod:`airflow.providers.google.cloud.operators.translate_speech`
-
@ -1029,25 +1029,25 @@ Google Marketing Platform
- Sensor
* - `Analytics360 <https://analytics.google.com/>`__
- :doc:`How to use <howto/operator/gcp/analytics>`
- :doc:`How to use <howto/operator/google/marketing_platform/analytics>`
- :mod:`airflow.providers.google.marketing_platform.hooks.analytics`
- :mod:`airflow.providers.google.marketing_platform.operators.analytics`
-
* - `Google Campaign Manager <https://developers.google.com/doubleclick-advertisers>`__
- :doc:`How to use <howto/operator/gcp/campaign_manager>`
- :doc:`How to use <howto/operator/google/marketing_platform/campaign_manager>`
- :mod:`airflow.providers.google.marketing_platform.hooks.campaign_manager`
- :mod:`airflow.providers.google.marketing_platform.operators.campaign_manager`
- :mod:`airflow.providers.google.marketing_platform.sensors.campaign_manager`
* - `Google Display&Video 360 <https://marketingplatform.google.com/about/display-video-360/>`__
- :doc:`How to use <howto/operator/gcp/display_video>`
- :doc:`How to use <howto/operator/google/marketing_platform/display_video>`
- :mod:`airflow.providers.google.marketing_platform.hooks.display_video`
- :mod:`airflow.providers.google.marketing_platform.operators.display_video`
- :mod:`airflow.providers.google.marketing_platform.sensors.display_video`
* - `Google Search Ads 360 <https://marketingplatform.google.com/about/search-ads-360/>`__
- :doc:`How to use <howto/operator/gcp/search_ads>`
- :doc:`How to use <howto/operator/google/marketing_platform/search_ads>`
- :mod:`airflow.providers.google.marketing_platform.hooks.search_ads`
- :mod:`airflow.providers.google.marketing_platform.operators.search_ads`
- :mod:`airflow.providers.google.marketing_platform.sensors.search_ads`
@ -1064,7 +1064,7 @@ Other Google operators and hooks
- Operator
* - `Google Ads <https://ads.google.com/home/>`__
- :doc:`How to use <howto/operator/gcp/ads>`
- :doc:`How to use <howto/operator/google/ads>`
- :mod:`airflow.providers.google.ads.hooks.ads`
- :mod:`airflow.providers.google.ads.operators.ads`
@ -1074,12 +1074,12 @@ Other Google operators and hooks
-
* - `Cloud Firestore <https://firebase.google.com/docs/firestore>`__
- :doc:`How to use <howto/operator/gcp/firestore>`
- :doc:`How to use <howto/operator/google/firebase/firestore>`
- :mod:`airflow.providers.google.firebase.hooks.firestore`
- :mod:`airflow.providers.google.firebase.operators.firestore`
* - `Google Spreadsheet <https://www.google.com/intl/en/sheets/about/>`__
- :doc:`How to use <howto/operator/gcp/sheets>`
- :doc:`How to use <howto/operator/google/suite/sheets>`
- :mod:`airflow.providers.google.suite.hooks.sheets`
- :mod:`airflow.providers.google.suite.operators.sheets`
@ -1272,7 +1272,7 @@ These integrations allow you to perform various operations within various servic
* - `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
- `Google Drive <https://www.google.com/drive/>`__
- :doc:`How to use <howto/operator/gcp/gcs_to_gdrive>`
- :doc:`How to use <howto/operator/google/transfer/gcs_to_gdrive>`
- :mod:`airflow.providers.google.suite.transfers.gcs_to_gdrive`
* - `Vertica <https://www.vertica.com/>`__

67
docs/redirects.txt Normal file
Просмотреть файл

@ -0,0 +1,67 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# New URL for Google Guide index
howto/operator/gcp/index.rst howto/operator/google/index.rst
# New URLs for Google Guides
howto/operator/gcp/ads.rst howto/operator/google/ads.rst
howto/operator/gcp/analytics.rst howto/operator/google/marketing_platform/analytics.rst
howto/operator/gcp/automl.rst howto/operator/google/cloud/automl.rst
howto/operator/gcp/bigquery.rst howto/operator/google/cloud/bigquery.rst
howto/operator/gcp/bigquery_dts.rst howto/operator/google/cloud/bigquery_dts.rst
howto/operator/gcp/bigtable.rst howto/operator/google/cloud/bigtable.rst
howto/operator/gcp/campaign_manager.rst howto/operator/google/marketing_platform/campaign_manager.rst
howto/operator/gcp/cloud_build.rst howto/operator/google/cloud/cloud_build.rst
howto/operator/gcp/cloud_memorystore.rst howto/operator/google/cloud/cloud_memorystore.rst
howto/operator/gcp/cloud_sql.rst howto/operator/google/cloud/cloud_sql.rst
howto/operator/gcp/cloud_storage_transfer_service.rst howto/operator/google/cloud/cloud_storage_transfer_service.rst
howto/operator/gcp/compute.rst howto/operator/google/cloud/compute.rst
howto/operator/gcp/datacatalog.rst howto/operator/google/cloud/datacatalog.rst
howto/operator/gcp/datafusion.rst howto/operator/google/cloud/datafusion.rst
howto/operator/gcp/dataproc.rst howto/operator/google/cloud/dataproc.rst
howto/operator/gcp/display_video.rst howto/operator/google/marketing_platform/display_video.rst
howto/operator/gcp/facebook_ads_to_gcs.rst howto/operator/google/transfer/facebook_ads_to_gcs.rst
howto/operator/gcp/firestore.rst howto/operator/google/firebase/firestore.rst
howto/operator/gcp/functions.rst howto/operator/google/cloud/functions.rst
howto/operator/gcp/gcs.rst howto/operator/google/cloud/gcs.rst
howto/operator/gcp/gcs_to_gcs.rst howto/operator/google/transfer/gcs_to_gcs.rst
howto/operator/gcp/gcs_to_gdrive.rst howto/operator/google/transfer/gcs_to_gdrive.rst
howto/operator/gcp/gcs_to_local.rst howto/operator/google/transfer/gcs_to_local.rst
howto/operator/gcp/gcs_to_sftp.rst howto/operator/google/transfer/gcs_to_sftp.rst
howto/operator/gcp/gcs_to_sheets.rst howto/operator/google/transfer/gcs_to_sheets.rst
howto/operator/gcp/index.rst howto/operator/google/cloud/index.rst
howto/operator/gcp/kubernetes_engine.rst howto/operator/google/cloud/kubernetes_engine.rst
howto/operator/gcp/life_sciences.rst howto/operator/google/cloud/life_sciences.rst
howto/operator/gcp/local_to_gcs.rst howto/operator/google/transfer/local_to_gcs.rst
howto/operator/gcp/natural_language.rst howto/operator/google/cloud/natural_language.rst
howto/operator/gcp/presto_to_gcs.rst howto/operator/google/transfer/presto_to_gcs.rst
howto/operator/gcp/pubsub.rst howto/operator/google/cloud/pubsub.rst
howto/operator/gcp/search_ads.rst howto/operator/google/marketing_platform/search_ads.rst
howto/operator/gcp/sftp_to_gcs.rst howto/operator/google/transfer/sftp_to_gcs.rst
howto/operator/gcp/sheets.rst howto/operator/google/suite/sheets.rst
howto/operator/gcp/sheets_to_gcs.rst howto/operator/google/transfer/sheets_to_gcs.rst
howto/operator/gcp/spanner.rst howto/operator/google/cloud/spanner.rst
howto/operator/gcp/speech_to_text.rst howto/operator/google/cloud/speech_to_text.rst
howto/operator/gcp/stackdriver.rst howto/operator/google/cloud/stackdriver.rst
howto/operator/gcp/text_to_speech.rst howto/operator/google/cloud/text_to_speech.rst
howto/operator/gcp/translate.rst howto/operator/google/cloud/translate.rst
howto/operator/gcp/translate_speech.rst howto/operator/google/cloud/translate_speech.rst
howto/operator/gcp/video_intelligence.rst howto/operator/google/cloud/video_intelligence.rst
howto/operator/gcp/vision.rst howto/operator/google/cloud/vision.rst
# Departments without a separate index yet
howto/operator/google/suite/index.rst howto/operator/google/index.rst
howto/operator/google/firebase/index.rst howto/operator/google/index.rst