From 9b1c7532931eb0fe0bbc56925c2678032b5518bb Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 1 Sep 2021 21:48:07 +0000 Subject: [PATCH 1/2] docs: Align session length with public documentation feat: Align ReadRows timeout with other versions of the API. Committer: @emkornfield PiperOrigin-RevId: 394308710 Source-Link: https://github.com/googleapis/googleapis/commit/1549a69af196ca062870211083f2856b0870d45c Source-Link: https://github.com/googleapis/googleapis-gen/commit/40b5e7dd36d0b4a67f4cb41ad2e2f670700404ca --- owl-bot-staging/v1/.coveragerc | 17 + owl-bot-staging/v1/MANIFEST.in | 2 + owl-bot-staging/v1/README.rst | 49 + .../bigquery_storage_v1/big_query_read.rst | 6 + .../v1/docs/bigquery_storage_v1/services.rst | 6 + .../v1/docs/bigquery_storage_v1/types.rst | 7 + owl-bot-staging/v1/docs/conf.py | 376 +++ owl-bot-staging/v1/docs/index.rst | 7 + .../google/cloud/bigquery_storage/__init__.py | 53 + .../v1/google/cloud/bigquery_storage/py.typed | 2 + .../cloud/bigquery_storage_v1/__init__.py | 54 + .../bigquery_storage_v1/gapic_metadata.json | 53 + .../google/cloud/bigquery_storage_v1/py.typed | 2 + .../bigquery_storage_v1/services/__init__.py | 15 + .../services/big_query_read/__init__.py | 22 + .../services/big_query_read/async_client.py | 474 ++++ .../services/big_query_read/client.py | 664 +++++ .../big_query_read/transports/__init__.py | 33 + .../big_query_read/transports/base.py | 219 ++ .../big_query_read/transports/grpc.py | 347 +++ .../big_query_read/transports/grpc_asyncio.py | 351 +++ .../bigquery_storage_v1/types/__init__.py | 56 + .../cloud/bigquery_storage_v1/types/arrow.py | 87 + .../cloud/bigquery_storage_v1/types/avro.py | 61 + .../bigquery_storage_v1/types/storage.py | 275 ++ .../cloud/bigquery_storage_v1/types/stream.py | 194 ++ owl-bot-staging/v1/mypy.ini | 3 + owl-bot-staging/v1/noxfile.py | 132 + .../fixup_bigquery_storage_v1_keywords.py | 178 ++ owl-bot-staging/v1/setup.py | 54 + owl-bot-staging/v1/tests/__init__.py | 16 + owl-bot-staging/v1/tests/unit/__init__.py | 16 + .../v1/tests/unit/gapic/__init__.py | 16 + .../gapic/bigquery_storage_v1/__init__.py | 16 + .../test_big_query_read.py | 1629 ++++++++++++ owl-bot-staging/v1beta2/.coveragerc | 17 + owl-bot-staging/v1beta2/MANIFEST.in | 2 + owl-bot-staging/v1beta2/README.rst | 49 + .../big_query_read.rst | 6 + .../big_query_write.rst | 6 + .../bigquery_storage_v1beta2/services.rst | 7 + .../docs/bigquery_storage_v1beta2/types.rst | 7 + owl-bot-staging/v1beta2/docs/conf.py | 376 +++ owl-bot-staging/v1beta2/docs/index.rst | 7 + .../google/cloud/bigquery_storage/__init__.py | 89 + .../google/cloud/bigquery_storage/py.typed | 2 + .../bigquery_storage_v1beta2/__init__.py | 90 + .../gapic_metadata.json | 127 + .../cloud/bigquery_storage_v1beta2/py.typed | 2 + .../services/__init__.py | 15 + .../services/big_query_read/__init__.py | 22 + .../services/big_query_read/async_client.py | 476 ++++ .../services/big_query_read/client.py | 666 +++++ .../big_query_read/transports/__init__.py | 33 + .../big_query_read/transports/base.py | 219 ++ .../big_query_read/transports/grpc.py | 349 +++ .../big_query_read/transports/grpc_asyncio.py | 353 +++ .../services/big_query_write/__init__.py | 22 + .../services/big_query_write/async_client.py | 678 +++++ .../services/big_query_write/client.py | 831 ++++++ .../big_query_write/transports/__init__.py | 33 + .../big_query_write/transports/base.py | 284 ++ .../big_query_write/transports/grpc.py | 420 +++ .../transports/grpc_asyncio.py | 424 +++ .../types/__init__.py | 92 + .../bigquery_storage_v1beta2/types/arrow.py | 80 + .../bigquery_storage_v1beta2/types/avro.py | 55 + .../types/protobuf.py | 68 + .../bigquery_storage_v1beta2/types/storage.py | 618 +++++ .../bigquery_storage_v1beta2/types/stream.py | 253 ++ .../bigquery_storage_v1beta2/types/table.py | 113 + owl-bot-staging/v1beta2/mypy.ini | 3 + owl-bot-staging/v1beta2/noxfile.py | 132 + ...fixup_bigquery_storage_v1beta2_keywords.py | 184 ++ owl-bot-staging/v1beta2/setup.py | 54 + owl-bot-staging/v1beta2/tests/__init__.py | 16 + .../v1beta2/tests/unit/__init__.py | 16 + .../v1beta2/tests/unit/gapic/__init__.py | 16 + .../bigquery_storage_v1beta2/__init__.py | 16 + .../test_big_query_read.py | 1629 ++++++++++++ .../test_big_query_write.py | 2294 +++++++++++++++++ 81 files changed, 16743 insertions(+) create mode 100644 owl-bot-staging/v1/.coveragerc create mode 100644 owl-bot-staging/v1/MANIFEST.in create mode 100644 owl-bot-staging/v1/README.rst create mode 100644 owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst create mode 100644 owl-bot-staging/v1/docs/bigquery_storage_v1/services.rst create mode 100644 owl-bot-staging/v1/docs/bigquery_storage_v1/types.rst create mode 100644 owl-bot-staging/v1/docs/conf.py create mode 100644 owl-bot-staging/v1/docs/index.rst create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py create mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py create mode 100644 owl-bot-staging/v1/mypy.ini create mode 100644 owl-bot-staging/v1/noxfile.py create mode 100644 owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py create mode 100644 owl-bot-staging/v1/setup.py create mode 100644 owl-bot-staging/v1/tests/__init__.py create mode 100644 owl-bot-staging/v1/tests/unit/__init__.py create mode 100644 owl-bot-staging/v1/tests/unit/gapic/__init__.py create mode 100644 owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py create mode 100644 owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py create mode 100644 owl-bot-staging/v1beta2/.coveragerc create mode 100644 owl-bot-staging/v1beta2/MANIFEST.in create mode 100644 owl-bot-staging/v1beta2/README.rst create mode 100644 owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst create mode 100644 owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst create mode 100644 owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services.rst create mode 100644 owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types.rst create mode 100644 owl-bot-staging/v1beta2/docs/conf.py create mode 100644 owl-bot-staging/v1beta2/docs/index.rst create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py create mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py create mode 100644 owl-bot-staging/v1beta2/mypy.ini create mode 100644 owl-bot-staging/v1beta2/noxfile.py create mode 100644 owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py create mode 100644 owl-bot-staging/v1beta2/setup.py create mode 100644 owl-bot-staging/v1beta2/tests/__init__.py create mode 100644 owl-bot-staging/v1beta2/tests/unit/__init__.py create mode 100644 owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py create mode 100644 owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py create mode 100644 owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py create mode 100644 owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py diff --git a/owl-bot-staging/v1/.coveragerc b/owl-bot-staging/v1/.coveragerc new file mode 100644 index 00000000..19fadc8b --- /dev/null +++ b/owl-bot-staging/v1/.coveragerc @@ -0,0 +1,17 @@ +[run] +branch = True + +[report] +show_missing = True +omit = + google/cloud/bigquery_storage/__init__.py +exclude_lines = + # Re-enable the standard pragma + pragma: NO COVER + # Ignore debug-only repr + def __repr__ + # Ignore pkg_resources exceptions. + # This is added at the module level as a safeguard for if someone + # generates the code and tries to run it without pip installing. This + # makes it virtually impossible to test properly. + except pkg_resources.DistributionNotFound diff --git a/owl-bot-staging/v1/MANIFEST.in b/owl-bot-staging/v1/MANIFEST.in new file mode 100644 index 00000000..88a0105e --- /dev/null +++ b/owl-bot-staging/v1/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include google/cloud/bigquery_storage *.py +recursive-include google/cloud/bigquery_storage_v1 *.py diff --git a/owl-bot-staging/v1/README.rst b/owl-bot-staging/v1/README.rst new file mode 100644 index 00000000..13a45c14 --- /dev/null +++ b/owl-bot-staging/v1/README.rst @@ -0,0 +1,49 @@ +Python Client for Google Cloud Bigquery Storage API +================================================= + +Quick Start +----------- + +In order to use this library, you first need to go through the following steps: + +1. `Select or create a Cloud Platform project.`_ +2. `Enable billing for your project.`_ +3. Enable the Google Cloud Bigquery Storage API. +4. `Setup Authentication.`_ + +.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project +.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project +.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html + +Installation +~~~~~~~~~~~~ + +Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to +create isolated Python environments. The basic problem it addresses is one of +dependencies and versions, and indirectly permissions. + +With `virtualenv`_, it's possible to install this library without needing system +install permissions, and without clashing with the installed system +dependencies. + +.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ + + +Mac/Linux +^^^^^^^^^ + +.. code-block:: console + + python3 -m venv + source /bin/activate + /bin/pip install /path/to/library + + +Windows +^^^^^^^ + +.. code-block:: console + + python3 -m venv + \Scripts\activate + \Scripts\pip.exe install \path\to\library diff --git a/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst b/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst new file mode 100644 index 00000000..74f0a5fa --- /dev/null +++ b/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst @@ -0,0 +1,6 @@ +BigQueryRead +------------------------------ + +.. automodule:: google.cloud.bigquery_storage_v1.services.big_query_read + :members: + :inherited-members: diff --git a/owl-bot-staging/v1/docs/bigquery_storage_v1/services.rst b/owl-bot-staging/v1/docs/bigquery_storage_v1/services.rst new file mode 100644 index 00000000..3f6cff3d --- /dev/null +++ b/owl-bot-staging/v1/docs/bigquery_storage_v1/services.rst @@ -0,0 +1,6 @@ +Services for Google Cloud Bigquery Storage v1 API +================================================= +.. toctree:: + :maxdepth: 2 + + big_query_read diff --git a/owl-bot-staging/v1/docs/bigquery_storage_v1/types.rst b/owl-bot-staging/v1/docs/bigquery_storage_v1/types.rst new file mode 100644 index 00000000..28b5db72 --- /dev/null +++ b/owl-bot-staging/v1/docs/bigquery_storage_v1/types.rst @@ -0,0 +1,7 @@ +Types for Google Cloud Bigquery Storage v1 API +============================================== + +.. automodule:: google.cloud.bigquery_storage_v1.types + :members: + :undoc-members: + :show-inheritance: diff --git a/owl-bot-staging/v1/docs/conf.py b/owl-bot-staging/v1/docs/conf.py new file mode 100644 index 00000000..e198bf7a --- /dev/null +++ b/owl-bot-staging/v1/docs/conf.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# google-cloud-bigquery-storage documentation build configuration file +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath("..")) + +__version__ = "0.1.0" + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +needs_sphinx = "1.6.3" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", +] + +# autodoc/autosummary flags +autoclass_content = "both" +autodoc_default_flags = ["members"] +autosummary_generate = True + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# Allow markdown includes (so releases.md can include CHANGLEOG.md) +# http://www.sphinx-doc.org/en/master/markdown.html +source_parsers = {".md": "recommonmark.parser.CommonMarkParser"} + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +source_suffix = [".rst", ".md"] + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = u"google-cloud-bigquery-storage" +copyright = u"2020, Google, LLC" +author = u"Google APIs" # TODO: autogenerate this bit + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The full version, including alpha/beta/rc tags. +release = __version__ +# The short X.Y version. +version = ".".join(release.split(".")[0:2]) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ["_build"] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "alabaster" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "description": "Google Cloud Client Libraries for Python", + "github_user": "googleapis", + "github_repo": "google-cloud-python", + "github_banner": True, + "font_family": "'Roboto', Georgia, sans", + "head_font_family": "'Roboto', Georgia, serif", + "code_font_family": "'Roboto Mono', 'Consolas', monospace", +} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = "google-cloud-bigquery-storage-doc" + +# -- Options for warnings ------------------------------------------------------ + + +suppress_warnings = [ + # Temporarily suppress this to avoid "more than one target found for + # cross-reference" warning, which are intractable for us to avoid while in + # a mono-repo. + # See https://github.com/sphinx-doc/sphinx/blob + # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843 + "ref.python" +] + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', + # Latex figure (float) alignment + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + master_doc, + "google-cloud-bigquery-storage.tex", + u"google-cloud-bigquery-storage Documentation", + author, + "manual", + ) +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ( + master_doc, + "google-cloud-bigquery-storage", + u"Google Cloud Bigquery Storage Documentation", + [author], + 1, + ) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "google-cloud-bigquery-storage", + u"google-cloud-bigquery-storage Documentation", + author, + "google-cloud-bigquery-storage", + "GAPIC library for Google Cloud Bigquery Storage API", + "APIs", + ) +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + "python": ("http://python.readthedocs.org/en/latest/", None), + "gax": ("https://gax-python.readthedocs.org/en/latest/", None), + "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), + "google-gax": ("https://gax-python.readthedocs.io/en/latest/", None), + "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None), + "grpc": ("https://grpc.io/grpc/python/", None), + "requests": ("http://requests.kennethreitz.org/en/stable/", None), + "proto": ("https://proto-plus-python.readthedocs.io/en/stable", None), + "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), +} + + +# Napoleon settings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = True +napoleon_use_admonition_for_examples = False +napoleon_use_admonition_for_notes = False +napoleon_use_admonition_for_references = False +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True diff --git a/owl-bot-staging/v1/docs/index.rst b/owl-bot-staging/v1/docs/index.rst new file mode 100644 index 00000000..a5e3a707 --- /dev/null +++ b/owl-bot-staging/v1/docs/index.rst @@ -0,0 +1,7 @@ +API Reference +------------- +.. toctree:: + :maxdepth: 2 + + bigquery_storage_v1/services + bigquery_storage_v1/types diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py new file mode 100644 index 00000000..281b811f --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from google.cloud.bigquery_storage_v1.services.big_query_read.client import BigQueryReadClient +from google.cloud.bigquery_storage_v1.services.big_query_read.async_client import BigQueryReadAsyncClient + +from google.cloud.bigquery_storage_v1.types.arrow import ArrowRecordBatch +from google.cloud.bigquery_storage_v1.types.arrow import ArrowSchema +from google.cloud.bigquery_storage_v1.types.arrow import ArrowSerializationOptions +from google.cloud.bigquery_storage_v1.types.avro import AvroRows +from google.cloud.bigquery_storage_v1.types.avro import AvroSchema +from google.cloud.bigquery_storage_v1.types.storage import CreateReadSessionRequest +from google.cloud.bigquery_storage_v1.types.storage import ReadRowsRequest +from google.cloud.bigquery_storage_v1.types.storage import ReadRowsResponse +from google.cloud.bigquery_storage_v1.types.storage import SplitReadStreamRequest +from google.cloud.bigquery_storage_v1.types.storage import SplitReadStreamResponse +from google.cloud.bigquery_storage_v1.types.storage import StreamStats +from google.cloud.bigquery_storage_v1.types.storage import ThrottleState +from google.cloud.bigquery_storage_v1.types.stream import ReadSession +from google.cloud.bigquery_storage_v1.types.stream import ReadStream +from google.cloud.bigquery_storage_v1.types.stream import DataFormat + +__all__ = ('BigQueryReadClient', + 'BigQueryReadAsyncClient', + 'ArrowRecordBatch', + 'ArrowSchema', + 'ArrowSerializationOptions', + 'AvroRows', + 'AvroSchema', + 'CreateReadSessionRequest', + 'ReadRowsRequest', + 'ReadRowsResponse', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'StreamStats', + 'ThrottleState', + 'ReadSession', + 'ReadStream', + 'DataFormat', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed b/owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed new file mode 100644 index 00000000..e71b4749 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py new file mode 100644 index 00000000..423f54ae --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .services.big_query_read import BigQueryReadClient +from .services.big_query_read import BigQueryReadAsyncClient + +from .types.arrow import ArrowRecordBatch +from .types.arrow import ArrowSchema +from .types.arrow import ArrowSerializationOptions +from .types.avro import AvroRows +from .types.avro import AvroSchema +from .types.storage import CreateReadSessionRequest +from .types.storage import ReadRowsRequest +from .types.storage import ReadRowsResponse +from .types.storage import SplitReadStreamRequest +from .types.storage import SplitReadStreamResponse +from .types.storage import StreamStats +from .types.storage import ThrottleState +from .types.stream import ReadSession +from .types.stream import ReadStream +from .types.stream import DataFormat + +__all__ = ( + 'BigQueryReadAsyncClient', +'ArrowRecordBatch', +'ArrowSchema', +'ArrowSerializationOptions', +'AvroRows', +'AvroSchema', +'BigQueryReadClient', +'CreateReadSessionRequest', +'DataFormat', +'ReadRowsRequest', +'ReadRowsResponse', +'ReadSession', +'ReadStream', +'SplitReadStreamRequest', +'SplitReadStreamResponse', +'StreamStats', +'ThrottleState', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json new file mode 100644 index 00000000..3311e568 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json @@ -0,0 +1,53 @@ + { + "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", + "language": "python", + "libraryPackage": "google.cloud.bigquery_storage_v1", + "protoPackage": "google.cloud.bigquery.storage.v1", + "schema": "1.0", + "services": { + "BigQueryRead": { + "clients": { + "grpc": { + "libraryClient": "BigQueryReadClient", + "rpcs": { + "CreateReadSession": { + "methods": [ + "create_read_session" + ] + }, + "ReadRows": { + "methods": [ + "read_rows" + ] + }, + "SplitReadStream": { + "methods": [ + "split_read_stream" + ] + } + } + }, + "grpc-async": { + "libraryClient": "BigQueryReadAsyncClient", + "rpcs": { + "CreateReadSession": { + "methods": [ + "create_read_session" + ] + }, + "ReadRows": { + "methods": [ + "read_rows" + ] + }, + "SplitReadStream": { + "methods": [ + "split_read_stream" + ] + } + } + } + } + } + } +} diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed new file mode 100644 index 00000000..e71b4749 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py new file mode 100644 index 00000000..4de65971 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py new file mode 100644 index 00000000..13be8ad1 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import BigQueryReadClient +from .async_client import BigQueryReadAsyncClient + +__all__ = ( + 'BigQueryReadClient', + 'BigQueryReadAsyncClient', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py new file mode 100644 index 00000000..f9a960ea --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py @@ -0,0 +1,474 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import functools +import re +from typing import Dict, AsyncIterable, Awaitable, Sequence, Tuple, Type, Union +import pkg_resources + +import google.api_core.client_options as ClientOptions # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.protobuf import timestamp_pb2 # type: ignore +from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport +from .client import BigQueryReadClient + + +class BigQueryReadAsyncClient: + """BigQuery Read API. + The Read API can be used to read data from BigQuery. + """ + + _client: BigQueryReadClient + + DEFAULT_ENDPOINT = BigQueryReadClient.DEFAULT_ENDPOINT + DEFAULT_MTLS_ENDPOINT = BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + + read_session_path = staticmethod(BigQueryReadClient.read_session_path) + parse_read_session_path = staticmethod(BigQueryReadClient.parse_read_session_path) + read_stream_path = staticmethod(BigQueryReadClient.read_stream_path) + parse_read_stream_path = staticmethod(BigQueryReadClient.parse_read_stream_path) + table_path = staticmethod(BigQueryReadClient.table_path) + parse_table_path = staticmethod(BigQueryReadClient.parse_table_path) + common_billing_account_path = staticmethod(BigQueryReadClient.common_billing_account_path) + parse_common_billing_account_path = staticmethod(BigQueryReadClient.parse_common_billing_account_path) + common_folder_path = staticmethod(BigQueryReadClient.common_folder_path) + parse_common_folder_path = staticmethod(BigQueryReadClient.parse_common_folder_path) + common_organization_path = staticmethod(BigQueryReadClient.common_organization_path) + parse_common_organization_path = staticmethod(BigQueryReadClient.parse_common_organization_path) + common_project_path = staticmethod(BigQueryReadClient.common_project_path) + parse_common_project_path = staticmethod(BigQueryReadClient.parse_common_project_path) + common_location_path = staticmethod(BigQueryReadClient.common_location_path) + parse_common_location_path = staticmethod(BigQueryReadClient.parse_common_location_path) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_info.__func__(BigQueryReadAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_file.__func__(BigQueryReadAsyncClient, filename, *args, **kwargs) # type: ignore + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryReadTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryReadTransport: The transport used by the client instance. + """ + return self._client.transport + + get_transport_class = functools.partial(type(BigQueryReadClient).get_transport_class, type(BigQueryReadClient)) + + def __init__(self, *, + credentials: ga_credentials.Credentials = None, + transport: Union[str, BigQueryReadTransport] = "grpc_asyncio", + client_options: ClientOptions = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query read client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Union[str, ~.BigQueryReadTransport]): The + transport to use. If set to None, a transport is chosen + automatically. + client_options (ClientOptions): Custom options for the client. It + won't take effect if a ``transport`` instance is provided. + (1) The ``api_endpoint`` property can be used to override the + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT + environment variable can also be used to override the endpoint: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client = BigQueryReadClient( + credentials=credentials, + transport=transport, + client_options=client_options, + client_info=client_info, + + ) + + async def create_read_session(self, + request: storage.CreateReadSessionRequest = None, + *, + parent: str = None, + read_session: stream.ReadSession = None, + max_stream_count: int = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.ReadSession: + r"""Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a push- + down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 24 hours after they + are created and do not require manual clean-up by the + caller. + + Args: + request (:class:`google.cloud.bigquery_storage_v1.types.CreateReadSessionRequest`): + The request object. Request message for + `CreateReadSession`. + parent (:class:`str`): + Required. The request project that owns the session, in + the form of ``projects/{project_id}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + read_session (:class:`google.cloud.bigquery_storage_v1.types.ReadSession`): + Required. Session to be created. + This corresponds to the ``read_session`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + max_stream_count (:class:`int`): + Max initial number of streams. If + unset or zero, the server will provide a + value of streams so as to produce + reasonable throughput. Must be non- + negative. The number of streams may be + lower than the requested number, + depending on the amount parallelism that + is reasonable for the table. Error will + be returned if the max count is greater + than the current system max limit of + 1,000. + + Streams must be read starting from + offset 0. + + This corresponds to the ``max_stream_count`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.ReadSession: + Information about the ReadSession. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, read_session, max_stream_count]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + request = storage.CreateReadSessionRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if read_session is not None: + request.read_session = read_session + if max_stream_count is not None: + request.max_stream_count = max_stream_count + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.create_read_session, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_session.table", request.read_session.table), + )), + ) + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def read_rows(self, + request: storage.ReadRowsRequest = None, + *, + read_stream: str = None, + offset: int = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Awaitable[AsyncIterable[storage.ReadRowsResponse]]: + r"""Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Args: + request (:class:`google.cloud.bigquery_storage_v1.types.ReadRowsRequest`): + The request object. Request message for `ReadRows`. + read_stream (:class:`str`): + Required. Stream to read rows from. + This corresponds to the ``read_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + offset (:class:`int`): + The offset requested must be less + than the last row read from Read. + Requesting a larger offset is undefined. + If not specified, start reading from + offset zero. + + This corresponds to the ``offset`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + AsyncIterable[google.cloud.bigquery_storage_v1.types.ReadRowsResponse]: + Response from calling ReadRows may include row data, progress and + throttling information. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([read_stream, offset]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + request = storage.ReadRowsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if read_stream is not None: + request.read_stream = read_stream + if offset is not None: + request.offset = offset + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.read_rows, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_stream", request.read_stream), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def split_read_stream(self, + request: storage.SplitReadStreamRequest = None, + *, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.SplitReadStreamResponse: + r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Args: + request (:class:`google.cloud.bigquery_storage_v1.types.SplitReadStreamRequest`): + The request object. Request message for + `SplitReadStream`. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.SplitReadStreamResponse: + Response message for SplitReadStream. + """ + # Create or coerce a protobuf request object. + request = storage.SplitReadStreamRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.split_read_stream, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + + + + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google-cloud-bigquery-storage", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + + +__all__ = ( + "BigQueryReadAsyncClient", +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py new file mode 100644 index 00000000..5c421b48 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py @@ -0,0 +1,664 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from distutils import util +import os +import re +from typing import Callable, Dict, Optional, Iterable, Sequence, Tuple, Type, Union +import pkg_resources + +from google.api_core import client_options as client_options_lib # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.protobuf import timestamp_pb2 # type: ignore +from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .transports.grpc import BigQueryReadGrpcTransport +from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport + + +class BigQueryReadClientMeta(type): + """Metaclass for the BigQueryRead client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] + _transport_registry["grpc"] = BigQueryReadGrpcTransport + _transport_registry["grpc_asyncio"] = BigQueryReadGrpcAsyncIOTransport + + def get_transport_class(cls, + label: str = None, + ) -> Type[BigQueryReadTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class BigQueryReadClient(metaclass=BigQueryReadClientMeta): + """BigQuery Read API. + The Read API can be used to read data from BigQuery. + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file( + filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryReadTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryReadTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def read_session_path(project: str,location: str,session: str,) -> str: + """Returns a fully-qualified read_session string.""" + return "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) + + @staticmethod + def parse_read_session_path(path: str) -> Dict[str,str]: + """Parses a read_session path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def read_stream_path(project: str,location: str,session: str,stream: str,) -> str: + """Returns a fully-qualified read_stream string.""" + return "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) + + @staticmethod + def parse_read_stream_path(path: str) -> Dict[str,str]: + """Parses a read_stream path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)/streams/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def table_path(project: str,dataset: str,table: str,) -> str: + """Returns a fully-qualified table string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + + @staticmethod + def parse_table_path(path: str) -> Dict[str,str]: + """Parses a table path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_billing_account_path(billing_account: str, ) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str,str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str, ) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder, ) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str,str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str, ) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization, ) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str,str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str, ) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project, ) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str,str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str, ) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format(project=project, location=location, ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str,str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Union[str, BigQueryReadTransport, None] = None, + client_options: Optional[client_options_lib.ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query read client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Union[str, BigQueryReadTransport]): The + transport to use. If set to None, a transport is chosen + automatically. + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. It won't take effect if a ``transport`` instance is provided. + (1) The ``api_endpoint`` property can be used to override the + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT + environment variable can also be used to override the endpoint: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + if isinstance(client_options, dict): + client_options = client_options_lib.from_dict(client_options) + if client_options is None: + client_options = client_options_lib.ClientOptions() + + # Create SSL credentials for mutual TLS if needed. + use_client_cert = bool(util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"))) + + client_cert_source_func = None + is_mtls = False + if use_client_cert: + if client_options.client_cert_source: + is_mtls = True + client_cert_source_func = client_options.client_cert_source + else: + is_mtls = mtls.has_default_client_cert_source() + if is_mtls: + client_cert_source_func = mtls.default_client_cert_source() + else: + client_cert_source_func = None + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + else: + use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_mtls_env == "never": + api_endpoint = self.DEFAULT_ENDPOINT + elif use_mtls_env == "always": + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + elif use_mtls_env == "auto": + if is_mtls: + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = self.DEFAULT_ENDPOINT + else: + raise MutualTLSChannelError( + "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " + "values: never, auto, always" + ) + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + if isinstance(transport, BigQueryReadTransport): + # transport is a BigQueryReadTransport instance. + if credentials or client_options.credentials_file: + raise ValueError("When providing a transport instance, " + "provide its credentials directly.") + if client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = transport + else: + Transport = type(self).get_transport_class(transport) + self._transport = Transport( + credentials=credentials, + credentials_file=client_options.credentials_file, + host=api_endpoint, + scopes=client_options.scopes, + client_cert_source_for_mtls=client_cert_source_func, + quota_project_id=client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=( + Transport == type(self).get_transport_class("grpc") + or Transport == type(self).get_transport_class("grpc_asyncio") + ), + ) + + def create_read_session(self, + request: storage.CreateReadSessionRequest = None, + *, + parent: str = None, + read_session: stream.ReadSession = None, + max_stream_count: int = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.ReadSession: + r"""Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a push- + down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 24 hours after they + are created and do not require manual clean-up by the + caller. + + Args: + request (google.cloud.bigquery_storage_v1.types.CreateReadSessionRequest): + The request object. Request message for + `CreateReadSession`. + parent (str): + Required. The request project that owns the session, in + the form of ``projects/{project_id}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + read_session (google.cloud.bigquery_storage_v1.types.ReadSession): + Required. Session to be created. + This corresponds to the ``read_session`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + max_stream_count (int): + Max initial number of streams. If + unset or zero, the server will provide a + value of streams so as to produce + reasonable throughput. Must be non- + negative. The number of streams may be + lower than the requested number, + depending on the amount parallelism that + is reasonable for the table. Error will + be returned if the max count is greater + than the current system max limit of + 1,000. + + Streams must be read starting from + offset 0. + + This corresponds to the ``max_stream_count`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.ReadSession: + Information about the ReadSession. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, read_session, max_stream_count]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # Minor optimization to avoid making a copy if the user passes + # in a storage.CreateReadSessionRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.CreateReadSessionRequest): + request = storage.CreateReadSessionRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if read_session is not None: + request.read_session = read_session + if max_stream_count is not None: + request.max_stream_count = max_stream_count + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.create_read_session] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_session.table", request.read_session.table), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def read_rows(self, + request: storage.ReadRowsRequest = None, + *, + read_stream: str = None, + offset: int = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Iterable[storage.ReadRowsResponse]: + r"""Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Args: + request (google.cloud.bigquery_storage_v1.types.ReadRowsRequest): + The request object. Request message for `ReadRows`. + read_stream (str): + Required. Stream to read rows from. + This corresponds to the ``read_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + offset (int): + The offset requested must be less + than the last row read from Read. + Requesting a larger offset is undefined. + If not specified, start reading from + offset zero. + + This corresponds to the ``offset`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + Iterable[google.cloud.bigquery_storage_v1.types.ReadRowsResponse]: + Response from calling ReadRows may include row data, progress and + throttling information. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([read_stream, offset]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # Minor optimization to avoid making a copy if the user passes + # in a storage.ReadRowsRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.ReadRowsRequest): + request = storage.ReadRowsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if read_stream is not None: + request.read_stream = read_stream + if offset is not None: + request.offset = offset + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.read_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_stream", request.read_stream), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def split_read_stream(self, + request: storage.SplitReadStreamRequest = None, + *, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.SplitReadStreamResponse: + r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Args: + request (google.cloud.bigquery_storage_v1.types.SplitReadStreamRequest): + The request object. Request message for + `SplitReadStream`. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.SplitReadStreamResponse: + Response message for SplitReadStream. + """ + # Create or coerce a protobuf request object. + # Minor optimization to avoid making a copy if the user passes + # in a storage.SplitReadStreamRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.SplitReadStreamRequest): + request = storage.SplitReadStreamRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.split_read_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + + + + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google-cloud-bigquery-storage", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + + +__all__ = ( + "BigQueryReadClient", +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py new file mode 100644 index 00000000..f51ba420 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import BigQueryReadTransport +from .grpc import BigQueryReadGrpcTransport +from .grpc_asyncio import BigQueryReadGrpcAsyncIOTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] +_transport_registry['grpc'] = BigQueryReadGrpcTransport +_transport_registry['grpc_asyncio'] = BigQueryReadGrpcAsyncIOTransport + +__all__ = ( + 'BigQueryReadTransport', + 'BigQueryReadGrpcTransport', + 'BigQueryReadGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py new file mode 100644 index 00000000..245f49f0 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py @@ -0,0 +1,219 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union +import packaging.version +import pkg_resources + +import google.auth # type: ignore +import google.api_core # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + 'google-cloud-bigquery-storage', + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + +try: + # google.auth.__version__ was added in 1.26.0 + _GOOGLE_AUTH_VERSION = google.auth.__version__ +except AttributeError: + try: # try pkg_resources if it is available + _GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version + except pkg_resources.DistributionNotFound: # pragma: NO COVER + _GOOGLE_AUTH_VERSION = None + + +class BigQueryReadTransport(abc.ABC): + """Abstract transport class for BigQueryRead.""" + + AUTH_SCOPES = ( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', + ) + + DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' + def __init__( + self, *, + host: str = DEFAULT_HOST, + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ':' not in host: + host += ':443' + self._host = host + + scopes_kwargs = self._get_scopes_kwargs(self._host, scopes) + + # Save the scopes. + self._scopes = scopes + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, + **scopes_kwargs, + quota_project_id=quota_project_id + ) + + elif credentials is None: + credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) + + # If the credentials is service account credentials, then always try to use self signed JWT. + if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + # TODO(busunkim): This method is in the base transport + # to avoid duplicating code across the transport classes. These functions + # should be deleted once the minimum required versions of google-auth is increased. + + # TODO: Remove this function once google-auth >= 1.25.0 is required + @classmethod + def _get_scopes_kwargs(cls, host: str, scopes: Optional[Sequence[str]]) -> Dict[str, Optional[Sequence[str]]]: + """Returns scopes kwargs to pass to google-auth methods depending on the google-auth version""" + + scopes_kwargs = {} + + if _GOOGLE_AUTH_VERSION and ( + packaging.version.parse(_GOOGLE_AUTH_VERSION) + >= packaging.version.parse("1.25.0") + ): + scopes_kwargs = {"scopes": scopes, "default_scopes": cls.AUTH_SCOPES} + else: + scopes_kwargs = {"scopes": scopes or cls.AUTH_SCOPES} + + return scopes_kwargs + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.create_read_session: gapic_v1.method.wrap_method( + self.create_read_session, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.read_rows: gapic_v1.method.wrap_method( + self.read_rows, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.split_read_stream: gapic_v1.method.wrap_method( + self.split_read_stream, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + Union[ + stream.ReadSession, + Awaitable[stream.ReadSession] + ]]: + raise NotImplementedError() + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + Union[ + storage.ReadRowsResponse, + Awaitable[storage.ReadRowsResponse] + ]]: + raise NotImplementedError() + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + Union[ + storage.SplitReadStreamResponse, + Awaitable[storage.SplitReadStreamResponse] + ]]: + raise NotImplementedError() + + +__all__ = ( + 'BigQueryReadTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py new file mode 100644 index 00000000..60c6d695 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py @@ -0,0 +1,347 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import grpc_helpers # type: ignore +from google.api_core import gapic_v1 # type: ignore +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO + + +class BigQueryReadGrpcTransport(BigQueryReadTransport): + """gRPC backend transport for BigQueryRead. + + BigQuery Read API. + The Read API can be used to read data from BigQuery. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + _stubs: Dict[str, Callable] + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: str = None, + scopes: Sequence[str] = None, + channel: grpc.Channel = None, + api_mtls_endpoint: str = None, + client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if ``channel`` is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if ``channel`` is provided. + channel (Optional[grpc.Channel]): A ``Channel`` instance through + which to make calls. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or applicatin default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if channel: + # Ignore credentials if a channel was passed. + credentials = False + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + ) + + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: str = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> grpc.Channel: + """Create and return a gRPC channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + grpc.Channel: A gRPC channel object. + + Raises: + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + + return grpc_helpers.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + @property + def grpc_channel(self) -> grpc.Channel: + """Return the channel designed to connect to this service. + """ + return self._grpc_channel + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + stream.ReadSession]: + r"""Return a callable for the create read session method over gRPC. + + Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a push- + down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 24 hours after they + are created and do not require manual clean-up by the + caller. + + Returns: + Callable[[~.CreateReadSessionRequest], + ~.ReadSession]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_read_session' not in self._stubs: + self._stubs['create_read_session'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryRead/CreateReadSession', + request_serializer=storage.CreateReadSessionRequest.serialize, + response_deserializer=stream.ReadSession.deserialize, + ) + return self._stubs['create_read_session'] + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + storage.ReadRowsResponse]: + r"""Return a callable for the read rows method over gRPC. + + Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Returns: + Callable[[~.ReadRowsRequest], + ~.ReadRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'read_rows' not in self._stubs: + self._stubs['read_rows'] = self.grpc_channel.unary_stream( + '/google.cloud.bigquery.storage.v1.BigQueryRead/ReadRows', + request_serializer=storage.ReadRowsRequest.serialize, + response_deserializer=storage.ReadRowsResponse.deserialize, + ) + return self._stubs['read_rows'] + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + storage.SplitReadStreamResponse]: + r"""Return a callable for the split read stream method over gRPC. + + Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Returns: + Callable[[~.SplitReadStreamRequest], + ~.SplitReadStreamResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'split_read_stream' not in self._stubs: + self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryRead/SplitReadStream', + request_serializer=storage.SplitReadStreamRequest.serialize, + response_deserializer=storage.SplitReadStreamResponse.deserialize, + ) + return self._stubs['split_read_stream'] + + +__all__ = ( + 'BigQueryReadGrpcTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py new file mode 100644 index 00000000..d2f7225f --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py @@ -0,0 +1,351 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import gapic_v1 # type: ignore +from google.api_core import grpc_helpers_async # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +import packaging.version + +import grpc # type: ignore +from grpc.experimental import aio # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .grpc import BigQueryReadGrpcTransport + + +class BigQueryReadGrpcAsyncIOTransport(BigQueryReadTransport): + """gRPC AsyncIO backend transport for BigQueryRead. + + BigQuery Read API. + The Read API can be used to read data from BigQuery. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + + _grpc_channel: aio.Channel + _stubs: Dict[str, Callable] = {} + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> aio.Channel: + """Create and return a gRPC AsyncIO channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + aio.Channel: A gRPC AsyncIO channel object. + """ + + return grpc_helpers_async.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: aio.Channel = None, + api_mtls_endpoint: str = None, + client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, + quota_project_id=None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if ``channel`` is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + channel (Optional[aio.Channel]): A ``Channel`` instance through + which to make calls. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or applicatin default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if channel: + # Ignore credentials if a channel was passed. + credentials = False + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + ) + + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @property + def grpc_channel(self) -> aio.Channel: + """Create the channel designed to connect to this service. + + This property caches on the instance; repeated calls return + the same channel. + """ + # Return the channel from cache. + return self._grpc_channel + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + Awaitable[stream.ReadSession]]: + r"""Return a callable for the create read session method over gRPC. + + Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a push- + down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 24 hours after they + are created and do not require manual clean-up by the + caller. + + Returns: + Callable[[~.CreateReadSessionRequest], + Awaitable[~.ReadSession]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_read_session' not in self._stubs: + self._stubs['create_read_session'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryRead/CreateReadSession', + request_serializer=storage.CreateReadSessionRequest.serialize, + response_deserializer=stream.ReadSession.deserialize, + ) + return self._stubs['create_read_session'] + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + Awaitable[storage.ReadRowsResponse]]: + r"""Return a callable for the read rows method over gRPC. + + Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Returns: + Callable[[~.ReadRowsRequest], + Awaitable[~.ReadRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'read_rows' not in self._stubs: + self._stubs['read_rows'] = self.grpc_channel.unary_stream( + '/google.cloud.bigquery.storage.v1.BigQueryRead/ReadRows', + request_serializer=storage.ReadRowsRequest.serialize, + response_deserializer=storage.ReadRowsResponse.deserialize, + ) + return self._stubs['read_rows'] + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + Awaitable[storage.SplitReadStreamResponse]]: + r"""Return a callable for the split read stream method over gRPC. + + Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Returns: + Callable[[~.SplitReadStreamRequest], + Awaitable[~.SplitReadStreamResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'split_read_stream' not in self._stubs: + self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryRead/SplitReadStream', + request_serializer=storage.SplitReadStreamRequest.serialize, + response_deserializer=storage.SplitReadStreamResponse.deserialize, + ) + return self._stubs['split_read_stream'] + + +__all__ = ( + 'BigQueryReadGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py new file mode 100644 index 00000000..030085d0 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .arrow import ( + ArrowRecordBatch, + ArrowSchema, + ArrowSerializationOptions, +) +from .avro import ( + AvroRows, + AvroSchema, +) +from .storage import ( + CreateReadSessionRequest, + ReadRowsRequest, + ReadRowsResponse, + SplitReadStreamRequest, + SplitReadStreamResponse, + StreamStats, + ThrottleState, +) +from .stream import ( + ReadSession, + ReadStream, + DataFormat, +) + +__all__ = ( + 'ArrowRecordBatch', + 'ArrowSchema', + 'ArrowSerializationOptions', + 'AvroRows', + 'AvroSchema', + 'CreateReadSessionRequest', + 'ReadRowsRequest', + 'ReadRowsResponse', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'StreamStats', + 'ThrottleState', + 'ReadSession', + 'ReadStream', + 'DataFormat', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py new file mode 100644 index 00000000..391088fe --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + 'ArrowSchema', + 'ArrowRecordBatch', + 'ArrowSerializationOptions', + }, +) + + +class ArrowSchema(proto.Message): + r"""Arrow schema as specified in + https://arrow.apache.org/docs/python/api/datatypes.html and + serialized to bytes using IPC: + https://arrow.apache.org/docs/format/Columnar.html#serialization- + and-interprocess-communication-ipc + See code samples on how this message can be deserialized. + + Attributes: + serialized_schema (bytes): + IPC serialized Arrow schema. + """ + + serialized_schema = proto.Field( + proto.BYTES, + number=1, + ) + + +class ArrowRecordBatch(proto.Message): + r"""Arrow RecordBatch. + Attributes: + serialized_record_batch (bytes): + IPC-serialized Arrow RecordBatch. + row_count (int): + The count of rows in ``serialized_record_batch``. + """ + + serialized_record_batch = proto.Field( + proto.BYTES, + number=1, + ) + row_count = proto.Field( + proto.INT64, + number=2, + ) + + +class ArrowSerializationOptions(proto.Message): + r"""Contains options specific to Arrow Serialization. + Attributes: + buffer_compression (google.cloud.bigquery_storage_v1.types.ArrowSerializationOptions.CompressionCodec): + The compression codec to use for Arrow + buffers in serialized record batches. + """ + class CompressionCodec(proto.Enum): + r"""Compression codec's supported by Arrow.""" + COMPRESSION_UNSPECIFIED = 0 + LZ4_FRAME = 1 + ZSTD = 2 + + buffer_compression = proto.Field( + proto.ENUM, + number=2, + enum=CompressionCodec, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py new file mode 100644 index 00000000..023fa87a --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + 'AvroSchema', + 'AvroRows', + }, +) + + +class AvroSchema(proto.Message): + r"""Avro schema. + Attributes: + schema (str): + Json serialized schema, as described at + https://avro.apache.org/docs/1.8.1/spec.html. + """ + + schema = proto.Field( + proto.STRING, + number=1, + ) + + +class AvroRows(proto.Message): + r"""Avro rows. + Attributes: + serialized_binary_rows (bytes): + Binary serialized rows in a block. + row_count (int): + The count of rows in the returning block. + """ + + serialized_binary_rows = proto.Field( + proto.BYTES, + number=1, + ) + row_count = proto.Field( + proto.INT64, + number=2, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py new file mode 100644 index 00000000..1d8641c7 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import stream + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + 'CreateReadSessionRequest', + 'ReadRowsRequest', + 'ThrottleState', + 'StreamStats', + 'ReadRowsResponse', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + }, +) + + +class CreateReadSessionRequest(proto.Message): + r"""Request message for ``CreateReadSession``. + Attributes: + parent (str): + Required. The request project that owns the session, in the + form of ``projects/{project_id}``. + read_session (google.cloud.bigquery_storage_v1.types.ReadSession): + Required. Session to be created. + max_stream_count (int): + Max initial number of streams. If unset or + zero, the server will provide a value of streams + so as to produce reasonable throughput. Must be + non-negative. The number of streams may be lower + than the requested number, depending on the + amount parallelism that is reasonable for the + table. Error will be returned if the max count + is greater than the current system max limit of + 1,000. + + Streams must be read starting from offset 0. + """ + + parent = proto.Field( + proto.STRING, + number=1, + ) + read_session = proto.Field( + proto.MESSAGE, + number=2, + message=stream.ReadSession, + ) + max_stream_count = proto.Field( + proto.INT32, + number=3, + ) + + +class ReadRowsRequest(proto.Message): + r"""Request message for ``ReadRows``. + Attributes: + read_stream (str): + Required. Stream to read rows from. + offset (int): + The offset requested must be less than the + last row read from Read. Requesting a larger + offset is undefined. If not specified, start + reading from offset zero. + """ + + read_stream = proto.Field( + proto.STRING, + number=1, + ) + offset = proto.Field( + proto.INT64, + number=2, + ) + + +class ThrottleState(proto.Message): + r"""Information on if the current connection is being throttled. + Attributes: + throttle_percent (int): + How much this connection is being throttled. + Zero means no throttling, 100 means fully + throttled. + """ + + throttle_percent = proto.Field( + proto.INT32, + number=1, + ) + + +class StreamStats(proto.Message): + r"""Estimated stream statistics for a given Stream. + Attributes: + progress (google.cloud.bigquery_storage_v1.types.StreamStats.Progress): + Represents the progress of the current + stream. + """ + + class Progress(proto.Message): + r""" + Attributes: + at_response_start (float): + The fraction of rows assigned to the stream that have been + processed by the server so far, not including the rows in + the current response message. + + This value, along with ``at_response_end``, can be used to + interpolate the progress made as the rows in the message are + being processed using the following formula: + ``at_response_start + (at_response_end - at_response_start) * rows_processed_from_response / rows_in_response``. + + Note that if a filter is provided, the ``at_response_end`` + value of the previous response may not necessarily be equal + to the ``at_response_start`` value of the current response. + at_response_end (float): + Similar to ``at_response_start``, except that this value + includes the rows in the current response. + """ + + at_response_start = proto.Field( + proto.DOUBLE, + number=1, + ) + at_response_end = proto.Field( + proto.DOUBLE, + number=2, + ) + + progress = proto.Field( + proto.MESSAGE, + number=2, + message=Progress, + ) + + +class ReadRowsResponse(proto.Message): + r"""Response from calling ``ReadRows`` may include row data, progress + and throttling information. + + Attributes: + avro_rows (google.cloud.bigquery_storage_v1.types.AvroRows): + Serialized row data in AVRO format. + arrow_record_batch (google.cloud.bigquery_storage_v1.types.ArrowRecordBatch): + Serialized row data in Arrow RecordBatch + format. + row_count (int): + Number of serialized rows in the rows block. + stats (google.cloud.bigquery_storage_v1.types.StreamStats): + Statistics for the stream. + throttle_state (google.cloud.bigquery_storage_v1.types.ThrottleState): + Throttling state. If unset, the latest + response still describes the current throttling + status. + avro_schema (google.cloud.bigquery_storage_v1.types.AvroSchema): + Output only. Avro schema. + arrow_schema (google.cloud.bigquery_storage_v1.types.ArrowSchema): + Output only. Arrow schema. + """ + + avro_rows = proto.Field( + proto.MESSAGE, + number=3, + oneof='rows', + message=avro.AvroRows, + ) + arrow_record_batch = proto.Field( + proto.MESSAGE, + number=4, + oneof='rows', + message=arrow.ArrowRecordBatch, + ) + row_count = proto.Field( + proto.INT64, + number=6, + ) + stats = proto.Field( + proto.MESSAGE, + number=2, + message='StreamStats', + ) + throttle_state = proto.Field( + proto.MESSAGE, + number=5, + message='ThrottleState', + ) + avro_schema = proto.Field( + proto.MESSAGE, + number=7, + oneof='schema', + message=avro.AvroSchema, + ) + arrow_schema = proto.Field( + proto.MESSAGE, + number=8, + oneof='schema', + message=arrow.ArrowSchema, + ) + + +class SplitReadStreamRequest(proto.Message): + r"""Request message for ``SplitReadStream``. + Attributes: + name (str): + Required. Name of the stream to split. + fraction (float): + A value in the range (0.0, 1.0) that + specifies the fractional point at which the + original stream should be split. The actual + split point is evaluated on pre-filtered rows, + so if a filter is provided, then there is no + guarantee that the division of the rows between + the new child streams will be proportional to + this fractional value. Additionally, because the + server-side unit for assigning data is + collections of rows, this fraction will always + map to a data storage boundary on the server + side. + """ + + name = proto.Field( + proto.STRING, + number=1, + ) + fraction = proto.Field( + proto.DOUBLE, + number=2, + ) + + +class SplitReadStreamResponse(proto.Message): + r"""Response message for ``SplitReadStream``. + Attributes: + primary_stream (google.cloud.bigquery_storage_v1.types.ReadStream): + Primary stream, which contains the beginning portion of + \|original_stream|. An empty value indicates that the + original stream can no longer be split. + remainder_stream (google.cloud.bigquery_storage_v1.types.ReadStream): + Remainder stream, which contains the tail of + \|original_stream|. An empty value indicates that the + original stream can no longer be split. + """ + + primary_stream = proto.Field( + proto.MESSAGE, + number=1, + message=stream.ReadStream, + ) + remainder_stream = proto.Field( + proto.MESSAGE, + number=2, + message=stream.ReadStream, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py new file mode 100644 index 00000000..9835d161 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import avro +from google.protobuf import timestamp_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + 'DataFormat', + 'ReadSession', + 'ReadStream', + }, +) + + +class DataFormat(proto.Enum): + r"""Data format for input or output data.""" + DATA_FORMAT_UNSPECIFIED = 0 + AVRO = 1 + ARROW = 2 + + +class ReadSession(proto.Message): + r"""Information about the ReadSession. + Attributes: + name (str): + Output only. Unique identifier for the session, in the form + ``projects/{project_id}/locations/{location}/sessions/{session_id}``. + expire_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Time at which the session becomes invalid. + After this time, subsequent requests to read this Session + will return errors. The expire_time is automatically + assigned and currently cannot be specified or updated. + data_format (google.cloud.bigquery_storage_v1.types.DataFormat): + Immutable. Data format of the output data. + avro_schema (google.cloud.bigquery_storage_v1.types.AvroSchema): + Output only. Avro schema. + arrow_schema (google.cloud.bigquery_storage_v1.types.ArrowSchema): + Output only. Arrow schema. + table (str): + Immutable. Table that this ReadSession is reading from, in + the form + ``projects/{project_id}/datasets/{dataset_id}/tables/{table_id}`` + table_modifiers (google.cloud.bigquery_storage_v1.types.ReadSession.TableModifiers): + Optional. Any modifiers which are applied + when reading from the specified table. + read_options (google.cloud.bigquery_storage_v1.types.ReadSession.TableReadOptions): + Optional. Read options for this session (e.g. + column selection, filters). + streams (Sequence[google.cloud.bigquery_storage_v1.types.ReadStream]): + Output only. A list of streams created with the session. + + At least one stream is created with the session. In the + future, larger request_stream_count values *may* result in + this list being unpopulated, in that case, the user will + need to use a List method to get the streams instead, which + is not yet available. + """ + + class TableModifiers(proto.Message): + r"""Additional attributes when reading a table. + Attributes: + snapshot_time (google.protobuf.timestamp_pb2.Timestamp): + The snapshot time of the table. If not set, + interpreted as now. + """ + + snapshot_time = proto.Field( + proto.MESSAGE, + number=1, + message=timestamp_pb2.Timestamp, + ) + + class TableReadOptions(proto.Message): + r"""Options dictating how we read a table. + Attributes: + selected_fields (Sequence[str]): + Names of the fields in the table that should be read. If + empty, all fields will be read. If the specified field is a + nested field, all the sub-fields in the field will be + selected. The output field order is unrelated to the order + of fields in selected_fields. + row_restriction (str): + SQL text filtering statement, similar to a WHERE clause in a + query. Aggregates are not supported. + + Examples: "int_field > 5" "date_field = CAST('2014-9-27' as + DATE)" "nullable_field is not NULL" "st_equals(geo_field, + st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 + AND 5.0" + + Restricted to a maximum length for 1 MB. + arrow_serialization_options (google.cloud.bigquery_storage_v1.types.ArrowSerializationOptions): + Optional. Options specific to the Apache + Arrow output format. + """ + + selected_fields = proto.RepeatedField( + proto.STRING, + number=1, + ) + row_restriction = proto.Field( + proto.STRING, + number=2, + ) + arrow_serialization_options = proto.Field( + proto.MESSAGE, + number=3, + oneof='output_format_serialization_options', + message=arrow.ArrowSerializationOptions, + ) + + name = proto.Field( + proto.STRING, + number=1, + ) + expire_time = proto.Field( + proto.MESSAGE, + number=2, + message=timestamp_pb2.Timestamp, + ) + data_format = proto.Field( + proto.ENUM, + number=3, + enum='DataFormat', + ) + avro_schema = proto.Field( + proto.MESSAGE, + number=4, + oneof='schema', + message=avro.AvroSchema, + ) + arrow_schema = proto.Field( + proto.MESSAGE, + number=5, + oneof='schema', + message=arrow.ArrowSchema, + ) + table = proto.Field( + proto.STRING, + number=6, + ) + table_modifiers = proto.Field( + proto.MESSAGE, + number=7, + message=TableModifiers, + ) + read_options = proto.Field( + proto.MESSAGE, + number=8, + message=TableReadOptions, + ) + streams = proto.RepeatedField( + proto.MESSAGE, + number=10, + message='ReadStream', + ) + + +class ReadStream(proto.Message): + r"""Information about a single stream that gets data out of the storage + system. Most of the information about ``ReadStream`` instances is + aggregated, making ``ReadStream`` lightweight. + + Attributes: + name (str): + Output only. Name of the stream, in the form + ``projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}``. + """ + + name = proto.Field( + proto.STRING, + number=1, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/mypy.ini b/owl-bot-staging/v1/mypy.ini new file mode 100644 index 00000000..4505b485 --- /dev/null +++ b/owl-bot-staging/v1/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +python_version = 3.6 +namespace_packages = True diff --git a/owl-bot-staging/v1/noxfile.py b/owl-bot-staging/v1/noxfile.py new file mode 100644 index 00000000..877e91ab --- /dev/null +++ b/owl-bot-staging/v1/noxfile.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import pathlib +import shutil +import subprocess +import sys + + +import nox # type: ignore + +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt" +PACKAGE_NAME = subprocess.check_output([sys.executable, "setup.py", "--name"], encoding="utf-8") + + +nox.sessions = [ + "unit", + "cover", + "mypy", + "check_lower_bounds" + # exclude update_lower_bounds from default + "docs", +] + +@nox.session(python=['3.6', '3.7', '3.8', '3.9']) +def unit(session): + """Run the unit test suite.""" + + session.install('coverage', 'pytest', 'pytest-cov', 'asyncmock', 'pytest-asyncio') + session.install('-e', '.') + + session.run( + 'py.test', + '--quiet', + '--cov=google/cloud/bigquery_storage_v1/', + '--cov-config=.coveragerc', + '--cov-report=term', + '--cov-report=html', + os.path.join('tests', 'unit', ''.join(session.posargs)) + ) + + +@nox.session(python='3.7') +def cover(session): + """Run the final coverage report. + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=100") + + session.run("coverage", "erase") + + +@nox.session(python=['3.6', '3.7']) +def mypy(session): + """Run the type checker.""" + session.install('mypy', 'types-pkg_resources') + session.install('.') + session.run( + 'mypy', + '--explicit-package-bases', + 'google', + ) + + +@nox.session +def update_lower_bounds(session): + """Update lower bounds in constraints.txt to match setup.py""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'update', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + + +@nox.session +def check_lower_bounds(session): + """Check lower bounds in setup.py are reflected in constraints file""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'check', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + +@nox.session(python='3.6') +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".") + session.install("sphinx<3.0.0", "alabaster", "recommonmark") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) diff --git a/owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py b/owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py new file mode 100644 index 00000000..107af58b --- /dev/null +++ b/owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py @@ -0,0 +1,178 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import os +import libcst as cst +import pathlib +import sys +from typing import (Any, Callable, Dict, List, Sequence, Tuple) + + +def partition( + predicate: Callable[[Any], bool], + iterator: Sequence[Any] +) -> Tuple[List[Any], List[Any]]: + """A stable, out-of-place partition.""" + results = ([], []) + + for i in iterator: + results[int(predicate(i))].append(i) + + # Returns trueList, falseList + return results[1], results[0] + + +class bigquery_storageCallTransformer(cst.CSTTransformer): + CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') + METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { + 'create_read_session': ('parent', 'read_session', 'max_stream_count', ), + 'read_rows': ('read_stream', 'offset', ), + 'split_read_stream': ('name', 'fraction', ), + } + + def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: + try: + key = original.func.attr.value + kword_params = self.METHOD_TO_PARAMS[key] + except (AttributeError, KeyError): + # Either not a method from the API or too convoluted to be sure. + return updated + + # If the existing code is valid, keyword args come after positional args. + # Therefore, all positional args must map to the first parameters. + args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) + if any(k.keyword.value == "request" for k in kwargs): + # We've already fixed this file, don't fix it again. + return updated + + kwargs, ctrl_kwargs = partition( + lambda a: not a.keyword.value in self.CTRL_PARAMS, + kwargs + ) + + args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] + ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) + for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) + + request_arg = cst.Arg( + value=cst.Dict([ + cst.DictElement( + cst.SimpleString("'{}'".format(name)), +cst.Element(value=arg.value) + ) + # Note: the args + kwargs looks silly, but keep in mind that + # the control parameters had to be stripped out, and that + # those could have been passed positionally or by keyword. + for name, arg in zip(kword_params, args + kwargs)]), + keyword=cst.Name("request") + ) + + return updated.with_changes( + args=[request_arg] + ctrl_kwargs + ) + + +def fix_files( + in_dir: pathlib.Path, + out_dir: pathlib.Path, + *, + transformer=bigquery_storageCallTransformer(), +): + """Duplicate the input dir to the output dir, fixing file method calls. + + Preconditions: + * in_dir is a real directory + * out_dir is a real, empty directory + """ + pyfile_gen = ( + pathlib.Path(os.path.join(root, f)) + for root, _, files in os.walk(in_dir) + for f in files if os.path.splitext(f)[1] == ".py" + ) + + for fpath in pyfile_gen: + with open(fpath, 'r') as f: + src = f.read() + + # Parse the code and insert method call fixes. + tree = cst.parse_module(src) + updated = tree.visit(transformer) + + # Create the path and directory structure for the new file. + updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) + updated_path.parent.mkdir(parents=True, exist_ok=True) + + # Generate the updated source file at the corresponding path. + with open(updated_path, 'w') as f: + f.write(updated.code) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="""Fix up source that uses the bigquery_storage client library. + +The existing sources are NOT overwritten but are copied to output_dir with changes made. + +Note: This tool operates at a best-effort level at converting positional + parameters in client method calls to keyword based parameters. + Cases where it WILL FAIL include + A) * or ** expansion in a method call. + B) Calls via function or method alias (includes free function calls) + C) Indirect or dispatched calls (e.g. the method is looked up dynamically) + + These all constitute false negatives. The tool will also detect false + positives when an API method shares a name with another method. +""") + parser.add_argument( + '-d', + '--input-directory', + required=True, + dest='input_dir', + help='the input directory to walk for python files to fix up', + ) + parser.add_argument( + '-o', + '--output-directory', + required=True, + dest='output_dir', + help='the directory to output files fixed via un-flattening', + ) + args = parser.parse_args() + input_dir = pathlib.Path(args.input_dir) + output_dir = pathlib.Path(args.output_dir) + if not input_dir.is_dir(): + print( + f"input directory '{input_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if not output_dir.is_dir(): + print( + f"output directory '{output_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if os.listdir(output_dir): + print( + f"output directory '{output_dir}' is not empty", + file=sys.stderr, + ) + sys.exit(-1) + + fix_files(input_dir, output_dir) diff --git a/owl-bot-staging/v1/setup.py b/owl-bot-staging/v1/setup.py new file mode 100644 index 00000000..591e1fba --- /dev/null +++ b/owl-bot-staging/v1/setup.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +import os +import setuptools # type: ignore + +version = '0.1.0' + +package_root = os.path.abspath(os.path.dirname(__file__)) + +readme_filename = os.path.join(package_root, 'README.rst') +with io.open(readme_filename, encoding='utf-8') as readme_file: + readme = readme_file.read() + +setuptools.setup( + name='google-cloud-bigquery-storage', + version=version, + long_description=readme, + packages=setuptools.PEP420PackageFinder.find(), + namespace_packages=('google', 'google.cloud'), + platforms='Posix; MacOS X; Windows', + include_package_data=True, + install_requires=( + 'google-api-core[grpc] >= 1.27.0, < 3.0.0dev', + 'libcst >= 0.2.5', + 'proto-plus >= 1.15.0', + 'packaging >= 14.3', ), + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Topic :: Internet', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + zip_safe=False, +) diff --git a/owl-bot-staging/v1/tests/__init__.py b/owl-bot-staging/v1/tests/__init__.py new file mode 100644 index 00000000..b54a5fcc --- /dev/null +++ b/owl-bot-staging/v1/tests/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1/tests/unit/__init__.py b/owl-bot-staging/v1/tests/unit/__init__.py new file mode 100644 index 00000000..b54a5fcc --- /dev/null +++ b/owl-bot-staging/v1/tests/unit/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1/tests/unit/gapic/__init__.py b/owl-bot-staging/v1/tests/unit/gapic/__init__.py new file mode 100644 index 00000000..b54a5fcc --- /dev/null +++ b/owl-bot-staging/v1/tests/unit/gapic/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py new file mode 100644 index 00000000..b54a5fcc --- /dev/null +++ b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py new file mode 100644 index 00000000..8fe8065d --- /dev/null +++ b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py @@ -0,0 +1,1629 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import mock +import packaging.version + +import grpc +from grpc.experimental import aio +import math +import pytest +from proto.marshal.rules.dates import DurationRule, TimestampRule + + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers +from google.api_core import grpc_helpers_async +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.cloud.bigquery_storage_v1.services.big_query_read import BigQueryReadAsyncClient +from google.cloud.bigquery_storage_v1.services.big_query_read import BigQueryReadClient +from google.cloud.bigquery_storage_v1.services.big_query_read import transports +from google.cloud.bigquery_storage_v1.services.big_query_read.transports.base import _GOOGLE_AUTH_VERSION +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.oauth2 import service_account +from google.protobuf import timestamp_pb2 # type: ignore +import google.auth + + +# TODO(busunkim): Once google-auth >= 1.25.0 is required transitively +# through google-api-core: +# - Delete the auth "less than" test cases +# - Delete these pytest markers (Make the "greater than or equal to" tests the default). +requires_google_auth_lt_1_25_0 = pytest.mark.skipif( + packaging.version.parse(_GOOGLE_AUTH_VERSION) >= packaging.version.parse("1.25.0"), + reason="This test requires google-auth < 1.25.0", +) +requires_google_auth_gte_1_25_0 = pytest.mark.skipif( + packaging.version.parse(_GOOGLE_AUTH_VERSION) < packaging.version.parse("1.25.0"), + reason="This test requires google-auth >= 1.25.0", +) + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert BigQueryReadClient._get_default_mtls_endpoint(None) is None + assert BigQueryReadClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi + + +@pytest.mark.parametrize("client_class", [ + BigQueryReadClient, + BigQueryReadAsyncClient, +]) +def test_big_query_read_client_from_service_account_info(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == 'bigquerystorage.googleapis.com:443' + + +@pytest.mark.parametrize("transport_class,transport_name", [ + (transports.BigQueryReadGrpcTransport, "grpc"), + (transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_read_client_service_account_always_use_jwt(transport_class, transport_name): + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class", [ + BigQueryReadClient, + BigQueryReadAsyncClient, +]) +def test_big_query_read_client_from_service_account_file(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == 'bigquerystorage.googleapis.com:443' + + +def test_big_query_read_client_get_transport_class(): + transport = BigQueryReadClient.get_transport_class() + available_transports = [ + transports.BigQueryReadGrpcTransport, + ] + assert transport in available_transports + + transport = BigQueryReadClient.get_transport_class("grpc") + assert transport == transports.BigQueryReadGrpcTransport + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +@mock.patch.object(BigQueryReadClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadAsyncClient)) +def test_big_query_read_client_client_options(client_class, transport_class, transport_name): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError): + client = client_class() + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError): + client = client_class() + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "true"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "true"), + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "false"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "false"), +]) +@mock.patch.object(BigQueryReadClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadAsyncClient)) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_big_query_read_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client.DEFAULT_ENDPOINT + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): + if use_client_cert_env == "false": + expected_host = client.DEFAULT_ENDPOINT + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_read_client_client_options_scopes(client_class, transport_class, transport_name): + # Check the case scopes are provided. + options = client_options.ClientOptions( + scopes=["1", "2"], + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_read_client_client_options_credentials_file(client_class, transport_class, transport_name): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +def test_big_query_read_client_client_options_from_dict(): + with mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadGrpcTransport.__init__') as grpc_transport: + grpc_transport.return_value = None + client = BigQueryReadClient( + client_options={'api_endpoint': 'squid.clam.whelk'} + ) + grpc_transport.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +def test_create_read_session(transport: str = 'grpc', request_type=storage.CreateReadSessionRequest): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession( + name='name_value', + data_format=stream.DataFormat.AVRO, + table='table_value', + avro_schema=avro.AvroSchema(schema='schema_value'), + ) + response = client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateReadSessionRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.ReadSession) + assert response.name == 'name_value' + assert response.data_format == stream.DataFormat.AVRO + assert response.table == 'table_value' + + +def test_create_read_session_from_dict(): + test_create_read_session(request_type=dict) + + +def test_create_read_session_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + client.create_read_session() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateReadSessionRequest() + + +@pytest.mark.asyncio +async def test_create_read_session_async(transport: str = 'grpc_asyncio', request_type=storage.CreateReadSessionRequest): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession( + name='name_value', + data_format=stream.DataFormat.AVRO, + table='table_value', + )) + response = await client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateReadSessionRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.ReadSession) + assert response.name == 'name_value' + assert response.data_format == stream.DataFormat.AVRO + assert response.table == 'table_value' + + +@pytest.mark.asyncio +async def test_create_read_session_async_from_dict(): + await test_create_read_session_async(request_type=dict) + + +def test_create_read_session_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateReadSessionRequest() + + request.read_session.table = 'read_session.table/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value = stream.ReadSession() + client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_session.table=read_session.table/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_create_read_session_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateReadSessionRequest() + + request.read_session.table = 'read_session.table/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) + await client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_session.table=read_session.table/value', + ) in kw['metadata'] + + +def test_create_read_session_flattened(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.create_read_session( + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].parent == 'parent_value' + assert args[0].read_session == stream.ReadSession(name='name_value') + assert args[0].max_stream_count == 1721 + + +def test_create_read_session_flattened_error(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.create_read_session( + storage.CreateReadSessionRequest(), + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + +@pytest.mark.asyncio +async def test_create_read_session_flattened_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.create_read_session( + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].parent == 'parent_value' + assert args[0].read_session == stream.ReadSession(name='name_value') + assert args[0].max_stream_count == 1721 + + +@pytest.mark.asyncio +async def test_create_read_session_flattened_error_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.create_read_session( + storage.CreateReadSessionRequest(), + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + +def test_read_rows(transport: str = 'grpc', request_type=storage.ReadRowsRequest): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + response = client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.ReadRowsRequest() + + # Establish that the response is the type that we expect. + for message in response: + assert isinstance(message, storage.ReadRowsResponse) + + +def test_read_rows_from_dict(): + test_read_rows(request_type=dict) + + +def test_read_rows_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + client.read_rows() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.ReadRowsRequest() + + +@pytest.mark.asyncio +async def test_read_rows_async(transport: str = 'grpc_asyncio', request_type=storage.ReadRowsRequest): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) + response = await client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.ReadRowsRequest() + + # Establish that the response is the type that we expect. + message = await response.read() + assert isinstance(message, storage.ReadRowsResponse) + + +@pytest.mark.asyncio +async def test_read_rows_async_from_dict(): + await test_read_rows_async(request_type=dict) + + +def test_read_rows_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.ReadRowsRequest() + + request.read_stream = 'read_stream/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value = iter([storage.ReadRowsResponse()]) + client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_stream=read_stream/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_read_rows_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.ReadRowsRequest() + + request.read_stream = 'read_stream/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) + await client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_stream=read_stream/value', + ) in kw['metadata'] + + +def test_read_rows_flattened(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.read_rows( + read_stream='read_stream_value', + offset=647, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].read_stream == 'read_stream_value' + assert args[0].offset == 647 + + +def test_read_rows_flattened_error(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.read_rows( + storage.ReadRowsRequest(), + read_stream='read_stream_value', + offset=647, + ) + + +@pytest.mark.asyncio +async def test_read_rows_flattened_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.read_rows( + read_stream='read_stream_value', + offset=647, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].read_stream == 'read_stream_value' + assert args[0].offset == 647 + + +@pytest.mark.asyncio +async def test_read_rows_flattened_error_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.read_rows( + storage.ReadRowsRequest(), + read_stream='read_stream_value', + offset=647, + ) + + +def test_split_read_stream(transport: str = 'grpc', request_type=storage.SplitReadStreamRequest): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.SplitReadStreamResponse( + ) + response = client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.SplitReadStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.SplitReadStreamResponse) + + +def test_split_read_stream_from_dict(): + test_split_read_stream(request_type=dict) + + +def test_split_read_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + client.split_read_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.SplitReadStreamRequest() + + +@pytest.mark.asyncio +async def test_split_read_stream_async(transport: str = 'grpc_asyncio', request_type=storage.SplitReadStreamRequest): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse( + )) + response = await client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.SplitReadStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.SplitReadStreamResponse) + + +@pytest.mark.asyncio +async def test_split_read_stream_async_from_dict(): + await test_split_read_stream_async(request_type=dict) + + +def test_split_read_stream_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.SplitReadStreamRequest() + + request.name = 'name/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value = storage.SplitReadStreamResponse() + client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_split_read_stream_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.SplitReadStreamRequest() + + request.name = 'name/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse()) + await client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name/value', + ) in kw['metadata'] + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options={"scopes": ["1", "2"]}, + transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = BigQueryReadClient(transport=transport) + assert client.transport is transport + +def test_transport_get_channel(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + + transport = transports.BigQueryReadGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + +@pytest.mark.parametrize("transport_class", [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, +]) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + +def test_transport_grpc_default(): + # A client should use the gRPC transport by default. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert isinstance( + client.transport, + transports.BigQueryReadGrpcTransport, + ) + +def test_big_query_read_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transport = transports.BigQueryReadTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json" + ) + + +def test_big_query_read_base_transport(): + # Instantiate the base transport. + with mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport.__init__') as Transport: + Transport.return_value = None + transport = transports.BigQueryReadTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + 'create_read_session', + 'read_rows', + 'split_read_stream', + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + +@requires_google_auth_gte_1_25_0 +def test_big_query_read_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryReadTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +@requires_google_auth_lt_1_25_0 +def test_big_query_read_base_transport_with_credentials_file_old_google_auth(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryReadTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', + ), + quota_project_id="octopus", + ) + + +def test_big_query_read_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryReadTransport() + adc.assert_called_once() + + +@requires_google_auth_gte_1_25_0 +def test_big_query_read_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryReadClient() + adc.assert_called_once_with( + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id=None, + ) + + +@requires_google_auth_lt_1_25_0 +def test_big_query_read_auth_adc_old_google_auth(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryReadClient() + adc.assert_called_once_with( + scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.readonly', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id=None, + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, + ], +) +@requires_google_auth_gte_1_25_0 +def test_big_query_read_transport_auth_adc(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + adc.assert_called_once_with( + scopes=["1", "2"], + default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.readonly', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, + ], +) +@requires_google_auth_lt_1_25_0 +def test_big_query_read_transport_auth_adc_old_google_auth(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus") + adc.assert_called_once_with(scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.BigQueryReadGrpcTransport, grpc_helpers), + (transports.BigQueryReadGrpcAsyncIOTransport, grpc_helpers_async) + ], +) +def test_big_query_read_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class( + quota_project_id="octopus", + scopes=["1", "2"] + ) + + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=["1", "2"], + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_grpc_transport_client_cert_source_for_mtls( + transport_class +): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, + private_key=expected_key + ) + + +def test_big_query_read_host_no_port(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), + ) + assert client.transport._host == 'bigquerystorage.googleapis.com:443' + + +def test_big_query_read_host_with_port(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), + ) + assert client.transport._host == 'bigquerystorage.googleapis.com:8000' + +def test_big_query_read_grpc_transport_channel(): + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryReadGrpcTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +def test_big_query_read_grpc_asyncio_transport_channel(): + channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryReadGrpcAsyncIOTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_transport_channel_mtls_with_client_cert_source( + transport_class +): + with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_ssl_cred = mock.Mock() + grpc_ssl_channel_cred.return_value = mock_ssl_cred + + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + + cred = ga_credentials.AnonymousCredentials() + with pytest.warns(DeprecationWarning): + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (cred, None) + transport = transport_class( + host="squid.clam.whelk", + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=client_cert_source_callback, + ) + adc.assert_called_once() + + grpc_ssl_channel_cred.assert_called_once_with( + certificate_chain=b"cert bytes", private_key=b"key bytes" + ) + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + assert transport._ssl_channel_credentials == mock_ssl_cred + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_transport_channel_mtls_with_adc( + transport_class +): + mock_ssl_cred = mock.Mock() + with mock.patch.multiple( + "google.auth.transport.grpc.SslCredentials", + __init__=mock.Mock(return_value=None), + ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), + ): + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + mock_cred = mock.Mock() + + with pytest.warns(DeprecationWarning): + transport = transport_class( + host="squid.clam.whelk", + credentials=mock_cred, + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=None, + ) + + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=mock_cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + + +def test_read_session_path(): + project = "squid" + location = "clam" + session = "whelk" + expected = "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) + actual = BigQueryReadClient.read_session_path(project, location, session) + assert expected == actual + + +def test_parse_read_session_path(): + expected = { + "project": "octopus", + "location": "oyster", + "session": "nudibranch", + } + path = BigQueryReadClient.read_session_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_read_session_path(path) + assert expected == actual + +def test_read_stream_path(): + project = "cuttlefish" + location = "mussel" + session = "winkle" + stream = "nautilus" + expected = "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) + actual = BigQueryReadClient.read_stream_path(project, location, session, stream) + assert expected == actual + + +def test_parse_read_stream_path(): + expected = { + "project": "scallop", + "location": "abalone", + "session": "squid", + "stream": "clam", + } + path = BigQueryReadClient.read_stream_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_read_stream_path(path) + assert expected == actual + +def test_table_path(): + project = "whelk" + dataset = "octopus" + table = "oyster" + expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + actual = BigQueryReadClient.table_path(project, dataset, table) + assert expected == actual + + +def test_parse_table_path(): + expected = { + "project": "nudibranch", + "dataset": "cuttlefish", + "table": "mussel", + } + path = BigQueryReadClient.table_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_table_path(path) + assert expected == actual + +def test_common_billing_account_path(): + billing_account = "winkle" + expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + actual = BigQueryReadClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "nautilus", + } + path = BigQueryReadClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_billing_account_path(path) + assert expected == actual + +def test_common_folder_path(): + folder = "scallop" + expected = "folders/{folder}".format(folder=folder, ) + actual = BigQueryReadClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "abalone", + } + path = BigQueryReadClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_folder_path(path) + assert expected == actual + +def test_common_organization_path(): + organization = "squid" + expected = "organizations/{organization}".format(organization=organization, ) + actual = BigQueryReadClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "clam", + } + path = BigQueryReadClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_organization_path(path) + assert expected == actual + +def test_common_project_path(): + project = "whelk" + expected = "projects/{project}".format(project=project, ) + actual = BigQueryReadClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "octopus", + } + path = BigQueryReadClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_project_path(path) + assert expected == actual + +def test_common_location_path(): + project = "oyster" + location = "nudibranch" + expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) + actual = BigQueryReadClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "cuttlefish", + "location": "mussel", + } + path = BigQueryReadClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_withDEFAULT_CLIENT_INFO(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: + transport_class = BigQueryReadClient.get_transport_class() + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) diff --git a/owl-bot-staging/v1beta2/.coveragerc b/owl-bot-staging/v1beta2/.coveragerc new file mode 100644 index 00000000..19fadc8b --- /dev/null +++ b/owl-bot-staging/v1beta2/.coveragerc @@ -0,0 +1,17 @@ +[run] +branch = True + +[report] +show_missing = True +omit = + google/cloud/bigquery_storage/__init__.py +exclude_lines = + # Re-enable the standard pragma + pragma: NO COVER + # Ignore debug-only repr + def __repr__ + # Ignore pkg_resources exceptions. + # This is added at the module level as a safeguard for if someone + # generates the code and tries to run it without pip installing. This + # makes it virtually impossible to test properly. + except pkg_resources.DistributionNotFound diff --git a/owl-bot-staging/v1beta2/MANIFEST.in b/owl-bot-staging/v1beta2/MANIFEST.in new file mode 100644 index 00000000..6b94fe3e --- /dev/null +++ b/owl-bot-staging/v1beta2/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include google/cloud/bigquery_storage *.py +recursive-include google/cloud/bigquery_storage_v1beta2 *.py diff --git a/owl-bot-staging/v1beta2/README.rst b/owl-bot-staging/v1beta2/README.rst new file mode 100644 index 00000000..13a45c14 --- /dev/null +++ b/owl-bot-staging/v1beta2/README.rst @@ -0,0 +1,49 @@ +Python Client for Google Cloud Bigquery Storage API +================================================= + +Quick Start +----------- + +In order to use this library, you first need to go through the following steps: + +1. `Select or create a Cloud Platform project.`_ +2. `Enable billing for your project.`_ +3. Enable the Google Cloud Bigquery Storage API. +4. `Setup Authentication.`_ + +.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project +.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project +.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html + +Installation +~~~~~~~~~~~~ + +Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to +create isolated Python environments. The basic problem it addresses is one of +dependencies and versions, and indirectly permissions. + +With `virtualenv`_, it's possible to install this library without needing system +install permissions, and without clashing with the installed system +dependencies. + +.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ + + +Mac/Linux +^^^^^^^^^ + +.. code-block:: console + + python3 -m venv + source /bin/activate + /bin/pip install /path/to/library + + +Windows +^^^^^^^ + +.. code-block:: console + + python3 -m venv + \Scripts\activate + \Scripts\pip.exe install \path\to\library diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst new file mode 100644 index 00000000..10b3714b --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst @@ -0,0 +1,6 @@ +BigQueryRead +------------------------------ + +.. automodule:: google.cloud.bigquery_storage_v1beta2.services.big_query_read + :members: + :inherited-members: diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst new file mode 100644 index 00000000..c685994d --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst @@ -0,0 +1,6 @@ +BigQueryWrite +------------------------------- + +.. automodule:: google.cloud.bigquery_storage_v1beta2.services.big_query_write + :members: + :inherited-members: diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services.rst new file mode 100644 index 00000000..67dae0ab --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services.rst @@ -0,0 +1,7 @@ +Services for Google Cloud Bigquery Storage v1beta2 API +====================================================== +.. toctree:: + :maxdepth: 2 + + big_query_read + big_query_write diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types.rst new file mode 100644 index 00000000..995806da --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types.rst @@ -0,0 +1,7 @@ +Types for Google Cloud Bigquery Storage v1beta2 API +=================================================== + +.. automodule:: google.cloud.bigquery_storage_v1beta2.types + :members: + :undoc-members: + :show-inheritance: diff --git a/owl-bot-staging/v1beta2/docs/conf.py b/owl-bot-staging/v1beta2/docs/conf.py new file mode 100644 index 00000000..e198bf7a --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/conf.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# google-cloud-bigquery-storage documentation build configuration file +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath("..")) + +__version__ = "0.1.0" + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +needs_sphinx = "1.6.3" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", +] + +# autodoc/autosummary flags +autoclass_content = "both" +autodoc_default_flags = ["members"] +autosummary_generate = True + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# Allow markdown includes (so releases.md can include CHANGLEOG.md) +# http://www.sphinx-doc.org/en/master/markdown.html +source_parsers = {".md": "recommonmark.parser.CommonMarkParser"} + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +source_suffix = [".rst", ".md"] + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = u"google-cloud-bigquery-storage" +copyright = u"2020, Google, LLC" +author = u"Google APIs" # TODO: autogenerate this bit + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The full version, including alpha/beta/rc tags. +release = __version__ +# The short X.Y version. +version = ".".join(release.split(".")[0:2]) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ["_build"] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "alabaster" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "description": "Google Cloud Client Libraries for Python", + "github_user": "googleapis", + "github_repo": "google-cloud-python", + "github_banner": True, + "font_family": "'Roboto', Georgia, sans", + "head_font_family": "'Roboto', Georgia, serif", + "code_font_family": "'Roboto Mono', 'Consolas', monospace", +} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = "google-cloud-bigquery-storage-doc" + +# -- Options for warnings ------------------------------------------------------ + + +suppress_warnings = [ + # Temporarily suppress this to avoid "more than one target found for + # cross-reference" warning, which are intractable for us to avoid while in + # a mono-repo. + # See https://github.com/sphinx-doc/sphinx/blob + # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843 + "ref.python" +] + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', + # Latex figure (float) alignment + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + master_doc, + "google-cloud-bigquery-storage.tex", + u"google-cloud-bigquery-storage Documentation", + author, + "manual", + ) +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ( + master_doc, + "google-cloud-bigquery-storage", + u"Google Cloud Bigquery Storage Documentation", + [author], + 1, + ) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "google-cloud-bigquery-storage", + u"google-cloud-bigquery-storage Documentation", + author, + "google-cloud-bigquery-storage", + "GAPIC library for Google Cloud Bigquery Storage API", + "APIs", + ) +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + "python": ("http://python.readthedocs.org/en/latest/", None), + "gax": ("https://gax-python.readthedocs.org/en/latest/", None), + "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), + "google-gax": ("https://gax-python.readthedocs.io/en/latest/", None), + "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None), + "grpc": ("https://grpc.io/grpc/python/", None), + "requests": ("http://requests.kennethreitz.org/en/stable/", None), + "proto": ("https://proto-plus-python.readthedocs.io/en/stable", None), + "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), +} + + +# Napoleon settings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = True +napoleon_use_admonition_for_examples = False +napoleon_use_admonition_for_notes = False +napoleon_use_admonition_for_references = False +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True diff --git a/owl-bot-staging/v1beta2/docs/index.rst b/owl-bot-staging/v1beta2/docs/index.rst new file mode 100644 index 00000000..ef2e07e2 --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/index.rst @@ -0,0 +1,7 @@ +API Reference +------------- +.. toctree:: + :maxdepth: 2 + + bigquery_storage_v1beta2/services + bigquery_storage_v1beta2/types diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py new file mode 100644 index 00000000..55ef84ca --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from google.cloud.bigquery_storage_v1beta2.services.big_query_read.client import BigQueryReadClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_read.async_client import BigQueryReadAsyncClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_write.client import BigQueryWriteClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_write.async_client import BigQueryWriteAsyncClient + +from google.cloud.bigquery_storage_v1beta2.types.arrow import ArrowRecordBatch +from google.cloud.bigquery_storage_v1beta2.types.arrow import ArrowSchema +from google.cloud.bigquery_storage_v1beta2.types.arrow import ArrowSerializationOptions +from google.cloud.bigquery_storage_v1beta2.types.avro import AvroRows +from google.cloud.bigquery_storage_v1beta2.types.avro import AvroSchema +from google.cloud.bigquery_storage_v1beta2.types.protobuf import ProtoRows +from google.cloud.bigquery_storage_v1beta2.types.protobuf import ProtoSchema +from google.cloud.bigquery_storage_v1beta2.types.storage import AppendRowsRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import AppendRowsResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import BatchCommitWriteStreamsRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import BatchCommitWriteStreamsResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import CreateReadSessionRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import CreateWriteStreamRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import FinalizeWriteStreamRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import FinalizeWriteStreamResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import FlushRowsRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import FlushRowsResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import GetWriteStreamRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import ReadRowsRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import ReadRowsResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import SplitReadStreamRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import SplitReadStreamResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import StorageError +from google.cloud.bigquery_storage_v1beta2.types.storage import StreamStats +from google.cloud.bigquery_storage_v1beta2.types.storage import ThrottleState +from google.cloud.bigquery_storage_v1beta2.types.stream import ReadSession +from google.cloud.bigquery_storage_v1beta2.types.stream import ReadStream +from google.cloud.bigquery_storage_v1beta2.types.stream import WriteStream +from google.cloud.bigquery_storage_v1beta2.types.stream import DataFormat +from google.cloud.bigquery_storage_v1beta2.types.table import TableFieldSchema +from google.cloud.bigquery_storage_v1beta2.types.table import TableSchema + +__all__ = ('BigQueryReadClient', + 'BigQueryReadAsyncClient', + 'BigQueryWriteClient', + 'BigQueryWriteAsyncClient', + 'ArrowRecordBatch', + 'ArrowSchema', + 'ArrowSerializationOptions', + 'AvroRows', + 'AvroSchema', + 'ProtoRows', + 'ProtoSchema', + 'AppendRowsRequest', + 'AppendRowsResponse', + 'BatchCommitWriteStreamsRequest', + 'BatchCommitWriteStreamsResponse', + 'CreateReadSessionRequest', + 'CreateWriteStreamRequest', + 'FinalizeWriteStreamRequest', + 'FinalizeWriteStreamResponse', + 'FlushRowsRequest', + 'FlushRowsResponse', + 'GetWriteStreamRequest', + 'ReadRowsRequest', + 'ReadRowsResponse', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'StorageError', + 'StreamStats', + 'ThrottleState', + 'ReadSession', + 'ReadStream', + 'WriteStream', + 'DataFormat', + 'TableFieldSchema', + 'TableSchema', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed new file mode 100644 index 00000000..e71b4749 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py new file mode 100644 index 00000000..c3136dce --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .services.big_query_read import BigQueryReadClient +from .services.big_query_read import BigQueryReadAsyncClient +from .services.big_query_write import BigQueryWriteClient +from .services.big_query_write import BigQueryWriteAsyncClient + +from .types.arrow import ArrowRecordBatch +from .types.arrow import ArrowSchema +from .types.arrow import ArrowSerializationOptions +from .types.avro import AvroRows +from .types.avro import AvroSchema +from .types.protobuf import ProtoRows +from .types.protobuf import ProtoSchema +from .types.storage import AppendRowsRequest +from .types.storage import AppendRowsResponse +from .types.storage import BatchCommitWriteStreamsRequest +from .types.storage import BatchCommitWriteStreamsResponse +from .types.storage import CreateReadSessionRequest +from .types.storage import CreateWriteStreamRequest +from .types.storage import FinalizeWriteStreamRequest +from .types.storage import FinalizeWriteStreamResponse +from .types.storage import FlushRowsRequest +from .types.storage import FlushRowsResponse +from .types.storage import GetWriteStreamRequest +from .types.storage import ReadRowsRequest +from .types.storage import ReadRowsResponse +from .types.storage import SplitReadStreamRequest +from .types.storage import SplitReadStreamResponse +from .types.storage import StorageError +from .types.storage import StreamStats +from .types.storage import ThrottleState +from .types.stream import ReadSession +from .types.stream import ReadStream +from .types.stream import WriteStream +from .types.stream import DataFormat +from .types.table import TableFieldSchema +from .types.table import TableSchema + +__all__ = ( + 'BigQueryReadAsyncClient', + 'BigQueryWriteAsyncClient', +'AppendRowsRequest', +'AppendRowsResponse', +'ArrowRecordBatch', +'ArrowSchema', +'ArrowSerializationOptions', +'AvroRows', +'AvroSchema', +'BatchCommitWriteStreamsRequest', +'BatchCommitWriteStreamsResponse', +'BigQueryReadClient', +'BigQueryWriteClient', +'CreateReadSessionRequest', +'CreateWriteStreamRequest', +'DataFormat', +'FinalizeWriteStreamRequest', +'FinalizeWriteStreamResponse', +'FlushRowsRequest', +'FlushRowsResponse', +'GetWriteStreamRequest', +'ProtoRows', +'ProtoSchema', +'ReadRowsRequest', +'ReadRowsResponse', +'ReadSession', +'ReadStream', +'SplitReadStreamRequest', +'SplitReadStreamResponse', +'StorageError', +'StreamStats', +'TableFieldSchema', +'TableSchema', +'ThrottleState', +'WriteStream', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json new file mode 100644 index 00000000..5aba90d6 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json @@ -0,0 +1,127 @@ + { + "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", + "language": "python", + "libraryPackage": "google.cloud.bigquery_storage_v1beta2", + "protoPackage": "google.cloud.bigquery.storage.v1beta2", + "schema": "1.0", + "services": { + "BigQueryRead": { + "clients": { + "grpc": { + "libraryClient": "BigQueryReadClient", + "rpcs": { + "CreateReadSession": { + "methods": [ + "create_read_session" + ] + }, + "ReadRows": { + "methods": [ + "read_rows" + ] + }, + "SplitReadStream": { + "methods": [ + "split_read_stream" + ] + } + } + }, + "grpc-async": { + "libraryClient": "BigQueryReadAsyncClient", + "rpcs": { + "CreateReadSession": { + "methods": [ + "create_read_session" + ] + }, + "ReadRows": { + "methods": [ + "read_rows" + ] + }, + "SplitReadStream": { + "methods": [ + "split_read_stream" + ] + } + } + } + } + }, + "BigQueryWrite": { + "clients": { + "grpc": { + "libraryClient": "BigQueryWriteClient", + "rpcs": { + "AppendRows": { + "methods": [ + "append_rows" + ] + }, + "BatchCommitWriteStreams": { + "methods": [ + "batch_commit_write_streams" + ] + }, + "CreateWriteStream": { + "methods": [ + "create_write_stream" + ] + }, + "FinalizeWriteStream": { + "methods": [ + "finalize_write_stream" + ] + }, + "FlushRows": { + "methods": [ + "flush_rows" + ] + }, + "GetWriteStream": { + "methods": [ + "get_write_stream" + ] + } + } + }, + "grpc-async": { + "libraryClient": "BigQueryWriteAsyncClient", + "rpcs": { + "AppendRows": { + "methods": [ + "append_rows" + ] + }, + "BatchCommitWriteStreams": { + "methods": [ + "batch_commit_write_streams" + ] + }, + "CreateWriteStream": { + "methods": [ + "create_write_stream" + ] + }, + "FinalizeWriteStream": { + "methods": [ + "finalize_write_stream" + ] + }, + "FlushRows": { + "methods": [ + "flush_rows" + ] + }, + "GetWriteStream": { + "methods": [ + "get_write_stream" + ] + } + } + } + } + } + } +} diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed new file mode 100644 index 00000000..e71b4749 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py new file mode 100644 index 00000000..4de65971 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py new file mode 100644 index 00000000..13be8ad1 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import BigQueryReadClient +from .async_client import BigQueryReadAsyncClient + +__all__ = ( + 'BigQueryReadClient', + 'BigQueryReadAsyncClient', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py new file mode 100644 index 00000000..881c717b --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py @@ -0,0 +1,476 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import functools +import re +from typing import Dict, AsyncIterable, Awaitable, Sequence, Tuple, Type, Union +import pkg_resources + +import google.api_core.client_options as ClientOptions # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import arrow +from google.cloud.bigquery_storage_v1beta2.types import avro +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.protobuf import timestamp_pb2 # type: ignore +from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport +from .client import BigQueryReadClient + + +class BigQueryReadAsyncClient: + """BigQuery Read API. + The Read API can be used to read data from BigQuery. + New code should use the v1 Read API going forward, if they don't + use Write API at the same time. + """ + + _client: BigQueryReadClient + + DEFAULT_ENDPOINT = BigQueryReadClient.DEFAULT_ENDPOINT + DEFAULT_MTLS_ENDPOINT = BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + + read_session_path = staticmethod(BigQueryReadClient.read_session_path) + parse_read_session_path = staticmethod(BigQueryReadClient.parse_read_session_path) + read_stream_path = staticmethod(BigQueryReadClient.read_stream_path) + parse_read_stream_path = staticmethod(BigQueryReadClient.parse_read_stream_path) + table_path = staticmethod(BigQueryReadClient.table_path) + parse_table_path = staticmethod(BigQueryReadClient.parse_table_path) + common_billing_account_path = staticmethod(BigQueryReadClient.common_billing_account_path) + parse_common_billing_account_path = staticmethod(BigQueryReadClient.parse_common_billing_account_path) + common_folder_path = staticmethod(BigQueryReadClient.common_folder_path) + parse_common_folder_path = staticmethod(BigQueryReadClient.parse_common_folder_path) + common_organization_path = staticmethod(BigQueryReadClient.common_organization_path) + parse_common_organization_path = staticmethod(BigQueryReadClient.parse_common_organization_path) + common_project_path = staticmethod(BigQueryReadClient.common_project_path) + parse_common_project_path = staticmethod(BigQueryReadClient.parse_common_project_path) + common_location_path = staticmethod(BigQueryReadClient.common_location_path) + parse_common_location_path = staticmethod(BigQueryReadClient.parse_common_location_path) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_info.__func__(BigQueryReadAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_file.__func__(BigQueryReadAsyncClient, filename, *args, **kwargs) # type: ignore + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryReadTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryReadTransport: The transport used by the client instance. + """ + return self._client.transport + + get_transport_class = functools.partial(type(BigQueryReadClient).get_transport_class, type(BigQueryReadClient)) + + def __init__(self, *, + credentials: ga_credentials.Credentials = None, + transport: Union[str, BigQueryReadTransport] = "grpc_asyncio", + client_options: ClientOptions = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query read client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Union[str, ~.BigQueryReadTransport]): The + transport to use. If set to None, a transport is chosen + automatically. + client_options (ClientOptions): Custom options for the client. It + won't take effect if a ``transport`` instance is provided. + (1) The ``api_endpoint`` property can be used to override the + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT + environment variable can also be used to override the endpoint: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client = BigQueryReadClient( + credentials=credentials, + transport=transport, + client_options=client_options, + client_info=client_info, + + ) + + async def create_read_session(self, + request: storage.CreateReadSessionRequest = None, + *, + parent: str = None, + read_session: stream.ReadSession = None, + max_stream_count: int = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.ReadSession: + r"""Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a push- + down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + Args: + request (:class:`google.cloud.bigquery_storage_v1beta2.types.CreateReadSessionRequest`): + The request object. Request message for + `CreateReadSession`. + parent (:class:`str`): + Required. The request project that owns the session, in + the form of ``projects/{project_id}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + read_session (:class:`google.cloud.bigquery_storage_v1beta2.types.ReadSession`): + Required. Session to be created. + This corresponds to the ``read_session`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + max_stream_count (:class:`int`): + Max initial number of streams. If + unset or zero, the server will provide a + value of streams so as to produce + reasonable throughput. Must be non- + negative. The number of streams may be + lower than the requested number, + depending on the amount parallelism that + is reasonable for the table. Error will + be returned if the max count is greater + than the current system max limit of + 1,000. + + Streams must be read starting from + offset 0. + + This corresponds to the ``max_stream_count`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.ReadSession: + Information about the ReadSession. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, read_session, max_stream_count]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + request = storage.CreateReadSessionRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if read_session is not None: + request.read_session = read_session + if max_stream_count is not None: + request.max_stream_count = max_stream_count + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.create_read_session, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_session.table", request.read_session.table), + )), + ) + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def read_rows(self, + request: storage.ReadRowsRequest = None, + *, + read_stream: str = None, + offset: int = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Awaitable[AsyncIterable[storage.ReadRowsResponse]]: + r"""Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Args: + request (:class:`google.cloud.bigquery_storage_v1beta2.types.ReadRowsRequest`): + The request object. Request message for `ReadRows`. + read_stream (:class:`str`): + Required. Stream to read rows from. + This corresponds to the ``read_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + offset (:class:`int`): + The offset requested must be less + than the last row read from Read. + Requesting a larger offset is undefined. + If not specified, start reading from + offset zero. + + This corresponds to the ``offset`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + AsyncIterable[google.cloud.bigquery_storage_v1beta2.types.ReadRowsResponse]: + Response from calling ReadRows may include row data, progress and + throttling information. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([read_stream, offset]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + request = storage.ReadRowsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if read_stream is not None: + request.read_stream = read_stream + if offset is not None: + request.offset = offset + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.read_rows, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_stream", request.read_stream), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def split_read_stream(self, + request: storage.SplitReadStreamRequest = None, + *, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.SplitReadStreamResponse: + r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Args: + request (:class:`google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamRequest`): + The request object. Request message for + `SplitReadStream`. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamResponse: + + """ + # Create or coerce a protobuf request object. + request = storage.SplitReadStreamRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.split_read_stream, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + + + + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google-cloud-bigquery-storage", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + + +__all__ = ( + "BigQueryReadAsyncClient", +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py new file mode 100644 index 00000000..8945a2f1 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py @@ -0,0 +1,666 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from distutils import util +import os +import re +from typing import Callable, Dict, Optional, Iterable, Sequence, Tuple, Type, Union +import pkg_resources + +from google.api_core import client_options as client_options_lib # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import arrow +from google.cloud.bigquery_storage_v1beta2.types import avro +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.protobuf import timestamp_pb2 # type: ignore +from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .transports.grpc import BigQueryReadGrpcTransport +from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport + + +class BigQueryReadClientMeta(type): + """Metaclass for the BigQueryRead client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] + _transport_registry["grpc"] = BigQueryReadGrpcTransport + _transport_registry["grpc_asyncio"] = BigQueryReadGrpcAsyncIOTransport + + def get_transport_class(cls, + label: str = None, + ) -> Type[BigQueryReadTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class BigQueryReadClient(metaclass=BigQueryReadClientMeta): + """BigQuery Read API. + The Read API can be used to read data from BigQuery. + New code should use the v1 Read API going forward, if they don't + use Write API at the same time. + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file( + filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryReadTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryReadTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def read_session_path(project: str,location: str,session: str,) -> str: + """Returns a fully-qualified read_session string.""" + return "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) + + @staticmethod + def parse_read_session_path(path: str) -> Dict[str,str]: + """Parses a read_session path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def read_stream_path(project: str,location: str,session: str,stream: str,) -> str: + """Returns a fully-qualified read_stream string.""" + return "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) + + @staticmethod + def parse_read_stream_path(path: str) -> Dict[str,str]: + """Parses a read_stream path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)/streams/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def table_path(project: str,dataset: str,table: str,) -> str: + """Returns a fully-qualified table string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + + @staticmethod + def parse_table_path(path: str) -> Dict[str,str]: + """Parses a table path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_billing_account_path(billing_account: str, ) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str,str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str, ) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder, ) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str,str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str, ) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization, ) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str,str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str, ) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project, ) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str,str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str, ) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format(project=project, location=location, ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str,str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Union[str, BigQueryReadTransport, None] = None, + client_options: Optional[client_options_lib.ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query read client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Union[str, BigQueryReadTransport]): The + transport to use. If set to None, a transport is chosen + automatically. + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. It won't take effect if a ``transport`` instance is provided. + (1) The ``api_endpoint`` property can be used to override the + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT + environment variable can also be used to override the endpoint: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + if isinstance(client_options, dict): + client_options = client_options_lib.from_dict(client_options) + if client_options is None: + client_options = client_options_lib.ClientOptions() + + # Create SSL credentials for mutual TLS if needed. + use_client_cert = bool(util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"))) + + client_cert_source_func = None + is_mtls = False + if use_client_cert: + if client_options.client_cert_source: + is_mtls = True + client_cert_source_func = client_options.client_cert_source + else: + is_mtls = mtls.has_default_client_cert_source() + if is_mtls: + client_cert_source_func = mtls.default_client_cert_source() + else: + client_cert_source_func = None + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + else: + use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_mtls_env == "never": + api_endpoint = self.DEFAULT_ENDPOINT + elif use_mtls_env == "always": + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + elif use_mtls_env == "auto": + if is_mtls: + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = self.DEFAULT_ENDPOINT + else: + raise MutualTLSChannelError( + "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " + "values: never, auto, always" + ) + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + if isinstance(transport, BigQueryReadTransport): + # transport is a BigQueryReadTransport instance. + if credentials or client_options.credentials_file: + raise ValueError("When providing a transport instance, " + "provide its credentials directly.") + if client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = transport + else: + Transport = type(self).get_transport_class(transport) + self._transport = Transport( + credentials=credentials, + credentials_file=client_options.credentials_file, + host=api_endpoint, + scopes=client_options.scopes, + client_cert_source_for_mtls=client_cert_source_func, + quota_project_id=client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=( + Transport == type(self).get_transport_class("grpc") + or Transport == type(self).get_transport_class("grpc_asyncio") + ), + ) + + def create_read_session(self, + request: storage.CreateReadSessionRequest = None, + *, + parent: str = None, + read_session: stream.ReadSession = None, + max_stream_count: int = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.ReadSession: + r"""Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a push- + down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + Args: + request (google.cloud.bigquery_storage_v1beta2.types.CreateReadSessionRequest): + The request object. Request message for + `CreateReadSession`. + parent (str): + Required. The request project that owns the session, in + the form of ``projects/{project_id}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + read_session (google.cloud.bigquery_storage_v1beta2.types.ReadSession): + Required. Session to be created. + This corresponds to the ``read_session`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + max_stream_count (int): + Max initial number of streams. If + unset or zero, the server will provide a + value of streams so as to produce + reasonable throughput. Must be non- + negative. The number of streams may be + lower than the requested number, + depending on the amount parallelism that + is reasonable for the table. Error will + be returned if the max count is greater + than the current system max limit of + 1,000. + + Streams must be read starting from + offset 0. + + This corresponds to the ``max_stream_count`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.ReadSession: + Information about the ReadSession. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, read_session, max_stream_count]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # Minor optimization to avoid making a copy if the user passes + # in a storage.CreateReadSessionRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.CreateReadSessionRequest): + request = storage.CreateReadSessionRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if read_session is not None: + request.read_session = read_session + if max_stream_count is not None: + request.max_stream_count = max_stream_count + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.create_read_session] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_session.table", request.read_session.table), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def read_rows(self, + request: storage.ReadRowsRequest = None, + *, + read_stream: str = None, + offset: int = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Iterable[storage.ReadRowsResponse]: + r"""Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Args: + request (google.cloud.bigquery_storage_v1beta2.types.ReadRowsRequest): + The request object. Request message for `ReadRows`. + read_stream (str): + Required. Stream to read rows from. + This corresponds to the ``read_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + offset (int): + The offset requested must be less + than the last row read from Read. + Requesting a larger offset is undefined. + If not specified, start reading from + offset zero. + + This corresponds to the ``offset`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + Iterable[google.cloud.bigquery_storage_v1beta2.types.ReadRowsResponse]: + Response from calling ReadRows may include row data, progress and + throttling information. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([read_stream, offset]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # Minor optimization to avoid making a copy if the user passes + # in a storage.ReadRowsRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.ReadRowsRequest): + request = storage.ReadRowsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if read_stream is not None: + request.read_stream = read_stream + if offset is not None: + request.offset = offset + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.read_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_stream", request.read_stream), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def split_read_stream(self, + request: storage.SplitReadStreamRequest = None, + *, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.SplitReadStreamResponse: + r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Args: + request (google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamRequest): + The request object. Request message for + `SplitReadStream`. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamResponse: + + """ + # Create or coerce a protobuf request object. + # Minor optimization to avoid making a copy if the user passes + # in a storage.SplitReadStreamRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.SplitReadStreamRequest): + request = storage.SplitReadStreamRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.split_read_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + + + + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google-cloud-bigquery-storage", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + + +__all__ = ( + "BigQueryReadClient", +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py new file mode 100644 index 00000000..f51ba420 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import BigQueryReadTransport +from .grpc import BigQueryReadGrpcTransport +from .grpc_asyncio import BigQueryReadGrpcAsyncIOTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] +_transport_registry['grpc'] = BigQueryReadGrpcTransport +_transport_registry['grpc_asyncio'] = BigQueryReadGrpcAsyncIOTransport + +__all__ = ( + 'BigQueryReadTransport', + 'BigQueryReadGrpcTransport', + 'BigQueryReadGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py new file mode 100644 index 00000000..bcedfdea --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py @@ -0,0 +1,219 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union +import packaging.version +import pkg_resources + +import google.auth # type: ignore +import google.api_core # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + 'google-cloud-bigquery-storage', + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + +try: + # google.auth.__version__ was added in 1.26.0 + _GOOGLE_AUTH_VERSION = google.auth.__version__ +except AttributeError: + try: # try pkg_resources if it is available + _GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version + except pkg_resources.DistributionNotFound: # pragma: NO COVER + _GOOGLE_AUTH_VERSION = None + + +class BigQueryReadTransport(abc.ABC): + """Abstract transport class for BigQueryRead.""" + + AUTH_SCOPES = ( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', + ) + + DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' + def __init__( + self, *, + host: str = DEFAULT_HOST, + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ':' not in host: + host += ':443' + self._host = host + + scopes_kwargs = self._get_scopes_kwargs(self._host, scopes) + + # Save the scopes. + self._scopes = scopes + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, + **scopes_kwargs, + quota_project_id=quota_project_id + ) + + elif credentials is None: + credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) + + # If the credentials is service account credentials, then always try to use self signed JWT. + if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + # TODO(busunkim): This method is in the base transport + # to avoid duplicating code across the transport classes. These functions + # should be deleted once the minimum required versions of google-auth is increased. + + # TODO: Remove this function once google-auth >= 1.25.0 is required + @classmethod + def _get_scopes_kwargs(cls, host: str, scopes: Optional[Sequence[str]]) -> Dict[str, Optional[Sequence[str]]]: + """Returns scopes kwargs to pass to google-auth methods depending on the google-auth version""" + + scopes_kwargs = {} + + if _GOOGLE_AUTH_VERSION and ( + packaging.version.parse(_GOOGLE_AUTH_VERSION) + >= packaging.version.parse("1.25.0") + ): + scopes_kwargs = {"scopes": scopes, "default_scopes": cls.AUTH_SCOPES} + else: + scopes_kwargs = {"scopes": scopes or cls.AUTH_SCOPES} + + return scopes_kwargs + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.create_read_session: gapic_v1.method.wrap_method( + self.create_read_session, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.read_rows: gapic_v1.method.wrap_method( + self.read_rows, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.split_read_stream: gapic_v1.method.wrap_method( + self.split_read_stream, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + Union[ + stream.ReadSession, + Awaitable[stream.ReadSession] + ]]: + raise NotImplementedError() + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + Union[ + storage.ReadRowsResponse, + Awaitable[storage.ReadRowsResponse] + ]]: + raise NotImplementedError() + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + Union[ + storage.SplitReadStreamResponse, + Awaitable[storage.SplitReadStreamResponse] + ]]: + raise NotImplementedError() + + +__all__ = ( + 'BigQueryReadTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py new file mode 100644 index 00000000..78987836 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py @@ -0,0 +1,349 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import grpc_helpers # type: ignore +from google.api_core import gapic_v1 # type: ignore +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO + + +class BigQueryReadGrpcTransport(BigQueryReadTransport): + """gRPC backend transport for BigQueryRead. + + BigQuery Read API. + The Read API can be used to read data from BigQuery. + New code should use the v1 Read API going forward, if they don't + use Write API at the same time. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + _stubs: Dict[str, Callable] + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: str = None, + scopes: Sequence[str] = None, + channel: grpc.Channel = None, + api_mtls_endpoint: str = None, + client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if ``channel`` is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if ``channel`` is provided. + channel (Optional[grpc.Channel]): A ``Channel`` instance through + which to make calls. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or applicatin default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if channel: + # Ignore credentials if a channel was passed. + credentials = False + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + ) + + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: str = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> grpc.Channel: + """Create and return a gRPC channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + grpc.Channel: A gRPC channel object. + + Raises: + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + + return grpc_helpers.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + @property + def grpc_channel(self) -> grpc.Channel: + """Return the channel designed to connect to this service. + """ + return self._grpc_channel + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + stream.ReadSession]: + r"""Return a callable for the create read session method over gRPC. + + Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a push- + down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + Returns: + Callable[[~.CreateReadSessionRequest], + ~.ReadSession]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_read_session' not in self._stubs: + self._stubs['create_read_session'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/CreateReadSession', + request_serializer=storage.CreateReadSessionRequest.serialize, + response_deserializer=stream.ReadSession.deserialize, + ) + return self._stubs['create_read_session'] + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + storage.ReadRowsResponse]: + r"""Return a callable for the read rows method over gRPC. + + Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Returns: + Callable[[~.ReadRowsRequest], + ~.ReadRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'read_rows' not in self._stubs: + self._stubs['read_rows'] = self.grpc_channel.unary_stream( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/ReadRows', + request_serializer=storage.ReadRowsRequest.serialize, + response_deserializer=storage.ReadRowsResponse.deserialize, + ) + return self._stubs['read_rows'] + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + storage.SplitReadStreamResponse]: + r"""Return a callable for the split read stream method over gRPC. + + Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Returns: + Callable[[~.SplitReadStreamRequest], + ~.SplitReadStreamResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'split_read_stream' not in self._stubs: + self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/SplitReadStream', + request_serializer=storage.SplitReadStreamRequest.serialize, + response_deserializer=storage.SplitReadStreamResponse.deserialize, + ) + return self._stubs['split_read_stream'] + + +__all__ = ( + 'BigQueryReadGrpcTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py new file mode 100644 index 00000000..25c6013b --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import gapic_v1 # type: ignore +from google.api_core import grpc_helpers_async # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +import packaging.version + +import grpc # type: ignore +from grpc.experimental import aio # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .grpc import BigQueryReadGrpcTransport + + +class BigQueryReadGrpcAsyncIOTransport(BigQueryReadTransport): + """gRPC AsyncIO backend transport for BigQueryRead. + + BigQuery Read API. + The Read API can be used to read data from BigQuery. + New code should use the v1 Read API going forward, if they don't + use Write API at the same time. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + + _grpc_channel: aio.Channel + _stubs: Dict[str, Callable] = {} + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> aio.Channel: + """Create and return a gRPC AsyncIO channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + aio.Channel: A gRPC AsyncIO channel object. + """ + + return grpc_helpers_async.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: aio.Channel = None, + api_mtls_endpoint: str = None, + client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, + quota_project_id=None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if ``channel`` is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + channel (Optional[aio.Channel]): A ``Channel`` instance through + which to make calls. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or applicatin default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if channel: + # Ignore credentials if a channel was passed. + credentials = False + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + ) + + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @property + def grpc_channel(self) -> aio.Channel: + """Create the channel designed to connect to this service. + + This property caches on the instance; repeated calls return + the same channel. + """ + # Return the channel from cache. + return self._grpc_channel + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + Awaitable[stream.ReadSession]]: + r"""Return a callable for the create read session method over gRPC. + + Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a push- + down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + Returns: + Callable[[~.CreateReadSessionRequest], + Awaitable[~.ReadSession]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_read_session' not in self._stubs: + self._stubs['create_read_session'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/CreateReadSession', + request_serializer=storage.CreateReadSessionRequest.serialize, + response_deserializer=stream.ReadSession.deserialize, + ) + return self._stubs['create_read_session'] + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + Awaitable[storage.ReadRowsResponse]]: + r"""Return a callable for the read rows method over gRPC. + + Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Returns: + Callable[[~.ReadRowsRequest], + Awaitable[~.ReadRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'read_rows' not in self._stubs: + self._stubs['read_rows'] = self.grpc_channel.unary_stream( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/ReadRows', + request_serializer=storage.ReadRowsRequest.serialize, + response_deserializer=storage.ReadRowsResponse.deserialize, + ) + return self._stubs['read_rows'] + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + Awaitable[storage.SplitReadStreamResponse]]: + r"""Return a callable for the split read stream method over gRPC. + + Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Returns: + Callable[[~.SplitReadStreamRequest], + Awaitable[~.SplitReadStreamResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'split_read_stream' not in self._stubs: + self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/SplitReadStream', + request_serializer=storage.SplitReadStreamRequest.serialize, + response_deserializer=storage.SplitReadStreamResponse.deserialize, + ) + return self._stubs['split_read_stream'] + + +__all__ = ( + 'BigQueryReadGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py new file mode 100644 index 00000000..753336d9 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import BigQueryWriteClient +from .async_client import BigQueryWriteAsyncClient + +__all__ = ( + 'BigQueryWriteClient', + 'BigQueryWriteAsyncClient', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py new file mode 100644 index 00000000..deb26b14 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py @@ -0,0 +1,678 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import functools +import re +from typing import Dict, AsyncIterable, Awaitable, AsyncIterator, Sequence, Tuple, Type, Union +import pkg_resources + +import google.api_core.client_options as ClientOptions # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.cloud.bigquery_storage_v1beta2.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +from .transports.base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .transports.grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport +from .client import BigQueryWriteClient + + +class BigQueryWriteAsyncClient: + """BigQuery Write API. + The Write API can be used to write data to BigQuery. + """ + + _client: BigQueryWriteClient + + DEFAULT_ENDPOINT = BigQueryWriteClient.DEFAULT_ENDPOINT + DEFAULT_MTLS_ENDPOINT = BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + + table_path = staticmethod(BigQueryWriteClient.table_path) + parse_table_path = staticmethod(BigQueryWriteClient.parse_table_path) + write_stream_path = staticmethod(BigQueryWriteClient.write_stream_path) + parse_write_stream_path = staticmethod(BigQueryWriteClient.parse_write_stream_path) + common_billing_account_path = staticmethod(BigQueryWriteClient.common_billing_account_path) + parse_common_billing_account_path = staticmethod(BigQueryWriteClient.parse_common_billing_account_path) + common_folder_path = staticmethod(BigQueryWriteClient.common_folder_path) + parse_common_folder_path = staticmethod(BigQueryWriteClient.parse_common_folder_path) + common_organization_path = staticmethod(BigQueryWriteClient.common_organization_path) + parse_common_organization_path = staticmethod(BigQueryWriteClient.parse_common_organization_path) + common_project_path = staticmethod(BigQueryWriteClient.common_project_path) + parse_common_project_path = staticmethod(BigQueryWriteClient.parse_common_project_path) + common_location_path = staticmethod(BigQueryWriteClient.common_location_path) + parse_common_location_path = staticmethod(BigQueryWriteClient.parse_common_location_path) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteAsyncClient: The constructed client. + """ + return BigQueryWriteClient.from_service_account_info.__func__(BigQueryWriteAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteAsyncClient: The constructed client. + """ + return BigQueryWriteClient.from_service_account_file.__func__(BigQueryWriteAsyncClient, filename, *args, **kwargs) # type: ignore + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryWriteTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryWriteTransport: The transport used by the client instance. + """ + return self._client.transport + + get_transport_class = functools.partial(type(BigQueryWriteClient).get_transport_class, type(BigQueryWriteClient)) + + def __init__(self, *, + credentials: ga_credentials.Credentials = None, + transport: Union[str, BigQueryWriteTransport] = "grpc_asyncio", + client_options: ClientOptions = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query write client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Union[str, ~.BigQueryWriteTransport]): The + transport to use. If set to None, a transport is chosen + automatically. + client_options (ClientOptions): Custom options for the client. It + won't take effect if a ``transport`` instance is provided. + (1) The ``api_endpoint`` property can be used to override the + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT + environment variable can also be used to override the endpoint: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client = BigQueryWriteClient( + credentials=credentials, + transport=transport, + client_options=client_options, + client_info=client_info, + + ) + + async def create_write_stream(self, + request: storage.CreateWriteStreamRequest = None, + *, + parent: str = None, + write_stream: stream.WriteStream = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Creates a write stream to the given table. Additionally, every + table has a special COMMITTED stream named '_default' to which + data can be written. This stream doesn't need to be created + using CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Args: + request (:class:`google.cloud.bigquery_storage_v1beta2.types.CreateWriteStreamRequest`): + The request object. Request message for + `CreateWriteStream`. + parent (:class:`str`): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + write_stream (:class:`google.cloud.bigquery_storage_v1beta2.types.WriteStream`): + Required. Stream to be created. + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, write_stream]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + request = storage.CreateWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.create_write_stream, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def append_rows(self, + requests: AsyncIterator[storage.AppendRowsRequest] = None, + *, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Awaitable[AsyncIterable[storage.AppendRowsResponse]]: + r"""Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provids an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC stream. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains the offset at which the append happened. + Responses are received in the same order in which requests are + sent. There will be one response for each successful request. If + the ``offset`` is not set in response, it means append didn't + happen due to some errors. If one request fails, all the + subsequent requests will also fail until a success request is + made again. + + If the stream is of ``PENDING`` type, data will only be + available for read operations after the stream is committed. + + Args: + requests (AsyncIterator[`google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest`]): + The request object AsyncIterator. Request message for `AppendRows`. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + AsyncIterable[google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse]: + Response message for AppendRows. + """ + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.append_rows, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + )), + ) + + # Send the request. + response = rpc( + requests, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def get_write_stream(self, + request: storage.GetWriteStreamRequest = None, + *, + name: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Gets a write stream. + + Args: + request (:class:`google.cloud.bigquery_storage_v1beta2.types.GetWriteStreamRequest`): + The request object. Request message for + `GetWriteStreamRequest`. + name (:class:`str`): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + request = storage.GetWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.get_write_stream, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def finalize_write_stream(self, + request: storage.FinalizeWriteStreamRequest = None, + *, + name: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FinalizeWriteStreamResponse: + r"""Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Args: + request (:class:`google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamRequest`): + The request object. Request message for invoking + `FinalizeWriteStream`. + name (:class:`str`): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamResponse: + Response message for FinalizeWriteStream. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + request = storage.FinalizeWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.finalize_write_stream, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def batch_commit_write_streams(self, + request: storage.BatchCommitWriteStreamsRequest = None, + *, + parent: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.BatchCommitWriteStreamsResponse: + r"""Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. Streams must be finalized before + commit and cannot be committed multiple times. Once a stream is + committed, data in the stream becomes available for read + operations. + + Args: + request (:class:`google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsRequest`): + The request object. Request message for + `BatchCommitWriteStreams`. + parent (:class:`str`): + Required. Parent table that all the streams should + belong to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsResponse: + Response message for BatchCommitWriteStreams. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + request = storage.BatchCommitWriteStreamsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.batch_commit_write_streams, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def flush_rows(self, + request: storage.FlushRowsRequest = None, + *, + write_stream: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FlushRowsResponse: + r"""Flushes rows to a BUFFERED stream. If users are appending rows + to BUFFERED stream, flush operation is required in order for the + rows to become available for reading. A Flush operation flushes + up to any previously flushed offset in a BUFFERED stream, to the + offset specified in the request. Flush is not supported on the + \_default stream, since it is not BUFFERED. + + Args: + request (:class:`google.cloud.bigquery_storage_v1beta2.types.FlushRowsRequest`): + The request object. Request message for `FlushRows`. + write_stream (:class:`str`): + Required. The stream that is the + target of the flush operation. + + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.FlushRowsResponse: + Respond message for FlushRows. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([write_stream]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + request = storage.FlushRowsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.flush_rows, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("write_stream", request.write_stream), + )), + ) + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + + + + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google-cloud-bigquery-storage", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + + +__all__ = ( + "BigQueryWriteAsyncClient", +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py new file mode 100644 index 00000000..74329a11 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py @@ -0,0 +1,831 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from distutils import util +import os +import re +from typing import Callable, Dict, Optional, Iterable, Iterator, Sequence, Tuple, Type, Union +import pkg_resources + +from google.api_core import client_options as client_options_lib # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.cloud.bigquery_storage_v1beta2.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +from .transports.base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .transports.grpc import BigQueryWriteGrpcTransport +from .transports.grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport + + +class BigQueryWriteClientMeta(type): + """Metaclass for the BigQueryWrite client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryWriteTransport]] + _transport_registry["grpc"] = BigQueryWriteGrpcTransport + _transport_registry["grpc_asyncio"] = BigQueryWriteGrpcAsyncIOTransport + + def get_transport_class(cls, + label: str = None, + ) -> Type[BigQueryWriteTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class BigQueryWriteClient(metaclass=BigQueryWriteClientMeta): + """BigQuery Write API. + The Write API can be used to write data to BigQuery. + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file( + filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryWriteTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryWriteTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def table_path(project: str,dataset: str,table: str,) -> str: + """Returns a fully-qualified table string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + + @staticmethod + def parse_table_path(path: str) -> Dict[str,str]: + """Parses a table path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def write_stream_path(project: str,dataset: str,table: str,stream: str,) -> str: + """Returns a fully-qualified write_stream string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}".format(project=project, dataset=dataset, table=table, stream=stream, ) + + @staticmethod + def parse_write_stream_path(path: str) -> Dict[str,str]: + """Parses a write_stream path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)/streams/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_billing_account_path(billing_account: str, ) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str,str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str, ) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder, ) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str,str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str, ) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization, ) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str,str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str, ) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project, ) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str,str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str, ) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format(project=project, location=location, ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str,str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Union[str, BigQueryWriteTransport, None] = None, + client_options: Optional[client_options_lib.ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query write client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Union[str, BigQueryWriteTransport]): The + transport to use. If set to None, a transport is chosen + automatically. + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. It won't take effect if a ``transport`` instance is provided. + (1) The ``api_endpoint`` property can be used to override the + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT + environment variable can also be used to override the endpoint: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + if isinstance(client_options, dict): + client_options = client_options_lib.from_dict(client_options) + if client_options is None: + client_options = client_options_lib.ClientOptions() + + # Create SSL credentials for mutual TLS if needed. + use_client_cert = bool(util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"))) + + client_cert_source_func = None + is_mtls = False + if use_client_cert: + if client_options.client_cert_source: + is_mtls = True + client_cert_source_func = client_options.client_cert_source + else: + is_mtls = mtls.has_default_client_cert_source() + if is_mtls: + client_cert_source_func = mtls.default_client_cert_source() + else: + client_cert_source_func = None + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + else: + use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_mtls_env == "never": + api_endpoint = self.DEFAULT_ENDPOINT + elif use_mtls_env == "always": + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + elif use_mtls_env == "auto": + if is_mtls: + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = self.DEFAULT_ENDPOINT + else: + raise MutualTLSChannelError( + "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " + "values: never, auto, always" + ) + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + if isinstance(transport, BigQueryWriteTransport): + # transport is a BigQueryWriteTransport instance. + if credentials or client_options.credentials_file: + raise ValueError("When providing a transport instance, " + "provide its credentials directly.") + if client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = transport + else: + Transport = type(self).get_transport_class(transport) + self._transport = Transport( + credentials=credentials, + credentials_file=client_options.credentials_file, + host=api_endpoint, + scopes=client_options.scopes, + client_cert_source_for_mtls=client_cert_source_func, + quota_project_id=client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=( + Transport == type(self).get_transport_class("grpc") + or Transport == type(self).get_transport_class("grpc_asyncio") + ), + ) + + def create_write_stream(self, + request: storage.CreateWriteStreamRequest = None, + *, + parent: str = None, + write_stream: stream.WriteStream = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Creates a write stream to the given table. Additionally, every + table has a special COMMITTED stream named '_default' to which + data can be written. This stream doesn't need to be created + using CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Args: + request (google.cloud.bigquery_storage_v1beta2.types.CreateWriteStreamRequest): + The request object. Request message for + `CreateWriteStream`. + parent (str): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + write_stream (google.cloud.bigquery_storage_v1beta2.types.WriteStream): + Required. Stream to be created. + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, write_stream]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # Minor optimization to avoid making a copy if the user passes + # in a storage.CreateWriteStreamRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.CreateWriteStreamRequest): + request = storage.CreateWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.create_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def append_rows(self, + requests: Iterator[storage.AppendRowsRequest] = None, + *, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Iterable[storage.AppendRowsResponse]: + r"""Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provids an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC stream. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains the offset at which the append happened. + Responses are received in the same order in which requests are + sent. There will be one response for each successful request. If + the ``offset`` is not set in response, it means append didn't + happen due to some errors. If one request fails, all the + subsequent requests will also fail until a success request is + made again. + + If the stream is of ``PENDING`` type, data will only be + available for read operations after the stream is committed. + + Args: + requests (Iterator[google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest]): + The request object iterator. Request message for `AppendRows`. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + Iterable[google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse]: + Response message for AppendRows. + """ + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.append_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + )), + ) + + # Send the request. + response = rpc( + requests, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def get_write_stream(self, + request: storage.GetWriteStreamRequest = None, + *, + name: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Gets a write stream. + + Args: + request (google.cloud.bigquery_storage_v1beta2.types.GetWriteStreamRequest): + The request object. Request message for + `GetWriteStreamRequest`. + name (str): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # Minor optimization to avoid making a copy if the user passes + # in a storage.GetWriteStreamRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.GetWriteStreamRequest): + request = storage.GetWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.get_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def finalize_write_stream(self, + request: storage.FinalizeWriteStreamRequest = None, + *, + name: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FinalizeWriteStreamResponse: + r"""Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Args: + request (google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamRequest): + The request object. Request message for invoking + `FinalizeWriteStream`. + name (str): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamResponse: + Response message for FinalizeWriteStream. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # Minor optimization to avoid making a copy if the user passes + # in a storage.FinalizeWriteStreamRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.FinalizeWriteStreamRequest): + request = storage.FinalizeWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.finalize_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def batch_commit_write_streams(self, + request: storage.BatchCommitWriteStreamsRequest = None, + *, + parent: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.BatchCommitWriteStreamsResponse: + r"""Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. Streams must be finalized before + commit and cannot be committed multiple times. Once a stream is + committed, data in the stream becomes available for read + operations. + + Args: + request (google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsRequest): + The request object. Request message for + `BatchCommitWriteStreams`. + parent (str): + Required. Parent table that all the streams should + belong to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsResponse: + Response message for BatchCommitWriteStreams. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # Minor optimization to avoid making a copy if the user passes + # in a storage.BatchCommitWriteStreamsRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.BatchCommitWriteStreamsRequest): + request = storage.BatchCommitWriteStreamsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.batch_commit_write_streams] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def flush_rows(self, + request: storage.FlushRowsRequest = None, + *, + write_stream: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FlushRowsResponse: + r"""Flushes rows to a BUFFERED stream. If users are appending rows + to BUFFERED stream, flush operation is required in order for the + rows to become available for reading. A Flush operation flushes + up to any previously flushed offset in a BUFFERED stream, to the + offset specified in the request. Flush is not supported on the + \_default stream, since it is not BUFFERED. + + Args: + request (google.cloud.bigquery_storage_v1beta2.types.FlushRowsRequest): + The request object. Request message for `FlushRows`. + write_stream (str): + Required. The stream that is the + target of the flush operation. + + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.FlushRowsResponse: + Respond message for FlushRows. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([write_stream]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # Minor optimization to avoid making a copy if the user passes + # in a storage.FlushRowsRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.FlushRowsRequest): + request = storage.FlushRowsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.flush_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("write_stream", request.write_stream), + )), + ) + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + + + + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google-cloud-bigquery-storage", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + + +__all__ = ( + "BigQueryWriteClient", +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py new file mode 100644 index 00000000..bf916a92 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import BigQueryWriteTransport +from .grpc import BigQueryWriteGrpcTransport +from .grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryWriteTransport]] +_transport_registry['grpc'] = BigQueryWriteGrpcTransport +_transport_registry['grpc_asyncio'] = BigQueryWriteGrpcAsyncIOTransport + +__all__ = ( + 'BigQueryWriteTransport', + 'BigQueryWriteGrpcTransport', + 'BigQueryWriteGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py new file mode 100644 index 00000000..cce7ef3e --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py @@ -0,0 +1,284 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union +import packaging.version +import pkg_resources + +import google.auth # type: ignore +import google.api_core # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + 'google-cloud-bigquery-storage', + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + +try: + # google.auth.__version__ was added in 1.26.0 + _GOOGLE_AUTH_VERSION = google.auth.__version__ +except AttributeError: + try: # try pkg_resources if it is available + _GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version + except pkg_resources.DistributionNotFound: # pragma: NO COVER + _GOOGLE_AUTH_VERSION = None + + +class BigQueryWriteTransport(abc.ABC): + """Abstract transport class for BigQueryWrite.""" + + AUTH_SCOPES = ( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', + ) + + DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' + def __init__( + self, *, + host: str = DEFAULT_HOST, + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ':' not in host: + host += ':443' + self._host = host + + scopes_kwargs = self._get_scopes_kwargs(self._host, scopes) + + # Save the scopes. + self._scopes = scopes + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, + **scopes_kwargs, + quota_project_id=quota_project_id + ) + + elif credentials is None: + credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) + + # If the credentials is service account credentials, then always try to use self signed JWT. + if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + # TODO(busunkim): This method is in the base transport + # to avoid duplicating code across the transport classes. These functions + # should be deleted once the minimum required versions of google-auth is increased. + + # TODO: Remove this function once google-auth >= 1.25.0 is required + @classmethod + def _get_scopes_kwargs(cls, host: str, scopes: Optional[Sequence[str]]) -> Dict[str, Optional[Sequence[str]]]: + """Returns scopes kwargs to pass to google-auth methods depending on the google-auth version""" + + scopes_kwargs = {} + + if _GOOGLE_AUTH_VERSION and ( + packaging.version.parse(_GOOGLE_AUTH_VERSION) + >= packaging.version.parse("1.25.0") + ): + scopes_kwargs = {"scopes": scopes, "default_scopes": cls.AUTH_SCOPES} + else: + scopes_kwargs = {"scopes": scopes or cls.AUTH_SCOPES} + + return scopes_kwargs + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.create_write_stream: gapic_v1.method.wrap_method( + self.create_write_stream, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.append_rows: gapic_v1.method.wrap_method( + self.append_rows, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.get_write_stream: gapic_v1.method.wrap_method( + self.get_write_stream, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.finalize_write_stream: gapic_v1.method.wrap_method( + self.finalize_write_stream, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.batch_commit_write_streams: gapic_v1.method.wrap_method( + self.batch_commit_write_streams, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.flush_rows: gapic_v1.method.wrap_method( + self.flush_rows, + default_retry=retries.Retry( +initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + @property + def create_write_stream(self) -> Callable[ + [storage.CreateWriteStreamRequest], + Union[ + stream.WriteStream, + Awaitable[stream.WriteStream] + ]]: + raise NotImplementedError() + + @property + def append_rows(self) -> Callable[ + [storage.AppendRowsRequest], + Union[ + storage.AppendRowsResponse, + Awaitable[storage.AppendRowsResponse] + ]]: + raise NotImplementedError() + + @property + def get_write_stream(self) -> Callable[ + [storage.GetWriteStreamRequest], + Union[ + stream.WriteStream, + Awaitable[stream.WriteStream] + ]]: + raise NotImplementedError() + + @property + def finalize_write_stream(self) -> Callable[ + [storage.FinalizeWriteStreamRequest], + Union[ + storage.FinalizeWriteStreamResponse, + Awaitable[storage.FinalizeWriteStreamResponse] + ]]: + raise NotImplementedError() + + @property + def batch_commit_write_streams(self) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + Union[ + storage.BatchCommitWriteStreamsResponse, + Awaitable[storage.BatchCommitWriteStreamsResponse] + ]]: + raise NotImplementedError() + + @property + def flush_rows(self) -> Callable[ + [storage.FlushRowsRequest], + Union[ + storage.FlushRowsResponse, + Awaitable[storage.FlushRowsResponse] + ]]: + raise NotImplementedError() + + +__all__ = ( + 'BigQueryWriteTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py new file mode 100644 index 00000000..8fa44995 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py @@ -0,0 +1,420 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import grpc_helpers # type: ignore +from google.api_core import gapic_v1 # type: ignore +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from .base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO + + +class BigQueryWriteGrpcTransport(BigQueryWriteTransport): + """gRPC backend transport for BigQueryWrite. + + BigQuery Write API. + The Write API can be used to write data to BigQuery. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + _stubs: Dict[str, Callable] + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: str = None, + scopes: Sequence[str] = None, + channel: grpc.Channel = None, + api_mtls_endpoint: str = None, + client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if ``channel`` is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if ``channel`` is provided. + channel (Optional[grpc.Channel]): A ``Channel`` instance through + which to make calls. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or applicatin default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if channel: + # Ignore credentials if a channel was passed. + credentials = False + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + ) + + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: str = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> grpc.Channel: + """Create and return a gRPC channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + grpc.Channel: A gRPC channel object. + + Raises: + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + + return grpc_helpers.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + @property + def grpc_channel(self) -> grpc.Channel: + """Return the channel designed to connect to this service. + """ + return self._grpc_channel + + @property + def create_write_stream(self) -> Callable[ + [storage.CreateWriteStreamRequest], + stream.WriteStream]: + r"""Return a callable for the create write stream method over gRPC. + + Creates a write stream to the given table. Additionally, every + table has a special COMMITTED stream named '_default' to which + data can be written. This stream doesn't need to be created + using CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Returns: + Callable[[~.CreateWriteStreamRequest], + ~.WriteStream]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_write_stream' not in self._stubs: + self._stubs['create_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/CreateWriteStream', + request_serializer=storage.CreateWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['create_write_stream'] + + @property + def append_rows(self) -> Callable[ + [storage.AppendRowsRequest], + storage.AppendRowsResponse]: + r"""Return a callable for the append rows method over gRPC. + + Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provids an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC stream. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains the offset at which the append happened. + Responses are received in the same order in which requests are + sent. There will be one response for each successful request. If + the ``offset`` is not set in response, it means append didn't + happen due to some errors. If one request fails, all the + subsequent requests will also fail until a success request is + made again. + + If the stream is of ``PENDING`` type, data will only be + available for read operations after the stream is committed. + + Returns: + Callable[[~.AppendRowsRequest], + ~.AppendRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'append_rows' not in self._stubs: + self._stubs['append_rows'] = self.grpc_channel.stream_stream( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/AppendRows', + request_serializer=storage.AppendRowsRequest.serialize, + response_deserializer=storage.AppendRowsResponse.deserialize, + ) + return self._stubs['append_rows'] + + @property + def get_write_stream(self) -> Callable[ + [storage.GetWriteStreamRequest], + stream.WriteStream]: + r"""Return a callable for the get write stream method over gRPC. + + Gets a write stream. + + Returns: + Callable[[~.GetWriteStreamRequest], + ~.WriteStream]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'get_write_stream' not in self._stubs: + self._stubs['get_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/GetWriteStream', + request_serializer=storage.GetWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['get_write_stream'] + + @property + def finalize_write_stream(self) -> Callable[ + [storage.FinalizeWriteStreamRequest], + storage.FinalizeWriteStreamResponse]: + r"""Return a callable for the finalize write stream method over gRPC. + + Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Returns: + Callable[[~.FinalizeWriteStreamRequest], + ~.FinalizeWriteStreamResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'finalize_write_stream' not in self._stubs: + self._stubs['finalize_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FinalizeWriteStream', + request_serializer=storage.FinalizeWriteStreamRequest.serialize, + response_deserializer=storage.FinalizeWriteStreamResponse.deserialize, + ) + return self._stubs['finalize_write_stream'] + + @property + def batch_commit_write_streams(self) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + storage.BatchCommitWriteStreamsResponse]: + r"""Return a callable for the batch commit write streams method over gRPC. + + Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. Streams must be finalized before + commit and cannot be committed multiple times. Once a stream is + committed, data in the stream becomes available for read + operations. + + Returns: + Callable[[~.BatchCommitWriteStreamsRequest], + ~.BatchCommitWriteStreamsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_commit_write_streams' not in self._stubs: + self._stubs['batch_commit_write_streams'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/BatchCommitWriteStreams', + request_serializer=storage.BatchCommitWriteStreamsRequest.serialize, + response_deserializer=storage.BatchCommitWriteStreamsResponse.deserialize, + ) + return self._stubs['batch_commit_write_streams'] + + @property + def flush_rows(self) -> Callable[ + [storage.FlushRowsRequest], + storage.FlushRowsResponse]: + r"""Return a callable for the flush rows method over gRPC. + + Flushes rows to a BUFFERED stream. If users are appending rows + to BUFFERED stream, flush operation is required in order for the + rows to become available for reading. A Flush operation flushes + up to any previously flushed offset in a BUFFERED stream, to the + offset specified in the request. Flush is not supported on the + \_default stream, since it is not BUFFERED. + + Returns: + Callable[[~.FlushRowsRequest], + ~.FlushRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'flush_rows' not in self._stubs: + self._stubs['flush_rows'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FlushRows', + request_serializer=storage.FlushRowsRequest.serialize, + response_deserializer=storage.FlushRowsResponse.deserialize, + ) + return self._stubs['flush_rows'] + + +__all__ = ( + 'BigQueryWriteGrpcTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py new file mode 100644 index 00000000..d5b164f0 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py @@ -0,0 +1,424 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import gapic_v1 # type: ignore +from google.api_core import grpc_helpers_async # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +import packaging.version + +import grpc # type: ignore +from grpc.experimental import aio # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from .base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .grpc import BigQueryWriteGrpcTransport + + +class BigQueryWriteGrpcAsyncIOTransport(BigQueryWriteTransport): + """gRPC AsyncIO backend transport for BigQueryWrite. + + BigQuery Write API. + The Write API can be used to write data to BigQuery. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + + _grpc_channel: aio.Channel + _stubs: Dict[str, Callable] = {} + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> aio.Channel: + """Create and return a gRPC AsyncIO channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + aio.Channel: A gRPC AsyncIO channel object. + """ + + return grpc_helpers_async.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: aio.Channel = None, + api_mtls_endpoint: str = None, + client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, + quota_project_id=None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if ``channel`` is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + channel (Optional[aio.Channel]): A ``Channel`` instance through + which to make calls. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or applicatin default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if channel: + # Ignore credentials if a channel was passed. + credentials = False + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + ) + + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @property + def grpc_channel(self) -> aio.Channel: + """Create the channel designed to connect to this service. + + This property caches on the instance; repeated calls return + the same channel. + """ + # Return the channel from cache. + return self._grpc_channel + + @property + def create_write_stream(self) -> Callable[ + [storage.CreateWriteStreamRequest], + Awaitable[stream.WriteStream]]: + r"""Return a callable for the create write stream method over gRPC. + + Creates a write stream to the given table. Additionally, every + table has a special COMMITTED stream named '_default' to which + data can be written. This stream doesn't need to be created + using CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Returns: + Callable[[~.CreateWriteStreamRequest], + Awaitable[~.WriteStream]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_write_stream' not in self._stubs: + self._stubs['create_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/CreateWriteStream', + request_serializer=storage.CreateWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['create_write_stream'] + + @property + def append_rows(self) -> Callable[ + [storage.AppendRowsRequest], + Awaitable[storage.AppendRowsResponse]]: + r"""Return a callable for the append rows method over gRPC. + + Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provids an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC stream. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains the offset at which the append happened. + Responses are received in the same order in which requests are + sent. There will be one response for each successful request. If + the ``offset`` is not set in response, it means append didn't + happen due to some errors. If one request fails, all the + subsequent requests will also fail until a success request is + made again. + + If the stream is of ``PENDING`` type, data will only be + available for read operations after the stream is committed. + + Returns: + Callable[[~.AppendRowsRequest], + Awaitable[~.AppendRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'append_rows' not in self._stubs: + self._stubs['append_rows'] = self.grpc_channel.stream_stream( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/AppendRows', + request_serializer=storage.AppendRowsRequest.serialize, + response_deserializer=storage.AppendRowsResponse.deserialize, + ) + return self._stubs['append_rows'] + + @property + def get_write_stream(self) -> Callable[ + [storage.GetWriteStreamRequest], + Awaitable[stream.WriteStream]]: + r"""Return a callable for the get write stream method over gRPC. + + Gets a write stream. + + Returns: + Callable[[~.GetWriteStreamRequest], + Awaitable[~.WriteStream]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'get_write_stream' not in self._stubs: + self._stubs['get_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/GetWriteStream', + request_serializer=storage.GetWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['get_write_stream'] + + @property + def finalize_write_stream(self) -> Callable[ + [storage.FinalizeWriteStreamRequest], + Awaitable[storage.FinalizeWriteStreamResponse]]: + r"""Return a callable for the finalize write stream method over gRPC. + + Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Returns: + Callable[[~.FinalizeWriteStreamRequest], + Awaitable[~.FinalizeWriteStreamResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'finalize_write_stream' not in self._stubs: + self._stubs['finalize_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FinalizeWriteStream', + request_serializer=storage.FinalizeWriteStreamRequest.serialize, + response_deserializer=storage.FinalizeWriteStreamResponse.deserialize, + ) + return self._stubs['finalize_write_stream'] + + @property + def batch_commit_write_streams(self) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + Awaitable[storage.BatchCommitWriteStreamsResponse]]: + r"""Return a callable for the batch commit write streams method over gRPC. + + Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. Streams must be finalized before + commit and cannot be committed multiple times. Once a stream is + committed, data in the stream becomes available for read + operations. + + Returns: + Callable[[~.BatchCommitWriteStreamsRequest], + Awaitable[~.BatchCommitWriteStreamsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_commit_write_streams' not in self._stubs: + self._stubs['batch_commit_write_streams'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/BatchCommitWriteStreams', + request_serializer=storage.BatchCommitWriteStreamsRequest.serialize, + response_deserializer=storage.BatchCommitWriteStreamsResponse.deserialize, + ) + return self._stubs['batch_commit_write_streams'] + + @property + def flush_rows(self) -> Callable[ + [storage.FlushRowsRequest], + Awaitable[storage.FlushRowsResponse]]: + r"""Return a callable for the flush rows method over gRPC. + + Flushes rows to a BUFFERED stream. If users are appending rows + to BUFFERED stream, flush operation is required in order for the + rows to become available for reading. A Flush operation flushes + up to any previously flushed offset in a BUFFERED stream, to the + offset specified in the request. Flush is not supported on the + \_default stream, since it is not BUFFERED. + + Returns: + Callable[[~.FlushRowsRequest], + Awaitable[~.FlushRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'flush_rows' not in self._stubs: + self._stubs['flush_rows'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FlushRows', + request_serializer=storage.FlushRowsRequest.serialize, + response_deserializer=storage.FlushRowsResponse.deserialize, + ) + return self._stubs['flush_rows'] + + +__all__ = ( + 'BigQueryWriteGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py new file mode 100644 index 00000000..b73a912f --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .arrow import ( + ArrowRecordBatch, + ArrowSchema, + ArrowSerializationOptions, +) +from .avro import ( + AvroRows, + AvroSchema, +) +from .protobuf import ( + ProtoRows, + ProtoSchema, +) +from .storage import ( + AppendRowsRequest, + AppendRowsResponse, + BatchCommitWriteStreamsRequest, + BatchCommitWriteStreamsResponse, + CreateReadSessionRequest, + CreateWriteStreamRequest, + FinalizeWriteStreamRequest, + FinalizeWriteStreamResponse, + FlushRowsRequest, + FlushRowsResponse, + GetWriteStreamRequest, + ReadRowsRequest, + ReadRowsResponse, + SplitReadStreamRequest, + SplitReadStreamResponse, + StorageError, + StreamStats, + ThrottleState, +) +from .stream import ( + ReadSession, + ReadStream, + WriteStream, + DataFormat, +) +from .table import ( + TableFieldSchema, + TableSchema, +) + +__all__ = ( + 'ArrowRecordBatch', + 'ArrowSchema', + 'ArrowSerializationOptions', + 'AvroRows', + 'AvroSchema', + 'ProtoRows', + 'ProtoSchema', + 'AppendRowsRequest', + 'AppendRowsResponse', + 'BatchCommitWriteStreamsRequest', + 'BatchCommitWriteStreamsResponse', + 'CreateReadSessionRequest', + 'CreateWriteStreamRequest', + 'FinalizeWriteStreamRequest', + 'FinalizeWriteStreamResponse', + 'FlushRowsRequest', + 'FlushRowsResponse', + 'GetWriteStreamRequest', + 'ReadRowsRequest', + 'ReadRowsResponse', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'StorageError', + 'StreamStats', + 'ThrottleState', + 'ReadSession', + 'ReadStream', + 'WriteStream', + 'DataFormat', + 'TableFieldSchema', + 'TableSchema', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py new file mode 100644 index 00000000..7bf84070 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'ArrowSchema', + 'ArrowRecordBatch', + 'ArrowSerializationOptions', + }, +) + + +class ArrowSchema(proto.Message): + r"""Arrow schema as specified in + https://arrow.apache.org/docs/python/api/datatypes.html and + serialized to bytes using IPC: + https://arrow.apache.org/docs/format/Columnar.html#serialization- + and-interprocess-communication-ipc + See code samples on how this message can be deserialized. + + Attributes: + serialized_schema (bytes): + IPC serialized Arrow schema. + """ + + serialized_schema = proto.Field( + proto.BYTES, + number=1, + ) + + +class ArrowRecordBatch(proto.Message): + r"""Arrow RecordBatch. + Attributes: + serialized_record_batch (bytes): + IPC-serialized Arrow RecordBatch. + """ + + serialized_record_batch = proto.Field( + proto.BYTES, + number=1, + ) + + +class ArrowSerializationOptions(proto.Message): + r"""Contains options specific to Arrow Serialization. + Attributes: + format_ (google.cloud.bigquery_storage_v1beta2.types.ArrowSerializationOptions.Format): + The Arrow IPC format to use. + """ + class Format(proto.Enum): + r"""The IPC format to use when serializing Arrow streams.""" + FORMAT_UNSPECIFIED = 0 + ARROW_0_14 = 1 + ARROW_0_15 = 2 + + format_ = proto.Field( + proto.ENUM, + number=1, + enum=Format, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py new file mode 100644 index 00000000..e982615c --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'AvroSchema', + 'AvroRows', + }, +) + + +class AvroSchema(proto.Message): + r"""Avro schema. + Attributes: + schema (str): + Json serialized schema, as described at + https://avro.apache.org/docs/1.8.1/spec.html. + """ + + schema = proto.Field( + proto.STRING, + number=1, + ) + + +class AvroRows(proto.Message): + r"""Avro rows. + Attributes: + serialized_binary_rows (bytes): + Binary serialized rows in a block. + """ + + serialized_binary_rows = proto.Field( + proto.BYTES, + number=1, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py new file mode 100644 index 00000000..07175d85 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + +from google.protobuf import descriptor_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'ProtoSchema', + 'ProtoRows', + }, +) + + +class ProtoSchema(proto.Message): + r"""ProtoSchema describes the schema of the serialized protocol + buffer data rows. + + Attributes: + proto_descriptor (google.protobuf.descriptor_pb2.DescriptorProto): + Descriptor for input message. The descriptor + has to be self contained, including all the + nested types, excepted for proto buffer well + known types + (https://developers.google.com/protocol- + buffers/docs/reference/google.protobuf). + """ + + proto_descriptor = proto.Field( + proto.MESSAGE, + number=1, + message=descriptor_pb2.DescriptorProto, + ) + + +class ProtoRows(proto.Message): + r""" + Attributes: + serialized_rows (Sequence[bytes]): + A sequence of rows serialized as a Protocol + Buffer. + See https://developers.google.com/protocol- + buffers/docs/overview for more information on + deserializing this field. + """ + + serialized_rows = proto.RepeatedField( + proto.BYTES, + number=1, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py new file mode 100644 index 00000000..b7d65dfc --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py @@ -0,0 +1,618 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import arrow +from google.cloud.bigquery_storage_v1beta2.types import avro +from google.cloud.bigquery_storage_v1beta2.types import protobuf +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.cloud.bigquery_storage_v1beta2.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'CreateReadSessionRequest', + 'ReadRowsRequest', + 'ThrottleState', + 'StreamStats', + 'ReadRowsResponse', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'CreateWriteStreamRequest', + 'AppendRowsRequest', + 'AppendRowsResponse', + 'GetWriteStreamRequest', + 'BatchCommitWriteStreamsRequest', + 'BatchCommitWriteStreamsResponse', + 'FinalizeWriteStreamRequest', + 'FinalizeWriteStreamResponse', + 'FlushRowsRequest', + 'FlushRowsResponse', + 'StorageError', + }, +) + + +class CreateReadSessionRequest(proto.Message): + r"""Request message for ``CreateReadSession``. + Attributes: + parent (str): + Required. The request project that owns the session, in the + form of ``projects/{project_id}``. + read_session (google.cloud.bigquery_storage_v1beta2.types.ReadSession): + Required. Session to be created. + max_stream_count (int): + Max initial number of streams. If unset or + zero, the server will provide a value of streams + so as to produce reasonable throughput. Must be + non-negative. The number of streams may be lower + than the requested number, depending on the + amount parallelism that is reasonable for the + table. Error will be returned if the max count + is greater than the current system max limit of + 1,000. + + Streams must be read starting from offset 0. + """ + + parent = proto.Field( + proto.STRING, + number=1, + ) + read_session = proto.Field( + proto.MESSAGE, + number=2, + message=stream.ReadSession, + ) + max_stream_count = proto.Field( + proto.INT32, + number=3, + ) + + +class ReadRowsRequest(proto.Message): + r"""Request message for ``ReadRows``. + Attributes: + read_stream (str): + Required. Stream to read rows from. + offset (int): + The offset requested must be less than the + last row read from Read. Requesting a larger + offset is undefined. If not specified, start + reading from offset zero. + """ + + read_stream = proto.Field( + proto.STRING, + number=1, + ) + offset = proto.Field( + proto.INT64, + number=2, + ) + + +class ThrottleState(proto.Message): + r"""Information on if the current connection is being throttled. + Attributes: + throttle_percent (int): + How much this connection is being throttled. + Zero means no throttling, 100 means fully + throttled. + """ + + throttle_percent = proto.Field( + proto.INT32, + number=1, + ) + + +class StreamStats(proto.Message): + r"""Estimated stream statistics for a given Stream. + Attributes: + progress (google.cloud.bigquery_storage_v1beta2.types.StreamStats.Progress): + Represents the progress of the current + stream. + """ + + class Progress(proto.Message): + r""" + Attributes: + at_response_start (float): + The fraction of rows assigned to the stream that have been + processed by the server so far, not including the rows in + the current response message. + + This value, along with ``at_response_end``, can be used to + interpolate the progress made as the rows in the message are + being processed using the following formula: + ``at_response_start + (at_response_end - at_response_start) * rows_processed_from_response / rows_in_response``. + + Note that if a filter is provided, the ``at_response_end`` + value of the previous response may not necessarily be equal + to the ``at_response_start`` value of the current response. + at_response_end (float): + Similar to ``at_response_start``, except that this value + includes the rows in the current response. + """ + + at_response_start = proto.Field( + proto.DOUBLE, + number=1, + ) + at_response_end = proto.Field( + proto.DOUBLE, + number=2, + ) + + progress = proto.Field( + proto.MESSAGE, + number=2, + message=Progress, + ) + + +class ReadRowsResponse(proto.Message): + r"""Response from calling ``ReadRows`` may include row data, progress + and throttling information. + + Attributes: + avro_rows (google.cloud.bigquery_storage_v1beta2.types.AvroRows): + Serialized row data in AVRO format. + arrow_record_batch (google.cloud.bigquery_storage_v1beta2.types.ArrowRecordBatch): + Serialized row data in Arrow RecordBatch + format. + row_count (int): + Number of serialized rows in the rows block. + stats (google.cloud.bigquery_storage_v1beta2.types.StreamStats): + Statistics for the stream. + throttle_state (google.cloud.bigquery_storage_v1beta2.types.ThrottleState): + Throttling state. If unset, the latest + response still describes the current throttling + status. + avro_schema (google.cloud.bigquery_storage_v1beta2.types.AvroSchema): + Output only. Avro schema. + arrow_schema (google.cloud.bigquery_storage_v1beta2.types.ArrowSchema): + Output only. Arrow schema. + """ + + avro_rows = proto.Field( + proto.MESSAGE, + number=3, + oneof='rows', + message=avro.AvroRows, + ) + arrow_record_batch = proto.Field( + proto.MESSAGE, + number=4, + oneof='rows', + message=arrow.ArrowRecordBatch, + ) + row_count = proto.Field( + proto.INT64, + number=6, + ) + stats = proto.Field( + proto.MESSAGE, + number=2, + message='StreamStats', + ) + throttle_state = proto.Field( + proto.MESSAGE, + number=5, + message='ThrottleState', + ) + avro_schema = proto.Field( + proto.MESSAGE, + number=7, + oneof='schema', + message=avro.AvroSchema, + ) + arrow_schema = proto.Field( + proto.MESSAGE, + number=8, + oneof='schema', + message=arrow.ArrowSchema, + ) + + +class SplitReadStreamRequest(proto.Message): + r"""Request message for ``SplitReadStream``. + Attributes: + name (str): + Required. Name of the stream to split. + fraction (float): + A value in the range (0.0, 1.0) that + specifies the fractional point at which the + original stream should be split. The actual + split point is evaluated on pre-filtered rows, + so if a filter is provided, then there is no + guarantee that the division of the rows between + the new child streams will be proportional to + this fractional value. Additionally, because the + server-side unit for assigning data is + collections of rows, this fraction will always + map to a data storage boundary on the server + side. + """ + + name = proto.Field( + proto.STRING, + number=1, + ) + fraction = proto.Field( + proto.DOUBLE, + number=2, + ) + + +class SplitReadStreamResponse(proto.Message): + r""" + Attributes: + primary_stream (google.cloud.bigquery_storage_v1beta2.types.ReadStream): + Primary stream, which contains the beginning portion of + \|original_stream|. An empty value indicates that the + original stream can no longer be split. + remainder_stream (google.cloud.bigquery_storage_v1beta2.types.ReadStream): + Remainder stream, which contains the tail of + \|original_stream|. An empty value indicates that the + original stream can no longer be split. + """ + + primary_stream = proto.Field( + proto.MESSAGE, + number=1, + message=stream.ReadStream, + ) + remainder_stream = proto.Field( + proto.MESSAGE, + number=2, + message=stream.ReadStream, + ) + + +class CreateWriteStreamRequest(proto.Message): + r"""Request message for ``CreateWriteStream``. + Attributes: + parent (str): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + write_stream (google.cloud.bigquery_storage_v1beta2.types.WriteStream): + Required. Stream to be created. + """ + + parent = proto.Field( + proto.STRING, + number=1, + ) + write_stream = proto.Field( + proto.MESSAGE, + number=2, + message=stream.WriteStream, + ) + + +class AppendRowsRequest(proto.Message): + r"""Request message for ``AppendRows``. + Attributes: + write_stream (str): + Required. The stream that is the target of the append + operation. This value must be specified for the initial + request. If subsequent requests specify the stream name, it + must equal to the value provided in the first request. To + write to the \_default stream, populate this field with a + string in the format + ``projects/{project}/datasets/{dataset}/tables/{table}/_default``. + offset (google.protobuf.wrappers_pb2.Int64Value): + If present, the write is only performed if the next append + offset is same as the provided value. If not present, the + write is performed at the current end of stream. Specifying + a value for this field is not allowed when calling + AppendRows for the '_default' stream. + proto_rows (google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest.ProtoData): + Rows in proto format. + trace_id (str): + Id set by client to annotate its identity. + Only initial request setting is respected. + """ + + class ProtoData(proto.Message): + r"""Proto schema and data. + Attributes: + writer_schema (google.cloud.bigquery_storage_v1beta2.types.ProtoSchema): + Proto schema used to serialize the data. + rows (google.cloud.bigquery_storage_v1beta2.types.ProtoRows): + Serialized row data in protobuf message + format. + """ + + writer_schema = proto.Field( + proto.MESSAGE, + number=1, + message=protobuf.ProtoSchema, + ) + rows = proto.Field( + proto.MESSAGE, + number=2, + message=protobuf.ProtoRows, + ) + + write_stream = proto.Field( + proto.STRING, + number=1, + ) + offset = proto.Field( + proto.MESSAGE, + number=2, + message=wrappers_pb2.Int64Value, + ) + proto_rows = proto.Field( + proto.MESSAGE, + number=4, + oneof='rows', + message=ProtoData, + ) + trace_id = proto.Field( + proto.STRING, + number=6, + ) + + +class AppendRowsResponse(proto.Message): + r"""Response message for ``AppendRows``. + Attributes: + append_result (google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse.AppendResult): + Result if the append is successful. + error (google.rpc.status_pb2.Status): + Error returned when problems were encountered. If present, + it indicates rows were not accepted into the system. Users + can retry or continue with other append requests within the + same connection. + + Additional information about error signalling: + + ALREADY_EXISTS: Happens when an append specified an offset, + and the backend already has received data at this offset. + Typically encountered in retry scenarios, and can be + ignored. + + OUT_OF_RANGE: Returned when the specified offset in the + stream is beyond the current end of the stream. + + INVALID_ARGUMENT: Indicates a malformed request or data. + + ABORTED: Request processing is aborted because of prior + failures. The request can be retried if previous failure is + addressed. + + INTERNAL: Indicates server side error(s) that can be + retried. + updated_schema (google.cloud.bigquery_storage_v1beta2.types.TableSchema): + If backend detects a schema update, pass it + to user so that user can use it to input new + type of message. It will be empty when no schema + updates have occurred. + """ + + class AppendResult(proto.Message): + r"""AppendResult is returned for successful append requests. + Attributes: + offset (google.protobuf.wrappers_pb2.Int64Value): + The row offset at which the last append + occurred. The offset will not be set if + appending using default streams. + """ + + offset = proto.Field( + proto.MESSAGE, + number=1, + message=wrappers_pb2.Int64Value, + ) + + append_result = proto.Field( + proto.MESSAGE, + number=1, + oneof='response', + message=AppendResult, + ) + error = proto.Field( + proto.MESSAGE, + number=2, + oneof='response', + message=status_pb2.Status, + ) + updated_schema = proto.Field( + proto.MESSAGE, + number=3, + message=table.TableSchema, + ) + + +class GetWriteStreamRequest(proto.Message): + r"""Request message for ``GetWriteStreamRequest``. + Attributes: + name (str): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + """ + + name = proto.Field( + proto.STRING, + number=1, + ) + + +class BatchCommitWriteStreamsRequest(proto.Message): + r"""Request message for ``BatchCommitWriteStreams``. + Attributes: + parent (str): + Required. Parent table that all the streams should belong + to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + write_streams (Sequence[str]): + Required. The group of streams that will be + committed atomically. + """ + + parent = proto.Field( + proto.STRING, + number=1, + ) + write_streams = proto.RepeatedField( + proto.STRING, + number=2, + ) + + +class BatchCommitWriteStreamsResponse(proto.Message): + r"""Response message for ``BatchCommitWriteStreams``. + Attributes: + commit_time (google.protobuf.timestamp_pb2.Timestamp): + The time at which streams were committed in microseconds + granularity. This field will only exist when there are no + stream errors. **Note** if this field is not set, it means + the commit was not successful. + stream_errors (Sequence[google.cloud.bigquery_storage_v1beta2.types.StorageError]): + Stream level error if commit failed. Only + streams with error will be in the list. + If empty, there is no error and all streams are + committed successfully. If non empty, certain + streams have errors and ZERO stream is committed + due to atomicity guarantee. + """ + + commit_time = proto.Field( + proto.MESSAGE, + number=1, + message=timestamp_pb2.Timestamp, + ) + stream_errors = proto.RepeatedField( + proto.MESSAGE, + number=2, + message='StorageError', + ) + + +class FinalizeWriteStreamRequest(proto.Message): + r"""Request message for invoking ``FinalizeWriteStream``. + Attributes: + name (str): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + """ + + name = proto.Field( + proto.STRING, + number=1, + ) + + +class FinalizeWriteStreamResponse(proto.Message): + r"""Response message for ``FinalizeWriteStream``. + Attributes: + row_count (int): + Number of rows in the finalized stream. + """ + + row_count = proto.Field( + proto.INT64, + number=1, + ) + + +class FlushRowsRequest(proto.Message): + r"""Request message for ``FlushRows``. + Attributes: + write_stream (str): + Required. The stream that is the target of + the flush operation. + offset (google.protobuf.wrappers_pb2.Int64Value): + Ending offset of the flush operation. Rows + before this offset(including this offset) will + be flushed. + """ + + write_stream = proto.Field( + proto.STRING, + number=1, + ) + offset = proto.Field( + proto.MESSAGE, + number=2, + message=wrappers_pb2.Int64Value, + ) + + +class FlushRowsResponse(proto.Message): + r"""Respond message for ``FlushRows``. + Attributes: + offset (int): + The rows before this offset (including this + offset) are flushed. + """ + + offset = proto.Field( + proto.INT64, + number=1, + ) + + +class StorageError(proto.Message): + r"""Structured custom BigQuery Storage error message. The error + can be attached as error details in the returned rpc Status. In + particular, the use of error codes allows more structured error + handling, and reduces the need to evaluate unstructured error + text strings. + + Attributes: + code (google.cloud.bigquery_storage_v1beta2.types.StorageError.StorageErrorCode): + BigQuery Storage specific error code. + entity (str): + Name of the failed entity. + error_message (str): + Message that describes the error. + """ + class StorageErrorCode(proto.Enum): + r"""Error code for ``StorageError``.""" + STORAGE_ERROR_CODE_UNSPECIFIED = 0 + TABLE_NOT_FOUND = 1 + STREAM_ALREADY_COMMITTED = 2 + STREAM_NOT_FOUND = 3 + INVALID_STREAM_TYPE = 4 + INVALID_STREAM_STATE = 5 + STREAM_FINALIZED = 6 + + code = proto.Field( + proto.ENUM, + number=1, + enum=StorageErrorCode, + ) + entity = proto.Field( + proto.STRING, + number=2, + ) + error_message = proto.Field( + proto.STRING, + number=3, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py new file mode 100644 index 00000000..028c48ac --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import arrow +from google.cloud.bigquery_storage_v1beta2.types import avro +from google.cloud.bigquery_storage_v1beta2.types import table as gcbs_table +from google.protobuf import timestamp_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'DataFormat', + 'ReadSession', + 'ReadStream', + 'WriteStream', + }, +) + + +class DataFormat(proto.Enum): + r"""Data format for input or output data.""" + DATA_FORMAT_UNSPECIFIED = 0 + AVRO = 1 + ARROW = 2 + + +class ReadSession(proto.Message): + r"""Information about the ReadSession. + Attributes: + name (str): + Output only. Unique identifier for the session, in the form + ``projects/{project_id}/locations/{location}/sessions/{session_id}``. + expire_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Time at which the session becomes invalid. + After this time, subsequent requests to read this Session + will return errors. The expire_time is automatically + assigned and currently cannot be specified or updated. + data_format (google.cloud.bigquery_storage_v1beta2.types.DataFormat): + Immutable. Data format of the output data. + avro_schema (google.cloud.bigquery_storage_v1beta2.types.AvroSchema): + Output only. Avro schema. + arrow_schema (google.cloud.bigquery_storage_v1beta2.types.ArrowSchema): + Output only. Arrow schema. + table (str): + Immutable. Table that this ReadSession is reading from, in + the form + \`projects/{project_id}/datasets/{dataset_id}/tables/{table_id} + table_modifiers (google.cloud.bigquery_storage_v1beta2.types.ReadSession.TableModifiers): + Optional. Any modifiers which are applied + when reading from the specified table. + read_options (google.cloud.bigquery_storage_v1beta2.types.ReadSession.TableReadOptions): + Optional. Read options for this session (e.g. + column selection, filters). + streams (Sequence[google.cloud.bigquery_storage_v1beta2.types.ReadStream]): + Output only. A list of streams created with the session. + + At least one stream is created with the session. In the + future, larger request_stream_count values *may* result in + this list being unpopulated, in that case, the user will + need to use a List method to get the streams instead, which + is not yet available. + """ + + class TableModifiers(proto.Message): + r"""Additional attributes when reading a table. + Attributes: + snapshot_time (google.protobuf.timestamp_pb2.Timestamp): + The snapshot time of the table. If not set, + interpreted as now. + """ + + snapshot_time = proto.Field( + proto.MESSAGE, + number=1, + message=timestamp_pb2.Timestamp, + ) + + class TableReadOptions(proto.Message): + r"""Options dictating how we read a table. + Attributes: + selected_fields (Sequence[str]): + Names of the fields in the table that should be read. If + empty, all fields will be read. If the specified field is a + nested field, all the sub-fields in the field will be + selected. The output field order is unrelated to the order + of fields in selected_fields. + row_restriction (str): + SQL text filtering statement, similar to a WHERE clause in a + query. Aggregates are not supported. + + Examples: "int_field > 5" "date_field = CAST('2014-9-27' as + DATE)" "nullable_field is not NULL" "st_equals(geo_field, + st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 + AND 5.0" + + Restricted to a maximum length for 1 MB. + arrow_serialization_options (google.cloud.bigquery_storage_v1beta2.types.ArrowSerializationOptions): + Optional. Options specific to the Apache + Arrow output format. + """ + + selected_fields = proto.RepeatedField( + proto.STRING, + number=1, + ) + row_restriction = proto.Field( + proto.STRING, + number=2, + ) + arrow_serialization_options = proto.Field( + proto.MESSAGE, + number=3, + message=arrow.ArrowSerializationOptions, + ) + + name = proto.Field( + proto.STRING, + number=1, + ) + expire_time = proto.Field( + proto.MESSAGE, + number=2, + message=timestamp_pb2.Timestamp, + ) + data_format = proto.Field( + proto.ENUM, + number=3, + enum='DataFormat', + ) + avro_schema = proto.Field( + proto.MESSAGE, + number=4, + oneof='schema', + message=avro.AvroSchema, + ) + arrow_schema = proto.Field( + proto.MESSAGE, + number=5, + oneof='schema', + message=arrow.ArrowSchema, + ) + table = proto.Field( + proto.STRING, + number=6, + ) + table_modifiers = proto.Field( + proto.MESSAGE, + number=7, + message=TableModifiers, + ) + read_options = proto.Field( + proto.MESSAGE, + number=8, + message=TableReadOptions, + ) + streams = proto.RepeatedField( + proto.MESSAGE, + number=10, + message='ReadStream', + ) + + +class ReadStream(proto.Message): + r"""Information about a single stream that gets data out of the storage + system. Most of the information about ``ReadStream`` instances is + aggregated, making ``ReadStream`` lightweight. + + Attributes: + name (str): + Output only. Name of the stream, in the form + ``projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}``. + """ + + name = proto.Field( + proto.STRING, + number=1, + ) + + +class WriteStream(proto.Message): + r"""Information about a single stream that gets data inside the + storage system. + + Attributes: + name (str): + Output only. Name of the stream, in the form + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + type_ (google.cloud.bigquery_storage_v1beta2.types.WriteStream.Type): + Immutable. Type of the stream. + create_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Create time of the stream. For the \_default + stream, this is the creation_time of the table. + commit_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Commit time of the stream. If a stream is of + ``COMMITTED`` type, then it will have a commit_time same as + ``create_time``. If the stream is of ``PENDING`` type, + commit_time being empty means it is not committed. + table_schema (google.cloud.bigquery_storage_v1beta2.types.TableSchema): + Output only. The schema of the destination table. It is only + returned in ``CreateWriteStream`` response. Caller should + generate data that's compatible with this schema to send in + initial ``AppendRowsRequest``. The table schema could go out + of date during the life time of the stream. + """ + class Type(proto.Enum): + r"""Type enum of the stream.""" + TYPE_UNSPECIFIED = 0 + COMMITTED = 1 + PENDING = 2 + BUFFERED = 3 + + name = proto.Field( + proto.STRING, + number=1, + ) + type_ = proto.Field( + proto.ENUM, + number=2, + enum=Type, + ) + create_time = proto.Field( + proto.MESSAGE, + number=3, + message=timestamp_pb2.Timestamp, + ) + commit_time = proto.Field( + proto.MESSAGE, + number=4, + message=timestamp_pb2.Timestamp, + ) + table_schema = proto.Field( + proto.MESSAGE, + number=5, + message=gcbs_table.TableSchema, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py new file mode 100644 index 00000000..2c32504e --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'TableSchema', + 'TableFieldSchema', + }, +) + + +class TableSchema(proto.Message): + r"""Schema of a table + Attributes: + fields (Sequence[google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema]): + Describes the fields in a table. + """ + + fields = proto.RepeatedField( + proto.MESSAGE, + number=1, + message='TableFieldSchema', + ) + + +class TableFieldSchema(proto.Message): + r"""A field in TableSchema + Attributes: + name (str): + Required. The field name. The name must contain only letters + (a-z, A-Z), numbers (0-9), or underscores (_), and must + start with a letter or underscore. The maximum length is 128 + characters. + type_ (google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema.Type): + Required. The field data type. + mode (google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema.Mode): + Optional. The field mode. The default value + is NULLABLE. + fields (Sequence[google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema]): + Optional. Describes the nested schema fields + if the type property is set to STRUCT. + description (str): + Optional. The field description. The maximum + length is 1,024 characters. + """ + class Type(proto.Enum): + r"""""" + TYPE_UNSPECIFIED = 0 + STRING = 1 + INT64 = 2 + DOUBLE = 3 + STRUCT = 4 + BYTES = 5 + BOOL = 6 + TIMESTAMP = 7 + DATE = 8 + TIME = 9 + DATETIME = 10 + GEOGRAPHY = 11 + NUMERIC = 12 + BIGNUMERIC = 13 + INTERVAL = 14 + JSON = 15 + + class Mode(proto.Enum): + r"""""" + MODE_UNSPECIFIED = 0 + NULLABLE = 1 + REQUIRED = 2 + REPEATED = 3 + + name = proto.Field( + proto.STRING, + number=1, + ) + type_ = proto.Field( + proto.ENUM, + number=2, + enum=Type, + ) + mode = proto.Field( + proto.ENUM, + number=3, + enum=Mode, + ) + fields = proto.RepeatedField( + proto.MESSAGE, + number=4, + message='TableFieldSchema', + ) + description = proto.Field( + proto.STRING, + number=6, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/mypy.ini b/owl-bot-staging/v1beta2/mypy.ini new file mode 100644 index 00000000..4505b485 --- /dev/null +++ b/owl-bot-staging/v1beta2/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +python_version = 3.6 +namespace_packages = True diff --git a/owl-bot-staging/v1beta2/noxfile.py b/owl-bot-staging/v1beta2/noxfile.py new file mode 100644 index 00000000..4b78e039 --- /dev/null +++ b/owl-bot-staging/v1beta2/noxfile.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import pathlib +import shutil +import subprocess +import sys + + +import nox # type: ignore + +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt" +PACKAGE_NAME = subprocess.check_output([sys.executable, "setup.py", "--name"], encoding="utf-8") + + +nox.sessions = [ + "unit", + "cover", + "mypy", + "check_lower_bounds" + # exclude update_lower_bounds from default + "docs", +] + +@nox.session(python=['3.6', '3.7', '3.8', '3.9']) +def unit(session): + """Run the unit test suite.""" + + session.install('coverage', 'pytest', 'pytest-cov', 'asyncmock', 'pytest-asyncio') + session.install('-e', '.') + + session.run( + 'py.test', + '--quiet', + '--cov=google/cloud/bigquery_storage_v1beta2/', + '--cov-config=.coveragerc', + '--cov-report=term', + '--cov-report=html', + os.path.join('tests', 'unit', ''.join(session.posargs)) + ) + + +@nox.session(python='3.7') +def cover(session): + """Run the final coverage report. + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=100") + + session.run("coverage", "erase") + + +@nox.session(python=['3.6', '3.7']) +def mypy(session): + """Run the type checker.""" + session.install('mypy', 'types-pkg_resources') + session.install('.') + session.run( + 'mypy', + '--explicit-package-bases', + 'google', + ) + + +@nox.session +def update_lower_bounds(session): + """Update lower bounds in constraints.txt to match setup.py""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'update', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + + +@nox.session +def check_lower_bounds(session): + """Check lower bounds in setup.py are reflected in constraints file""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'check', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + +@nox.session(python='3.6') +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".") + session.install("sphinx<3.0.0", "alabaster", "recommonmark") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) diff --git a/owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py b/owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py new file mode 100644 index 00000000..9ba710dc --- /dev/null +++ b/owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py @@ -0,0 +1,184 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import os +import libcst as cst +import pathlib +import sys +from typing import (Any, Callable, Dict, List, Sequence, Tuple) + + +def partition( + predicate: Callable[[Any], bool], + iterator: Sequence[Any] +) -> Tuple[List[Any], List[Any]]: + """A stable, out-of-place partition.""" + results = ([], []) + + for i in iterator: + results[int(predicate(i))].append(i) + + # Returns trueList, falseList + return results[1], results[0] + + +class bigquery_storageCallTransformer(cst.CSTTransformer): + CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') + METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { + 'append_rows': ('write_stream', 'offset', 'proto_rows', 'trace_id', ), + 'batch_commit_write_streams': ('parent', 'write_streams', ), + 'create_read_session': ('parent', 'read_session', 'max_stream_count', ), + 'create_write_stream': ('parent', 'write_stream', ), + 'finalize_write_stream': ('name', ), + 'flush_rows': ('write_stream', 'offset', ), + 'get_write_stream': ('name', ), + 'read_rows': ('read_stream', 'offset', ), + 'split_read_stream': ('name', 'fraction', ), + } + + def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: + try: + key = original.func.attr.value + kword_params = self.METHOD_TO_PARAMS[key] + except (AttributeError, KeyError): + # Either not a method from the API or too convoluted to be sure. + return updated + + # If the existing code is valid, keyword args come after positional args. + # Therefore, all positional args must map to the first parameters. + args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) + if any(k.keyword.value == "request" for k in kwargs): + # We've already fixed this file, don't fix it again. + return updated + + kwargs, ctrl_kwargs = partition( + lambda a: not a.keyword.value in self.CTRL_PARAMS, + kwargs + ) + + args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] + ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) + for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) + + request_arg = cst.Arg( + value=cst.Dict([ + cst.DictElement( + cst.SimpleString("'{}'".format(name)), +cst.Element(value=arg.value) + ) + # Note: the args + kwargs looks silly, but keep in mind that + # the control parameters had to be stripped out, and that + # those could have been passed positionally or by keyword. + for name, arg in zip(kword_params, args + kwargs)]), + keyword=cst.Name("request") + ) + + return updated.with_changes( + args=[request_arg] + ctrl_kwargs + ) + + +def fix_files( + in_dir: pathlib.Path, + out_dir: pathlib.Path, + *, + transformer=bigquery_storageCallTransformer(), +): + """Duplicate the input dir to the output dir, fixing file method calls. + + Preconditions: + * in_dir is a real directory + * out_dir is a real, empty directory + """ + pyfile_gen = ( + pathlib.Path(os.path.join(root, f)) + for root, _, files in os.walk(in_dir) + for f in files if os.path.splitext(f)[1] == ".py" + ) + + for fpath in pyfile_gen: + with open(fpath, 'r') as f: + src = f.read() + + # Parse the code and insert method call fixes. + tree = cst.parse_module(src) + updated = tree.visit(transformer) + + # Create the path and directory structure for the new file. + updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) + updated_path.parent.mkdir(parents=True, exist_ok=True) + + # Generate the updated source file at the corresponding path. + with open(updated_path, 'w') as f: + f.write(updated.code) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="""Fix up source that uses the bigquery_storage client library. + +The existing sources are NOT overwritten but are copied to output_dir with changes made. + +Note: This tool operates at a best-effort level at converting positional + parameters in client method calls to keyword based parameters. + Cases where it WILL FAIL include + A) * or ** expansion in a method call. + B) Calls via function or method alias (includes free function calls) + C) Indirect or dispatched calls (e.g. the method is looked up dynamically) + + These all constitute false negatives. The tool will also detect false + positives when an API method shares a name with another method. +""") + parser.add_argument( + '-d', + '--input-directory', + required=True, + dest='input_dir', + help='the input directory to walk for python files to fix up', + ) + parser.add_argument( + '-o', + '--output-directory', + required=True, + dest='output_dir', + help='the directory to output files fixed via un-flattening', + ) + args = parser.parse_args() + input_dir = pathlib.Path(args.input_dir) + output_dir = pathlib.Path(args.output_dir) + if not input_dir.is_dir(): + print( + f"input directory '{input_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if not output_dir.is_dir(): + print( + f"output directory '{output_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if os.listdir(output_dir): + print( + f"output directory '{output_dir}' is not empty", + file=sys.stderr, + ) + sys.exit(-1) + + fix_files(input_dir, output_dir) diff --git a/owl-bot-staging/v1beta2/setup.py b/owl-bot-staging/v1beta2/setup.py new file mode 100644 index 00000000..591e1fba --- /dev/null +++ b/owl-bot-staging/v1beta2/setup.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +import os +import setuptools # type: ignore + +version = '0.1.0' + +package_root = os.path.abspath(os.path.dirname(__file__)) + +readme_filename = os.path.join(package_root, 'README.rst') +with io.open(readme_filename, encoding='utf-8') as readme_file: + readme = readme_file.read() + +setuptools.setup( + name='google-cloud-bigquery-storage', + version=version, + long_description=readme, + packages=setuptools.PEP420PackageFinder.find(), + namespace_packages=('google', 'google.cloud'), + platforms='Posix; MacOS X; Windows', + include_package_data=True, + install_requires=( + 'google-api-core[grpc] >= 1.27.0, < 3.0.0dev', + 'libcst >= 0.2.5', + 'proto-plus >= 1.15.0', + 'packaging >= 14.3', ), + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Topic :: Internet', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + zip_safe=False, +) diff --git a/owl-bot-staging/v1beta2/tests/__init__.py b/owl-bot-staging/v1beta2/tests/__init__.py new file mode 100644 index 00000000..b54a5fcc --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1beta2/tests/unit/__init__.py b/owl-bot-staging/v1beta2/tests/unit/__init__.py new file mode 100644 index 00000000..b54a5fcc --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/unit/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py b/owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py new file mode 100644 index 00000000..b54a5fcc --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py new file mode 100644 index 00000000..b54a5fcc --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py new file mode 100644 index 00000000..1e3ad444 --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py @@ -0,0 +1,1629 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import mock +import packaging.version + +import grpc +from grpc.experimental import aio +import math +import pytest +from proto.marshal.rules.dates import DurationRule, TimestampRule + + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers +from google.api_core import grpc_helpers_async +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.cloud.bigquery_storage_v1beta2.services.big_query_read import BigQueryReadAsyncClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_read import BigQueryReadClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_read import transports +from google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.base import _GOOGLE_AUTH_VERSION +from google.cloud.bigquery_storage_v1beta2.types import arrow +from google.cloud.bigquery_storage_v1beta2.types import avro +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.oauth2 import service_account +from google.protobuf import timestamp_pb2 # type: ignore +import google.auth + + +# TODO(busunkim): Once google-auth >= 1.25.0 is required transitively +# through google-api-core: +# - Delete the auth "less than" test cases +# - Delete these pytest markers (Make the "greater than or equal to" tests the default). +requires_google_auth_lt_1_25_0 = pytest.mark.skipif( + packaging.version.parse(_GOOGLE_AUTH_VERSION) >= packaging.version.parse("1.25.0"), + reason="This test requires google-auth < 1.25.0", +) +requires_google_auth_gte_1_25_0 = pytest.mark.skipif( + packaging.version.parse(_GOOGLE_AUTH_VERSION) < packaging.version.parse("1.25.0"), + reason="This test requires google-auth >= 1.25.0", +) + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert BigQueryReadClient._get_default_mtls_endpoint(None) is None + assert BigQueryReadClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi + + +@pytest.mark.parametrize("client_class", [ + BigQueryReadClient, + BigQueryReadAsyncClient, +]) +def test_big_query_read_client_from_service_account_info(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == 'bigquerystorage.googleapis.com:443' + + +@pytest.mark.parametrize("transport_class,transport_name", [ + (transports.BigQueryReadGrpcTransport, "grpc"), + (transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_read_client_service_account_always_use_jwt(transport_class, transport_name): + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class", [ + BigQueryReadClient, + BigQueryReadAsyncClient, +]) +def test_big_query_read_client_from_service_account_file(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == 'bigquerystorage.googleapis.com:443' + + +def test_big_query_read_client_get_transport_class(): + transport = BigQueryReadClient.get_transport_class() + available_transports = [ + transports.BigQueryReadGrpcTransport, + ] + assert transport in available_transports + + transport = BigQueryReadClient.get_transport_class("grpc") + assert transport == transports.BigQueryReadGrpcTransport + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +@mock.patch.object(BigQueryReadClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadAsyncClient)) +def test_big_query_read_client_client_options(client_class, transport_class, transport_name): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError): + client = client_class() + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError): + client = client_class() + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "true"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "true"), + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "false"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "false"), +]) +@mock.patch.object(BigQueryReadClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadAsyncClient)) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_big_query_read_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client.DEFAULT_ENDPOINT + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): + if use_client_cert_env == "false": + expected_host = client.DEFAULT_ENDPOINT + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_read_client_client_options_scopes(client_class, transport_class, transport_name): + # Check the case scopes are provided. + options = client_options.ClientOptions( + scopes=["1", "2"], + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_read_client_client_options_credentials_file(client_class, transport_class, transport_name): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +def test_big_query_read_client_client_options_from_dict(): + with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadGrpcTransport.__init__') as grpc_transport: + grpc_transport.return_value = None + client = BigQueryReadClient( + client_options={'api_endpoint': 'squid.clam.whelk'} + ) + grpc_transport.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +def test_create_read_session(transport: str = 'grpc', request_type=storage.CreateReadSessionRequest): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession( + name='name_value', + data_format=stream.DataFormat.AVRO, + table='table_value', + avro_schema=avro.AvroSchema(schema='schema_value'), + ) + response = client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateReadSessionRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.ReadSession) + assert response.name == 'name_value' + assert response.data_format == stream.DataFormat.AVRO + assert response.table == 'table_value' + + +def test_create_read_session_from_dict(): + test_create_read_session(request_type=dict) + + +def test_create_read_session_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + client.create_read_session() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateReadSessionRequest() + + +@pytest.mark.asyncio +async def test_create_read_session_async(transport: str = 'grpc_asyncio', request_type=storage.CreateReadSessionRequest): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession( + name='name_value', + data_format=stream.DataFormat.AVRO, + table='table_value', + )) + response = await client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateReadSessionRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.ReadSession) + assert response.name == 'name_value' + assert response.data_format == stream.DataFormat.AVRO + assert response.table == 'table_value' + + +@pytest.mark.asyncio +async def test_create_read_session_async_from_dict(): + await test_create_read_session_async(request_type=dict) + + +def test_create_read_session_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateReadSessionRequest() + + request.read_session.table = 'read_session.table/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value = stream.ReadSession() + client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_session.table=read_session.table/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_create_read_session_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateReadSessionRequest() + + request.read_session.table = 'read_session.table/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) + await client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_session.table=read_session.table/value', + ) in kw['metadata'] + + +def test_create_read_session_flattened(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.create_read_session( + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].parent == 'parent_value' + assert args[0].read_session == stream.ReadSession(name='name_value') + assert args[0].max_stream_count == 1721 + + +def test_create_read_session_flattened_error(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.create_read_session( + storage.CreateReadSessionRequest(), + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + +@pytest.mark.asyncio +async def test_create_read_session_flattened_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.create_read_session( + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].parent == 'parent_value' + assert args[0].read_session == stream.ReadSession(name='name_value') + assert args[0].max_stream_count == 1721 + + +@pytest.mark.asyncio +async def test_create_read_session_flattened_error_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.create_read_session( + storage.CreateReadSessionRequest(), + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + +def test_read_rows(transport: str = 'grpc', request_type=storage.ReadRowsRequest): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + response = client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.ReadRowsRequest() + + # Establish that the response is the type that we expect. + for message in response: + assert isinstance(message, storage.ReadRowsResponse) + + +def test_read_rows_from_dict(): + test_read_rows(request_type=dict) + + +def test_read_rows_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + client.read_rows() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.ReadRowsRequest() + + +@pytest.mark.asyncio +async def test_read_rows_async(transport: str = 'grpc_asyncio', request_type=storage.ReadRowsRequest): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) + response = await client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.ReadRowsRequest() + + # Establish that the response is the type that we expect. + message = await response.read() + assert isinstance(message, storage.ReadRowsResponse) + + +@pytest.mark.asyncio +async def test_read_rows_async_from_dict(): + await test_read_rows_async(request_type=dict) + + +def test_read_rows_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.ReadRowsRequest() + + request.read_stream = 'read_stream/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value = iter([storage.ReadRowsResponse()]) + client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_stream=read_stream/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_read_rows_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.ReadRowsRequest() + + request.read_stream = 'read_stream/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) + await client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_stream=read_stream/value', + ) in kw['metadata'] + + +def test_read_rows_flattened(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.read_rows( + read_stream='read_stream_value', + offset=647, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].read_stream == 'read_stream_value' + assert args[0].offset == 647 + + +def test_read_rows_flattened_error(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.read_rows( + storage.ReadRowsRequest(), + read_stream='read_stream_value', + offset=647, + ) + + +@pytest.mark.asyncio +async def test_read_rows_flattened_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.read_rows( + read_stream='read_stream_value', + offset=647, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].read_stream == 'read_stream_value' + assert args[0].offset == 647 + + +@pytest.mark.asyncio +async def test_read_rows_flattened_error_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.read_rows( + storage.ReadRowsRequest(), + read_stream='read_stream_value', + offset=647, + ) + + +def test_split_read_stream(transport: str = 'grpc', request_type=storage.SplitReadStreamRequest): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.SplitReadStreamResponse( + ) + response = client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.SplitReadStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.SplitReadStreamResponse) + + +def test_split_read_stream_from_dict(): + test_split_read_stream(request_type=dict) + + +def test_split_read_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + client.split_read_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.SplitReadStreamRequest() + + +@pytest.mark.asyncio +async def test_split_read_stream_async(transport: str = 'grpc_asyncio', request_type=storage.SplitReadStreamRequest): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse( + )) + response = await client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.SplitReadStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.SplitReadStreamResponse) + + +@pytest.mark.asyncio +async def test_split_read_stream_async_from_dict(): + await test_split_read_stream_async(request_type=dict) + + +def test_split_read_stream_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.SplitReadStreamRequest() + + request.name = 'name/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value = storage.SplitReadStreamResponse() + client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_split_read_stream_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.SplitReadStreamRequest() + + request.name = 'name/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse()) + await client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name/value', + ) in kw['metadata'] + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options={"scopes": ["1", "2"]}, + transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = BigQueryReadClient(transport=transport) + assert client.transport is transport + +def test_transport_get_channel(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + + transport = transports.BigQueryReadGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + +@pytest.mark.parametrize("transport_class", [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, +]) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + +def test_transport_grpc_default(): + # A client should use the gRPC transport by default. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert isinstance( + client.transport, + transports.BigQueryReadGrpcTransport, + ) + +def test_big_query_read_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transport = transports.BigQueryReadTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json" + ) + + +def test_big_query_read_base_transport(): + # Instantiate the base transport. + with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport.__init__') as Transport: + Transport.return_value = None + transport = transports.BigQueryReadTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + 'create_read_session', + 'read_rows', + 'split_read_stream', + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + +@requires_google_auth_gte_1_25_0 +def test_big_query_read_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryReadTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +@requires_google_auth_lt_1_25_0 +def test_big_query_read_base_transport_with_credentials_file_old_google_auth(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryReadTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', + ), + quota_project_id="octopus", + ) + + +def test_big_query_read_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryReadTransport() + adc.assert_called_once() + + +@requires_google_auth_gte_1_25_0 +def test_big_query_read_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryReadClient() + adc.assert_called_once_with( + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id=None, + ) + + +@requires_google_auth_lt_1_25_0 +def test_big_query_read_auth_adc_old_google_auth(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryReadClient() + adc.assert_called_once_with( + scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.readonly', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id=None, + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, + ], +) +@requires_google_auth_gte_1_25_0 +def test_big_query_read_transport_auth_adc(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + adc.assert_called_once_with( + scopes=["1", "2"], + default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.readonly', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, + ], +) +@requires_google_auth_lt_1_25_0 +def test_big_query_read_transport_auth_adc_old_google_auth(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus") + adc.assert_called_once_with(scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.BigQueryReadGrpcTransport, grpc_helpers), + (transports.BigQueryReadGrpcAsyncIOTransport, grpc_helpers_async) + ], +) +def test_big_query_read_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class( + quota_project_id="octopus", + scopes=["1", "2"] + ) + + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.readonly', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=["1", "2"], + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_grpc_transport_client_cert_source_for_mtls( + transport_class +): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, + private_key=expected_key + ) + + +def test_big_query_read_host_no_port(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), + ) + assert client.transport._host == 'bigquerystorage.googleapis.com:443' + + +def test_big_query_read_host_with_port(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), + ) + assert client.transport._host == 'bigquerystorage.googleapis.com:8000' + +def test_big_query_read_grpc_transport_channel(): + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryReadGrpcTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +def test_big_query_read_grpc_asyncio_transport_channel(): + channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryReadGrpcAsyncIOTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_transport_channel_mtls_with_client_cert_source( + transport_class +): + with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_ssl_cred = mock.Mock() + grpc_ssl_channel_cred.return_value = mock_ssl_cred + + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + + cred = ga_credentials.AnonymousCredentials() + with pytest.warns(DeprecationWarning): + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (cred, None) + transport = transport_class( + host="squid.clam.whelk", + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=client_cert_source_callback, + ) + adc.assert_called_once() + + grpc_ssl_channel_cred.assert_called_once_with( + certificate_chain=b"cert bytes", private_key=b"key bytes" + ) + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + assert transport._ssl_channel_credentials == mock_ssl_cred + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_transport_channel_mtls_with_adc( + transport_class +): + mock_ssl_cred = mock.Mock() + with mock.patch.multiple( + "google.auth.transport.grpc.SslCredentials", + __init__=mock.Mock(return_value=None), + ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), + ): + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + mock_cred = mock.Mock() + + with pytest.warns(DeprecationWarning): + transport = transport_class( + host="squid.clam.whelk", + credentials=mock_cred, + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=None, + ) + + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=mock_cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + + +def test_read_session_path(): + project = "squid" + location = "clam" + session = "whelk" + expected = "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) + actual = BigQueryReadClient.read_session_path(project, location, session) + assert expected == actual + + +def test_parse_read_session_path(): + expected = { + "project": "octopus", + "location": "oyster", + "session": "nudibranch", + } + path = BigQueryReadClient.read_session_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_read_session_path(path) + assert expected == actual + +def test_read_stream_path(): + project = "cuttlefish" + location = "mussel" + session = "winkle" + stream = "nautilus" + expected = "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) + actual = BigQueryReadClient.read_stream_path(project, location, session, stream) + assert expected == actual + + +def test_parse_read_stream_path(): + expected = { + "project": "scallop", + "location": "abalone", + "session": "squid", + "stream": "clam", + } + path = BigQueryReadClient.read_stream_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_read_stream_path(path) + assert expected == actual + +def test_table_path(): + project = "whelk" + dataset = "octopus" + table = "oyster" + expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + actual = BigQueryReadClient.table_path(project, dataset, table) + assert expected == actual + + +def test_parse_table_path(): + expected = { + "project": "nudibranch", + "dataset": "cuttlefish", + "table": "mussel", + } + path = BigQueryReadClient.table_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_table_path(path) + assert expected == actual + +def test_common_billing_account_path(): + billing_account = "winkle" + expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + actual = BigQueryReadClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "nautilus", + } + path = BigQueryReadClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_billing_account_path(path) + assert expected == actual + +def test_common_folder_path(): + folder = "scallop" + expected = "folders/{folder}".format(folder=folder, ) + actual = BigQueryReadClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "abalone", + } + path = BigQueryReadClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_folder_path(path) + assert expected == actual + +def test_common_organization_path(): + organization = "squid" + expected = "organizations/{organization}".format(organization=organization, ) + actual = BigQueryReadClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "clam", + } + path = BigQueryReadClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_organization_path(path) + assert expected == actual + +def test_common_project_path(): + project = "whelk" + expected = "projects/{project}".format(project=project, ) + actual = BigQueryReadClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "octopus", + } + path = BigQueryReadClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_project_path(path) + assert expected == actual + +def test_common_location_path(): + project = "oyster" + location = "nudibranch" + expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) + actual = BigQueryReadClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "cuttlefish", + "location": "mussel", + } + path = BigQueryReadClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_withDEFAULT_CLIENT_INFO(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: + transport_class = BigQueryReadClient.get_transport_class() + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py new file mode 100644 index 00000000..e49950ab --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py @@ -0,0 +1,2294 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import mock +import packaging.version + +import grpc +from grpc.experimental import aio +import math +import pytest +from proto.marshal.rules.dates import DurationRule, TimestampRule + + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers +from google.api_core import grpc_helpers_async +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.cloud.bigquery_storage_v1beta2.services.big_query_write import BigQueryWriteAsyncClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_write import BigQueryWriteClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_write import transports +from google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.base import _GOOGLE_AUTH_VERSION +from google.cloud.bigquery_storage_v1beta2.types import protobuf +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.cloud.bigquery_storage_v1beta2.types import table +from google.oauth2 import service_account +from google.protobuf import descriptor_pb2 # type: ignore +from google.protobuf import timestamp_pb2 # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +import google.auth + + +# TODO(busunkim): Once google-auth >= 1.25.0 is required transitively +# through google-api-core: +# - Delete the auth "less than" test cases +# - Delete these pytest markers (Make the "greater than or equal to" tests the default). +requires_google_auth_lt_1_25_0 = pytest.mark.skipif( + packaging.version.parse(_GOOGLE_AUTH_VERSION) >= packaging.version.parse("1.25.0"), + reason="This test requires google-auth < 1.25.0", +) +requires_google_auth_gte_1_25_0 = pytest.mark.skipif( + packaging.version.parse(_GOOGLE_AUTH_VERSION) < packaging.version.parse("1.25.0"), + reason="This test requires google-auth >= 1.25.0", +) + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert BigQueryWriteClient._get_default_mtls_endpoint(None) is None + assert BigQueryWriteClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi + + +@pytest.mark.parametrize("client_class", [ + BigQueryWriteClient, + BigQueryWriteAsyncClient, +]) +def test_big_query_write_client_from_service_account_info(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == 'bigquerystorage.googleapis.com:443' + + +@pytest.mark.parametrize("transport_class,transport_name", [ + (transports.BigQueryWriteGrpcTransport, "grpc"), + (transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_write_client_service_account_always_use_jwt(transport_class, transport_name): + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class", [ + BigQueryWriteClient, + BigQueryWriteAsyncClient, +]) +def test_big_query_write_client_from_service_account_file(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == 'bigquerystorage.googleapis.com:443' + + +def test_big_query_write_client_get_transport_class(): + transport = BigQueryWriteClient.get_transport_class() + available_transports = [ + transports.BigQueryWriteGrpcTransport, + ] + assert transport in available_transports + + transport = BigQueryWriteClient.get_transport_class("grpc") + assert transport == transports.BigQueryWriteGrpcTransport + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), +]) +@mock.patch.object(BigQueryWriteClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteAsyncClient)) +def test_big_query_write_client_client_options(client_class, transport_class, transport_name): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(BigQueryWriteClient, 'get_transport_class') as gtc: + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(BigQueryWriteClient, 'get_transport_class') as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError): + client = client_class() + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError): + client = client_class() + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", "true"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", "true"), + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", "false"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", "false"), +]) +@mock.patch.object(BigQueryWriteClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteAsyncClient)) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_big_query_write_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client.DEFAULT_ENDPOINT + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): + if use_client_cert_env == "false": + expected_host = client.DEFAULT_ENDPOINT + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_write_client_client_options_scopes(client_class, transport_class, transport_name): + # Check the case scopes are provided. + options = client_options.ClientOptions( + scopes=["1", "2"], + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_write_client_client_options_credentials_file(client_class, transport_class, transport_name): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +def test_big_query_write_client_client_options_from_dict(): + with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteGrpcTransport.__init__') as grpc_transport: + grpc_transport.return_value = None + client = BigQueryWriteClient( + client_options={'api_endpoint': 'squid.clam.whelk'} + ) + grpc_transport.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +def test_create_write_stream(transport: str = 'grpc', request_type=storage.CreateWriteStreamRequest): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + ) + response = client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +def test_create_write_stream_from_dict(): + test_create_write_stream(request_type=dict) + + +def test_create_write_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + client.create_write_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateWriteStreamRequest() + + +@pytest.mark.asyncio +async def test_create_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.CreateWriteStreamRequest): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + )) + response = await client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +@pytest.mark.asyncio +async def test_create_write_stream_async_from_dict(): + await test_create_write_stream_async(request_type=dict) + + +def test_create_write_stream_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateWriteStreamRequest() + + request.parent = 'parent/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + call.return_value = stream.WriteStream() + client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_create_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateWriteStreamRequest() + + request.parent = 'parent/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + await client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent/value', + ) in kw['metadata'] + + +def test_create_write_stream_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.create_write_stream( + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].parent == 'parent_value' + assert args[0].write_stream == stream.WriteStream(name='name_value') + + +def test_create_write_stream_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.create_write_stream( + storage.CreateWriteStreamRequest(), + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + + +@pytest.mark.asyncio +async def test_create_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.create_write_stream( + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].parent == 'parent_value' + assert args[0].write_stream == stream.WriteStream(name='name_value') + + +@pytest.mark.asyncio +async def test_create_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.create_write_stream( + storage.CreateWriteStreamRequest(), + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + + +def test_append_rows(transport: str = 'grpc', request_type=storage.AppendRowsRequest): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + requests = [request] + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.append_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.AppendRowsResponse()]) + response = client.append_rows(iter(requests)) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert next(args[0]) == request + + # Establish that the response is the type that we expect. + for message in response: + assert isinstance(message, storage.AppendRowsResponse) + + +def test_append_rows_from_dict(): + test_append_rows(request_type=dict) + + +@pytest.mark.asyncio +async def test_append_rows_async(transport: str = 'grpc_asyncio', request_type=storage.AppendRowsRequest): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + requests = [request] + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.append_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.StreamStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.AppendRowsResponse()]) + response = await client.append_rows(iter(requests)) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert next(args[0]) == request + + # Establish that the response is the type that we expect. + message = await response.read() + assert isinstance(message, storage.AppendRowsResponse) + + +@pytest.mark.asyncio +async def test_append_rows_async_from_dict(): + await test_append_rows_async(request_type=dict) + + +def test_append_rows_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.append_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.AppendRowsResponse()]) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.append_rows( + write_stream='write_stream_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].write_stream == 'write_stream_value' + + +def test_append_rows_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.append_rows( + storage.AppendRowsRequest(), + write_stream='write_stream_value', + ) + + +@pytest.mark.asyncio +async def test_append_rows_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.append_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.AppendRowsResponse()]) + + call.return_value = mock.Mock(aio.StreamStreamCall, autospec=True) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.append_rows( + write_stream='write_stream_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].write_stream == 'write_stream_value' + + +@pytest.mark.asyncio +async def test_append_rows_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.append_rows( + storage.AppendRowsRequest(), + write_stream='write_stream_value', + ) + + +def test_get_write_stream(transport: str = 'grpc', request_type=storage.GetWriteStreamRequest): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + ) + response = client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.GetWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +def test_get_write_stream_from_dict(): + test_get_write_stream(request_type=dict) + + +def test_get_write_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + client.get_write_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.GetWriteStreamRequest() + + +@pytest.mark.asyncio +async def test_get_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.GetWriteStreamRequest): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + )) + response = await client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.GetWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +@pytest.mark.asyncio +async def test_get_write_stream_async_from_dict(): + await test_get_write_stream_async(request_type=dict) + + +def test_get_write_stream_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.GetWriteStreamRequest() + + request.name = 'name/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + call.return_value = stream.WriteStream() + client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_get_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.GetWriteStreamRequest() + + request.name = 'name/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + await client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name/value', + ) in kw['metadata'] + + +def test_get_write_stream_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.get_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].name == 'name_value' + + +def test_get_write_stream_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.get_write_stream( + storage.GetWriteStreamRequest(), + name='name_value', + ) + + +@pytest.mark.asyncio +async def test_get_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.get_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].name == 'name_value' + + +@pytest.mark.asyncio +async def test_get_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.get_write_stream( + storage.GetWriteStreamRequest(), + name='name_value', + ) + + +def test_finalize_write_stream(transport: str = 'grpc', request_type=storage.FinalizeWriteStreamRequest): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse( + row_count=992, + ) + response = client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FinalizeWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FinalizeWriteStreamResponse) + assert response.row_count == 992 + + +def test_finalize_write_stream_from_dict(): + test_finalize_write_stream(request_type=dict) + + +def test_finalize_write_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + client.finalize_write_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FinalizeWriteStreamRequest() + + +@pytest.mark.asyncio +async def test_finalize_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.FinalizeWriteStreamRequest): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse( + row_count=992, + )) + response = await client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FinalizeWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FinalizeWriteStreamResponse) + assert response.row_count == 992 + + +@pytest.mark.asyncio +async def test_finalize_write_stream_async_from_dict(): + await test_finalize_write_stream_async(request_type=dict) + + +def test_finalize_write_stream_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FinalizeWriteStreamRequest() + + request.name = 'name/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + call.return_value = storage.FinalizeWriteStreamResponse() + client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_finalize_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FinalizeWriteStreamRequest() + + request.name = 'name/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse()) + await client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name/value', + ) in kw['metadata'] + + +def test_finalize_write_stream_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.finalize_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].name == 'name_value' + + +def test_finalize_write_stream_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.finalize_write_stream( + storage.FinalizeWriteStreamRequest(), + name='name_value', + ) + + +@pytest.mark.asyncio +async def test_finalize_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.finalize_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].name == 'name_value' + + +@pytest.mark.asyncio +async def test_finalize_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.finalize_write_stream( + storage.FinalizeWriteStreamRequest(), + name='name_value', + ) + + +def test_batch_commit_write_streams(transport: str = 'grpc', request_type=storage.BatchCommitWriteStreamsRequest): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse( + ) + response = client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.BatchCommitWriteStreamsRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.BatchCommitWriteStreamsResponse) + + +def test_batch_commit_write_streams_from_dict(): + test_batch_commit_write_streams(request_type=dict) + + +def test_batch_commit_write_streams_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + client.batch_commit_write_streams() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.BatchCommitWriteStreamsRequest() + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_async(transport: str = 'grpc_asyncio', request_type=storage.BatchCommitWriteStreamsRequest): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse( + )) + response = await client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.BatchCommitWriteStreamsRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.BatchCommitWriteStreamsResponse) + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_async_from_dict(): + await test_batch_commit_write_streams_async(request_type=dict) + + +def test_batch_commit_write_streams_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.BatchCommitWriteStreamsRequest() + + request.parent = 'parent/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + call.return_value = storage.BatchCommitWriteStreamsResponse() + client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.BatchCommitWriteStreamsRequest() + + request.parent = 'parent/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse()) + await client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent/value', + ) in kw['metadata'] + + +def test_batch_commit_write_streams_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.batch_commit_write_streams( + parent='parent_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].parent == 'parent_value' + + +def test_batch_commit_write_streams_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.batch_commit_write_streams( + storage.BatchCommitWriteStreamsRequest(), + parent='parent_value', + ) + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.batch_commit_write_streams( + parent='parent_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].parent == 'parent_value' + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.batch_commit_write_streams( + storage.BatchCommitWriteStreamsRequest(), + parent='parent_value', + ) + + +def test_flush_rows(transport: str = 'grpc', request_type=storage.FlushRowsRequest): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse( + offset=647, + ) + response = client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FlushRowsRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FlushRowsResponse) + assert response.offset == 647 + + +def test_flush_rows_from_dict(): + test_flush_rows(request_type=dict) + + +def test_flush_rows_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + client.flush_rows() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FlushRowsRequest() + + +@pytest.mark.asyncio +async def test_flush_rows_async(transport: str = 'grpc_asyncio', request_type=storage.FlushRowsRequest): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse( + offset=647, + )) + response = await client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FlushRowsRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FlushRowsResponse) + assert response.offset == 647 + + +@pytest.mark.asyncio +async def test_flush_rows_async_from_dict(): + await test_flush_rows_async(request_type=dict) + + +def test_flush_rows_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FlushRowsRequest() + + request.write_stream = 'write_stream/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + call.return_value = storage.FlushRowsResponse() + client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'write_stream=write_stream/value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_flush_rows_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FlushRowsRequest() + + request.write_stream = 'write_stream/value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse()) + await client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'write_stream=write_stream/value', + ) in kw['metadata'] + + +def test_flush_rows_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.flush_rows( + write_stream='write_stream_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].write_stream == 'write_stream_value' + + +def test_flush_rows_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.flush_rows( + storage.FlushRowsRequest(), + write_stream='write_stream_value', + ) + + +@pytest.mark.asyncio +async def test_flush_rows_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.flush_rows( + write_stream='write_stream_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].write_stream == 'write_stream_value' + + +@pytest.mark.asyncio +async def test_flush_rows_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.flush_rows( + storage.FlushRowsRequest(), + write_stream='write_stream_value', + ) + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options={"scopes": ["1", "2"]}, + transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = BigQueryWriteClient(transport=transport) + assert client.transport is transport + +def test_transport_get_channel(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + + transport = transports.BigQueryWriteGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + +@pytest.mark.parametrize("transport_class", [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, +]) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + +def test_transport_grpc_default(): + # A client should use the gRPC transport by default. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert isinstance( + client.transport, + transports.BigQueryWriteGrpcTransport, + ) + +def test_big_query_write_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transport = transports.BigQueryWriteTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json" + ) + + +def test_big_query_write_base_transport(): + # Instantiate the base transport. + with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport.__init__') as Transport: + Transport.return_value = None + transport = transports.BigQueryWriteTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + 'create_write_stream', + 'append_rows', + 'get_write_stream', + 'finalize_write_stream', + 'batch_commit_write_streams', + 'flush_rows', + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + +@requires_google_auth_gte_1_25_0 +def test_big_query_write_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryWriteTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +@requires_google_auth_lt_1_25_0 +def test_big_query_write_base_transport_with_credentials_file_old_google_auth(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryWriteTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', + ), + quota_project_id="octopus", + ) + + +def test_big_query_write_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryWriteTransport() + adc.assert_called_once() + + +@requires_google_auth_gte_1_25_0 +def test_big_query_write_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryWriteClient() + adc.assert_called_once_with( + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id=None, + ) + + +@requires_google_auth_lt_1_25_0 +def test_big_query_write_auth_adc_old_google_auth(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryWriteClient() + adc.assert_called_once_with( + scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.insertdata', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id=None, + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +@requires_google_auth_gte_1_25_0 +def test_big_query_write_transport_auth_adc(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + adc.assert_called_once_with( + scopes=["1", "2"], + default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.insertdata', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +@requires_google_auth_lt_1_25_0 +def test_big_query_write_transport_auth_adc_old_google_auth(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus") + adc.assert_called_once_with(scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.BigQueryWriteGrpcTransport, grpc_helpers), + (transports.BigQueryWriteGrpcAsyncIOTransport, grpc_helpers_async) + ], +) +def test_big_query_write_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class( + quota_project_id="octopus", + scopes=["1", "2"] + ) + + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=["1", "2"], + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) +def test_big_query_write_grpc_transport_client_cert_source_for_mtls( + transport_class +): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, + private_key=expected_key + ) + + +def test_big_query_write_host_no_port(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), + ) + assert client.transport._host == 'bigquerystorage.googleapis.com:443' + + +def test_big_query_write_host_with_port(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), + ) + assert client.transport._host == 'bigquerystorage.googleapis.com:8000' + +def test_big_query_write_grpc_transport_channel(): + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryWriteGrpcTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +def test_big_query_write_grpc_asyncio_transport_channel(): + channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryWriteGrpcAsyncIOTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) +def test_big_query_write_transport_channel_mtls_with_client_cert_source( + transport_class +): + with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_ssl_cred = mock.Mock() + grpc_ssl_channel_cred.return_value = mock_ssl_cred + + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + + cred = ga_credentials.AnonymousCredentials() + with pytest.warns(DeprecationWarning): + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (cred, None) + transport = transport_class( + host="squid.clam.whelk", + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=client_cert_source_callback, + ) + adc.assert_called_once() + + grpc_ssl_channel_cred.assert_called_once_with( + certificate_chain=b"cert bytes", private_key=b"key bytes" + ) + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + assert transport._ssl_channel_credentials == mock_ssl_cred + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) +def test_big_query_write_transport_channel_mtls_with_adc( + transport_class +): + mock_ssl_cred = mock.Mock() + with mock.patch.multiple( + "google.auth.transport.grpc.SslCredentials", + __init__=mock.Mock(return_value=None), + ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), + ): + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + mock_cred = mock.Mock() + + with pytest.warns(DeprecationWarning): + transport = transport_class( + host="squid.clam.whelk", + credentials=mock_cred, + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=None, + ) + + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=mock_cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + + +def test_table_path(): + project = "squid" + dataset = "clam" + table = "whelk" + expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + actual = BigQueryWriteClient.table_path(project, dataset, table) + assert expected == actual + + +def test_parse_table_path(): + expected = { + "project": "octopus", + "dataset": "oyster", + "table": "nudibranch", + } + path = BigQueryWriteClient.table_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_table_path(path) + assert expected == actual + +def test_write_stream_path(): + project = "cuttlefish" + dataset = "mussel" + table = "winkle" + stream = "nautilus" + expected = "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}".format(project=project, dataset=dataset, table=table, stream=stream, ) + actual = BigQueryWriteClient.write_stream_path(project, dataset, table, stream) + assert expected == actual + + +def test_parse_write_stream_path(): + expected = { + "project": "scallop", + "dataset": "abalone", + "table": "squid", + "stream": "clam", + } + path = BigQueryWriteClient.write_stream_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_write_stream_path(path) + assert expected == actual + +def test_common_billing_account_path(): + billing_account = "whelk" + expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + actual = BigQueryWriteClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "octopus", + } + path = BigQueryWriteClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_billing_account_path(path) + assert expected == actual + +def test_common_folder_path(): + folder = "oyster" + expected = "folders/{folder}".format(folder=folder, ) + actual = BigQueryWriteClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "nudibranch", + } + path = BigQueryWriteClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_folder_path(path) + assert expected == actual + +def test_common_organization_path(): + organization = "cuttlefish" + expected = "organizations/{organization}".format(organization=organization, ) + actual = BigQueryWriteClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "mussel", + } + path = BigQueryWriteClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_organization_path(path) + assert expected == actual + +def test_common_project_path(): + project = "winkle" + expected = "projects/{project}".format(project=project, ) + actual = BigQueryWriteClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "nautilus", + } + path = BigQueryWriteClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_project_path(path) + assert expected == actual + +def test_common_location_path(): + project = "scallop" + location = "abalone" + expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) + actual = BigQueryWriteClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "squid", + "location": "clam", + } + path = BigQueryWriteClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_withDEFAULT_CLIENT_INFO(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object(transports.BigQueryWriteTransport, '_prep_wrapped_messages') as prep: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object(transports.BigQueryWriteTransport, '_prep_wrapped_messages') as prep: + transport_class = BigQueryWriteClient.get_transport_class() + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) From 7a23fab36968c176435c3f1dfe7229e0e292822e Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 1 Sep 2021 21:51:30 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../services/big_query_read/async_client.py | 2 +- .../services/big_query_read/client.py | 2 +- .../big_query_read/transports/grpc.py | 2 +- .../big_query_read/transports/grpc_asyncio.py | 2 +- .../services/big_query_write/async_client.py | 4 +- .../big_query_write/transports/base.py | 4 +- owl-bot-staging/v1/.coveragerc | 17 - owl-bot-staging/v1/MANIFEST.in | 2 - owl-bot-staging/v1/README.rst | 49 - .../bigquery_storage_v1/big_query_read.rst | 6 - .../v1/docs/bigquery_storage_v1/services.rst | 6 - .../v1/docs/bigquery_storage_v1/types.rst | 7 - owl-bot-staging/v1/docs/conf.py | 376 --- owl-bot-staging/v1/docs/index.rst | 7 - .../google/cloud/bigquery_storage/__init__.py | 53 - .../v1/google/cloud/bigquery_storage/py.typed | 2 - .../cloud/bigquery_storage_v1/__init__.py | 54 - .../bigquery_storage_v1/gapic_metadata.json | 53 - .../google/cloud/bigquery_storage_v1/py.typed | 2 - .../bigquery_storage_v1/services/__init__.py | 15 - .../services/big_query_read/__init__.py | 22 - .../services/big_query_read/async_client.py | 474 ---- .../services/big_query_read/client.py | 664 ----- .../big_query_read/transports/__init__.py | 33 - .../big_query_read/transports/base.py | 219 -- .../big_query_read/transports/grpc.py | 347 --- .../big_query_read/transports/grpc_asyncio.py | 351 --- .../bigquery_storage_v1/types/__init__.py | 56 - .../cloud/bigquery_storage_v1/types/arrow.py | 87 - .../cloud/bigquery_storage_v1/types/avro.py | 61 - .../bigquery_storage_v1/types/storage.py | 275 -- .../cloud/bigquery_storage_v1/types/stream.py | 194 -- owl-bot-staging/v1/mypy.ini | 3 - owl-bot-staging/v1/noxfile.py | 132 - .../fixup_bigquery_storage_v1_keywords.py | 178 -- owl-bot-staging/v1/setup.py | 54 - owl-bot-staging/v1/tests/__init__.py | 16 - owl-bot-staging/v1/tests/unit/__init__.py | 16 - .../v1/tests/unit/gapic/__init__.py | 16 - .../gapic/bigquery_storage_v1/__init__.py | 16 - .../test_big_query_read.py | 1629 ------------ owl-bot-staging/v1beta2/.coveragerc | 17 - owl-bot-staging/v1beta2/MANIFEST.in | 2 - owl-bot-staging/v1beta2/README.rst | 49 - .../big_query_read.rst | 6 - .../big_query_write.rst | 6 - .../bigquery_storage_v1beta2/services.rst | 7 - .../docs/bigquery_storage_v1beta2/types.rst | 7 - owl-bot-staging/v1beta2/docs/conf.py | 376 --- owl-bot-staging/v1beta2/docs/index.rst | 7 - .../google/cloud/bigquery_storage/__init__.py | 89 - .../google/cloud/bigquery_storage/py.typed | 2 - .../bigquery_storage_v1beta2/__init__.py | 90 - .../gapic_metadata.json | 127 - .../cloud/bigquery_storage_v1beta2/py.typed | 2 - .../services/__init__.py | 15 - .../services/big_query_read/__init__.py | 22 - .../services/big_query_read/async_client.py | 476 ---- .../services/big_query_read/client.py | 666 ----- .../big_query_read/transports/__init__.py | 33 - .../big_query_read/transports/base.py | 219 -- .../big_query_read/transports/grpc.py | 349 --- .../big_query_read/transports/grpc_asyncio.py | 353 --- .../services/big_query_write/__init__.py | 22 - .../services/big_query_write/async_client.py | 678 ----- .../services/big_query_write/client.py | 831 ------ .../big_query_write/transports/__init__.py | 33 - .../big_query_write/transports/base.py | 284 -- .../big_query_write/transports/grpc.py | 420 --- .../transports/grpc_asyncio.py | 424 --- .../types/__init__.py | 92 - .../bigquery_storage_v1beta2/types/arrow.py | 80 - .../bigquery_storage_v1beta2/types/avro.py | 55 - .../types/protobuf.py | 68 - .../bigquery_storage_v1beta2/types/storage.py | 618 ----- .../bigquery_storage_v1beta2/types/stream.py | 253 -- .../bigquery_storage_v1beta2/types/table.py | 113 - owl-bot-staging/v1beta2/mypy.ini | 3 - owl-bot-staging/v1beta2/noxfile.py | 132 - ...fixup_bigquery_storage_v1beta2_keywords.py | 184 -- owl-bot-staging/v1beta2/setup.py | 54 - owl-bot-staging/v1beta2/tests/__init__.py | 16 - .../v1beta2/tests/unit/__init__.py | 16 - .../v1beta2/tests/unit/gapic/__init__.py | 16 - .../bigquery_storage_v1beta2/__init__.py | 16 - .../test_big_query_read.py | 1629 ------------ .../test_big_query_write.py | 2294 ----------------- 87 files changed, 8 insertions(+), 16751 deletions(-) delete mode 100644 owl-bot-staging/v1/.coveragerc delete mode 100644 owl-bot-staging/v1/MANIFEST.in delete mode 100644 owl-bot-staging/v1/README.rst delete mode 100644 owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst delete mode 100644 owl-bot-staging/v1/docs/bigquery_storage_v1/services.rst delete mode 100644 owl-bot-staging/v1/docs/bigquery_storage_v1/types.rst delete mode 100644 owl-bot-staging/v1/docs/conf.py delete mode 100644 owl-bot-staging/v1/docs/index.rst delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py delete mode 100644 owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py delete mode 100644 owl-bot-staging/v1/mypy.ini delete mode 100644 owl-bot-staging/v1/noxfile.py delete mode 100644 owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py delete mode 100644 owl-bot-staging/v1/setup.py delete mode 100644 owl-bot-staging/v1/tests/__init__.py delete mode 100644 owl-bot-staging/v1/tests/unit/__init__.py delete mode 100644 owl-bot-staging/v1/tests/unit/gapic/__init__.py delete mode 100644 owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py delete mode 100644 owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py delete mode 100644 owl-bot-staging/v1beta2/.coveragerc delete mode 100644 owl-bot-staging/v1beta2/MANIFEST.in delete mode 100644 owl-bot-staging/v1beta2/README.rst delete mode 100644 owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst delete mode 100644 owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst delete mode 100644 owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services.rst delete mode 100644 owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types.rst delete mode 100644 owl-bot-staging/v1beta2/docs/conf.py delete mode 100644 owl-bot-staging/v1beta2/docs/index.rst delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py delete mode 100644 owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py delete mode 100644 owl-bot-staging/v1beta2/mypy.ini delete mode 100644 owl-bot-staging/v1beta2/noxfile.py delete mode 100644 owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py delete mode 100644 owl-bot-staging/v1beta2/setup.py delete mode 100644 owl-bot-staging/v1beta2/tests/__init__.py delete mode 100644 owl-bot-staging/v1beta2/tests/unit/__init__.py delete mode 100644 owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py delete mode 100644 owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py delete mode 100644 owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py delete mode 100644 owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py index b8fc974a..9a91feae 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py @@ -199,7 +199,7 @@ async def create_read_session( number of pre-filtered rows, so some filters can lead to lopsided assignments. - Read sessions automatically expire 24 hours after they + Read sessions automatically expire 6 hours after they are created and do not require manual clean-up by the caller. diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py index f1838685..35675b84 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py @@ -420,7 +420,7 @@ def create_read_session( number of pre-filtered rows, so some filters can lead to lopsided assignments. - Read sessions automatically expire 24 hours after they + Read sessions automatically expire 6 hours after they are created and do not require manual clean-up by the caller. diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py index 54f5fed1..42f4e733 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py @@ -255,7 +255,7 @@ def create_read_session( number of pre-filtered rows, so some filters can lead to lopsided assignments. - Read sessions automatically expire 24 hours after they + Read sessions automatically expire 6 hours after they are created and do not require manual clean-up by the caller. diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py index 7cb0784f..2b7433a3 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py @@ -258,7 +258,7 @@ def create_read_session( number of pre-filtered rows, so some filters can lead to lopsided assignments. - Read sessions automatically expire 24 hours after they + Read sessions automatically expire 6 hours after they are created and do not require manual clean-up by the caller. diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py index 497182fb..9c6ba354 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py @@ -331,9 +331,9 @@ def append_rows( core_exceptions.ResourceExhausted, core_exceptions.ServiceUnavailable, ), - deadline=None, + deadline=86400.0, ), - default_timeout=None, + default_timeout=86400.0, client_info=DEFAULT_CLIENT_INFO, ) diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py index 7286ce5b..78571ced 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py @@ -185,9 +185,9 @@ def _prep_wrapped_messages(self, client_info): core_exceptions.ResourceExhausted, core_exceptions.ServiceUnavailable, ), - deadline=None, + deadline=86400.0, ), - default_timeout=None, + default_timeout=86400.0, client_info=client_info, ), self.get_write_stream: gapic_v1.method.wrap_method( diff --git a/owl-bot-staging/v1/.coveragerc b/owl-bot-staging/v1/.coveragerc deleted file mode 100644 index 19fadc8b..00000000 --- a/owl-bot-staging/v1/.coveragerc +++ /dev/null @@ -1,17 +0,0 @@ -[run] -branch = True - -[report] -show_missing = True -omit = - google/cloud/bigquery_storage/__init__.py -exclude_lines = - # Re-enable the standard pragma - pragma: NO COVER - # Ignore debug-only repr - def __repr__ - # Ignore pkg_resources exceptions. - # This is added at the module level as a safeguard for if someone - # generates the code and tries to run it without pip installing. This - # makes it virtually impossible to test properly. - except pkg_resources.DistributionNotFound diff --git a/owl-bot-staging/v1/MANIFEST.in b/owl-bot-staging/v1/MANIFEST.in deleted file mode 100644 index 88a0105e..00000000 --- a/owl-bot-staging/v1/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -recursive-include google/cloud/bigquery_storage *.py -recursive-include google/cloud/bigquery_storage_v1 *.py diff --git a/owl-bot-staging/v1/README.rst b/owl-bot-staging/v1/README.rst deleted file mode 100644 index 13a45c14..00000000 --- a/owl-bot-staging/v1/README.rst +++ /dev/null @@ -1,49 +0,0 @@ -Python Client for Google Cloud Bigquery Storage API -================================================= - -Quick Start ------------ - -In order to use this library, you first need to go through the following steps: - -1. `Select or create a Cloud Platform project.`_ -2. `Enable billing for your project.`_ -3. Enable the Google Cloud Bigquery Storage API. -4. `Setup Authentication.`_ - -.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project -.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project -.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html - -Installation -~~~~~~~~~~~~ - -Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to -create isolated Python environments. The basic problem it addresses is one of -dependencies and versions, and indirectly permissions. - -With `virtualenv`_, it's possible to install this library without needing system -install permissions, and without clashing with the installed system -dependencies. - -.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ - - -Mac/Linux -^^^^^^^^^ - -.. code-block:: console - - python3 -m venv - source /bin/activate - /bin/pip install /path/to/library - - -Windows -^^^^^^^ - -.. code-block:: console - - python3 -m venv - \Scripts\activate - \Scripts\pip.exe install \path\to\library diff --git a/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst b/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst deleted file mode 100644 index 74f0a5fa..00000000 --- a/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst +++ /dev/null @@ -1,6 +0,0 @@ -BigQueryRead ------------------------------- - -.. automodule:: google.cloud.bigquery_storage_v1.services.big_query_read - :members: - :inherited-members: diff --git a/owl-bot-staging/v1/docs/bigquery_storage_v1/services.rst b/owl-bot-staging/v1/docs/bigquery_storage_v1/services.rst deleted file mode 100644 index 3f6cff3d..00000000 --- a/owl-bot-staging/v1/docs/bigquery_storage_v1/services.rst +++ /dev/null @@ -1,6 +0,0 @@ -Services for Google Cloud Bigquery Storage v1 API -================================================= -.. toctree:: - :maxdepth: 2 - - big_query_read diff --git a/owl-bot-staging/v1/docs/bigquery_storage_v1/types.rst b/owl-bot-staging/v1/docs/bigquery_storage_v1/types.rst deleted file mode 100644 index 28b5db72..00000000 --- a/owl-bot-staging/v1/docs/bigquery_storage_v1/types.rst +++ /dev/null @@ -1,7 +0,0 @@ -Types for Google Cloud Bigquery Storage v1 API -============================================== - -.. automodule:: google.cloud.bigquery_storage_v1.types - :members: - :undoc-members: - :show-inheritance: diff --git a/owl-bot-staging/v1/docs/conf.py b/owl-bot-staging/v1/docs/conf.py deleted file mode 100644 index e198bf7a..00000000 --- a/owl-bot-staging/v1/docs/conf.py +++ /dev/null @@ -1,376 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# google-cloud-bigquery-storage documentation build configuration file -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os -import shlex - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath("..")) - -__version__ = "0.1.0" - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = "1.6.3" - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.autosummary", - "sphinx.ext.intersphinx", - "sphinx.ext.coverage", - "sphinx.ext.napoleon", - "sphinx.ext.todo", - "sphinx.ext.viewcode", -] - -# autodoc/autosummary flags -autoclass_content = "both" -autodoc_default_flags = ["members"] -autosummary_generate = True - - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# Allow markdown includes (so releases.md can include CHANGLEOG.md) -# http://www.sphinx-doc.org/en/master/markdown.html -source_parsers = {".md": "recommonmark.parser.CommonMarkParser"} - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -source_suffix = [".rst", ".md"] - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = u"google-cloud-bigquery-storage" -copyright = u"2020, Google, LLC" -author = u"Google APIs" # TODO: autogenerate this bit - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The full version, including alpha/beta/rc tags. -release = __version__ -# The short X.Y version. -version = ".".join(release.split(".")[0:2]) - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ["_build"] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = "alabaster" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = { - "description": "Google Cloud Client Libraries for Python", - "github_user": "googleapis", - "github_repo": "google-cloud-python", - "github_banner": True, - "font_family": "'Roboto', Georgia, sans", - "head_font_family": "'Roboto', Georgia, serif", - "code_font_family": "'Roboto Mono', 'Consolas', monospace", -} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -# html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# Now only 'ja' uses this config value -# html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -# html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = "google-cloud-bigquery-storage-doc" - -# -- Options for warnings ------------------------------------------------------ - - -suppress_warnings = [ - # Temporarily suppress this to avoid "more than one target found for - # cross-reference" warning, which are intractable for us to avoid while in - # a mono-repo. - # See https://github.com/sphinx-doc/sphinx/blob - # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843 - "ref.python" -] - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # 'preamble': '', - # Latex figure (float) alignment - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ( - master_doc, - "google-cloud-bigquery-storage.tex", - u"google-cloud-bigquery-storage Documentation", - author, - "manual", - ) -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ( - master_doc, - "google-cloud-bigquery-storage", - u"Google Cloud Bigquery Storage Documentation", - [author], - 1, - ) -] - -# If true, show URL addresses after external links. -# man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - "google-cloud-bigquery-storage", - u"google-cloud-bigquery-storage Documentation", - author, - "google-cloud-bigquery-storage", - "GAPIC library for Google Cloud Bigquery Storage API", - "APIs", - ) -] - -# Documents to append as an appendix to all manuals. -# texinfo_appendices = [] - -# If false, no module index is generated. -# texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -# texinfo_no_detailmenu = False - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = { - "python": ("http://python.readthedocs.org/en/latest/", None), - "gax": ("https://gax-python.readthedocs.org/en/latest/", None), - "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), - "google-gax": ("https://gax-python.readthedocs.io/en/latest/", None), - "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None), - "grpc": ("https://grpc.io/grpc/python/", None), - "requests": ("http://requests.kennethreitz.org/en/stable/", None), - "proto": ("https://proto-plus-python.readthedocs.io/en/stable", None), - "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), -} - - -# Napoleon settings -napoleon_google_docstring = True -napoleon_numpy_docstring = True -napoleon_include_private_with_doc = False -napoleon_include_special_with_doc = True -napoleon_use_admonition_for_examples = False -napoleon_use_admonition_for_notes = False -napoleon_use_admonition_for_references = False -napoleon_use_ivar = False -napoleon_use_param = True -napoleon_use_rtype = True diff --git a/owl-bot-staging/v1/docs/index.rst b/owl-bot-staging/v1/docs/index.rst deleted file mode 100644 index a5e3a707..00000000 --- a/owl-bot-staging/v1/docs/index.rst +++ /dev/null @@ -1,7 +0,0 @@ -API Reference -------------- -.. toctree:: - :maxdepth: 2 - - bigquery_storage_v1/services - bigquery_storage_v1/types diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py deleted file mode 100644 index 281b811f..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from google.cloud.bigquery_storage_v1.services.big_query_read.client import BigQueryReadClient -from google.cloud.bigquery_storage_v1.services.big_query_read.async_client import BigQueryReadAsyncClient - -from google.cloud.bigquery_storage_v1.types.arrow import ArrowRecordBatch -from google.cloud.bigquery_storage_v1.types.arrow import ArrowSchema -from google.cloud.bigquery_storage_v1.types.arrow import ArrowSerializationOptions -from google.cloud.bigquery_storage_v1.types.avro import AvroRows -from google.cloud.bigquery_storage_v1.types.avro import AvroSchema -from google.cloud.bigquery_storage_v1.types.storage import CreateReadSessionRequest -from google.cloud.bigquery_storage_v1.types.storage import ReadRowsRequest -from google.cloud.bigquery_storage_v1.types.storage import ReadRowsResponse -from google.cloud.bigquery_storage_v1.types.storage import SplitReadStreamRequest -from google.cloud.bigquery_storage_v1.types.storage import SplitReadStreamResponse -from google.cloud.bigquery_storage_v1.types.storage import StreamStats -from google.cloud.bigquery_storage_v1.types.storage import ThrottleState -from google.cloud.bigquery_storage_v1.types.stream import ReadSession -from google.cloud.bigquery_storage_v1.types.stream import ReadStream -from google.cloud.bigquery_storage_v1.types.stream import DataFormat - -__all__ = ('BigQueryReadClient', - 'BigQueryReadAsyncClient', - 'ArrowRecordBatch', - 'ArrowSchema', - 'ArrowSerializationOptions', - 'AvroRows', - 'AvroSchema', - 'CreateReadSessionRequest', - 'ReadRowsRequest', - 'ReadRowsResponse', - 'SplitReadStreamRequest', - 'SplitReadStreamResponse', - 'StreamStats', - 'ThrottleState', - 'ReadSession', - 'ReadStream', - 'DataFormat', -) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed b/owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed deleted file mode 100644 index e71b4749..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed +++ /dev/null @@ -1,2 +0,0 @@ -# Marker file for PEP 561. -# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py deleted file mode 100644 index 423f54ae..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .services.big_query_read import BigQueryReadClient -from .services.big_query_read import BigQueryReadAsyncClient - -from .types.arrow import ArrowRecordBatch -from .types.arrow import ArrowSchema -from .types.arrow import ArrowSerializationOptions -from .types.avro import AvroRows -from .types.avro import AvroSchema -from .types.storage import CreateReadSessionRequest -from .types.storage import ReadRowsRequest -from .types.storage import ReadRowsResponse -from .types.storage import SplitReadStreamRequest -from .types.storage import SplitReadStreamResponse -from .types.storage import StreamStats -from .types.storage import ThrottleState -from .types.stream import ReadSession -from .types.stream import ReadStream -from .types.stream import DataFormat - -__all__ = ( - 'BigQueryReadAsyncClient', -'ArrowRecordBatch', -'ArrowSchema', -'ArrowSerializationOptions', -'AvroRows', -'AvroSchema', -'BigQueryReadClient', -'CreateReadSessionRequest', -'DataFormat', -'ReadRowsRequest', -'ReadRowsResponse', -'ReadSession', -'ReadStream', -'SplitReadStreamRequest', -'SplitReadStreamResponse', -'StreamStats', -'ThrottleState', -) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json deleted file mode 100644 index 3311e568..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json +++ /dev/null @@ -1,53 +0,0 @@ - { - "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", - "language": "python", - "libraryPackage": "google.cloud.bigquery_storage_v1", - "protoPackage": "google.cloud.bigquery.storage.v1", - "schema": "1.0", - "services": { - "BigQueryRead": { - "clients": { - "grpc": { - "libraryClient": "BigQueryReadClient", - "rpcs": { - "CreateReadSession": { - "methods": [ - "create_read_session" - ] - }, - "ReadRows": { - "methods": [ - "read_rows" - ] - }, - "SplitReadStream": { - "methods": [ - "split_read_stream" - ] - } - } - }, - "grpc-async": { - "libraryClient": "BigQueryReadAsyncClient", - "rpcs": { - "CreateReadSession": { - "methods": [ - "create_read_session" - ] - }, - "ReadRows": { - "methods": [ - "read_rows" - ] - }, - "SplitReadStream": { - "methods": [ - "split_read_stream" - ] - } - } - } - } - } - } -} diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed deleted file mode 100644 index e71b4749..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed +++ /dev/null @@ -1,2 +0,0 @@ -# Marker file for PEP 561. -# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py deleted file mode 100644 index 4de65971..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py deleted file mode 100644 index 13be8ad1..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from .client import BigQueryReadClient -from .async_client import BigQueryReadAsyncClient - -__all__ = ( - 'BigQueryReadClient', - 'BigQueryReadAsyncClient', -) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py deleted file mode 100644 index f9a960ea..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py +++ /dev/null @@ -1,474 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from collections import OrderedDict -import functools -import re -from typing import Dict, AsyncIterable, Awaitable, Sequence, Tuple, Type, Union -import pkg_resources - -import google.api_core.client_options as ClientOptions # type: ignore -from google.api_core import exceptions as core_exceptions # type: ignore -from google.api_core import gapic_v1 # type: ignore -from google.api_core import retry as retries # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.oauth2 import service_account # type: ignore - -from google.cloud.bigquery_storage_v1.types import arrow -from google.cloud.bigquery_storage_v1.types import avro -from google.cloud.bigquery_storage_v1.types import storage -from google.cloud.bigquery_storage_v1.types import stream -from google.protobuf import timestamp_pb2 # type: ignore -from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO -from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport -from .client import BigQueryReadClient - - -class BigQueryReadAsyncClient: - """BigQuery Read API. - The Read API can be used to read data from BigQuery. - """ - - _client: BigQueryReadClient - - DEFAULT_ENDPOINT = BigQueryReadClient.DEFAULT_ENDPOINT - DEFAULT_MTLS_ENDPOINT = BigQueryReadClient.DEFAULT_MTLS_ENDPOINT - - read_session_path = staticmethod(BigQueryReadClient.read_session_path) - parse_read_session_path = staticmethod(BigQueryReadClient.parse_read_session_path) - read_stream_path = staticmethod(BigQueryReadClient.read_stream_path) - parse_read_stream_path = staticmethod(BigQueryReadClient.parse_read_stream_path) - table_path = staticmethod(BigQueryReadClient.table_path) - parse_table_path = staticmethod(BigQueryReadClient.parse_table_path) - common_billing_account_path = staticmethod(BigQueryReadClient.common_billing_account_path) - parse_common_billing_account_path = staticmethod(BigQueryReadClient.parse_common_billing_account_path) - common_folder_path = staticmethod(BigQueryReadClient.common_folder_path) - parse_common_folder_path = staticmethod(BigQueryReadClient.parse_common_folder_path) - common_organization_path = staticmethod(BigQueryReadClient.common_organization_path) - parse_common_organization_path = staticmethod(BigQueryReadClient.parse_common_organization_path) - common_project_path = staticmethod(BigQueryReadClient.common_project_path) - parse_common_project_path = staticmethod(BigQueryReadClient.parse_common_project_path) - common_location_path = staticmethod(BigQueryReadClient.common_location_path) - parse_common_location_path = staticmethod(BigQueryReadClient.parse_common_location_path) - - @classmethod - def from_service_account_info(cls, info: dict, *args, **kwargs): - """Creates an instance of this client using the provided credentials - info. - - Args: - info (dict): The service account private key info. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryReadAsyncClient: The constructed client. - """ - return BigQueryReadClient.from_service_account_info.__func__(BigQueryReadAsyncClient, info, *args, **kwargs) # type: ignore - - @classmethod - def from_service_account_file(cls, filename: str, *args, **kwargs): - """Creates an instance of this client using the provided credentials - file. - - Args: - filename (str): The path to the service account private key json - file. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryReadAsyncClient: The constructed client. - """ - return BigQueryReadClient.from_service_account_file.__func__(BigQueryReadAsyncClient, filename, *args, **kwargs) # type: ignore - - from_service_account_json = from_service_account_file - - @property - def transport(self) -> BigQueryReadTransport: - """Returns the transport used by the client instance. - - Returns: - BigQueryReadTransport: The transport used by the client instance. - """ - return self._client.transport - - get_transport_class = functools.partial(type(BigQueryReadClient).get_transport_class, type(BigQueryReadClient)) - - def __init__(self, *, - credentials: ga_credentials.Credentials = None, - transport: Union[str, BigQueryReadTransport] = "grpc_asyncio", - client_options: ClientOptions = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - ) -> None: - """Instantiates the big query read client. - - Args: - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - transport (Union[str, ~.BigQueryReadTransport]): The - transport to use. If set to None, a transport is chosen - automatically. - client_options (ClientOptions): Custom options for the client. It - won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: - "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable - is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If - not provided, the default SSL client certificate will be used if - present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not - set, no client certificate will be used. - - Raises: - google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport - creation failed for any reason. - """ - self._client = BigQueryReadClient( - credentials=credentials, - transport=transport, - client_options=client_options, - client_info=client_info, - - ) - - async def create_read_session(self, - request: storage.CreateReadSessionRequest = None, - *, - parent: str = None, - read_session: stream.ReadSession = None, - max_stream_count: int = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> stream.ReadSession: - r"""Creates a new read session. A read session divides - the contents of a BigQuery table into one or more - streams, which can then be used to read data from the - table. The read session also specifies properties of the - data to be read, such as a list of columns or a push- - down filter describing the rows to be returned. - - A particular row can be read by at most one stream. When - the caller has reached the end of each stream in the - session, then all the data in the table has been read. - - Data is assigned to each stream such that roughly the - same number of rows can be read from each stream. - Because the server-side unit for assigning data is - collections of rows, the API does not guarantee that - each stream will return the same number or rows. - Additionally, the limits are enforced based on the - number of pre-filtered rows, so some filters can lead to - lopsided assignments. - - Read sessions automatically expire 24 hours after they - are created and do not require manual clean-up by the - caller. - - Args: - request (:class:`google.cloud.bigquery_storage_v1.types.CreateReadSessionRequest`): - The request object. Request message for - `CreateReadSession`. - parent (:class:`str`): - Required. The request project that owns the session, in - the form of ``projects/{project_id}``. - - This corresponds to the ``parent`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - read_session (:class:`google.cloud.bigquery_storage_v1.types.ReadSession`): - Required. Session to be created. - This corresponds to the ``read_session`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - max_stream_count (:class:`int`): - Max initial number of streams. If - unset or zero, the server will provide a - value of streams so as to produce - reasonable throughput. Must be non- - negative. The number of streams may be - lower than the requested number, - depending on the amount parallelism that - is reasonable for the table. Error will - be returned if the max count is greater - than the current system max limit of - 1,000. - - Streams must be read starting from - offset 0. - - This corresponds to the ``max_stream_count`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1.types.ReadSession: - Information about the ReadSession. - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([parent, read_session, max_stream_count]) - if request is not None and has_flattened_params: - raise ValueError("If the `request` argument is set, then none of " - "the individual field arguments should be set.") - - request = storage.CreateReadSessionRequest(request) - - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if parent is not None: - request.parent = parent - if read_session is not None: - request.read_session = read_session - if max_stream_count is not None: - request.max_stream_count = max_stream_count - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.create_read_session, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("read_session.table", request.read_session.table), - )), - ) - - # Send the request. - response = await rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def read_rows(self, - request: storage.ReadRowsRequest = None, - *, - read_stream: str = None, - offset: int = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> Awaitable[AsyncIterable[storage.ReadRowsResponse]]: - r"""Reads rows from the stream in the format prescribed - by the ReadSession. Each response contains one or more - table rows, up to a maximum of 100 MiB per response; - read requests which attempt to read individual rows - larger than 100 MiB will fail. - - Each request also returns a set of stream statistics - reflecting the current state of the stream. - - Args: - request (:class:`google.cloud.bigquery_storage_v1.types.ReadRowsRequest`): - The request object. Request message for `ReadRows`. - read_stream (:class:`str`): - Required. Stream to read rows from. - This corresponds to the ``read_stream`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - offset (:class:`int`): - The offset requested must be less - than the last row read from Read. - Requesting a larger offset is undefined. - If not specified, start reading from - offset zero. - - This corresponds to the ``offset`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - AsyncIterable[google.cloud.bigquery_storage_v1.types.ReadRowsResponse]: - Response from calling ReadRows may include row data, progress and - throttling information. - - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([read_stream, offset]) - if request is not None and has_flattened_params: - raise ValueError("If the `request` argument is set, then none of " - "the individual field arguments should be set.") - - request = storage.ReadRowsRequest(request) - - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if read_stream is not None: - request.read_stream = read_stream - if offset is not None: - request.offset = offset - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.read_rows, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.ServiceUnavailable, - ), - deadline=86400.0, - ), - default_timeout=86400.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("read_stream", request.read_stream), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - async def split_read_stream(self, - request: storage.SplitReadStreamRequest = None, - *, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> storage.SplitReadStreamResponse: - r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. - These ``ReadStream`` objects are referred to as the primary and - the residual streams of the split. The original ``ReadStream`` - can still be read from in the same manner as before. Both of the - returned ``ReadStream`` objects can also be read from, and the - rows returned by both child streams will be the same as the rows - read from the original stream. - - Moreover, the two child streams will be allocated back-to-back - in the original ``ReadStream``. Concretely, it is guaranteed - that for streams original, primary, and residual, that - original[0-j] = primary[0-j] and original[j-n] = residual[0-m] - once the streams have been read to completion. - - Args: - request (:class:`google.cloud.bigquery_storage_v1.types.SplitReadStreamRequest`): - The request object. Request message for - `SplitReadStream`. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1.types.SplitReadStreamResponse: - Response message for SplitReadStream. - """ - # Create or coerce a protobuf request object. - request = storage.SplitReadStreamRequest(request) - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.split_read_stream, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("name", request.name), - )), - ) - - # Send the request. - response = await rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - - - - -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - "google-cloud-bigquery-storage", - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() - - -__all__ = ( - "BigQueryReadAsyncClient", -) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py deleted file mode 100644 index 5c421b48..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py +++ /dev/null @@ -1,664 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from collections import OrderedDict -from distutils import util -import os -import re -from typing import Callable, Dict, Optional, Iterable, Sequence, Tuple, Type, Union -import pkg_resources - -from google.api_core import client_options as client_options_lib # type: ignore -from google.api_core import exceptions as core_exceptions # type: ignore -from google.api_core import gapic_v1 # type: ignore -from google.api_core import retry as retries # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.auth.transport import mtls # type: ignore -from google.auth.transport.grpc import SslCredentials # type: ignore -from google.auth.exceptions import MutualTLSChannelError # type: ignore -from google.oauth2 import service_account # type: ignore - -from google.cloud.bigquery_storage_v1.types import arrow -from google.cloud.bigquery_storage_v1.types import avro -from google.cloud.bigquery_storage_v1.types import storage -from google.cloud.bigquery_storage_v1.types import stream -from google.protobuf import timestamp_pb2 # type: ignore -from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO -from .transports.grpc import BigQueryReadGrpcTransport -from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport - - -class BigQueryReadClientMeta(type): - """Metaclass for the BigQueryRead client. - - This provides class-level methods for building and retrieving - support objects (e.g. transport) without polluting the client instance - objects. - """ - _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] - _transport_registry["grpc"] = BigQueryReadGrpcTransport - _transport_registry["grpc_asyncio"] = BigQueryReadGrpcAsyncIOTransport - - def get_transport_class(cls, - label: str = None, - ) -> Type[BigQueryReadTransport]: - """Returns an appropriate transport class. - - Args: - label: The name of the desired transport. If none is - provided, then the first transport in the registry is used. - - Returns: - The transport class to use. - """ - # If a specific transport is requested, return that one. - if label: - return cls._transport_registry[label] - - # No transport is requested; return the default (that is, the first one - # in the dictionary). - return next(iter(cls._transport_registry.values())) - - -class BigQueryReadClient(metaclass=BigQueryReadClientMeta): - """BigQuery Read API. - The Read API can be used to read data from BigQuery. - """ - - @staticmethod - def _get_default_mtls_endpoint(api_endpoint): - """Converts api endpoint to mTLS endpoint. - - Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to - "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. - Args: - api_endpoint (Optional[str]): the api endpoint to convert. - Returns: - str: converted mTLS api endpoint. - """ - if not api_endpoint: - return api_endpoint - - mtls_endpoint_re = re.compile( - r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" - ) - - m = mtls_endpoint_re.match(api_endpoint) - name, mtls, sandbox, googledomain = m.groups() - if mtls or not googledomain: - return api_endpoint - - if sandbox: - return api_endpoint.replace( - "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" - ) - - return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") - - DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" - DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore - DEFAULT_ENDPOINT - ) - - @classmethod - def from_service_account_info(cls, info: dict, *args, **kwargs): - """Creates an instance of this client using the provided credentials - info. - - Args: - info (dict): The service account private key info. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryReadClient: The constructed client. - """ - credentials = service_account.Credentials.from_service_account_info(info) - kwargs["credentials"] = credentials - return cls(*args, **kwargs) - - @classmethod - def from_service_account_file(cls, filename: str, *args, **kwargs): - """Creates an instance of this client using the provided credentials - file. - - Args: - filename (str): The path to the service account private key json - file. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryReadClient: The constructed client. - """ - credentials = service_account.Credentials.from_service_account_file( - filename) - kwargs["credentials"] = credentials - return cls(*args, **kwargs) - - from_service_account_json = from_service_account_file - - @property - def transport(self) -> BigQueryReadTransport: - """Returns the transport used by the client instance. - - Returns: - BigQueryReadTransport: The transport used by the client - instance. - """ - return self._transport - - @staticmethod - def read_session_path(project: str,location: str,session: str,) -> str: - """Returns a fully-qualified read_session string.""" - return "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) - - @staticmethod - def parse_read_session_path(path: str) -> Dict[str,str]: - """Parses a read_session path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def read_stream_path(project: str,location: str,session: str,stream: str,) -> str: - """Returns a fully-qualified read_stream string.""" - return "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) - - @staticmethod - def parse_read_stream_path(path: str) -> Dict[str,str]: - """Parses a read_stream path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)/streams/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def table_path(project: str,dataset: str,table: str,) -> str: - """Returns a fully-qualified table string.""" - return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) - - @staticmethod - def parse_table_path(path: str) -> Dict[str,str]: - """Parses a table path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_billing_account_path(billing_account: str, ) -> str: - """Returns a fully-qualified billing_account string.""" - return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) - - @staticmethod - def parse_common_billing_account_path(path: str) -> Dict[str,str]: - """Parse a billing_account path into its component segments.""" - m = re.match(r"^billingAccounts/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_folder_path(folder: str, ) -> str: - """Returns a fully-qualified folder string.""" - return "folders/{folder}".format(folder=folder, ) - - @staticmethod - def parse_common_folder_path(path: str) -> Dict[str,str]: - """Parse a folder path into its component segments.""" - m = re.match(r"^folders/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_organization_path(organization: str, ) -> str: - """Returns a fully-qualified organization string.""" - return "organizations/{organization}".format(organization=organization, ) - - @staticmethod - def parse_common_organization_path(path: str) -> Dict[str,str]: - """Parse a organization path into its component segments.""" - m = re.match(r"^organizations/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_project_path(project: str, ) -> str: - """Returns a fully-qualified project string.""" - return "projects/{project}".format(project=project, ) - - @staticmethod - def parse_common_project_path(path: str) -> Dict[str,str]: - """Parse a project path into its component segments.""" - m = re.match(r"^projects/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_location_path(project: str, location: str, ) -> str: - """Returns a fully-qualified location string.""" - return "projects/{project}/locations/{location}".format(project=project, location=location, ) - - @staticmethod - def parse_common_location_path(path: str) -> Dict[str,str]: - """Parse a location path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) - return m.groupdict() if m else {} - - def __init__(self, *, - credentials: Optional[ga_credentials.Credentials] = None, - transport: Union[str, BigQueryReadTransport, None] = None, - client_options: Optional[client_options_lib.ClientOptions] = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - ) -> None: - """Instantiates the big query read client. - - Args: - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - transport (Union[str, BigQueryReadTransport]): The - transport to use. If set to None, a transport is chosen - automatically. - client_options (google.api_core.client_options.ClientOptions): Custom options for the - client. It won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: - "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable - is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If - not provided, the default SSL client certificate will be used if - present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not - set, no client certificate will be used. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - - Raises: - google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport - creation failed for any reason. - """ - if isinstance(client_options, dict): - client_options = client_options_lib.from_dict(client_options) - if client_options is None: - client_options = client_options_lib.ClientOptions() - - # Create SSL credentials for mutual TLS if needed. - use_client_cert = bool(util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"))) - - client_cert_source_func = None - is_mtls = False - if use_client_cert: - if client_options.client_cert_source: - is_mtls = True - client_cert_source_func = client_options.client_cert_source - else: - is_mtls = mtls.has_default_client_cert_source() - if is_mtls: - client_cert_source_func = mtls.default_client_cert_source() - else: - client_cert_source_func = None - - # Figure out which api endpoint to use. - if client_options.api_endpoint is not None: - api_endpoint = client_options.api_endpoint - else: - use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") - if use_mtls_env == "never": - api_endpoint = self.DEFAULT_ENDPOINT - elif use_mtls_env == "always": - api_endpoint = self.DEFAULT_MTLS_ENDPOINT - elif use_mtls_env == "auto": - if is_mtls: - api_endpoint = self.DEFAULT_MTLS_ENDPOINT - else: - api_endpoint = self.DEFAULT_ENDPOINT - else: - raise MutualTLSChannelError( - "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " - "values: never, auto, always" - ) - - # Save or instantiate the transport. - # Ordinarily, we provide the transport, but allowing a custom transport - # instance provides an extensibility point for unusual situations. - if isinstance(transport, BigQueryReadTransport): - # transport is a BigQueryReadTransport instance. - if credentials or client_options.credentials_file: - raise ValueError("When providing a transport instance, " - "provide its credentials directly.") - if client_options.scopes: - raise ValueError( - "When providing a transport instance, provide its scopes " - "directly." - ) - self._transport = transport - else: - Transport = type(self).get_transport_class(transport) - self._transport = Transport( - credentials=credentials, - credentials_file=client_options.credentials_file, - host=api_endpoint, - scopes=client_options.scopes, - client_cert_source_for_mtls=client_cert_source_func, - quota_project_id=client_options.quota_project_id, - client_info=client_info, - always_use_jwt_access=( - Transport == type(self).get_transport_class("grpc") - or Transport == type(self).get_transport_class("grpc_asyncio") - ), - ) - - def create_read_session(self, - request: storage.CreateReadSessionRequest = None, - *, - parent: str = None, - read_session: stream.ReadSession = None, - max_stream_count: int = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> stream.ReadSession: - r"""Creates a new read session. A read session divides - the contents of a BigQuery table into one or more - streams, which can then be used to read data from the - table. The read session also specifies properties of the - data to be read, such as a list of columns or a push- - down filter describing the rows to be returned. - - A particular row can be read by at most one stream. When - the caller has reached the end of each stream in the - session, then all the data in the table has been read. - - Data is assigned to each stream such that roughly the - same number of rows can be read from each stream. - Because the server-side unit for assigning data is - collections of rows, the API does not guarantee that - each stream will return the same number or rows. - Additionally, the limits are enforced based on the - number of pre-filtered rows, so some filters can lead to - lopsided assignments. - - Read sessions automatically expire 24 hours after they - are created and do not require manual clean-up by the - caller. - - Args: - request (google.cloud.bigquery_storage_v1.types.CreateReadSessionRequest): - The request object. Request message for - `CreateReadSession`. - parent (str): - Required. The request project that owns the session, in - the form of ``projects/{project_id}``. - - This corresponds to the ``parent`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - read_session (google.cloud.bigquery_storage_v1.types.ReadSession): - Required. Session to be created. - This corresponds to the ``read_session`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - max_stream_count (int): - Max initial number of streams. If - unset or zero, the server will provide a - value of streams so as to produce - reasonable throughput. Must be non- - negative. The number of streams may be - lower than the requested number, - depending on the amount parallelism that - is reasonable for the table. Error will - be returned if the max count is greater - than the current system max limit of - 1,000. - - Streams must be read starting from - offset 0. - - This corresponds to the ``max_stream_count`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1.types.ReadSession: - Information about the ReadSession. - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([parent, read_session, max_stream_count]) - if request is not None and has_flattened_params: - raise ValueError('If the `request` argument is set, then none of ' - 'the individual field arguments should be set.') - - # Minor optimization to avoid making a copy if the user passes - # in a storage.CreateReadSessionRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.CreateReadSessionRequest): - request = storage.CreateReadSessionRequest(request) - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if parent is not None: - request.parent = parent - if read_session is not None: - request.read_session = read_session - if max_stream_count is not None: - request.max_stream_count = max_stream_count - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.create_read_session] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("read_session.table", request.read_session.table), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def read_rows(self, - request: storage.ReadRowsRequest = None, - *, - read_stream: str = None, - offset: int = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> Iterable[storage.ReadRowsResponse]: - r"""Reads rows from the stream in the format prescribed - by the ReadSession. Each response contains one or more - table rows, up to a maximum of 100 MiB per response; - read requests which attempt to read individual rows - larger than 100 MiB will fail. - - Each request also returns a set of stream statistics - reflecting the current state of the stream. - - Args: - request (google.cloud.bigquery_storage_v1.types.ReadRowsRequest): - The request object. Request message for `ReadRows`. - read_stream (str): - Required. Stream to read rows from. - This corresponds to the ``read_stream`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - offset (int): - The offset requested must be less - than the last row read from Read. - Requesting a larger offset is undefined. - If not specified, start reading from - offset zero. - - This corresponds to the ``offset`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - Iterable[google.cloud.bigquery_storage_v1.types.ReadRowsResponse]: - Response from calling ReadRows may include row data, progress and - throttling information. - - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([read_stream, offset]) - if request is not None and has_flattened_params: - raise ValueError('If the `request` argument is set, then none of ' - 'the individual field arguments should be set.') - - # Minor optimization to avoid making a copy if the user passes - # in a storage.ReadRowsRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.ReadRowsRequest): - request = storage.ReadRowsRequest(request) - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if read_stream is not None: - request.read_stream = read_stream - if offset is not None: - request.offset = offset - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.read_rows] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("read_stream", request.read_stream), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def split_read_stream(self, - request: storage.SplitReadStreamRequest = None, - *, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> storage.SplitReadStreamResponse: - r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. - These ``ReadStream`` objects are referred to as the primary and - the residual streams of the split. The original ``ReadStream`` - can still be read from in the same manner as before. Both of the - returned ``ReadStream`` objects can also be read from, and the - rows returned by both child streams will be the same as the rows - read from the original stream. - - Moreover, the two child streams will be allocated back-to-back - in the original ``ReadStream``. Concretely, it is guaranteed - that for streams original, primary, and residual, that - original[0-j] = primary[0-j] and original[j-n] = residual[0-m] - once the streams have been read to completion. - - Args: - request (google.cloud.bigquery_storage_v1.types.SplitReadStreamRequest): - The request object. Request message for - `SplitReadStream`. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1.types.SplitReadStreamResponse: - Response message for SplitReadStream. - """ - # Create or coerce a protobuf request object. - # Minor optimization to avoid making a copy if the user passes - # in a storage.SplitReadStreamRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.SplitReadStreamRequest): - request = storage.SplitReadStreamRequest(request) - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.split_read_stream] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("name", request.name), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - - - - -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - "google-cloud-bigquery-storage", - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() - - -__all__ = ( - "BigQueryReadClient", -) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py deleted file mode 100644 index f51ba420..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from collections import OrderedDict -from typing import Dict, Type - -from .base import BigQueryReadTransport -from .grpc import BigQueryReadGrpcTransport -from .grpc_asyncio import BigQueryReadGrpcAsyncIOTransport - - -# Compile a registry of transports. -_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] -_transport_registry['grpc'] = BigQueryReadGrpcTransport -_transport_registry['grpc_asyncio'] = BigQueryReadGrpcAsyncIOTransport - -__all__ = ( - 'BigQueryReadTransport', - 'BigQueryReadGrpcTransport', - 'BigQueryReadGrpcAsyncIOTransport', -) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py deleted file mode 100644 index 245f49f0..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py +++ /dev/null @@ -1,219 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import abc -from typing import Awaitable, Callable, Dict, Optional, Sequence, Union -import packaging.version -import pkg_resources - -import google.auth # type: ignore -import google.api_core # type: ignore -from google.api_core import exceptions as core_exceptions # type: ignore -from google.api_core import gapic_v1 # type: ignore -from google.api_core import retry as retries # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.oauth2 import service_account # type: ignore - -from google.cloud.bigquery_storage_v1.types import storage -from google.cloud.bigquery_storage_v1.types import stream - -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - 'google-cloud-bigquery-storage', - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() - -try: - # google.auth.__version__ was added in 1.26.0 - _GOOGLE_AUTH_VERSION = google.auth.__version__ -except AttributeError: - try: # try pkg_resources if it is available - _GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version - except pkg_resources.DistributionNotFound: # pragma: NO COVER - _GOOGLE_AUTH_VERSION = None - - -class BigQueryReadTransport(abc.ABC): - """Abstract transport class for BigQueryRead.""" - - AUTH_SCOPES = ( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', - ) - - DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' - def __init__( - self, *, - host: str = DEFAULT_HOST, - credentials: ga_credentials.Credentials = None, - credentials_file: Optional[str] = None, - scopes: Optional[Sequence[str]] = None, - quota_project_id: Optional[str] = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - always_use_jwt_access: Optional[bool] = False, - **kwargs, - ) -> None: - """Instantiate the transport. - - Args: - host (Optional[str]): - The hostname to connect to. - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is mutually exclusive with credentials. - scopes (Optional[Sequence[str]]): A list of scopes. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - always_use_jwt_access (Optional[bool]): Whether self signed JWT should - be used for service account credentials. - """ - # Save the hostname. Default to port 443 (HTTPS) if none is specified. - if ':' not in host: - host += ':443' - self._host = host - - scopes_kwargs = self._get_scopes_kwargs(self._host, scopes) - - # Save the scopes. - self._scopes = scopes - - # If no credentials are provided, then determine the appropriate - # defaults. - if credentials and credentials_file: - raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") - - if credentials_file is not None: - credentials, _ = google.auth.load_credentials_from_file( - credentials_file, - **scopes_kwargs, - quota_project_id=quota_project_id - ) - - elif credentials is None: - credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) - - # If the credentials is service account credentials, then always try to use self signed JWT. - if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): - credentials = credentials.with_always_use_jwt_access(True) - - # Save the credentials. - self._credentials = credentials - - # TODO(busunkim): This method is in the base transport - # to avoid duplicating code across the transport classes. These functions - # should be deleted once the minimum required versions of google-auth is increased. - - # TODO: Remove this function once google-auth >= 1.25.0 is required - @classmethod - def _get_scopes_kwargs(cls, host: str, scopes: Optional[Sequence[str]]) -> Dict[str, Optional[Sequence[str]]]: - """Returns scopes kwargs to pass to google-auth methods depending on the google-auth version""" - - scopes_kwargs = {} - - if _GOOGLE_AUTH_VERSION and ( - packaging.version.parse(_GOOGLE_AUTH_VERSION) - >= packaging.version.parse("1.25.0") - ): - scopes_kwargs = {"scopes": scopes, "default_scopes": cls.AUTH_SCOPES} - else: - scopes_kwargs = {"scopes": scopes or cls.AUTH_SCOPES} - - return scopes_kwargs - - def _prep_wrapped_messages(self, client_info): - # Precompute the wrapped methods. - self._wrapped_methods = { - self.create_read_session: gapic_v1.method.wrap_method( - self.create_read_session, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=client_info, - ), - self.read_rows: gapic_v1.method.wrap_method( - self.read_rows, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.ServiceUnavailable, - ), - deadline=86400.0, - ), - default_timeout=86400.0, - client_info=client_info, - ), - self.split_read_stream: gapic_v1.method.wrap_method( - self.split_read_stream, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=client_info, - ), - } - - @property - def create_read_session(self) -> Callable[ - [storage.CreateReadSessionRequest], - Union[ - stream.ReadSession, - Awaitable[stream.ReadSession] - ]]: - raise NotImplementedError() - - @property - def read_rows(self) -> Callable[ - [storage.ReadRowsRequest], - Union[ - storage.ReadRowsResponse, - Awaitable[storage.ReadRowsResponse] - ]]: - raise NotImplementedError() - - @property - def split_read_stream(self) -> Callable[ - [storage.SplitReadStreamRequest], - Union[ - storage.SplitReadStreamResponse, - Awaitable[storage.SplitReadStreamResponse] - ]]: - raise NotImplementedError() - - -__all__ = ( - 'BigQueryReadTransport', -) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py deleted file mode 100644 index 60c6d695..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py +++ /dev/null @@ -1,347 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import warnings -from typing import Callable, Dict, Optional, Sequence, Tuple, Union - -from google.api_core import grpc_helpers # type: ignore -from google.api_core import gapic_v1 # type: ignore -import google.auth # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.auth.transport.grpc import SslCredentials # type: ignore - -import grpc # type: ignore - -from google.cloud.bigquery_storage_v1.types import storage -from google.cloud.bigquery_storage_v1.types import stream -from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO - - -class BigQueryReadGrpcTransport(BigQueryReadTransport): - """gRPC backend transport for BigQueryRead. - - BigQuery Read API. - The Read API can be used to read data from BigQuery. - - This class defines the same methods as the primary client, so the - primary client can load the underlying transport implementation - and call it. - - It sends protocol buffers over the wire using gRPC (which is built on - top of HTTP/2); the ``grpcio`` package must be installed. - """ - _stubs: Dict[str, Callable] - - def __init__(self, *, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: str = None, - scopes: Sequence[str] = None, - channel: grpc.Channel = None, - api_mtls_endpoint: str = None, - client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, - ssl_channel_credentials: grpc.ChannelCredentials = None, - client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, - quota_project_id: Optional[str] = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - always_use_jwt_access: Optional[bool] = False, - ) -> None: - """Instantiate the transport. - - Args: - host (Optional[str]): - The hostname to connect to. - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - This argument is ignored if ``channel`` is provided. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. - scopes (Optional(Sequence[str])): A list of scopes. This argument is - ignored if ``channel`` is provided. - channel (Optional[grpc.Channel]): A ``Channel`` instance through - which to make calls. - api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. - If provided, it overrides the ``host`` argument and tries to create - a mutual TLS channel with client SSL credentials from - ``client_cert_source`` or applicatin default SSL credentials. - client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): - Deprecated. A callback to provide client SSL certificate bytes and - private key bytes, both in PEM format. It is ignored if - ``api_mtls_endpoint`` is None. - ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials - for grpc channel. It is ignored if ``channel`` is provided. - client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): - A callback to provide client certificate bytes and private key bytes, - both in PEM format. It is used to configure mutual TLS channel. It is - ignored if ``channel`` or ``ssl_channel_credentials`` is provided. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - always_use_jwt_access (Optional[bool]): Whether self signed JWT should - be used for service account credentials. - - Raises: - google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport - creation failed for any reason. - google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` - and ``credentials_file`` are passed. - """ - self._grpc_channel = None - self._ssl_channel_credentials = ssl_channel_credentials - self._stubs: Dict[str, Callable] = {} - - if api_mtls_endpoint: - warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) - if client_cert_source: - warnings.warn("client_cert_source is deprecated", DeprecationWarning) - - if channel: - # Ignore credentials if a channel was passed. - credentials = False - # If a channel was explicitly provided, set it. - self._grpc_channel = channel - self._ssl_channel_credentials = None - - else: - if api_mtls_endpoint: - host = api_mtls_endpoint - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - self._ssl_channel_credentials = SslCredentials().ssl_credentials - - else: - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - - # The base transport sets the host, credentials and scopes - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes, - quota_project_id=quota_project_id, - client_info=client_info, - always_use_jwt_access=always_use_jwt_access, - ) - - if not self._grpc_channel: - self._grpc_channel = type(self).create_channel( - self._host, - credentials=self._credentials, - credentials_file=credentials_file, - scopes=self._scopes, - ssl_credentials=self._ssl_channel_credentials, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - # Wrap messages. This must be done after self._grpc_channel exists - self._prep_wrapped_messages(client_info) - - @classmethod - def create_channel(cls, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: str = None, - scopes: Optional[Sequence[str]] = None, - quota_project_id: Optional[str] = None, - **kwargs) -> grpc.Channel: - """Create and return a gRPC channel object. - Args: - host (Optional[str]): The host for the channel to use. - credentials (Optional[~.Credentials]): The - authorization credentials to attach to requests. These - credentials identify this application to the service. If - none are specified, the client will attempt to ascertain - the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is mutually exclusive with credentials. - scopes (Optional[Sequence[str]]): A optional list of scopes needed for this - service. These are only used when credentials are not specified and - are passed to :func:`google.auth.default`. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - kwargs (Optional[dict]): Keyword arguments, which are passed to the - channel creation. - Returns: - grpc.Channel: A gRPC channel object. - - Raises: - google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` - and ``credentials_file`` are passed. - """ - - return grpc_helpers.create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - quota_project_id=quota_project_id, - default_scopes=cls.AUTH_SCOPES, - scopes=scopes, - default_host=cls.DEFAULT_HOST, - **kwargs - ) - - @property - def grpc_channel(self) -> grpc.Channel: - """Return the channel designed to connect to this service. - """ - return self._grpc_channel - - @property - def create_read_session(self) -> Callable[ - [storage.CreateReadSessionRequest], - stream.ReadSession]: - r"""Return a callable for the create read session method over gRPC. - - Creates a new read session. A read session divides - the contents of a BigQuery table into one or more - streams, which can then be used to read data from the - table. The read session also specifies properties of the - data to be read, such as a list of columns or a push- - down filter describing the rows to be returned. - - A particular row can be read by at most one stream. When - the caller has reached the end of each stream in the - session, then all the data in the table has been read. - - Data is assigned to each stream such that roughly the - same number of rows can be read from each stream. - Because the server-side unit for assigning data is - collections of rows, the API does not guarantee that - each stream will return the same number or rows. - Additionally, the limits are enforced based on the - number of pre-filtered rows, so some filters can lead to - lopsided assignments. - - Read sessions automatically expire 24 hours after they - are created and do not require manual clean-up by the - caller. - - Returns: - Callable[[~.CreateReadSessionRequest], - ~.ReadSession]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'create_read_session' not in self._stubs: - self._stubs['create_read_session'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1.BigQueryRead/CreateReadSession', - request_serializer=storage.CreateReadSessionRequest.serialize, - response_deserializer=stream.ReadSession.deserialize, - ) - return self._stubs['create_read_session'] - - @property - def read_rows(self) -> Callable[ - [storage.ReadRowsRequest], - storage.ReadRowsResponse]: - r"""Return a callable for the read rows method over gRPC. - - Reads rows from the stream in the format prescribed - by the ReadSession. Each response contains one or more - table rows, up to a maximum of 100 MiB per response; - read requests which attempt to read individual rows - larger than 100 MiB will fail. - - Each request also returns a set of stream statistics - reflecting the current state of the stream. - - Returns: - Callable[[~.ReadRowsRequest], - ~.ReadRowsResponse]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'read_rows' not in self._stubs: - self._stubs['read_rows'] = self.grpc_channel.unary_stream( - '/google.cloud.bigquery.storage.v1.BigQueryRead/ReadRows', - request_serializer=storage.ReadRowsRequest.serialize, - response_deserializer=storage.ReadRowsResponse.deserialize, - ) - return self._stubs['read_rows'] - - @property - def split_read_stream(self) -> Callable[ - [storage.SplitReadStreamRequest], - storage.SplitReadStreamResponse]: - r"""Return a callable for the split read stream method over gRPC. - - Splits a given ``ReadStream`` into two ``ReadStream`` objects. - These ``ReadStream`` objects are referred to as the primary and - the residual streams of the split. The original ``ReadStream`` - can still be read from in the same manner as before. Both of the - returned ``ReadStream`` objects can also be read from, and the - rows returned by both child streams will be the same as the rows - read from the original stream. - - Moreover, the two child streams will be allocated back-to-back - in the original ``ReadStream``. Concretely, it is guaranteed - that for streams original, primary, and residual, that - original[0-j] = primary[0-j] and original[j-n] = residual[0-m] - once the streams have been read to completion. - - Returns: - Callable[[~.SplitReadStreamRequest], - ~.SplitReadStreamResponse]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'split_read_stream' not in self._stubs: - self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1.BigQueryRead/SplitReadStream', - request_serializer=storage.SplitReadStreamRequest.serialize, - response_deserializer=storage.SplitReadStreamResponse.deserialize, - ) - return self._stubs['split_read_stream'] - - -__all__ = ( - 'BigQueryReadGrpcTransport', -) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py deleted file mode 100644 index d2f7225f..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py +++ /dev/null @@ -1,351 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import warnings -from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union - -from google.api_core import gapic_v1 # type: ignore -from google.api_core import grpc_helpers_async # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.auth.transport.grpc import SslCredentials # type: ignore -import packaging.version - -import grpc # type: ignore -from grpc.experimental import aio # type: ignore - -from google.cloud.bigquery_storage_v1.types import storage -from google.cloud.bigquery_storage_v1.types import stream -from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO -from .grpc import BigQueryReadGrpcTransport - - -class BigQueryReadGrpcAsyncIOTransport(BigQueryReadTransport): - """gRPC AsyncIO backend transport for BigQueryRead. - - BigQuery Read API. - The Read API can be used to read data from BigQuery. - - This class defines the same methods as the primary client, so the - primary client can load the underlying transport implementation - and call it. - - It sends protocol buffers over the wire using gRPC (which is built on - top of HTTP/2); the ``grpcio`` package must be installed. - """ - - _grpc_channel: aio.Channel - _stubs: Dict[str, Callable] = {} - - @classmethod - def create_channel(cls, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: Optional[str] = None, - scopes: Optional[Sequence[str]] = None, - quota_project_id: Optional[str] = None, - **kwargs) -> aio.Channel: - """Create and return a gRPC AsyncIO channel object. - Args: - host (Optional[str]): The host for the channel to use. - credentials (Optional[~.Credentials]): The - authorization credentials to attach to requests. These - credentials identify this application to the service. If - none are specified, the client will attempt to ascertain - the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. - scopes (Optional[Sequence[str]]): A optional list of scopes needed for this - service. These are only used when credentials are not specified and - are passed to :func:`google.auth.default`. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - kwargs (Optional[dict]): Keyword arguments, which are passed to the - channel creation. - Returns: - aio.Channel: A gRPC AsyncIO channel object. - """ - - return grpc_helpers_async.create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - quota_project_id=quota_project_id, - default_scopes=cls.AUTH_SCOPES, - scopes=scopes, - default_host=cls.DEFAULT_HOST, - **kwargs - ) - - def __init__(self, *, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: Optional[str] = None, - scopes: Optional[Sequence[str]] = None, - channel: aio.Channel = None, - api_mtls_endpoint: str = None, - client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, - ssl_channel_credentials: grpc.ChannelCredentials = None, - client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, - quota_project_id=None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - always_use_jwt_access: Optional[bool] = False, - ) -> None: - """Instantiate the transport. - - Args: - host (Optional[str]): - The hostname to connect to. - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - This argument is ignored if ``channel`` is provided. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. - scopes (Optional[Sequence[str]]): A optional list of scopes needed for this - service. These are only used when credentials are not specified and - are passed to :func:`google.auth.default`. - channel (Optional[aio.Channel]): A ``Channel`` instance through - which to make calls. - api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. - If provided, it overrides the ``host`` argument and tries to create - a mutual TLS channel with client SSL credentials from - ``client_cert_source`` or applicatin default SSL credentials. - client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): - Deprecated. A callback to provide client SSL certificate bytes and - private key bytes, both in PEM format. It is ignored if - ``api_mtls_endpoint`` is None. - ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials - for grpc channel. It is ignored if ``channel`` is provided. - client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): - A callback to provide client certificate bytes and private key bytes, - both in PEM format. It is used to configure mutual TLS channel. It is - ignored if ``channel`` or ``ssl_channel_credentials`` is provided. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - always_use_jwt_access (Optional[bool]): Whether self signed JWT should - be used for service account credentials. - - Raises: - google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport - creation failed for any reason. - google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` - and ``credentials_file`` are passed. - """ - self._grpc_channel = None - self._ssl_channel_credentials = ssl_channel_credentials - self._stubs: Dict[str, Callable] = {} - - if api_mtls_endpoint: - warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) - if client_cert_source: - warnings.warn("client_cert_source is deprecated", DeprecationWarning) - - if channel: - # Ignore credentials if a channel was passed. - credentials = False - # If a channel was explicitly provided, set it. - self._grpc_channel = channel - self._ssl_channel_credentials = None - else: - if api_mtls_endpoint: - host = api_mtls_endpoint - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - self._ssl_channel_credentials = SslCredentials().ssl_credentials - - else: - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - - # The base transport sets the host, credentials and scopes - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes, - quota_project_id=quota_project_id, - client_info=client_info, - always_use_jwt_access=always_use_jwt_access, - ) - - if not self._grpc_channel: - self._grpc_channel = type(self).create_channel( - self._host, - credentials=self._credentials, - credentials_file=credentials_file, - scopes=self._scopes, - ssl_credentials=self._ssl_channel_credentials, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - # Wrap messages. This must be done after self._grpc_channel exists - self._prep_wrapped_messages(client_info) - - @property - def grpc_channel(self) -> aio.Channel: - """Create the channel designed to connect to this service. - - This property caches on the instance; repeated calls return - the same channel. - """ - # Return the channel from cache. - return self._grpc_channel - - @property - def create_read_session(self) -> Callable[ - [storage.CreateReadSessionRequest], - Awaitable[stream.ReadSession]]: - r"""Return a callable for the create read session method over gRPC. - - Creates a new read session. A read session divides - the contents of a BigQuery table into one or more - streams, which can then be used to read data from the - table. The read session also specifies properties of the - data to be read, such as a list of columns or a push- - down filter describing the rows to be returned. - - A particular row can be read by at most one stream. When - the caller has reached the end of each stream in the - session, then all the data in the table has been read. - - Data is assigned to each stream such that roughly the - same number of rows can be read from each stream. - Because the server-side unit for assigning data is - collections of rows, the API does not guarantee that - each stream will return the same number or rows. - Additionally, the limits are enforced based on the - number of pre-filtered rows, so some filters can lead to - lopsided assignments. - - Read sessions automatically expire 24 hours after they - are created and do not require manual clean-up by the - caller. - - Returns: - Callable[[~.CreateReadSessionRequest], - Awaitable[~.ReadSession]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'create_read_session' not in self._stubs: - self._stubs['create_read_session'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1.BigQueryRead/CreateReadSession', - request_serializer=storage.CreateReadSessionRequest.serialize, - response_deserializer=stream.ReadSession.deserialize, - ) - return self._stubs['create_read_session'] - - @property - def read_rows(self) -> Callable[ - [storage.ReadRowsRequest], - Awaitable[storage.ReadRowsResponse]]: - r"""Return a callable for the read rows method over gRPC. - - Reads rows from the stream in the format prescribed - by the ReadSession. Each response contains one or more - table rows, up to a maximum of 100 MiB per response; - read requests which attempt to read individual rows - larger than 100 MiB will fail. - - Each request also returns a set of stream statistics - reflecting the current state of the stream. - - Returns: - Callable[[~.ReadRowsRequest], - Awaitable[~.ReadRowsResponse]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'read_rows' not in self._stubs: - self._stubs['read_rows'] = self.grpc_channel.unary_stream( - '/google.cloud.bigquery.storage.v1.BigQueryRead/ReadRows', - request_serializer=storage.ReadRowsRequest.serialize, - response_deserializer=storage.ReadRowsResponse.deserialize, - ) - return self._stubs['read_rows'] - - @property - def split_read_stream(self) -> Callable[ - [storage.SplitReadStreamRequest], - Awaitable[storage.SplitReadStreamResponse]]: - r"""Return a callable for the split read stream method over gRPC. - - Splits a given ``ReadStream`` into two ``ReadStream`` objects. - These ``ReadStream`` objects are referred to as the primary and - the residual streams of the split. The original ``ReadStream`` - can still be read from in the same manner as before. Both of the - returned ``ReadStream`` objects can also be read from, and the - rows returned by both child streams will be the same as the rows - read from the original stream. - - Moreover, the two child streams will be allocated back-to-back - in the original ``ReadStream``. Concretely, it is guaranteed - that for streams original, primary, and residual, that - original[0-j] = primary[0-j] and original[j-n] = residual[0-m] - once the streams have been read to completion. - - Returns: - Callable[[~.SplitReadStreamRequest], - Awaitable[~.SplitReadStreamResponse]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'split_read_stream' not in self._stubs: - self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1.BigQueryRead/SplitReadStream', - request_serializer=storage.SplitReadStreamRequest.serialize, - response_deserializer=storage.SplitReadStreamResponse.deserialize, - ) - return self._stubs['split_read_stream'] - - -__all__ = ( - 'BigQueryReadGrpcAsyncIOTransport', -) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py deleted file mode 100644 index 030085d0..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from .arrow import ( - ArrowRecordBatch, - ArrowSchema, - ArrowSerializationOptions, -) -from .avro import ( - AvroRows, - AvroSchema, -) -from .storage import ( - CreateReadSessionRequest, - ReadRowsRequest, - ReadRowsResponse, - SplitReadStreamRequest, - SplitReadStreamResponse, - StreamStats, - ThrottleState, -) -from .stream import ( - ReadSession, - ReadStream, - DataFormat, -) - -__all__ = ( - 'ArrowRecordBatch', - 'ArrowSchema', - 'ArrowSerializationOptions', - 'AvroRows', - 'AvroSchema', - 'CreateReadSessionRequest', - 'ReadRowsRequest', - 'ReadRowsResponse', - 'SplitReadStreamRequest', - 'SplitReadStreamResponse', - 'StreamStats', - 'ThrottleState', - 'ReadSession', - 'ReadStream', - 'DataFormat', -) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py deleted file mode 100644 index 391088fe..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package='google.cloud.bigquery.storage.v1', - manifest={ - 'ArrowSchema', - 'ArrowRecordBatch', - 'ArrowSerializationOptions', - }, -) - - -class ArrowSchema(proto.Message): - r"""Arrow schema as specified in - https://arrow.apache.org/docs/python/api/datatypes.html and - serialized to bytes using IPC: - https://arrow.apache.org/docs/format/Columnar.html#serialization- - and-interprocess-communication-ipc - See code samples on how this message can be deserialized. - - Attributes: - serialized_schema (bytes): - IPC serialized Arrow schema. - """ - - serialized_schema = proto.Field( - proto.BYTES, - number=1, - ) - - -class ArrowRecordBatch(proto.Message): - r"""Arrow RecordBatch. - Attributes: - serialized_record_batch (bytes): - IPC-serialized Arrow RecordBatch. - row_count (int): - The count of rows in ``serialized_record_batch``. - """ - - serialized_record_batch = proto.Field( - proto.BYTES, - number=1, - ) - row_count = proto.Field( - proto.INT64, - number=2, - ) - - -class ArrowSerializationOptions(proto.Message): - r"""Contains options specific to Arrow Serialization. - Attributes: - buffer_compression (google.cloud.bigquery_storage_v1.types.ArrowSerializationOptions.CompressionCodec): - The compression codec to use for Arrow - buffers in serialized record batches. - """ - class CompressionCodec(proto.Enum): - r"""Compression codec's supported by Arrow.""" - COMPRESSION_UNSPECIFIED = 0 - LZ4_FRAME = 1 - ZSTD = 2 - - buffer_compression = proto.Field( - proto.ENUM, - number=2, - enum=CompressionCodec, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py deleted file mode 100644 index 023fa87a..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package='google.cloud.bigquery.storage.v1', - manifest={ - 'AvroSchema', - 'AvroRows', - }, -) - - -class AvroSchema(proto.Message): - r"""Avro schema. - Attributes: - schema (str): - Json serialized schema, as described at - https://avro.apache.org/docs/1.8.1/spec.html. - """ - - schema = proto.Field( - proto.STRING, - number=1, - ) - - -class AvroRows(proto.Message): - r"""Avro rows. - Attributes: - serialized_binary_rows (bytes): - Binary serialized rows in a block. - row_count (int): - The count of rows in the returning block. - """ - - serialized_binary_rows = proto.Field( - proto.BYTES, - number=1, - ) - row_count = proto.Field( - proto.INT64, - number=2, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py deleted file mode 100644 index 1d8641c7..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py +++ /dev/null @@ -1,275 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - -from google.cloud.bigquery_storage_v1.types import arrow -from google.cloud.bigquery_storage_v1.types import avro -from google.cloud.bigquery_storage_v1.types import stream - - -__protobuf__ = proto.module( - package='google.cloud.bigquery.storage.v1', - manifest={ - 'CreateReadSessionRequest', - 'ReadRowsRequest', - 'ThrottleState', - 'StreamStats', - 'ReadRowsResponse', - 'SplitReadStreamRequest', - 'SplitReadStreamResponse', - }, -) - - -class CreateReadSessionRequest(proto.Message): - r"""Request message for ``CreateReadSession``. - Attributes: - parent (str): - Required. The request project that owns the session, in the - form of ``projects/{project_id}``. - read_session (google.cloud.bigquery_storage_v1.types.ReadSession): - Required. Session to be created. - max_stream_count (int): - Max initial number of streams. If unset or - zero, the server will provide a value of streams - so as to produce reasonable throughput. Must be - non-negative. The number of streams may be lower - than the requested number, depending on the - amount parallelism that is reasonable for the - table. Error will be returned if the max count - is greater than the current system max limit of - 1,000. - - Streams must be read starting from offset 0. - """ - - parent = proto.Field( - proto.STRING, - number=1, - ) - read_session = proto.Field( - proto.MESSAGE, - number=2, - message=stream.ReadSession, - ) - max_stream_count = proto.Field( - proto.INT32, - number=3, - ) - - -class ReadRowsRequest(proto.Message): - r"""Request message for ``ReadRows``. - Attributes: - read_stream (str): - Required. Stream to read rows from. - offset (int): - The offset requested must be less than the - last row read from Read. Requesting a larger - offset is undefined. If not specified, start - reading from offset zero. - """ - - read_stream = proto.Field( - proto.STRING, - number=1, - ) - offset = proto.Field( - proto.INT64, - number=2, - ) - - -class ThrottleState(proto.Message): - r"""Information on if the current connection is being throttled. - Attributes: - throttle_percent (int): - How much this connection is being throttled. - Zero means no throttling, 100 means fully - throttled. - """ - - throttle_percent = proto.Field( - proto.INT32, - number=1, - ) - - -class StreamStats(proto.Message): - r"""Estimated stream statistics for a given Stream. - Attributes: - progress (google.cloud.bigquery_storage_v1.types.StreamStats.Progress): - Represents the progress of the current - stream. - """ - - class Progress(proto.Message): - r""" - Attributes: - at_response_start (float): - The fraction of rows assigned to the stream that have been - processed by the server so far, not including the rows in - the current response message. - - This value, along with ``at_response_end``, can be used to - interpolate the progress made as the rows in the message are - being processed using the following formula: - ``at_response_start + (at_response_end - at_response_start) * rows_processed_from_response / rows_in_response``. - - Note that if a filter is provided, the ``at_response_end`` - value of the previous response may not necessarily be equal - to the ``at_response_start`` value of the current response. - at_response_end (float): - Similar to ``at_response_start``, except that this value - includes the rows in the current response. - """ - - at_response_start = proto.Field( - proto.DOUBLE, - number=1, - ) - at_response_end = proto.Field( - proto.DOUBLE, - number=2, - ) - - progress = proto.Field( - proto.MESSAGE, - number=2, - message=Progress, - ) - - -class ReadRowsResponse(proto.Message): - r"""Response from calling ``ReadRows`` may include row data, progress - and throttling information. - - Attributes: - avro_rows (google.cloud.bigquery_storage_v1.types.AvroRows): - Serialized row data in AVRO format. - arrow_record_batch (google.cloud.bigquery_storage_v1.types.ArrowRecordBatch): - Serialized row data in Arrow RecordBatch - format. - row_count (int): - Number of serialized rows in the rows block. - stats (google.cloud.bigquery_storage_v1.types.StreamStats): - Statistics for the stream. - throttle_state (google.cloud.bigquery_storage_v1.types.ThrottleState): - Throttling state. If unset, the latest - response still describes the current throttling - status. - avro_schema (google.cloud.bigquery_storage_v1.types.AvroSchema): - Output only. Avro schema. - arrow_schema (google.cloud.bigquery_storage_v1.types.ArrowSchema): - Output only. Arrow schema. - """ - - avro_rows = proto.Field( - proto.MESSAGE, - number=3, - oneof='rows', - message=avro.AvroRows, - ) - arrow_record_batch = proto.Field( - proto.MESSAGE, - number=4, - oneof='rows', - message=arrow.ArrowRecordBatch, - ) - row_count = proto.Field( - proto.INT64, - number=6, - ) - stats = proto.Field( - proto.MESSAGE, - number=2, - message='StreamStats', - ) - throttle_state = proto.Field( - proto.MESSAGE, - number=5, - message='ThrottleState', - ) - avro_schema = proto.Field( - proto.MESSAGE, - number=7, - oneof='schema', - message=avro.AvroSchema, - ) - arrow_schema = proto.Field( - proto.MESSAGE, - number=8, - oneof='schema', - message=arrow.ArrowSchema, - ) - - -class SplitReadStreamRequest(proto.Message): - r"""Request message for ``SplitReadStream``. - Attributes: - name (str): - Required. Name of the stream to split. - fraction (float): - A value in the range (0.0, 1.0) that - specifies the fractional point at which the - original stream should be split. The actual - split point is evaluated on pre-filtered rows, - so if a filter is provided, then there is no - guarantee that the division of the rows between - the new child streams will be proportional to - this fractional value. Additionally, because the - server-side unit for assigning data is - collections of rows, this fraction will always - map to a data storage boundary on the server - side. - """ - - name = proto.Field( - proto.STRING, - number=1, - ) - fraction = proto.Field( - proto.DOUBLE, - number=2, - ) - - -class SplitReadStreamResponse(proto.Message): - r"""Response message for ``SplitReadStream``. - Attributes: - primary_stream (google.cloud.bigquery_storage_v1.types.ReadStream): - Primary stream, which contains the beginning portion of - \|original_stream|. An empty value indicates that the - original stream can no longer be split. - remainder_stream (google.cloud.bigquery_storage_v1.types.ReadStream): - Remainder stream, which contains the tail of - \|original_stream|. An empty value indicates that the - original stream can no longer be split. - """ - - primary_stream = proto.Field( - proto.MESSAGE, - number=1, - message=stream.ReadStream, - ) - remainder_stream = proto.Field( - proto.MESSAGE, - number=2, - message=stream.ReadStream, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py deleted file mode 100644 index 9835d161..00000000 --- a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py +++ /dev/null @@ -1,194 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - -from google.cloud.bigquery_storage_v1.types import arrow -from google.cloud.bigquery_storage_v1.types import avro -from google.protobuf import timestamp_pb2 # type: ignore - - -__protobuf__ = proto.module( - package='google.cloud.bigquery.storage.v1', - manifest={ - 'DataFormat', - 'ReadSession', - 'ReadStream', - }, -) - - -class DataFormat(proto.Enum): - r"""Data format for input or output data.""" - DATA_FORMAT_UNSPECIFIED = 0 - AVRO = 1 - ARROW = 2 - - -class ReadSession(proto.Message): - r"""Information about the ReadSession. - Attributes: - name (str): - Output only. Unique identifier for the session, in the form - ``projects/{project_id}/locations/{location}/sessions/{session_id}``. - expire_time (google.protobuf.timestamp_pb2.Timestamp): - Output only. Time at which the session becomes invalid. - After this time, subsequent requests to read this Session - will return errors. The expire_time is automatically - assigned and currently cannot be specified or updated. - data_format (google.cloud.bigquery_storage_v1.types.DataFormat): - Immutable. Data format of the output data. - avro_schema (google.cloud.bigquery_storage_v1.types.AvroSchema): - Output only. Avro schema. - arrow_schema (google.cloud.bigquery_storage_v1.types.ArrowSchema): - Output only. Arrow schema. - table (str): - Immutable. Table that this ReadSession is reading from, in - the form - ``projects/{project_id}/datasets/{dataset_id}/tables/{table_id}`` - table_modifiers (google.cloud.bigquery_storage_v1.types.ReadSession.TableModifiers): - Optional. Any modifiers which are applied - when reading from the specified table. - read_options (google.cloud.bigquery_storage_v1.types.ReadSession.TableReadOptions): - Optional. Read options for this session (e.g. - column selection, filters). - streams (Sequence[google.cloud.bigquery_storage_v1.types.ReadStream]): - Output only. A list of streams created with the session. - - At least one stream is created with the session. In the - future, larger request_stream_count values *may* result in - this list being unpopulated, in that case, the user will - need to use a List method to get the streams instead, which - is not yet available. - """ - - class TableModifiers(proto.Message): - r"""Additional attributes when reading a table. - Attributes: - snapshot_time (google.protobuf.timestamp_pb2.Timestamp): - The snapshot time of the table. If not set, - interpreted as now. - """ - - snapshot_time = proto.Field( - proto.MESSAGE, - number=1, - message=timestamp_pb2.Timestamp, - ) - - class TableReadOptions(proto.Message): - r"""Options dictating how we read a table. - Attributes: - selected_fields (Sequence[str]): - Names of the fields in the table that should be read. If - empty, all fields will be read. If the specified field is a - nested field, all the sub-fields in the field will be - selected. The output field order is unrelated to the order - of fields in selected_fields. - row_restriction (str): - SQL text filtering statement, similar to a WHERE clause in a - query. Aggregates are not supported. - - Examples: "int_field > 5" "date_field = CAST('2014-9-27' as - DATE)" "nullable_field is not NULL" "st_equals(geo_field, - st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 - AND 5.0" - - Restricted to a maximum length for 1 MB. - arrow_serialization_options (google.cloud.bigquery_storage_v1.types.ArrowSerializationOptions): - Optional. Options specific to the Apache - Arrow output format. - """ - - selected_fields = proto.RepeatedField( - proto.STRING, - number=1, - ) - row_restriction = proto.Field( - proto.STRING, - number=2, - ) - arrow_serialization_options = proto.Field( - proto.MESSAGE, - number=3, - oneof='output_format_serialization_options', - message=arrow.ArrowSerializationOptions, - ) - - name = proto.Field( - proto.STRING, - number=1, - ) - expire_time = proto.Field( - proto.MESSAGE, - number=2, - message=timestamp_pb2.Timestamp, - ) - data_format = proto.Field( - proto.ENUM, - number=3, - enum='DataFormat', - ) - avro_schema = proto.Field( - proto.MESSAGE, - number=4, - oneof='schema', - message=avro.AvroSchema, - ) - arrow_schema = proto.Field( - proto.MESSAGE, - number=5, - oneof='schema', - message=arrow.ArrowSchema, - ) - table = proto.Field( - proto.STRING, - number=6, - ) - table_modifiers = proto.Field( - proto.MESSAGE, - number=7, - message=TableModifiers, - ) - read_options = proto.Field( - proto.MESSAGE, - number=8, - message=TableReadOptions, - ) - streams = proto.RepeatedField( - proto.MESSAGE, - number=10, - message='ReadStream', - ) - - -class ReadStream(proto.Message): - r"""Information about a single stream that gets data out of the storage - system. Most of the information about ``ReadStream`` instances is - aggregated, making ``ReadStream`` lightweight. - - Attributes: - name (str): - Output only. Name of the stream, in the form - ``projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}``. - """ - - name = proto.Field( - proto.STRING, - number=1, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/mypy.ini b/owl-bot-staging/v1/mypy.ini deleted file mode 100644 index 4505b485..00000000 --- a/owl-bot-staging/v1/mypy.ini +++ /dev/null @@ -1,3 +0,0 @@ -[mypy] -python_version = 3.6 -namespace_packages = True diff --git a/owl-bot-staging/v1/noxfile.py b/owl-bot-staging/v1/noxfile.py deleted file mode 100644 index 877e91ab..00000000 --- a/owl-bot-staging/v1/noxfile.py +++ /dev/null @@ -1,132 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import pathlib -import shutil -import subprocess -import sys - - -import nox # type: ignore - -CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() - -LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt" -PACKAGE_NAME = subprocess.check_output([sys.executable, "setup.py", "--name"], encoding="utf-8") - - -nox.sessions = [ - "unit", - "cover", - "mypy", - "check_lower_bounds" - # exclude update_lower_bounds from default - "docs", -] - -@nox.session(python=['3.6', '3.7', '3.8', '3.9']) -def unit(session): - """Run the unit test suite.""" - - session.install('coverage', 'pytest', 'pytest-cov', 'asyncmock', 'pytest-asyncio') - session.install('-e', '.') - - session.run( - 'py.test', - '--quiet', - '--cov=google/cloud/bigquery_storage_v1/', - '--cov-config=.coveragerc', - '--cov-report=term', - '--cov-report=html', - os.path.join('tests', 'unit', ''.join(session.posargs)) - ) - - -@nox.session(python='3.7') -def cover(session): - """Run the final coverage report. - This outputs the coverage report aggregating coverage from the unit - test runs (not system test runs), and then erases coverage data. - """ - session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=100") - - session.run("coverage", "erase") - - -@nox.session(python=['3.6', '3.7']) -def mypy(session): - """Run the type checker.""" - session.install('mypy', 'types-pkg_resources') - session.install('.') - session.run( - 'mypy', - '--explicit-package-bases', - 'google', - ) - - -@nox.session -def update_lower_bounds(session): - """Update lower bounds in constraints.txt to match setup.py""" - session.install('google-cloud-testutils') - session.install('.') - - session.run( - 'lower-bound-checker', - 'update', - '--package-name', - PACKAGE_NAME, - '--constraints-file', - str(LOWER_BOUND_CONSTRAINTS_FILE), - ) - - -@nox.session -def check_lower_bounds(session): - """Check lower bounds in setup.py are reflected in constraints file""" - session.install('google-cloud-testutils') - session.install('.') - - session.run( - 'lower-bound-checker', - 'check', - '--package-name', - PACKAGE_NAME, - '--constraints-file', - str(LOWER_BOUND_CONSTRAINTS_FILE), - ) - -@nox.session(python='3.6') -def docs(session): - """Build the docs for this library.""" - - session.install("-e", ".") - session.install("sphinx<3.0.0", "alabaster", "recommonmark") - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run( - "sphinx-build", - "-W", # warnings as errors - "-T", # show full traceback on exception - "-N", # no colors - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) diff --git a/owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py b/owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py deleted file mode 100644 index 107af58b..00000000 --- a/owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py +++ /dev/null @@ -1,178 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import os -import libcst as cst -import pathlib -import sys -from typing import (Any, Callable, Dict, List, Sequence, Tuple) - - -def partition( - predicate: Callable[[Any], bool], - iterator: Sequence[Any] -) -> Tuple[List[Any], List[Any]]: - """A stable, out-of-place partition.""" - results = ([], []) - - for i in iterator: - results[int(predicate(i))].append(i) - - # Returns trueList, falseList - return results[1], results[0] - - -class bigquery_storageCallTransformer(cst.CSTTransformer): - CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') - METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { - 'create_read_session': ('parent', 'read_session', 'max_stream_count', ), - 'read_rows': ('read_stream', 'offset', ), - 'split_read_stream': ('name', 'fraction', ), - } - - def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: - try: - key = original.func.attr.value - kword_params = self.METHOD_TO_PARAMS[key] - except (AttributeError, KeyError): - # Either not a method from the API or too convoluted to be sure. - return updated - - # If the existing code is valid, keyword args come after positional args. - # Therefore, all positional args must map to the first parameters. - args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) - if any(k.keyword.value == "request" for k in kwargs): - # We've already fixed this file, don't fix it again. - return updated - - kwargs, ctrl_kwargs = partition( - lambda a: not a.keyword.value in self.CTRL_PARAMS, - kwargs - ) - - args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] - ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) - for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) - - request_arg = cst.Arg( - value=cst.Dict([ - cst.DictElement( - cst.SimpleString("'{}'".format(name)), -cst.Element(value=arg.value) - ) - # Note: the args + kwargs looks silly, but keep in mind that - # the control parameters had to be stripped out, and that - # those could have been passed positionally or by keyword. - for name, arg in zip(kword_params, args + kwargs)]), - keyword=cst.Name("request") - ) - - return updated.with_changes( - args=[request_arg] + ctrl_kwargs - ) - - -def fix_files( - in_dir: pathlib.Path, - out_dir: pathlib.Path, - *, - transformer=bigquery_storageCallTransformer(), -): - """Duplicate the input dir to the output dir, fixing file method calls. - - Preconditions: - * in_dir is a real directory - * out_dir is a real, empty directory - """ - pyfile_gen = ( - pathlib.Path(os.path.join(root, f)) - for root, _, files in os.walk(in_dir) - for f in files if os.path.splitext(f)[1] == ".py" - ) - - for fpath in pyfile_gen: - with open(fpath, 'r') as f: - src = f.read() - - # Parse the code and insert method call fixes. - tree = cst.parse_module(src) - updated = tree.visit(transformer) - - # Create the path and directory structure for the new file. - updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) - updated_path.parent.mkdir(parents=True, exist_ok=True) - - # Generate the updated source file at the corresponding path. - with open(updated_path, 'w') as f: - f.write(updated.code) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="""Fix up source that uses the bigquery_storage client library. - -The existing sources are NOT overwritten but are copied to output_dir with changes made. - -Note: This tool operates at a best-effort level at converting positional - parameters in client method calls to keyword based parameters. - Cases where it WILL FAIL include - A) * or ** expansion in a method call. - B) Calls via function or method alias (includes free function calls) - C) Indirect or dispatched calls (e.g. the method is looked up dynamically) - - These all constitute false negatives. The tool will also detect false - positives when an API method shares a name with another method. -""") - parser.add_argument( - '-d', - '--input-directory', - required=True, - dest='input_dir', - help='the input directory to walk for python files to fix up', - ) - parser.add_argument( - '-o', - '--output-directory', - required=True, - dest='output_dir', - help='the directory to output files fixed via un-flattening', - ) - args = parser.parse_args() - input_dir = pathlib.Path(args.input_dir) - output_dir = pathlib.Path(args.output_dir) - if not input_dir.is_dir(): - print( - f"input directory '{input_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if not output_dir.is_dir(): - print( - f"output directory '{output_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if os.listdir(output_dir): - print( - f"output directory '{output_dir}' is not empty", - file=sys.stderr, - ) - sys.exit(-1) - - fix_files(input_dir, output_dir) diff --git a/owl-bot-staging/v1/setup.py b/owl-bot-staging/v1/setup.py deleted file mode 100644 index 591e1fba..00000000 --- a/owl-bot-staging/v1/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import io -import os -import setuptools # type: ignore - -version = '0.1.0' - -package_root = os.path.abspath(os.path.dirname(__file__)) - -readme_filename = os.path.join(package_root, 'README.rst') -with io.open(readme_filename, encoding='utf-8') as readme_file: - readme = readme_file.read() - -setuptools.setup( - name='google-cloud-bigquery-storage', - version=version, - long_description=readme, - packages=setuptools.PEP420PackageFinder.find(), - namespace_packages=('google', 'google.cloud'), - platforms='Posix; MacOS X; Windows', - include_package_data=True, - install_requires=( - 'google-api-core[grpc] >= 1.27.0, < 3.0.0dev', - 'libcst >= 0.2.5', - 'proto-plus >= 1.15.0', - 'packaging >= 14.3', ), - python_requires='>=3.6', - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Topic :: Internet', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - zip_safe=False, -) diff --git a/owl-bot-staging/v1/tests/__init__.py b/owl-bot-staging/v1/tests/__init__.py deleted file mode 100644 index b54a5fcc..00000000 --- a/owl-bot-staging/v1/tests/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ - -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/owl-bot-staging/v1/tests/unit/__init__.py b/owl-bot-staging/v1/tests/unit/__init__.py deleted file mode 100644 index b54a5fcc..00000000 --- a/owl-bot-staging/v1/tests/unit/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ - -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/owl-bot-staging/v1/tests/unit/gapic/__init__.py b/owl-bot-staging/v1/tests/unit/gapic/__init__.py deleted file mode 100644 index b54a5fcc..00000000 --- a/owl-bot-staging/v1/tests/unit/gapic/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ - -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py deleted file mode 100644 index b54a5fcc..00000000 --- a/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ - -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py deleted file mode 100644 index 8fe8065d..00000000 --- a/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py +++ /dev/null @@ -1,1629 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import mock -import packaging.version - -import grpc -from grpc.experimental import aio -import math -import pytest -from proto.marshal.rules.dates import DurationRule, TimestampRule - - -from google.api_core import client_options -from google.api_core import exceptions as core_exceptions -from google.api_core import gapic_v1 -from google.api_core import grpc_helpers -from google.api_core import grpc_helpers_async -from google.auth import credentials as ga_credentials -from google.auth.exceptions import MutualTLSChannelError -from google.cloud.bigquery_storage_v1.services.big_query_read import BigQueryReadAsyncClient -from google.cloud.bigquery_storage_v1.services.big_query_read import BigQueryReadClient -from google.cloud.bigquery_storage_v1.services.big_query_read import transports -from google.cloud.bigquery_storage_v1.services.big_query_read.transports.base import _GOOGLE_AUTH_VERSION -from google.cloud.bigquery_storage_v1.types import arrow -from google.cloud.bigquery_storage_v1.types import avro -from google.cloud.bigquery_storage_v1.types import storage -from google.cloud.bigquery_storage_v1.types import stream -from google.oauth2 import service_account -from google.protobuf import timestamp_pb2 # type: ignore -import google.auth - - -# TODO(busunkim): Once google-auth >= 1.25.0 is required transitively -# through google-api-core: -# - Delete the auth "less than" test cases -# - Delete these pytest markers (Make the "greater than or equal to" tests the default). -requires_google_auth_lt_1_25_0 = pytest.mark.skipif( - packaging.version.parse(_GOOGLE_AUTH_VERSION) >= packaging.version.parse("1.25.0"), - reason="This test requires google-auth < 1.25.0", -) -requires_google_auth_gte_1_25_0 = pytest.mark.skipif( - packaging.version.parse(_GOOGLE_AUTH_VERSION) < packaging.version.parse("1.25.0"), - reason="This test requires google-auth >= 1.25.0", -) - -def client_cert_source_callback(): - return b"cert bytes", b"key bytes" - - -# If default endpoint is localhost, then default mtls endpoint will be the same. -# This method modifies the default endpoint so the client can produce a different -# mtls endpoint for endpoint testing purposes. -def modify_default_endpoint(client): - return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT - - -def test__get_default_mtls_endpoint(): - api_endpoint = "example.googleapis.com" - api_mtls_endpoint = "example.mtls.googleapis.com" - sandbox_endpoint = "example.sandbox.googleapis.com" - sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" - non_googleapi = "api.example.com" - - assert BigQueryReadClient._get_default_mtls_endpoint(None) is None - assert BigQueryReadClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint - assert BigQueryReadClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint - assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint - assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint - assert BigQueryReadClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi - - -@pytest.mark.parametrize("client_class", [ - BigQueryReadClient, - BigQueryReadAsyncClient, -]) -def test_big_query_read_client_from_service_account_info(client_class): - creds = ga_credentials.AnonymousCredentials() - with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: - factory.return_value = creds - info = {"valid": True} - client = client_class.from_service_account_info(info) - assert client.transport._credentials == creds - assert isinstance(client, client_class) - - assert client.transport._host == 'bigquerystorage.googleapis.com:443' - - -@pytest.mark.parametrize("transport_class,transport_name", [ - (transports.BigQueryReadGrpcTransport, "grpc"), - (transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), -]) -def test_big_query_read_client_service_account_always_use_jwt(transport_class, transport_name): - with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: - creds = service_account.Credentials(None, None, None) - transport = transport_class(credentials=creds, always_use_jwt_access=True) - use_jwt.assert_called_once_with(True) - - with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: - creds = service_account.Credentials(None, None, None) - transport = transport_class(credentials=creds, always_use_jwt_access=False) - use_jwt.assert_not_called() - - -@pytest.mark.parametrize("client_class", [ - BigQueryReadClient, - BigQueryReadAsyncClient, -]) -def test_big_query_read_client_from_service_account_file(client_class): - creds = ga_credentials.AnonymousCredentials() - with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: - factory.return_value = creds - client = client_class.from_service_account_file("dummy/file/path.json") - assert client.transport._credentials == creds - assert isinstance(client, client_class) - - client = client_class.from_service_account_json("dummy/file/path.json") - assert client.transport._credentials == creds - assert isinstance(client, client_class) - - assert client.transport._host == 'bigquerystorage.googleapis.com:443' - - -def test_big_query_read_client_get_transport_class(): - transport = BigQueryReadClient.get_transport_class() - available_transports = [ - transports.BigQueryReadGrpcTransport, - ] - assert transport in available_transports - - transport = BigQueryReadClient.get_transport_class("grpc") - assert transport == transports.BigQueryReadGrpcTransport - - -@pytest.mark.parametrize("client_class,transport_class,transport_name", [ - (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), - (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), -]) -@mock.patch.object(BigQueryReadClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadClient)) -@mock.patch.object(BigQueryReadAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadAsyncClient)) -def test_big_query_read_client_client_options(client_class, transport_class, transport_name): - # Check that if channel is provided we won't create a new one. - with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: - transport = transport_class( - credentials=ga_credentials.AnonymousCredentials() - ) - client = client_class(transport=transport) - gtc.assert_not_called() - - # Check that if channel is provided via str we will create a new one. - with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: - client = client_class(transport=transport_name) - gtc.assert_called() - - # Check the case api_endpoint is provided. - options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host="squid.clam.whelk", - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is - # "never". - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is - # "always". - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_MTLS_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has - # unsupported value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): - with pytest.raises(MutualTLSChannelError): - client = client_class() - - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): - with pytest.raises(ValueError): - client = client_class() - - # Check the case quota_project_id is provided - options = client_options.ClientOptions(quota_project_id="octopus") - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id="octopus", - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - -@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ - (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "true"), - (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "true"), - (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "false"), - (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "false"), -]) -@mock.patch.object(BigQueryReadClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadClient)) -@mock.patch.object(BigQueryReadAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadAsyncClient)) -@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) -def test_big_query_read_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): - # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default - # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. - - # Check the case client_cert_source is provided. Whether client cert is used depends on - # GOOGLE_API_USE_CLIENT_CERTIFICATE value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): - options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - - if use_client_cert_env == "false": - expected_client_cert_source = None - expected_host = client.DEFAULT_ENDPOINT - else: - expected_client_cert_source = client_cert_source_callback - expected_host = client.DEFAULT_MTLS_ENDPOINT - - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=expected_host, - scopes=None, - client_cert_source_for_mtls=expected_client_cert_source, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case ADC client cert is provided. Whether client cert is used depends on - # GOOGLE_API_USE_CLIENT_CERTIFICATE value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): - with mock.patch.object(transport_class, '__init__') as patched: - with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): - with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): - if use_client_cert_env == "false": - expected_host = client.DEFAULT_ENDPOINT - expected_client_cert_source = None - else: - expected_host = client.DEFAULT_MTLS_ENDPOINT - expected_client_cert_source = client_cert_source_callback - - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=expected_host, - scopes=None, - client_cert_source_for_mtls=expected_client_cert_source, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case client_cert_source and ADC client cert are not provided. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): - with mock.patch.object(transport_class, '__init__') as patched: - with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - -@pytest.mark.parametrize("client_class,transport_class,transport_name", [ - (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), - (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), -]) -def test_big_query_read_client_client_options_scopes(client_class, transport_class, transport_name): - # Check the case scopes are provided. - options = client_options.ClientOptions( - scopes=["1", "2"], - ) - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=["1", "2"], - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - -@pytest.mark.parametrize("client_class,transport_class,transport_name", [ - (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), - (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), -]) -def test_big_query_read_client_client_options_credentials_file(client_class, transport_class, transport_name): - # Check the case credentials file is provided. - options = client_options.ClientOptions( - credentials_file="credentials.json" - ) - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file="credentials.json", - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - -def test_big_query_read_client_client_options_from_dict(): - with mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadGrpcTransport.__init__') as grpc_transport: - grpc_transport.return_value = None - client = BigQueryReadClient( - client_options={'api_endpoint': 'squid.clam.whelk'} - ) - grpc_transport.assert_called_once_with( - credentials=None, - credentials_file=None, - host="squid.clam.whelk", - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - -def test_create_read_session(transport: str = 'grpc', request_type=storage.CreateReadSessionRequest): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.ReadSession( - name='name_value', - data_format=stream.DataFormat.AVRO, - table='table_value', - avro_schema=avro.AvroSchema(schema='schema_value'), - ) - response = client.create_read_session(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.CreateReadSessionRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, stream.ReadSession) - assert response.name == 'name_value' - assert response.data_format == stream.DataFormat.AVRO - assert response.table == 'table_value' - - -def test_create_read_session_from_dict(): - test_create_read_session(request_type=dict) - - -def test_create_read_session_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - client.create_read_session() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.CreateReadSessionRequest() - - -@pytest.mark.asyncio -async def test_create_read_session_async(transport: str = 'grpc_asyncio', request_type=storage.CreateReadSessionRequest): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession( - name='name_value', - data_format=stream.DataFormat.AVRO, - table='table_value', - )) - response = await client.create_read_session(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.CreateReadSessionRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, stream.ReadSession) - assert response.name == 'name_value' - assert response.data_format == stream.DataFormat.AVRO - assert response.table == 'table_value' - - -@pytest.mark.asyncio -async def test_create_read_session_async_from_dict(): - await test_create_read_session_async(request_type=dict) - - -def test_create_read_session_field_headers(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.CreateReadSessionRequest() - - request.read_session.table = 'read_session.table/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - call.return_value = stream.ReadSession() - client.create_read_session(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'read_session.table=read_session.table/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_create_read_session_field_headers_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.CreateReadSessionRequest() - - request.read_session.table = 'read_session.table/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) - await client.create_read_session(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'read_session.table=read_session.table/value', - ) in kw['metadata'] - - -def test_create_read_session_flattened(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.ReadSession() - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - client.create_read_session( - parent='parent_value', - read_session=stream.ReadSession(name='name_value'), - max_stream_count=1721, - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0].parent == 'parent_value' - assert args[0].read_session == stream.ReadSession(name='name_value') - assert args[0].max_stream_count == 1721 - - -def test_create_read_session_flattened_error(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - client.create_read_session( - storage.CreateReadSessionRequest(), - parent='parent_value', - read_session=stream.ReadSession(name='name_value'), - max_stream_count=1721, - ) - - -@pytest.mark.asyncio -async def test_create_read_session_flattened_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.ReadSession() - - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - response = await client.create_read_session( - parent='parent_value', - read_session=stream.ReadSession(name='name_value'), - max_stream_count=1721, - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0].parent == 'parent_value' - assert args[0].read_session == stream.ReadSession(name='name_value') - assert args[0].max_stream_count == 1721 - - -@pytest.mark.asyncio -async def test_create_read_session_flattened_error_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - await client.create_read_session( - storage.CreateReadSessionRequest(), - parent='parent_value', - read_session=stream.ReadSession(name='name_value'), - max_stream_count=1721, - ) - - -def test_read_rows(transport: str = 'grpc', request_type=storage.ReadRowsRequest): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = iter([storage.ReadRowsResponse()]) - response = client.read_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.ReadRowsRequest() - - # Establish that the response is the type that we expect. - for message in response: - assert isinstance(message, storage.ReadRowsResponse) - - -def test_read_rows_from_dict(): - test_read_rows(request_type=dict) - - -def test_read_rows_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - client.read_rows() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.ReadRowsRequest() - - -@pytest.mark.asyncio -async def test_read_rows_async(transport: str = 'grpc_asyncio', request_type=storage.ReadRowsRequest): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) - call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) - response = await client.read_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.ReadRowsRequest() - - # Establish that the response is the type that we expect. - message = await response.read() - assert isinstance(message, storage.ReadRowsResponse) - - -@pytest.mark.asyncio -async def test_read_rows_async_from_dict(): - await test_read_rows_async(request_type=dict) - - -def test_read_rows_field_headers(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.ReadRowsRequest() - - request.read_stream = 'read_stream/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - call.return_value = iter([storage.ReadRowsResponse()]) - client.read_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'read_stream=read_stream/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_read_rows_field_headers_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.ReadRowsRequest() - - request.read_stream = 'read_stream/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) - call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) - await client.read_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'read_stream=read_stream/value', - ) in kw['metadata'] - - -def test_read_rows_flattened(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = iter([storage.ReadRowsResponse()]) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - client.read_rows( - read_stream='read_stream_value', - offset=647, - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0].read_stream == 'read_stream_value' - assert args[0].offset == 647 - - -def test_read_rows_flattened_error(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - client.read_rows( - storage.ReadRowsRequest(), - read_stream='read_stream_value', - offset=647, - ) - - -@pytest.mark.asyncio -async def test_read_rows_flattened_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = iter([storage.ReadRowsResponse()]) - - call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - response = await client.read_rows( - read_stream='read_stream_value', - offset=647, - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0].read_stream == 'read_stream_value' - assert args[0].offset == 647 - - -@pytest.mark.asyncio -async def test_read_rows_flattened_error_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - await client.read_rows( - storage.ReadRowsRequest(), - read_stream='read_stream_value', - offset=647, - ) - - -def test_split_read_stream(transport: str = 'grpc', request_type=storage.SplitReadStreamRequest): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.split_read_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.SplitReadStreamResponse( - ) - response = client.split_read_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.SplitReadStreamRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, storage.SplitReadStreamResponse) - - -def test_split_read_stream_from_dict(): - test_split_read_stream(request_type=dict) - - -def test_split_read_stream_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.split_read_stream), - '__call__') as call: - client.split_read_stream() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.SplitReadStreamRequest() - - -@pytest.mark.asyncio -async def test_split_read_stream_async(transport: str = 'grpc_asyncio', request_type=storage.SplitReadStreamRequest): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.split_read_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse( - )) - response = await client.split_read_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.SplitReadStreamRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, storage.SplitReadStreamResponse) - - -@pytest.mark.asyncio -async def test_split_read_stream_async_from_dict(): - await test_split_read_stream_async(request_type=dict) - - -def test_split_read_stream_field_headers(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.SplitReadStreamRequest() - - request.name = 'name/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.split_read_stream), - '__call__') as call: - call.return_value = storage.SplitReadStreamResponse() - client.split_read_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'name=name/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_split_read_stream_field_headers_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.SplitReadStreamRequest() - - request.name = 'name/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.split_read_stream), - '__call__') as call: - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse()) - await client.split_read_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'name=name/value', - ) in kw['metadata'] - - -def test_credentials_transport_error(): - # It is an error to provide credentials and a transport instance. - transport = transports.BigQueryReadGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # It is an error to provide a credentials file and a transport instance. - transport = transports.BigQueryReadGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): - client = BigQueryReadClient( - client_options={"credentials_file": "credentials.json"}, - transport=transport, - ) - - # It is an error to provide scopes and a transport instance. - transport = transports.BigQueryReadGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): - client = BigQueryReadClient( - client_options={"scopes": ["1", "2"]}, - transport=transport, - ) - - -def test_transport_instance(): - # A client may be instantiated with a custom transport instance. - transport = transports.BigQueryReadGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - client = BigQueryReadClient(transport=transport) - assert client.transport is transport - -def test_transport_get_channel(): - # A client may be instantiated with a custom transport instance. - transport = transports.BigQueryReadGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - channel = transport.grpc_channel - assert channel - - transport = transports.BigQueryReadGrpcAsyncIOTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - channel = transport.grpc_channel - assert channel - -@pytest.mark.parametrize("transport_class", [ - transports.BigQueryReadGrpcTransport, - transports.BigQueryReadGrpcAsyncIOTransport, -]) -def test_transport_adc(transport_class): - # Test default credentials are used if not provided. - with mock.patch.object(google.auth, 'default') as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport_class() - adc.assert_called_once() - -def test_transport_grpc_default(): - # A client should use the gRPC transport by default. - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - assert isinstance( - client.transport, - transports.BigQueryReadGrpcTransport, - ) - -def test_big_query_read_base_transport_error(): - # Passing both a credentials object and credentials_file should raise an error - with pytest.raises(core_exceptions.DuplicateCredentialArgs): - transport = transports.BigQueryReadTransport( - credentials=ga_credentials.AnonymousCredentials(), - credentials_file="credentials.json" - ) - - -def test_big_query_read_base_transport(): - # Instantiate the base transport. - with mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport.__init__') as Transport: - Transport.return_value = None - transport = transports.BigQueryReadTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Every method on the transport should just blindly - # raise NotImplementedError. - methods = ( - 'create_read_session', - 'read_rows', - 'split_read_stream', - ) - for method in methods: - with pytest.raises(NotImplementedError): - getattr(transport, method)(request=object()) - - -@requires_google_auth_gte_1_25_0 -def test_big_query_read_base_transport_with_credentials_file(): - # Instantiate the base transport with a credentials file - with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: - Transport.return_value = None - load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) - transport = transports.BigQueryReadTransport( - credentials_file="credentials.json", - quota_project_id="octopus", - ) - load_creds.assert_called_once_with("credentials.json", - scopes=None, - default_scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', -), - quota_project_id="octopus", - ) - - -@requires_google_auth_lt_1_25_0 -def test_big_query_read_base_transport_with_credentials_file_old_google_auth(): - # Instantiate the base transport with a credentials file - with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: - Transport.return_value = None - load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) - transport = transports.BigQueryReadTransport( - credentials_file="credentials.json", - quota_project_id="octopus", - ) - load_creds.assert_called_once_with("credentials.json", scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', - ), - quota_project_id="octopus", - ) - - -def test_big_query_read_base_transport_with_adc(): - # Test the default credentials are used if credentials and credentials_file are None. - with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: - Transport.return_value = None - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport = transports.BigQueryReadTransport() - adc.assert_called_once() - - -@requires_google_auth_gte_1_25_0 -def test_big_query_read_auth_adc(): - # If no credentials are provided, we should use ADC credentials. - with mock.patch.object(google.auth, 'default', autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - BigQueryReadClient() - adc.assert_called_once_with( - scopes=None, - default_scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', -), - quota_project_id=None, - ) - - -@requires_google_auth_lt_1_25_0 -def test_big_query_read_auth_adc_old_google_auth(): - # If no credentials are provided, we should use ADC credentials. - with mock.patch.object(google.auth, 'default', autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - BigQueryReadClient() - adc.assert_called_once_with( - scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.readonly', 'https://www.googleapis.com/auth/cloud-platform',), - quota_project_id=None, - ) - - -@pytest.mark.parametrize( - "transport_class", - [ - transports.BigQueryReadGrpcTransport, - transports.BigQueryReadGrpcAsyncIOTransport, - ], -) -@requires_google_auth_gte_1_25_0 -def test_big_query_read_transport_auth_adc(transport_class): - # If credentials and host are not provided, the transport class should use - # ADC credentials. - with mock.patch.object(google.auth, 'default', autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport_class(quota_project_id="octopus", scopes=["1", "2"]) - adc.assert_called_once_with( - scopes=["1", "2"], - default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.readonly', 'https://www.googleapis.com/auth/cloud-platform',), - quota_project_id="octopus", - ) - - -@pytest.mark.parametrize( - "transport_class", - [ - transports.BigQueryReadGrpcTransport, - transports.BigQueryReadGrpcAsyncIOTransport, - ], -) -@requires_google_auth_lt_1_25_0 -def test_big_query_read_transport_auth_adc_old_google_auth(transport_class): - # If credentials and host are not provided, the transport class should use - # ADC credentials. - with mock.patch.object(google.auth, "default", autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport_class(quota_project_id="octopus") - adc.assert_called_once_with(scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', -), - quota_project_id="octopus", - ) - - -@pytest.mark.parametrize( - "transport_class,grpc_helpers", - [ - (transports.BigQueryReadGrpcTransport, grpc_helpers), - (transports.BigQueryReadGrpcAsyncIOTransport, grpc_helpers_async) - ], -) -def test_big_query_read_transport_create_channel(transport_class, grpc_helpers): - # If credentials and host are not provided, the transport class should use - # ADC credentials. - with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( - grpc_helpers, "create_channel", autospec=True - ) as create_channel: - creds = ga_credentials.AnonymousCredentials() - adc.return_value = (creds, None) - transport_class( - quota_project_id="octopus", - scopes=["1", "2"] - ) - - create_channel.assert_called_with( - "bigquerystorage.googleapis.com:443", - credentials=creds, - credentials_file=None, - quota_project_id="octopus", - default_scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', -), - scopes=["1", "2"], - default_host="bigquerystorage.googleapis.com", - ssl_credentials=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - -@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) -def test_big_query_read_grpc_transport_client_cert_source_for_mtls( - transport_class -): - cred = ga_credentials.AnonymousCredentials() - - # Check ssl_channel_credentials is used if provided. - with mock.patch.object(transport_class, "create_channel") as mock_create_channel: - mock_ssl_channel_creds = mock.Mock() - transport_class( - host="squid.clam.whelk", - credentials=cred, - ssl_channel_credentials=mock_ssl_channel_creds - ) - mock_create_channel.assert_called_once_with( - "squid.clam.whelk:443", - credentials=cred, - credentials_file=None, - scopes=None, - ssl_credentials=mock_ssl_channel_creds, - quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls - # is used. - with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): - with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: - transport_class( - credentials=cred, - client_cert_source_for_mtls=client_cert_source_callback - ) - expected_cert, expected_key = client_cert_source_callback() - mock_ssl_cred.assert_called_once_with( - certificate_chain=expected_cert, - private_key=expected_key - ) - - -def test_big_query_read_host_no_port(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), - ) - assert client.transport._host == 'bigquerystorage.googleapis.com:443' - - -def test_big_query_read_host_with_port(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), - ) - assert client.transport._host == 'bigquerystorage.googleapis.com:8000' - -def test_big_query_read_grpc_transport_channel(): - channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) - - # Check that channel is used if provided. - transport = transports.BigQueryReadGrpcTransport( - host="squid.clam.whelk", - channel=channel, - ) - assert transport.grpc_channel == channel - assert transport._host == "squid.clam.whelk:443" - assert transport._ssl_channel_credentials == None - - -def test_big_query_read_grpc_asyncio_transport_channel(): - channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) - - # Check that channel is used if provided. - transport = transports.BigQueryReadGrpcAsyncIOTransport( - host="squid.clam.whelk", - channel=channel, - ) - assert transport.grpc_channel == channel - assert transport._host == "squid.clam.whelk:443" - assert transport._ssl_channel_credentials == None - - -# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are -# removed from grpc/grpc_asyncio transport constructor. -@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) -def test_big_query_read_transport_channel_mtls_with_client_cert_source( - transport_class -): - with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: - with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: - mock_ssl_cred = mock.Mock() - grpc_ssl_channel_cred.return_value = mock_ssl_cred - - mock_grpc_channel = mock.Mock() - grpc_create_channel.return_value = mock_grpc_channel - - cred = ga_credentials.AnonymousCredentials() - with pytest.warns(DeprecationWarning): - with mock.patch.object(google.auth, 'default') as adc: - adc.return_value = (cred, None) - transport = transport_class( - host="squid.clam.whelk", - api_mtls_endpoint="mtls.squid.clam.whelk", - client_cert_source=client_cert_source_callback, - ) - adc.assert_called_once() - - grpc_ssl_channel_cred.assert_called_once_with( - certificate_chain=b"cert bytes", private_key=b"key bytes" - ) - grpc_create_channel.assert_called_once_with( - "mtls.squid.clam.whelk:443", - credentials=cred, - credentials_file=None, - scopes=None, - ssl_credentials=mock_ssl_cred, - quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - assert transport.grpc_channel == mock_grpc_channel - assert transport._ssl_channel_credentials == mock_ssl_cred - - -# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are -# removed from grpc/grpc_asyncio transport constructor. -@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) -def test_big_query_read_transport_channel_mtls_with_adc( - transport_class -): - mock_ssl_cred = mock.Mock() - with mock.patch.multiple( - "google.auth.transport.grpc.SslCredentials", - __init__=mock.Mock(return_value=None), - ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), - ): - with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: - mock_grpc_channel = mock.Mock() - grpc_create_channel.return_value = mock_grpc_channel - mock_cred = mock.Mock() - - with pytest.warns(DeprecationWarning): - transport = transport_class( - host="squid.clam.whelk", - credentials=mock_cred, - api_mtls_endpoint="mtls.squid.clam.whelk", - client_cert_source=None, - ) - - grpc_create_channel.assert_called_once_with( - "mtls.squid.clam.whelk:443", - credentials=mock_cred, - credentials_file=None, - scopes=None, - ssl_credentials=mock_ssl_cred, - quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - assert transport.grpc_channel == mock_grpc_channel - - -def test_read_session_path(): - project = "squid" - location = "clam" - session = "whelk" - expected = "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) - actual = BigQueryReadClient.read_session_path(project, location, session) - assert expected == actual - - -def test_parse_read_session_path(): - expected = { - "project": "octopus", - "location": "oyster", - "session": "nudibranch", - } - path = BigQueryReadClient.read_session_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_read_session_path(path) - assert expected == actual - -def test_read_stream_path(): - project = "cuttlefish" - location = "mussel" - session = "winkle" - stream = "nautilus" - expected = "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) - actual = BigQueryReadClient.read_stream_path(project, location, session, stream) - assert expected == actual - - -def test_parse_read_stream_path(): - expected = { - "project": "scallop", - "location": "abalone", - "session": "squid", - "stream": "clam", - } - path = BigQueryReadClient.read_stream_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_read_stream_path(path) - assert expected == actual - -def test_table_path(): - project = "whelk" - dataset = "octopus" - table = "oyster" - expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) - actual = BigQueryReadClient.table_path(project, dataset, table) - assert expected == actual - - -def test_parse_table_path(): - expected = { - "project": "nudibranch", - "dataset": "cuttlefish", - "table": "mussel", - } - path = BigQueryReadClient.table_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_table_path(path) - assert expected == actual - -def test_common_billing_account_path(): - billing_account = "winkle" - expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) - actual = BigQueryReadClient.common_billing_account_path(billing_account) - assert expected == actual - - -def test_parse_common_billing_account_path(): - expected = { - "billing_account": "nautilus", - } - path = BigQueryReadClient.common_billing_account_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_common_billing_account_path(path) - assert expected == actual - -def test_common_folder_path(): - folder = "scallop" - expected = "folders/{folder}".format(folder=folder, ) - actual = BigQueryReadClient.common_folder_path(folder) - assert expected == actual - - -def test_parse_common_folder_path(): - expected = { - "folder": "abalone", - } - path = BigQueryReadClient.common_folder_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_common_folder_path(path) - assert expected == actual - -def test_common_organization_path(): - organization = "squid" - expected = "organizations/{organization}".format(organization=organization, ) - actual = BigQueryReadClient.common_organization_path(organization) - assert expected == actual - - -def test_parse_common_organization_path(): - expected = { - "organization": "clam", - } - path = BigQueryReadClient.common_organization_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_common_organization_path(path) - assert expected == actual - -def test_common_project_path(): - project = "whelk" - expected = "projects/{project}".format(project=project, ) - actual = BigQueryReadClient.common_project_path(project) - assert expected == actual - - -def test_parse_common_project_path(): - expected = { - "project": "octopus", - } - path = BigQueryReadClient.common_project_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_common_project_path(path) - assert expected == actual - -def test_common_location_path(): - project = "oyster" - location = "nudibranch" - expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) - actual = BigQueryReadClient.common_location_path(project, location) - assert expected == actual - - -def test_parse_common_location_path(): - expected = { - "project": "cuttlefish", - "location": "mussel", - } - path = BigQueryReadClient.common_location_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_common_location_path(path) - assert expected == actual - - -def test_client_withDEFAULT_CLIENT_INFO(): - client_info = gapic_v1.client_info.ClientInfo() - - with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - client_info=client_info, - ) - prep.assert_called_once_with(client_info) - - with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: - transport_class = BigQueryReadClient.get_transport_class() - transport = transport_class( - credentials=ga_credentials.AnonymousCredentials(), - client_info=client_info, - ) - prep.assert_called_once_with(client_info) diff --git a/owl-bot-staging/v1beta2/.coveragerc b/owl-bot-staging/v1beta2/.coveragerc deleted file mode 100644 index 19fadc8b..00000000 --- a/owl-bot-staging/v1beta2/.coveragerc +++ /dev/null @@ -1,17 +0,0 @@ -[run] -branch = True - -[report] -show_missing = True -omit = - google/cloud/bigquery_storage/__init__.py -exclude_lines = - # Re-enable the standard pragma - pragma: NO COVER - # Ignore debug-only repr - def __repr__ - # Ignore pkg_resources exceptions. - # This is added at the module level as a safeguard for if someone - # generates the code and tries to run it without pip installing. This - # makes it virtually impossible to test properly. - except pkg_resources.DistributionNotFound diff --git a/owl-bot-staging/v1beta2/MANIFEST.in b/owl-bot-staging/v1beta2/MANIFEST.in deleted file mode 100644 index 6b94fe3e..00000000 --- a/owl-bot-staging/v1beta2/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -recursive-include google/cloud/bigquery_storage *.py -recursive-include google/cloud/bigquery_storage_v1beta2 *.py diff --git a/owl-bot-staging/v1beta2/README.rst b/owl-bot-staging/v1beta2/README.rst deleted file mode 100644 index 13a45c14..00000000 --- a/owl-bot-staging/v1beta2/README.rst +++ /dev/null @@ -1,49 +0,0 @@ -Python Client for Google Cloud Bigquery Storage API -================================================= - -Quick Start ------------ - -In order to use this library, you first need to go through the following steps: - -1. `Select or create a Cloud Platform project.`_ -2. `Enable billing for your project.`_ -3. Enable the Google Cloud Bigquery Storage API. -4. `Setup Authentication.`_ - -.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project -.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project -.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html - -Installation -~~~~~~~~~~~~ - -Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to -create isolated Python environments. The basic problem it addresses is one of -dependencies and versions, and indirectly permissions. - -With `virtualenv`_, it's possible to install this library without needing system -install permissions, and without clashing with the installed system -dependencies. - -.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ - - -Mac/Linux -^^^^^^^^^ - -.. code-block:: console - - python3 -m venv - source /bin/activate - /bin/pip install /path/to/library - - -Windows -^^^^^^^ - -.. code-block:: console - - python3 -m venv - \Scripts\activate - \Scripts\pip.exe install \path\to\library diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst deleted file mode 100644 index 10b3714b..00000000 --- a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst +++ /dev/null @@ -1,6 +0,0 @@ -BigQueryRead ------------------------------- - -.. automodule:: google.cloud.bigquery_storage_v1beta2.services.big_query_read - :members: - :inherited-members: diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst deleted file mode 100644 index c685994d..00000000 --- a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst +++ /dev/null @@ -1,6 +0,0 @@ -BigQueryWrite -------------------------------- - -.. automodule:: google.cloud.bigquery_storage_v1beta2.services.big_query_write - :members: - :inherited-members: diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services.rst deleted file mode 100644 index 67dae0ab..00000000 --- a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services.rst +++ /dev/null @@ -1,7 +0,0 @@ -Services for Google Cloud Bigquery Storage v1beta2 API -====================================================== -.. toctree:: - :maxdepth: 2 - - big_query_read - big_query_write diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types.rst deleted file mode 100644 index 995806da..00000000 --- a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types.rst +++ /dev/null @@ -1,7 +0,0 @@ -Types for Google Cloud Bigquery Storage v1beta2 API -=================================================== - -.. automodule:: google.cloud.bigquery_storage_v1beta2.types - :members: - :undoc-members: - :show-inheritance: diff --git a/owl-bot-staging/v1beta2/docs/conf.py b/owl-bot-staging/v1beta2/docs/conf.py deleted file mode 100644 index e198bf7a..00000000 --- a/owl-bot-staging/v1beta2/docs/conf.py +++ /dev/null @@ -1,376 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# google-cloud-bigquery-storage documentation build configuration file -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os -import shlex - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath("..")) - -__version__ = "0.1.0" - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = "1.6.3" - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.autosummary", - "sphinx.ext.intersphinx", - "sphinx.ext.coverage", - "sphinx.ext.napoleon", - "sphinx.ext.todo", - "sphinx.ext.viewcode", -] - -# autodoc/autosummary flags -autoclass_content = "both" -autodoc_default_flags = ["members"] -autosummary_generate = True - - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# Allow markdown includes (so releases.md can include CHANGLEOG.md) -# http://www.sphinx-doc.org/en/master/markdown.html -source_parsers = {".md": "recommonmark.parser.CommonMarkParser"} - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -source_suffix = [".rst", ".md"] - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = u"google-cloud-bigquery-storage" -copyright = u"2020, Google, LLC" -author = u"Google APIs" # TODO: autogenerate this bit - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The full version, including alpha/beta/rc tags. -release = __version__ -# The short X.Y version. -version = ".".join(release.split(".")[0:2]) - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ["_build"] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = "alabaster" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = { - "description": "Google Cloud Client Libraries for Python", - "github_user": "googleapis", - "github_repo": "google-cloud-python", - "github_banner": True, - "font_family": "'Roboto', Georgia, sans", - "head_font_family": "'Roboto', Georgia, serif", - "code_font_family": "'Roboto Mono', 'Consolas', monospace", -} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -# html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# Now only 'ja' uses this config value -# html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -# html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = "google-cloud-bigquery-storage-doc" - -# -- Options for warnings ------------------------------------------------------ - - -suppress_warnings = [ - # Temporarily suppress this to avoid "more than one target found for - # cross-reference" warning, which are intractable for us to avoid while in - # a mono-repo. - # See https://github.com/sphinx-doc/sphinx/blob - # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843 - "ref.python" -] - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # 'preamble': '', - # Latex figure (float) alignment - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ( - master_doc, - "google-cloud-bigquery-storage.tex", - u"google-cloud-bigquery-storage Documentation", - author, - "manual", - ) -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ( - master_doc, - "google-cloud-bigquery-storage", - u"Google Cloud Bigquery Storage Documentation", - [author], - 1, - ) -] - -# If true, show URL addresses after external links. -# man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - "google-cloud-bigquery-storage", - u"google-cloud-bigquery-storage Documentation", - author, - "google-cloud-bigquery-storage", - "GAPIC library for Google Cloud Bigquery Storage API", - "APIs", - ) -] - -# Documents to append as an appendix to all manuals. -# texinfo_appendices = [] - -# If false, no module index is generated. -# texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -# texinfo_no_detailmenu = False - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = { - "python": ("http://python.readthedocs.org/en/latest/", None), - "gax": ("https://gax-python.readthedocs.org/en/latest/", None), - "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), - "google-gax": ("https://gax-python.readthedocs.io/en/latest/", None), - "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None), - "grpc": ("https://grpc.io/grpc/python/", None), - "requests": ("http://requests.kennethreitz.org/en/stable/", None), - "proto": ("https://proto-plus-python.readthedocs.io/en/stable", None), - "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), -} - - -# Napoleon settings -napoleon_google_docstring = True -napoleon_numpy_docstring = True -napoleon_include_private_with_doc = False -napoleon_include_special_with_doc = True -napoleon_use_admonition_for_examples = False -napoleon_use_admonition_for_notes = False -napoleon_use_admonition_for_references = False -napoleon_use_ivar = False -napoleon_use_param = True -napoleon_use_rtype = True diff --git a/owl-bot-staging/v1beta2/docs/index.rst b/owl-bot-staging/v1beta2/docs/index.rst deleted file mode 100644 index ef2e07e2..00000000 --- a/owl-bot-staging/v1beta2/docs/index.rst +++ /dev/null @@ -1,7 +0,0 @@ -API Reference -------------- -.. toctree:: - :maxdepth: 2 - - bigquery_storage_v1beta2/services - bigquery_storage_v1beta2/types diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py deleted file mode 100644 index 55ef84ca..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from google.cloud.bigquery_storage_v1beta2.services.big_query_read.client import BigQueryReadClient -from google.cloud.bigquery_storage_v1beta2.services.big_query_read.async_client import BigQueryReadAsyncClient -from google.cloud.bigquery_storage_v1beta2.services.big_query_write.client import BigQueryWriteClient -from google.cloud.bigquery_storage_v1beta2.services.big_query_write.async_client import BigQueryWriteAsyncClient - -from google.cloud.bigquery_storage_v1beta2.types.arrow import ArrowRecordBatch -from google.cloud.bigquery_storage_v1beta2.types.arrow import ArrowSchema -from google.cloud.bigquery_storage_v1beta2.types.arrow import ArrowSerializationOptions -from google.cloud.bigquery_storage_v1beta2.types.avro import AvroRows -from google.cloud.bigquery_storage_v1beta2.types.avro import AvroSchema -from google.cloud.bigquery_storage_v1beta2.types.protobuf import ProtoRows -from google.cloud.bigquery_storage_v1beta2.types.protobuf import ProtoSchema -from google.cloud.bigquery_storage_v1beta2.types.storage import AppendRowsRequest -from google.cloud.bigquery_storage_v1beta2.types.storage import AppendRowsResponse -from google.cloud.bigquery_storage_v1beta2.types.storage import BatchCommitWriteStreamsRequest -from google.cloud.bigquery_storage_v1beta2.types.storage import BatchCommitWriteStreamsResponse -from google.cloud.bigquery_storage_v1beta2.types.storage import CreateReadSessionRequest -from google.cloud.bigquery_storage_v1beta2.types.storage import CreateWriteStreamRequest -from google.cloud.bigquery_storage_v1beta2.types.storage import FinalizeWriteStreamRequest -from google.cloud.bigquery_storage_v1beta2.types.storage import FinalizeWriteStreamResponse -from google.cloud.bigquery_storage_v1beta2.types.storage import FlushRowsRequest -from google.cloud.bigquery_storage_v1beta2.types.storage import FlushRowsResponse -from google.cloud.bigquery_storage_v1beta2.types.storage import GetWriteStreamRequest -from google.cloud.bigquery_storage_v1beta2.types.storage import ReadRowsRequest -from google.cloud.bigquery_storage_v1beta2.types.storage import ReadRowsResponse -from google.cloud.bigquery_storage_v1beta2.types.storage import SplitReadStreamRequest -from google.cloud.bigquery_storage_v1beta2.types.storage import SplitReadStreamResponse -from google.cloud.bigquery_storage_v1beta2.types.storage import StorageError -from google.cloud.bigquery_storage_v1beta2.types.storage import StreamStats -from google.cloud.bigquery_storage_v1beta2.types.storage import ThrottleState -from google.cloud.bigquery_storage_v1beta2.types.stream import ReadSession -from google.cloud.bigquery_storage_v1beta2.types.stream import ReadStream -from google.cloud.bigquery_storage_v1beta2.types.stream import WriteStream -from google.cloud.bigquery_storage_v1beta2.types.stream import DataFormat -from google.cloud.bigquery_storage_v1beta2.types.table import TableFieldSchema -from google.cloud.bigquery_storage_v1beta2.types.table import TableSchema - -__all__ = ('BigQueryReadClient', - 'BigQueryReadAsyncClient', - 'BigQueryWriteClient', - 'BigQueryWriteAsyncClient', - 'ArrowRecordBatch', - 'ArrowSchema', - 'ArrowSerializationOptions', - 'AvroRows', - 'AvroSchema', - 'ProtoRows', - 'ProtoSchema', - 'AppendRowsRequest', - 'AppendRowsResponse', - 'BatchCommitWriteStreamsRequest', - 'BatchCommitWriteStreamsResponse', - 'CreateReadSessionRequest', - 'CreateWriteStreamRequest', - 'FinalizeWriteStreamRequest', - 'FinalizeWriteStreamResponse', - 'FlushRowsRequest', - 'FlushRowsResponse', - 'GetWriteStreamRequest', - 'ReadRowsRequest', - 'ReadRowsResponse', - 'SplitReadStreamRequest', - 'SplitReadStreamResponse', - 'StorageError', - 'StreamStats', - 'ThrottleState', - 'ReadSession', - 'ReadStream', - 'WriteStream', - 'DataFormat', - 'TableFieldSchema', - 'TableSchema', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed deleted file mode 100644 index e71b4749..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed +++ /dev/null @@ -1,2 +0,0 @@ -# Marker file for PEP 561. -# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py deleted file mode 100644 index c3136dce..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .services.big_query_read import BigQueryReadClient -from .services.big_query_read import BigQueryReadAsyncClient -from .services.big_query_write import BigQueryWriteClient -from .services.big_query_write import BigQueryWriteAsyncClient - -from .types.arrow import ArrowRecordBatch -from .types.arrow import ArrowSchema -from .types.arrow import ArrowSerializationOptions -from .types.avro import AvroRows -from .types.avro import AvroSchema -from .types.protobuf import ProtoRows -from .types.protobuf import ProtoSchema -from .types.storage import AppendRowsRequest -from .types.storage import AppendRowsResponse -from .types.storage import BatchCommitWriteStreamsRequest -from .types.storage import BatchCommitWriteStreamsResponse -from .types.storage import CreateReadSessionRequest -from .types.storage import CreateWriteStreamRequest -from .types.storage import FinalizeWriteStreamRequest -from .types.storage import FinalizeWriteStreamResponse -from .types.storage import FlushRowsRequest -from .types.storage import FlushRowsResponse -from .types.storage import GetWriteStreamRequest -from .types.storage import ReadRowsRequest -from .types.storage import ReadRowsResponse -from .types.storage import SplitReadStreamRequest -from .types.storage import SplitReadStreamResponse -from .types.storage import StorageError -from .types.storage import StreamStats -from .types.storage import ThrottleState -from .types.stream import ReadSession -from .types.stream import ReadStream -from .types.stream import WriteStream -from .types.stream import DataFormat -from .types.table import TableFieldSchema -from .types.table import TableSchema - -__all__ = ( - 'BigQueryReadAsyncClient', - 'BigQueryWriteAsyncClient', -'AppendRowsRequest', -'AppendRowsResponse', -'ArrowRecordBatch', -'ArrowSchema', -'ArrowSerializationOptions', -'AvroRows', -'AvroSchema', -'BatchCommitWriteStreamsRequest', -'BatchCommitWriteStreamsResponse', -'BigQueryReadClient', -'BigQueryWriteClient', -'CreateReadSessionRequest', -'CreateWriteStreamRequest', -'DataFormat', -'FinalizeWriteStreamRequest', -'FinalizeWriteStreamResponse', -'FlushRowsRequest', -'FlushRowsResponse', -'GetWriteStreamRequest', -'ProtoRows', -'ProtoSchema', -'ReadRowsRequest', -'ReadRowsResponse', -'ReadSession', -'ReadStream', -'SplitReadStreamRequest', -'SplitReadStreamResponse', -'StorageError', -'StreamStats', -'TableFieldSchema', -'TableSchema', -'ThrottleState', -'WriteStream', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json deleted file mode 100644 index 5aba90d6..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json +++ /dev/null @@ -1,127 +0,0 @@ - { - "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", - "language": "python", - "libraryPackage": "google.cloud.bigquery_storage_v1beta2", - "protoPackage": "google.cloud.bigquery.storage.v1beta2", - "schema": "1.0", - "services": { - "BigQueryRead": { - "clients": { - "grpc": { - "libraryClient": "BigQueryReadClient", - "rpcs": { - "CreateReadSession": { - "methods": [ - "create_read_session" - ] - }, - "ReadRows": { - "methods": [ - "read_rows" - ] - }, - "SplitReadStream": { - "methods": [ - "split_read_stream" - ] - } - } - }, - "grpc-async": { - "libraryClient": "BigQueryReadAsyncClient", - "rpcs": { - "CreateReadSession": { - "methods": [ - "create_read_session" - ] - }, - "ReadRows": { - "methods": [ - "read_rows" - ] - }, - "SplitReadStream": { - "methods": [ - "split_read_stream" - ] - } - } - } - } - }, - "BigQueryWrite": { - "clients": { - "grpc": { - "libraryClient": "BigQueryWriteClient", - "rpcs": { - "AppendRows": { - "methods": [ - "append_rows" - ] - }, - "BatchCommitWriteStreams": { - "methods": [ - "batch_commit_write_streams" - ] - }, - "CreateWriteStream": { - "methods": [ - "create_write_stream" - ] - }, - "FinalizeWriteStream": { - "methods": [ - "finalize_write_stream" - ] - }, - "FlushRows": { - "methods": [ - "flush_rows" - ] - }, - "GetWriteStream": { - "methods": [ - "get_write_stream" - ] - } - } - }, - "grpc-async": { - "libraryClient": "BigQueryWriteAsyncClient", - "rpcs": { - "AppendRows": { - "methods": [ - "append_rows" - ] - }, - "BatchCommitWriteStreams": { - "methods": [ - "batch_commit_write_streams" - ] - }, - "CreateWriteStream": { - "methods": [ - "create_write_stream" - ] - }, - "FinalizeWriteStream": { - "methods": [ - "finalize_write_stream" - ] - }, - "FlushRows": { - "methods": [ - "flush_rows" - ] - }, - "GetWriteStream": { - "methods": [ - "get_write_stream" - ] - } - } - } - } - } - } -} diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed deleted file mode 100644 index e71b4749..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed +++ /dev/null @@ -1,2 +0,0 @@ -# Marker file for PEP 561. -# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py deleted file mode 100644 index 4de65971..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py deleted file mode 100644 index 13be8ad1..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from .client import BigQueryReadClient -from .async_client import BigQueryReadAsyncClient - -__all__ = ( - 'BigQueryReadClient', - 'BigQueryReadAsyncClient', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py deleted file mode 100644 index 881c717b..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py +++ /dev/null @@ -1,476 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from collections import OrderedDict -import functools -import re -from typing import Dict, AsyncIterable, Awaitable, Sequence, Tuple, Type, Union -import pkg_resources - -import google.api_core.client_options as ClientOptions # type: ignore -from google.api_core import exceptions as core_exceptions # type: ignore -from google.api_core import gapic_v1 # type: ignore -from google.api_core import retry as retries # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.oauth2 import service_account # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import arrow -from google.cloud.bigquery_storage_v1beta2.types import avro -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream -from google.protobuf import timestamp_pb2 # type: ignore -from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO -from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport -from .client import BigQueryReadClient - - -class BigQueryReadAsyncClient: - """BigQuery Read API. - The Read API can be used to read data from BigQuery. - New code should use the v1 Read API going forward, if they don't - use Write API at the same time. - """ - - _client: BigQueryReadClient - - DEFAULT_ENDPOINT = BigQueryReadClient.DEFAULT_ENDPOINT - DEFAULT_MTLS_ENDPOINT = BigQueryReadClient.DEFAULT_MTLS_ENDPOINT - - read_session_path = staticmethod(BigQueryReadClient.read_session_path) - parse_read_session_path = staticmethod(BigQueryReadClient.parse_read_session_path) - read_stream_path = staticmethod(BigQueryReadClient.read_stream_path) - parse_read_stream_path = staticmethod(BigQueryReadClient.parse_read_stream_path) - table_path = staticmethod(BigQueryReadClient.table_path) - parse_table_path = staticmethod(BigQueryReadClient.parse_table_path) - common_billing_account_path = staticmethod(BigQueryReadClient.common_billing_account_path) - parse_common_billing_account_path = staticmethod(BigQueryReadClient.parse_common_billing_account_path) - common_folder_path = staticmethod(BigQueryReadClient.common_folder_path) - parse_common_folder_path = staticmethod(BigQueryReadClient.parse_common_folder_path) - common_organization_path = staticmethod(BigQueryReadClient.common_organization_path) - parse_common_organization_path = staticmethod(BigQueryReadClient.parse_common_organization_path) - common_project_path = staticmethod(BigQueryReadClient.common_project_path) - parse_common_project_path = staticmethod(BigQueryReadClient.parse_common_project_path) - common_location_path = staticmethod(BigQueryReadClient.common_location_path) - parse_common_location_path = staticmethod(BigQueryReadClient.parse_common_location_path) - - @classmethod - def from_service_account_info(cls, info: dict, *args, **kwargs): - """Creates an instance of this client using the provided credentials - info. - - Args: - info (dict): The service account private key info. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryReadAsyncClient: The constructed client. - """ - return BigQueryReadClient.from_service_account_info.__func__(BigQueryReadAsyncClient, info, *args, **kwargs) # type: ignore - - @classmethod - def from_service_account_file(cls, filename: str, *args, **kwargs): - """Creates an instance of this client using the provided credentials - file. - - Args: - filename (str): The path to the service account private key json - file. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryReadAsyncClient: The constructed client. - """ - return BigQueryReadClient.from_service_account_file.__func__(BigQueryReadAsyncClient, filename, *args, **kwargs) # type: ignore - - from_service_account_json = from_service_account_file - - @property - def transport(self) -> BigQueryReadTransport: - """Returns the transport used by the client instance. - - Returns: - BigQueryReadTransport: The transport used by the client instance. - """ - return self._client.transport - - get_transport_class = functools.partial(type(BigQueryReadClient).get_transport_class, type(BigQueryReadClient)) - - def __init__(self, *, - credentials: ga_credentials.Credentials = None, - transport: Union[str, BigQueryReadTransport] = "grpc_asyncio", - client_options: ClientOptions = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - ) -> None: - """Instantiates the big query read client. - - Args: - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - transport (Union[str, ~.BigQueryReadTransport]): The - transport to use. If set to None, a transport is chosen - automatically. - client_options (ClientOptions): Custom options for the client. It - won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: - "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable - is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If - not provided, the default SSL client certificate will be used if - present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not - set, no client certificate will be used. - - Raises: - google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport - creation failed for any reason. - """ - self._client = BigQueryReadClient( - credentials=credentials, - transport=transport, - client_options=client_options, - client_info=client_info, - - ) - - async def create_read_session(self, - request: storage.CreateReadSessionRequest = None, - *, - parent: str = None, - read_session: stream.ReadSession = None, - max_stream_count: int = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> stream.ReadSession: - r"""Creates a new read session. A read session divides - the contents of a BigQuery table into one or more - streams, which can then be used to read data from the - table. The read session also specifies properties of the - data to be read, such as a list of columns or a push- - down filter describing the rows to be returned. - - A particular row can be read by at most one stream. When - the caller has reached the end of each stream in the - session, then all the data in the table has been read. - - Data is assigned to each stream such that roughly the - same number of rows can be read from each stream. - Because the server-side unit for assigning data is - collections of rows, the API does not guarantee that - each stream will return the same number or rows. - Additionally, the limits are enforced based on the - number of pre-filtered rows, so some filters can lead to - lopsided assignments. - - Read sessions automatically expire 6 hours after they - are created and do not require manual clean-up by the - caller. - - Args: - request (:class:`google.cloud.bigquery_storage_v1beta2.types.CreateReadSessionRequest`): - The request object. Request message for - `CreateReadSession`. - parent (:class:`str`): - Required. The request project that owns the session, in - the form of ``projects/{project_id}``. - - This corresponds to the ``parent`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - read_session (:class:`google.cloud.bigquery_storage_v1beta2.types.ReadSession`): - Required. Session to be created. - This corresponds to the ``read_session`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - max_stream_count (:class:`int`): - Max initial number of streams. If - unset or zero, the server will provide a - value of streams so as to produce - reasonable throughput. Must be non- - negative. The number of streams may be - lower than the requested number, - depending on the amount parallelism that - is reasonable for the table. Error will - be returned if the max count is greater - than the current system max limit of - 1,000. - - Streams must be read starting from - offset 0. - - This corresponds to the ``max_stream_count`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.ReadSession: - Information about the ReadSession. - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([parent, read_session, max_stream_count]) - if request is not None and has_flattened_params: - raise ValueError("If the `request` argument is set, then none of " - "the individual field arguments should be set.") - - request = storage.CreateReadSessionRequest(request) - - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if parent is not None: - request.parent = parent - if read_session is not None: - request.read_session = read_session - if max_stream_count is not None: - request.max_stream_count = max_stream_count - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.create_read_session, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("read_session.table", request.read_session.table), - )), - ) - - # Send the request. - response = await rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def read_rows(self, - request: storage.ReadRowsRequest = None, - *, - read_stream: str = None, - offset: int = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> Awaitable[AsyncIterable[storage.ReadRowsResponse]]: - r"""Reads rows from the stream in the format prescribed - by the ReadSession. Each response contains one or more - table rows, up to a maximum of 100 MiB per response; - read requests which attempt to read individual rows - larger than 100 MiB will fail. - - Each request also returns a set of stream statistics - reflecting the current state of the stream. - - Args: - request (:class:`google.cloud.bigquery_storage_v1beta2.types.ReadRowsRequest`): - The request object. Request message for `ReadRows`. - read_stream (:class:`str`): - Required. Stream to read rows from. - This corresponds to the ``read_stream`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - offset (:class:`int`): - The offset requested must be less - than the last row read from Read. - Requesting a larger offset is undefined. - If not specified, start reading from - offset zero. - - This corresponds to the ``offset`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - AsyncIterable[google.cloud.bigquery_storage_v1beta2.types.ReadRowsResponse]: - Response from calling ReadRows may include row data, progress and - throttling information. - - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([read_stream, offset]) - if request is not None and has_flattened_params: - raise ValueError("If the `request` argument is set, then none of " - "the individual field arguments should be set.") - - request = storage.ReadRowsRequest(request) - - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if read_stream is not None: - request.read_stream = read_stream - if offset is not None: - request.offset = offset - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.read_rows, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.ServiceUnavailable, - ), - deadline=86400.0, - ), - default_timeout=86400.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("read_stream", request.read_stream), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - async def split_read_stream(self, - request: storage.SplitReadStreamRequest = None, - *, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> storage.SplitReadStreamResponse: - r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. - These ``ReadStream`` objects are referred to as the primary and - the residual streams of the split. The original ``ReadStream`` - can still be read from in the same manner as before. Both of the - returned ``ReadStream`` objects can also be read from, and the - rows returned by both child streams will be the same as the rows - read from the original stream. - - Moreover, the two child streams will be allocated back-to-back - in the original ``ReadStream``. Concretely, it is guaranteed - that for streams original, primary, and residual, that - original[0-j] = primary[0-j] and original[j-n] = residual[0-m] - once the streams have been read to completion. - - Args: - request (:class:`google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamRequest`): - The request object. Request message for - `SplitReadStream`. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamResponse: - - """ - # Create or coerce a protobuf request object. - request = storage.SplitReadStreamRequest(request) - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.split_read_stream, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("name", request.name), - )), - ) - - # Send the request. - response = await rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - - - - -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - "google-cloud-bigquery-storage", - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() - - -__all__ = ( - "BigQueryReadAsyncClient", -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py deleted file mode 100644 index 8945a2f1..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py +++ /dev/null @@ -1,666 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from collections import OrderedDict -from distutils import util -import os -import re -from typing import Callable, Dict, Optional, Iterable, Sequence, Tuple, Type, Union -import pkg_resources - -from google.api_core import client_options as client_options_lib # type: ignore -from google.api_core import exceptions as core_exceptions # type: ignore -from google.api_core import gapic_v1 # type: ignore -from google.api_core import retry as retries # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.auth.transport import mtls # type: ignore -from google.auth.transport.grpc import SslCredentials # type: ignore -from google.auth.exceptions import MutualTLSChannelError # type: ignore -from google.oauth2 import service_account # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import arrow -from google.cloud.bigquery_storage_v1beta2.types import avro -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream -from google.protobuf import timestamp_pb2 # type: ignore -from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO -from .transports.grpc import BigQueryReadGrpcTransport -from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport - - -class BigQueryReadClientMeta(type): - """Metaclass for the BigQueryRead client. - - This provides class-level methods for building and retrieving - support objects (e.g. transport) without polluting the client instance - objects. - """ - _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] - _transport_registry["grpc"] = BigQueryReadGrpcTransport - _transport_registry["grpc_asyncio"] = BigQueryReadGrpcAsyncIOTransport - - def get_transport_class(cls, - label: str = None, - ) -> Type[BigQueryReadTransport]: - """Returns an appropriate transport class. - - Args: - label: The name of the desired transport. If none is - provided, then the first transport in the registry is used. - - Returns: - The transport class to use. - """ - # If a specific transport is requested, return that one. - if label: - return cls._transport_registry[label] - - # No transport is requested; return the default (that is, the first one - # in the dictionary). - return next(iter(cls._transport_registry.values())) - - -class BigQueryReadClient(metaclass=BigQueryReadClientMeta): - """BigQuery Read API. - The Read API can be used to read data from BigQuery. - New code should use the v1 Read API going forward, if they don't - use Write API at the same time. - """ - - @staticmethod - def _get_default_mtls_endpoint(api_endpoint): - """Converts api endpoint to mTLS endpoint. - - Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to - "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. - Args: - api_endpoint (Optional[str]): the api endpoint to convert. - Returns: - str: converted mTLS api endpoint. - """ - if not api_endpoint: - return api_endpoint - - mtls_endpoint_re = re.compile( - r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" - ) - - m = mtls_endpoint_re.match(api_endpoint) - name, mtls, sandbox, googledomain = m.groups() - if mtls or not googledomain: - return api_endpoint - - if sandbox: - return api_endpoint.replace( - "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" - ) - - return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") - - DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" - DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore - DEFAULT_ENDPOINT - ) - - @classmethod - def from_service_account_info(cls, info: dict, *args, **kwargs): - """Creates an instance of this client using the provided credentials - info. - - Args: - info (dict): The service account private key info. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryReadClient: The constructed client. - """ - credentials = service_account.Credentials.from_service_account_info(info) - kwargs["credentials"] = credentials - return cls(*args, **kwargs) - - @classmethod - def from_service_account_file(cls, filename: str, *args, **kwargs): - """Creates an instance of this client using the provided credentials - file. - - Args: - filename (str): The path to the service account private key json - file. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryReadClient: The constructed client. - """ - credentials = service_account.Credentials.from_service_account_file( - filename) - kwargs["credentials"] = credentials - return cls(*args, **kwargs) - - from_service_account_json = from_service_account_file - - @property - def transport(self) -> BigQueryReadTransport: - """Returns the transport used by the client instance. - - Returns: - BigQueryReadTransport: The transport used by the client - instance. - """ - return self._transport - - @staticmethod - def read_session_path(project: str,location: str,session: str,) -> str: - """Returns a fully-qualified read_session string.""" - return "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) - - @staticmethod - def parse_read_session_path(path: str) -> Dict[str,str]: - """Parses a read_session path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def read_stream_path(project: str,location: str,session: str,stream: str,) -> str: - """Returns a fully-qualified read_stream string.""" - return "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) - - @staticmethod - def parse_read_stream_path(path: str) -> Dict[str,str]: - """Parses a read_stream path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)/streams/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def table_path(project: str,dataset: str,table: str,) -> str: - """Returns a fully-qualified table string.""" - return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) - - @staticmethod - def parse_table_path(path: str) -> Dict[str,str]: - """Parses a table path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_billing_account_path(billing_account: str, ) -> str: - """Returns a fully-qualified billing_account string.""" - return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) - - @staticmethod - def parse_common_billing_account_path(path: str) -> Dict[str,str]: - """Parse a billing_account path into its component segments.""" - m = re.match(r"^billingAccounts/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_folder_path(folder: str, ) -> str: - """Returns a fully-qualified folder string.""" - return "folders/{folder}".format(folder=folder, ) - - @staticmethod - def parse_common_folder_path(path: str) -> Dict[str,str]: - """Parse a folder path into its component segments.""" - m = re.match(r"^folders/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_organization_path(organization: str, ) -> str: - """Returns a fully-qualified organization string.""" - return "organizations/{organization}".format(organization=organization, ) - - @staticmethod - def parse_common_organization_path(path: str) -> Dict[str,str]: - """Parse a organization path into its component segments.""" - m = re.match(r"^organizations/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_project_path(project: str, ) -> str: - """Returns a fully-qualified project string.""" - return "projects/{project}".format(project=project, ) - - @staticmethod - def parse_common_project_path(path: str) -> Dict[str,str]: - """Parse a project path into its component segments.""" - m = re.match(r"^projects/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_location_path(project: str, location: str, ) -> str: - """Returns a fully-qualified location string.""" - return "projects/{project}/locations/{location}".format(project=project, location=location, ) - - @staticmethod - def parse_common_location_path(path: str) -> Dict[str,str]: - """Parse a location path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) - return m.groupdict() if m else {} - - def __init__(self, *, - credentials: Optional[ga_credentials.Credentials] = None, - transport: Union[str, BigQueryReadTransport, None] = None, - client_options: Optional[client_options_lib.ClientOptions] = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - ) -> None: - """Instantiates the big query read client. - - Args: - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - transport (Union[str, BigQueryReadTransport]): The - transport to use. If set to None, a transport is chosen - automatically. - client_options (google.api_core.client_options.ClientOptions): Custom options for the - client. It won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: - "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable - is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If - not provided, the default SSL client certificate will be used if - present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not - set, no client certificate will be used. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - - Raises: - google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport - creation failed for any reason. - """ - if isinstance(client_options, dict): - client_options = client_options_lib.from_dict(client_options) - if client_options is None: - client_options = client_options_lib.ClientOptions() - - # Create SSL credentials for mutual TLS if needed. - use_client_cert = bool(util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"))) - - client_cert_source_func = None - is_mtls = False - if use_client_cert: - if client_options.client_cert_source: - is_mtls = True - client_cert_source_func = client_options.client_cert_source - else: - is_mtls = mtls.has_default_client_cert_source() - if is_mtls: - client_cert_source_func = mtls.default_client_cert_source() - else: - client_cert_source_func = None - - # Figure out which api endpoint to use. - if client_options.api_endpoint is not None: - api_endpoint = client_options.api_endpoint - else: - use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") - if use_mtls_env == "never": - api_endpoint = self.DEFAULT_ENDPOINT - elif use_mtls_env == "always": - api_endpoint = self.DEFAULT_MTLS_ENDPOINT - elif use_mtls_env == "auto": - if is_mtls: - api_endpoint = self.DEFAULT_MTLS_ENDPOINT - else: - api_endpoint = self.DEFAULT_ENDPOINT - else: - raise MutualTLSChannelError( - "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " - "values: never, auto, always" - ) - - # Save or instantiate the transport. - # Ordinarily, we provide the transport, but allowing a custom transport - # instance provides an extensibility point for unusual situations. - if isinstance(transport, BigQueryReadTransport): - # transport is a BigQueryReadTransport instance. - if credentials or client_options.credentials_file: - raise ValueError("When providing a transport instance, " - "provide its credentials directly.") - if client_options.scopes: - raise ValueError( - "When providing a transport instance, provide its scopes " - "directly." - ) - self._transport = transport - else: - Transport = type(self).get_transport_class(transport) - self._transport = Transport( - credentials=credentials, - credentials_file=client_options.credentials_file, - host=api_endpoint, - scopes=client_options.scopes, - client_cert_source_for_mtls=client_cert_source_func, - quota_project_id=client_options.quota_project_id, - client_info=client_info, - always_use_jwt_access=( - Transport == type(self).get_transport_class("grpc") - or Transport == type(self).get_transport_class("grpc_asyncio") - ), - ) - - def create_read_session(self, - request: storage.CreateReadSessionRequest = None, - *, - parent: str = None, - read_session: stream.ReadSession = None, - max_stream_count: int = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> stream.ReadSession: - r"""Creates a new read session. A read session divides - the contents of a BigQuery table into one or more - streams, which can then be used to read data from the - table. The read session also specifies properties of the - data to be read, such as a list of columns or a push- - down filter describing the rows to be returned. - - A particular row can be read by at most one stream. When - the caller has reached the end of each stream in the - session, then all the data in the table has been read. - - Data is assigned to each stream such that roughly the - same number of rows can be read from each stream. - Because the server-side unit for assigning data is - collections of rows, the API does not guarantee that - each stream will return the same number or rows. - Additionally, the limits are enforced based on the - number of pre-filtered rows, so some filters can lead to - lopsided assignments. - - Read sessions automatically expire 6 hours after they - are created and do not require manual clean-up by the - caller. - - Args: - request (google.cloud.bigquery_storage_v1beta2.types.CreateReadSessionRequest): - The request object. Request message for - `CreateReadSession`. - parent (str): - Required. The request project that owns the session, in - the form of ``projects/{project_id}``. - - This corresponds to the ``parent`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - read_session (google.cloud.bigquery_storage_v1beta2.types.ReadSession): - Required. Session to be created. - This corresponds to the ``read_session`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - max_stream_count (int): - Max initial number of streams. If - unset or zero, the server will provide a - value of streams so as to produce - reasonable throughput. Must be non- - negative. The number of streams may be - lower than the requested number, - depending on the amount parallelism that - is reasonable for the table. Error will - be returned if the max count is greater - than the current system max limit of - 1,000. - - Streams must be read starting from - offset 0. - - This corresponds to the ``max_stream_count`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.ReadSession: - Information about the ReadSession. - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([parent, read_session, max_stream_count]) - if request is not None and has_flattened_params: - raise ValueError('If the `request` argument is set, then none of ' - 'the individual field arguments should be set.') - - # Minor optimization to avoid making a copy if the user passes - # in a storage.CreateReadSessionRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.CreateReadSessionRequest): - request = storage.CreateReadSessionRequest(request) - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if parent is not None: - request.parent = parent - if read_session is not None: - request.read_session = read_session - if max_stream_count is not None: - request.max_stream_count = max_stream_count - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.create_read_session] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("read_session.table", request.read_session.table), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def read_rows(self, - request: storage.ReadRowsRequest = None, - *, - read_stream: str = None, - offset: int = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> Iterable[storage.ReadRowsResponse]: - r"""Reads rows from the stream in the format prescribed - by the ReadSession. Each response contains one or more - table rows, up to a maximum of 100 MiB per response; - read requests which attempt to read individual rows - larger than 100 MiB will fail. - - Each request also returns a set of stream statistics - reflecting the current state of the stream. - - Args: - request (google.cloud.bigquery_storage_v1beta2.types.ReadRowsRequest): - The request object. Request message for `ReadRows`. - read_stream (str): - Required. Stream to read rows from. - This corresponds to the ``read_stream`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - offset (int): - The offset requested must be less - than the last row read from Read. - Requesting a larger offset is undefined. - If not specified, start reading from - offset zero. - - This corresponds to the ``offset`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - Iterable[google.cloud.bigquery_storage_v1beta2.types.ReadRowsResponse]: - Response from calling ReadRows may include row data, progress and - throttling information. - - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([read_stream, offset]) - if request is not None and has_flattened_params: - raise ValueError('If the `request` argument is set, then none of ' - 'the individual field arguments should be set.') - - # Minor optimization to avoid making a copy if the user passes - # in a storage.ReadRowsRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.ReadRowsRequest): - request = storage.ReadRowsRequest(request) - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if read_stream is not None: - request.read_stream = read_stream - if offset is not None: - request.offset = offset - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.read_rows] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("read_stream", request.read_stream), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def split_read_stream(self, - request: storage.SplitReadStreamRequest = None, - *, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> storage.SplitReadStreamResponse: - r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. - These ``ReadStream`` objects are referred to as the primary and - the residual streams of the split. The original ``ReadStream`` - can still be read from in the same manner as before. Both of the - returned ``ReadStream`` objects can also be read from, and the - rows returned by both child streams will be the same as the rows - read from the original stream. - - Moreover, the two child streams will be allocated back-to-back - in the original ``ReadStream``. Concretely, it is guaranteed - that for streams original, primary, and residual, that - original[0-j] = primary[0-j] and original[j-n] = residual[0-m] - once the streams have been read to completion. - - Args: - request (google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamRequest): - The request object. Request message for - `SplitReadStream`. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamResponse: - - """ - # Create or coerce a protobuf request object. - # Minor optimization to avoid making a copy if the user passes - # in a storage.SplitReadStreamRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.SplitReadStreamRequest): - request = storage.SplitReadStreamRequest(request) - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.split_read_stream] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("name", request.name), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - - - - -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - "google-cloud-bigquery-storage", - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() - - -__all__ = ( - "BigQueryReadClient", -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py deleted file mode 100644 index f51ba420..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from collections import OrderedDict -from typing import Dict, Type - -from .base import BigQueryReadTransport -from .grpc import BigQueryReadGrpcTransport -from .grpc_asyncio import BigQueryReadGrpcAsyncIOTransport - - -# Compile a registry of transports. -_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] -_transport_registry['grpc'] = BigQueryReadGrpcTransport -_transport_registry['grpc_asyncio'] = BigQueryReadGrpcAsyncIOTransport - -__all__ = ( - 'BigQueryReadTransport', - 'BigQueryReadGrpcTransport', - 'BigQueryReadGrpcAsyncIOTransport', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py deleted file mode 100644 index bcedfdea..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py +++ /dev/null @@ -1,219 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import abc -from typing import Awaitable, Callable, Dict, Optional, Sequence, Union -import packaging.version -import pkg_resources - -import google.auth # type: ignore -import google.api_core # type: ignore -from google.api_core import exceptions as core_exceptions # type: ignore -from google.api_core import gapic_v1 # type: ignore -from google.api_core import retry as retries # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.oauth2 import service_account # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream - -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - 'google-cloud-bigquery-storage', - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() - -try: - # google.auth.__version__ was added in 1.26.0 - _GOOGLE_AUTH_VERSION = google.auth.__version__ -except AttributeError: - try: # try pkg_resources if it is available - _GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version - except pkg_resources.DistributionNotFound: # pragma: NO COVER - _GOOGLE_AUTH_VERSION = None - - -class BigQueryReadTransport(abc.ABC): - """Abstract transport class for BigQueryRead.""" - - AUTH_SCOPES = ( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', - ) - - DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' - def __init__( - self, *, - host: str = DEFAULT_HOST, - credentials: ga_credentials.Credentials = None, - credentials_file: Optional[str] = None, - scopes: Optional[Sequence[str]] = None, - quota_project_id: Optional[str] = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - always_use_jwt_access: Optional[bool] = False, - **kwargs, - ) -> None: - """Instantiate the transport. - - Args: - host (Optional[str]): - The hostname to connect to. - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is mutually exclusive with credentials. - scopes (Optional[Sequence[str]]): A list of scopes. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - always_use_jwt_access (Optional[bool]): Whether self signed JWT should - be used for service account credentials. - """ - # Save the hostname. Default to port 443 (HTTPS) if none is specified. - if ':' not in host: - host += ':443' - self._host = host - - scopes_kwargs = self._get_scopes_kwargs(self._host, scopes) - - # Save the scopes. - self._scopes = scopes - - # If no credentials are provided, then determine the appropriate - # defaults. - if credentials and credentials_file: - raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") - - if credentials_file is not None: - credentials, _ = google.auth.load_credentials_from_file( - credentials_file, - **scopes_kwargs, - quota_project_id=quota_project_id - ) - - elif credentials is None: - credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) - - # If the credentials is service account credentials, then always try to use self signed JWT. - if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): - credentials = credentials.with_always_use_jwt_access(True) - - # Save the credentials. - self._credentials = credentials - - # TODO(busunkim): This method is in the base transport - # to avoid duplicating code across the transport classes. These functions - # should be deleted once the minimum required versions of google-auth is increased. - - # TODO: Remove this function once google-auth >= 1.25.0 is required - @classmethod - def _get_scopes_kwargs(cls, host: str, scopes: Optional[Sequence[str]]) -> Dict[str, Optional[Sequence[str]]]: - """Returns scopes kwargs to pass to google-auth methods depending on the google-auth version""" - - scopes_kwargs = {} - - if _GOOGLE_AUTH_VERSION and ( - packaging.version.parse(_GOOGLE_AUTH_VERSION) - >= packaging.version.parse("1.25.0") - ): - scopes_kwargs = {"scopes": scopes, "default_scopes": cls.AUTH_SCOPES} - else: - scopes_kwargs = {"scopes": scopes or cls.AUTH_SCOPES} - - return scopes_kwargs - - def _prep_wrapped_messages(self, client_info): - # Precompute the wrapped methods. - self._wrapped_methods = { - self.create_read_session: gapic_v1.method.wrap_method( - self.create_read_session, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=client_info, - ), - self.read_rows: gapic_v1.method.wrap_method( - self.read_rows, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.ServiceUnavailable, - ), - deadline=86400.0, - ), - default_timeout=86400.0, - client_info=client_info, - ), - self.split_read_stream: gapic_v1.method.wrap_method( - self.split_read_stream, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=client_info, - ), - } - - @property - def create_read_session(self) -> Callable[ - [storage.CreateReadSessionRequest], - Union[ - stream.ReadSession, - Awaitable[stream.ReadSession] - ]]: - raise NotImplementedError() - - @property - def read_rows(self) -> Callable[ - [storage.ReadRowsRequest], - Union[ - storage.ReadRowsResponse, - Awaitable[storage.ReadRowsResponse] - ]]: - raise NotImplementedError() - - @property - def split_read_stream(self) -> Callable[ - [storage.SplitReadStreamRequest], - Union[ - storage.SplitReadStreamResponse, - Awaitable[storage.SplitReadStreamResponse] - ]]: - raise NotImplementedError() - - -__all__ = ( - 'BigQueryReadTransport', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py deleted file mode 100644 index 78987836..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py +++ /dev/null @@ -1,349 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import warnings -from typing import Callable, Dict, Optional, Sequence, Tuple, Union - -from google.api_core import grpc_helpers # type: ignore -from google.api_core import gapic_v1 # type: ignore -import google.auth # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.auth.transport.grpc import SslCredentials # type: ignore - -import grpc # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream -from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO - - -class BigQueryReadGrpcTransport(BigQueryReadTransport): - """gRPC backend transport for BigQueryRead. - - BigQuery Read API. - The Read API can be used to read data from BigQuery. - New code should use the v1 Read API going forward, if they don't - use Write API at the same time. - - This class defines the same methods as the primary client, so the - primary client can load the underlying transport implementation - and call it. - - It sends protocol buffers over the wire using gRPC (which is built on - top of HTTP/2); the ``grpcio`` package must be installed. - """ - _stubs: Dict[str, Callable] - - def __init__(self, *, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: str = None, - scopes: Sequence[str] = None, - channel: grpc.Channel = None, - api_mtls_endpoint: str = None, - client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, - ssl_channel_credentials: grpc.ChannelCredentials = None, - client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, - quota_project_id: Optional[str] = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - always_use_jwt_access: Optional[bool] = False, - ) -> None: - """Instantiate the transport. - - Args: - host (Optional[str]): - The hostname to connect to. - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - This argument is ignored if ``channel`` is provided. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. - scopes (Optional(Sequence[str])): A list of scopes. This argument is - ignored if ``channel`` is provided. - channel (Optional[grpc.Channel]): A ``Channel`` instance through - which to make calls. - api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. - If provided, it overrides the ``host`` argument and tries to create - a mutual TLS channel with client SSL credentials from - ``client_cert_source`` or applicatin default SSL credentials. - client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): - Deprecated. A callback to provide client SSL certificate bytes and - private key bytes, both in PEM format. It is ignored if - ``api_mtls_endpoint`` is None. - ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials - for grpc channel. It is ignored if ``channel`` is provided. - client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): - A callback to provide client certificate bytes and private key bytes, - both in PEM format. It is used to configure mutual TLS channel. It is - ignored if ``channel`` or ``ssl_channel_credentials`` is provided. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - always_use_jwt_access (Optional[bool]): Whether self signed JWT should - be used for service account credentials. - - Raises: - google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport - creation failed for any reason. - google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` - and ``credentials_file`` are passed. - """ - self._grpc_channel = None - self._ssl_channel_credentials = ssl_channel_credentials - self._stubs: Dict[str, Callable] = {} - - if api_mtls_endpoint: - warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) - if client_cert_source: - warnings.warn("client_cert_source is deprecated", DeprecationWarning) - - if channel: - # Ignore credentials if a channel was passed. - credentials = False - # If a channel was explicitly provided, set it. - self._grpc_channel = channel - self._ssl_channel_credentials = None - - else: - if api_mtls_endpoint: - host = api_mtls_endpoint - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - self._ssl_channel_credentials = SslCredentials().ssl_credentials - - else: - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - - # The base transport sets the host, credentials and scopes - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes, - quota_project_id=quota_project_id, - client_info=client_info, - always_use_jwt_access=always_use_jwt_access, - ) - - if not self._grpc_channel: - self._grpc_channel = type(self).create_channel( - self._host, - credentials=self._credentials, - credentials_file=credentials_file, - scopes=self._scopes, - ssl_credentials=self._ssl_channel_credentials, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - # Wrap messages. This must be done after self._grpc_channel exists - self._prep_wrapped_messages(client_info) - - @classmethod - def create_channel(cls, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: str = None, - scopes: Optional[Sequence[str]] = None, - quota_project_id: Optional[str] = None, - **kwargs) -> grpc.Channel: - """Create and return a gRPC channel object. - Args: - host (Optional[str]): The host for the channel to use. - credentials (Optional[~.Credentials]): The - authorization credentials to attach to requests. These - credentials identify this application to the service. If - none are specified, the client will attempt to ascertain - the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is mutually exclusive with credentials. - scopes (Optional[Sequence[str]]): A optional list of scopes needed for this - service. These are only used when credentials are not specified and - are passed to :func:`google.auth.default`. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - kwargs (Optional[dict]): Keyword arguments, which are passed to the - channel creation. - Returns: - grpc.Channel: A gRPC channel object. - - Raises: - google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` - and ``credentials_file`` are passed. - """ - - return grpc_helpers.create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - quota_project_id=quota_project_id, - default_scopes=cls.AUTH_SCOPES, - scopes=scopes, - default_host=cls.DEFAULT_HOST, - **kwargs - ) - - @property - def grpc_channel(self) -> grpc.Channel: - """Return the channel designed to connect to this service. - """ - return self._grpc_channel - - @property - def create_read_session(self) -> Callable[ - [storage.CreateReadSessionRequest], - stream.ReadSession]: - r"""Return a callable for the create read session method over gRPC. - - Creates a new read session. A read session divides - the contents of a BigQuery table into one or more - streams, which can then be used to read data from the - table. The read session also specifies properties of the - data to be read, such as a list of columns or a push- - down filter describing the rows to be returned. - - A particular row can be read by at most one stream. When - the caller has reached the end of each stream in the - session, then all the data in the table has been read. - - Data is assigned to each stream such that roughly the - same number of rows can be read from each stream. - Because the server-side unit for assigning data is - collections of rows, the API does not guarantee that - each stream will return the same number or rows. - Additionally, the limits are enforced based on the - number of pre-filtered rows, so some filters can lead to - lopsided assignments. - - Read sessions automatically expire 6 hours after they - are created and do not require manual clean-up by the - caller. - - Returns: - Callable[[~.CreateReadSessionRequest], - ~.ReadSession]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'create_read_session' not in self._stubs: - self._stubs['create_read_session'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/CreateReadSession', - request_serializer=storage.CreateReadSessionRequest.serialize, - response_deserializer=stream.ReadSession.deserialize, - ) - return self._stubs['create_read_session'] - - @property - def read_rows(self) -> Callable[ - [storage.ReadRowsRequest], - storage.ReadRowsResponse]: - r"""Return a callable for the read rows method over gRPC. - - Reads rows from the stream in the format prescribed - by the ReadSession. Each response contains one or more - table rows, up to a maximum of 100 MiB per response; - read requests which attempt to read individual rows - larger than 100 MiB will fail. - - Each request also returns a set of stream statistics - reflecting the current state of the stream. - - Returns: - Callable[[~.ReadRowsRequest], - ~.ReadRowsResponse]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'read_rows' not in self._stubs: - self._stubs['read_rows'] = self.grpc_channel.unary_stream( - '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/ReadRows', - request_serializer=storage.ReadRowsRequest.serialize, - response_deserializer=storage.ReadRowsResponse.deserialize, - ) - return self._stubs['read_rows'] - - @property - def split_read_stream(self) -> Callable[ - [storage.SplitReadStreamRequest], - storage.SplitReadStreamResponse]: - r"""Return a callable for the split read stream method over gRPC. - - Splits a given ``ReadStream`` into two ``ReadStream`` objects. - These ``ReadStream`` objects are referred to as the primary and - the residual streams of the split. The original ``ReadStream`` - can still be read from in the same manner as before. Both of the - returned ``ReadStream`` objects can also be read from, and the - rows returned by both child streams will be the same as the rows - read from the original stream. - - Moreover, the two child streams will be allocated back-to-back - in the original ``ReadStream``. Concretely, it is guaranteed - that for streams original, primary, and residual, that - original[0-j] = primary[0-j] and original[j-n] = residual[0-m] - once the streams have been read to completion. - - Returns: - Callable[[~.SplitReadStreamRequest], - ~.SplitReadStreamResponse]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'split_read_stream' not in self._stubs: - self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/SplitReadStream', - request_serializer=storage.SplitReadStreamRequest.serialize, - response_deserializer=storage.SplitReadStreamResponse.deserialize, - ) - return self._stubs['split_read_stream'] - - -__all__ = ( - 'BigQueryReadGrpcTransport', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py deleted file mode 100644 index 25c6013b..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py +++ /dev/null @@ -1,353 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import warnings -from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union - -from google.api_core import gapic_v1 # type: ignore -from google.api_core import grpc_helpers_async # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.auth.transport.grpc import SslCredentials # type: ignore -import packaging.version - -import grpc # type: ignore -from grpc.experimental import aio # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream -from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO -from .grpc import BigQueryReadGrpcTransport - - -class BigQueryReadGrpcAsyncIOTransport(BigQueryReadTransport): - """gRPC AsyncIO backend transport for BigQueryRead. - - BigQuery Read API. - The Read API can be used to read data from BigQuery. - New code should use the v1 Read API going forward, if they don't - use Write API at the same time. - - This class defines the same methods as the primary client, so the - primary client can load the underlying transport implementation - and call it. - - It sends protocol buffers over the wire using gRPC (which is built on - top of HTTP/2); the ``grpcio`` package must be installed. - """ - - _grpc_channel: aio.Channel - _stubs: Dict[str, Callable] = {} - - @classmethod - def create_channel(cls, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: Optional[str] = None, - scopes: Optional[Sequence[str]] = None, - quota_project_id: Optional[str] = None, - **kwargs) -> aio.Channel: - """Create and return a gRPC AsyncIO channel object. - Args: - host (Optional[str]): The host for the channel to use. - credentials (Optional[~.Credentials]): The - authorization credentials to attach to requests. These - credentials identify this application to the service. If - none are specified, the client will attempt to ascertain - the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. - scopes (Optional[Sequence[str]]): A optional list of scopes needed for this - service. These are only used when credentials are not specified and - are passed to :func:`google.auth.default`. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - kwargs (Optional[dict]): Keyword arguments, which are passed to the - channel creation. - Returns: - aio.Channel: A gRPC AsyncIO channel object. - """ - - return grpc_helpers_async.create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - quota_project_id=quota_project_id, - default_scopes=cls.AUTH_SCOPES, - scopes=scopes, - default_host=cls.DEFAULT_HOST, - **kwargs - ) - - def __init__(self, *, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: Optional[str] = None, - scopes: Optional[Sequence[str]] = None, - channel: aio.Channel = None, - api_mtls_endpoint: str = None, - client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, - ssl_channel_credentials: grpc.ChannelCredentials = None, - client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, - quota_project_id=None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - always_use_jwt_access: Optional[bool] = False, - ) -> None: - """Instantiate the transport. - - Args: - host (Optional[str]): - The hostname to connect to. - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - This argument is ignored if ``channel`` is provided. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. - scopes (Optional[Sequence[str]]): A optional list of scopes needed for this - service. These are only used when credentials are not specified and - are passed to :func:`google.auth.default`. - channel (Optional[aio.Channel]): A ``Channel`` instance through - which to make calls. - api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. - If provided, it overrides the ``host`` argument and tries to create - a mutual TLS channel with client SSL credentials from - ``client_cert_source`` or applicatin default SSL credentials. - client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): - Deprecated. A callback to provide client SSL certificate bytes and - private key bytes, both in PEM format. It is ignored if - ``api_mtls_endpoint`` is None. - ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials - for grpc channel. It is ignored if ``channel`` is provided. - client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): - A callback to provide client certificate bytes and private key bytes, - both in PEM format. It is used to configure mutual TLS channel. It is - ignored if ``channel`` or ``ssl_channel_credentials`` is provided. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - always_use_jwt_access (Optional[bool]): Whether self signed JWT should - be used for service account credentials. - - Raises: - google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport - creation failed for any reason. - google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` - and ``credentials_file`` are passed. - """ - self._grpc_channel = None - self._ssl_channel_credentials = ssl_channel_credentials - self._stubs: Dict[str, Callable] = {} - - if api_mtls_endpoint: - warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) - if client_cert_source: - warnings.warn("client_cert_source is deprecated", DeprecationWarning) - - if channel: - # Ignore credentials if a channel was passed. - credentials = False - # If a channel was explicitly provided, set it. - self._grpc_channel = channel - self._ssl_channel_credentials = None - else: - if api_mtls_endpoint: - host = api_mtls_endpoint - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - self._ssl_channel_credentials = SslCredentials().ssl_credentials - - else: - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - - # The base transport sets the host, credentials and scopes - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes, - quota_project_id=quota_project_id, - client_info=client_info, - always_use_jwt_access=always_use_jwt_access, - ) - - if not self._grpc_channel: - self._grpc_channel = type(self).create_channel( - self._host, - credentials=self._credentials, - credentials_file=credentials_file, - scopes=self._scopes, - ssl_credentials=self._ssl_channel_credentials, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - # Wrap messages. This must be done after self._grpc_channel exists - self._prep_wrapped_messages(client_info) - - @property - def grpc_channel(self) -> aio.Channel: - """Create the channel designed to connect to this service. - - This property caches on the instance; repeated calls return - the same channel. - """ - # Return the channel from cache. - return self._grpc_channel - - @property - def create_read_session(self) -> Callable[ - [storage.CreateReadSessionRequest], - Awaitable[stream.ReadSession]]: - r"""Return a callable for the create read session method over gRPC. - - Creates a new read session. A read session divides - the contents of a BigQuery table into one or more - streams, which can then be used to read data from the - table. The read session also specifies properties of the - data to be read, such as a list of columns or a push- - down filter describing the rows to be returned. - - A particular row can be read by at most one stream. When - the caller has reached the end of each stream in the - session, then all the data in the table has been read. - - Data is assigned to each stream such that roughly the - same number of rows can be read from each stream. - Because the server-side unit for assigning data is - collections of rows, the API does not guarantee that - each stream will return the same number or rows. - Additionally, the limits are enforced based on the - number of pre-filtered rows, so some filters can lead to - lopsided assignments. - - Read sessions automatically expire 6 hours after they - are created and do not require manual clean-up by the - caller. - - Returns: - Callable[[~.CreateReadSessionRequest], - Awaitable[~.ReadSession]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'create_read_session' not in self._stubs: - self._stubs['create_read_session'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/CreateReadSession', - request_serializer=storage.CreateReadSessionRequest.serialize, - response_deserializer=stream.ReadSession.deserialize, - ) - return self._stubs['create_read_session'] - - @property - def read_rows(self) -> Callable[ - [storage.ReadRowsRequest], - Awaitable[storage.ReadRowsResponse]]: - r"""Return a callable for the read rows method over gRPC. - - Reads rows from the stream in the format prescribed - by the ReadSession. Each response contains one or more - table rows, up to a maximum of 100 MiB per response; - read requests which attempt to read individual rows - larger than 100 MiB will fail. - - Each request also returns a set of stream statistics - reflecting the current state of the stream. - - Returns: - Callable[[~.ReadRowsRequest], - Awaitable[~.ReadRowsResponse]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'read_rows' not in self._stubs: - self._stubs['read_rows'] = self.grpc_channel.unary_stream( - '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/ReadRows', - request_serializer=storage.ReadRowsRequest.serialize, - response_deserializer=storage.ReadRowsResponse.deserialize, - ) - return self._stubs['read_rows'] - - @property - def split_read_stream(self) -> Callable[ - [storage.SplitReadStreamRequest], - Awaitable[storage.SplitReadStreamResponse]]: - r"""Return a callable for the split read stream method over gRPC. - - Splits a given ``ReadStream`` into two ``ReadStream`` objects. - These ``ReadStream`` objects are referred to as the primary and - the residual streams of the split. The original ``ReadStream`` - can still be read from in the same manner as before. Both of the - returned ``ReadStream`` objects can also be read from, and the - rows returned by both child streams will be the same as the rows - read from the original stream. - - Moreover, the two child streams will be allocated back-to-back - in the original ``ReadStream``. Concretely, it is guaranteed - that for streams original, primary, and residual, that - original[0-j] = primary[0-j] and original[j-n] = residual[0-m] - once the streams have been read to completion. - - Returns: - Callable[[~.SplitReadStreamRequest], - Awaitable[~.SplitReadStreamResponse]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'split_read_stream' not in self._stubs: - self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/SplitReadStream', - request_serializer=storage.SplitReadStreamRequest.serialize, - response_deserializer=storage.SplitReadStreamResponse.deserialize, - ) - return self._stubs['split_read_stream'] - - -__all__ = ( - 'BigQueryReadGrpcAsyncIOTransport', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py deleted file mode 100644 index 753336d9..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from .client import BigQueryWriteClient -from .async_client import BigQueryWriteAsyncClient - -__all__ = ( - 'BigQueryWriteClient', - 'BigQueryWriteAsyncClient', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py deleted file mode 100644 index deb26b14..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py +++ /dev/null @@ -1,678 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from collections import OrderedDict -import functools -import re -from typing import Dict, AsyncIterable, Awaitable, AsyncIterator, Sequence, Tuple, Type, Union -import pkg_resources - -import google.api_core.client_options as ClientOptions # type: ignore -from google.api_core import exceptions as core_exceptions # type: ignore -from google.api_core import gapic_v1 # type: ignore -from google.api_core import retry as retries # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.oauth2 import service_account # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream -from google.cloud.bigquery_storage_v1beta2.types import table -from google.protobuf import timestamp_pb2 # type: ignore -from google.rpc import status_pb2 # type: ignore -from .transports.base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO -from .transports.grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport -from .client import BigQueryWriteClient - - -class BigQueryWriteAsyncClient: - """BigQuery Write API. - The Write API can be used to write data to BigQuery. - """ - - _client: BigQueryWriteClient - - DEFAULT_ENDPOINT = BigQueryWriteClient.DEFAULT_ENDPOINT - DEFAULT_MTLS_ENDPOINT = BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT - - table_path = staticmethod(BigQueryWriteClient.table_path) - parse_table_path = staticmethod(BigQueryWriteClient.parse_table_path) - write_stream_path = staticmethod(BigQueryWriteClient.write_stream_path) - parse_write_stream_path = staticmethod(BigQueryWriteClient.parse_write_stream_path) - common_billing_account_path = staticmethod(BigQueryWriteClient.common_billing_account_path) - parse_common_billing_account_path = staticmethod(BigQueryWriteClient.parse_common_billing_account_path) - common_folder_path = staticmethod(BigQueryWriteClient.common_folder_path) - parse_common_folder_path = staticmethod(BigQueryWriteClient.parse_common_folder_path) - common_organization_path = staticmethod(BigQueryWriteClient.common_organization_path) - parse_common_organization_path = staticmethod(BigQueryWriteClient.parse_common_organization_path) - common_project_path = staticmethod(BigQueryWriteClient.common_project_path) - parse_common_project_path = staticmethod(BigQueryWriteClient.parse_common_project_path) - common_location_path = staticmethod(BigQueryWriteClient.common_location_path) - parse_common_location_path = staticmethod(BigQueryWriteClient.parse_common_location_path) - - @classmethod - def from_service_account_info(cls, info: dict, *args, **kwargs): - """Creates an instance of this client using the provided credentials - info. - - Args: - info (dict): The service account private key info. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryWriteAsyncClient: The constructed client. - """ - return BigQueryWriteClient.from_service_account_info.__func__(BigQueryWriteAsyncClient, info, *args, **kwargs) # type: ignore - - @classmethod - def from_service_account_file(cls, filename: str, *args, **kwargs): - """Creates an instance of this client using the provided credentials - file. - - Args: - filename (str): The path to the service account private key json - file. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryWriteAsyncClient: The constructed client. - """ - return BigQueryWriteClient.from_service_account_file.__func__(BigQueryWriteAsyncClient, filename, *args, **kwargs) # type: ignore - - from_service_account_json = from_service_account_file - - @property - def transport(self) -> BigQueryWriteTransport: - """Returns the transport used by the client instance. - - Returns: - BigQueryWriteTransport: The transport used by the client instance. - """ - return self._client.transport - - get_transport_class = functools.partial(type(BigQueryWriteClient).get_transport_class, type(BigQueryWriteClient)) - - def __init__(self, *, - credentials: ga_credentials.Credentials = None, - transport: Union[str, BigQueryWriteTransport] = "grpc_asyncio", - client_options: ClientOptions = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - ) -> None: - """Instantiates the big query write client. - - Args: - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - transport (Union[str, ~.BigQueryWriteTransport]): The - transport to use. If set to None, a transport is chosen - automatically. - client_options (ClientOptions): Custom options for the client. It - won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: - "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable - is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If - not provided, the default SSL client certificate will be used if - present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not - set, no client certificate will be used. - - Raises: - google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport - creation failed for any reason. - """ - self._client = BigQueryWriteClient( - credentials=credentials, - transport=transport, - client_options=client_options, - client_info=client_info, - - ) - - async def create_write_stream(self, - request: storage.CreateWriteStreamRequest = None, - *, - parent: str = None, - write_stream: stream.WriteStream = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> stream.WriteStream: - r"""Creates a write stream to the given table. Additionally, every - table has a special COMMITTED stream named '_default' to which - data can be written. This stream doesn't need to be created - using CreateWriteStream. It is a stream that can be used - simultaneously by any number of clients. Data written to this - stream is considered committed as soon as an acknowledgement is - received. - - Args: - request (:class:`google.cloud.bigquery_storage_v1beta2.types.CreateWriteStreamRequest`): - The request object. Request message for - `CreateWriteStream`. - parent (:class:`str`): - Required. Reference to the table to which the stream - belongs, in the format of - ``projects/{project}/datasets/{dataset}/tables/{table}``. - - This corresponds to the ``parent`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - write_stream (:class:`google.cloud.bigquery_storage_v1beta2.types.WriteStream`): - Required. Stream to be created. - This corresponds to the ``write_stream`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.WriteStream: - Information about a single stream - that gets data inside the storage - system. - - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([parent, write_stream]) - if request is not None and has_flattened_params: - raise ValueError("If the `request` argument is set, then none of " - "the individual field arguments should be set.") - - request = storage.CreateWriteStreamRequest(request) - - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if parent is not None: - request.parent = parent - if write_stream is not None: - request.write_stream = write_stream - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.create_write_stream, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ResourceExhausted, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("parent", request.parent), - )), - ) - - # Send the request. - response = await rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def append_rows(self, - requests: AsyncIterator[storage.AppendRowsRequest] = None, - *, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> Awaitable[AsyncIterable[storage.AppendRowsResponse]]: - r"""Appends data to the given stream. - - If ``offset`` is specified, the ``offset`` is checked against - the end of stream. The server returns ``OUT_OF_RANGE`` in - ``AppendRowsResponse`` if an attempt is made to append to an - offset beyond the current end of the stream or - ``ALREADY_EXISTS`` if user provids an ``offset`` that has - already been written to. User can retry with adjusted offset - within the same RPC stream. If ``offset`` is not specified, - append happens at the end of the stream. - - The response contains the offset at which the append happened. - Responses are received in the same order in which requests are - sent. There will be one response for each successful request. If - the ``offset`` is not set in response, it means append didn't - happen due to some errors. If one request fails, all the - subsequent requests will also fail until a success request is - made again. - - If the stream is of ``PENDING`` type, data will only be - available for read operations after the stream is committed. - - Args: - requests (AsyncIterator[`google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest`]): - The request object AsyncIterator. Request message for `AppendRows`. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - AsyncIterable[google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse]: - Response message for AppendRows. - """ - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.append_rows, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.ResourceExhausted, - core_exceptions.ServiceUnavailable, - ), - deadline=86400.0, - ), - default_timeout=86400.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - )), - ) - - # Send the request. - response = rpc( - requests, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - async def get_write_stream(self, - request: storage.GetWriteStreamRequest = None, - *, - name: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> stream.WriteStream: - r"""Gets a write stream. - - Args: - request (:class:`google.cloud.bigquery_storage_v1beta2.types.GetWriteStreamRequest`): - The request object. Request message for - `GetWriteStreamRequest`. - name (:class:`str`): - Required. Name of the stream to get, in the form of - ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. - - This corresponds to the ``name`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.WriteStream: - Information about a single stream - that gets data inside the storage - system. - - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([name]) - if request is not None and has_flattened_params: - raise ValueError("If the `request` argument is set, then none of " - "the individual field arguments should be set.") - - request = storage.GetWriteStreamRequest(request) - - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if name is not None: - request.name = name - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.get_write_stream, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("name", request.name), - )), - ) - - # Send the request. - response = await rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - async def finalize_write_stream(self, - request: storage.FinalizeWriteStreamRequest = None, - *, - name: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> storage.FinalizeWriteStreamResponse: - r"""Finalize a write stream so that no new data can be appended to - the stream. Finalize is not supported on the '_default' stream. - - Args: - request (:class:`google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamRequest`): - The request object. Request message for invoking - `FinalizeWriteStream`. - name (:class:`str`): - Required. Name of the stream to finalize, in the form of - ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. - - This corresponds to the ``name`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamResponse: - Response message for FinalizeWriteStream. - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([name]) - if request is not None and has_flattened_params: - raise ValueError("If the `request` argument is set, then none of " - "the individual field arguments should be set.") - - request = storage.FinalizeWriteStreamRequest(request) - - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if name is not None: - request.name = name - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.finalize_write_stream, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("name", request.name), - )), - ) - - # Send the request. - response = await rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - async def batch_commit_write_streams(self, - request: storage.BatchCommitWriteStreamsRequest = None, - *, - parent: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> storage.BatchCommitWriteStreamsResponse: - r"""Atomically commits a group of ``PENDING`` streams that belong to - the same ``parent`` table. Streams must be finalized before - commit and cannot be committed multiple times. Once a stream is - committed, data in the stream becomes available for read - operations. - - Args: - request (:class:`google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsRequest`): - The request object. Request message for - `BatchCommitWriteStreams`. - parent (:class:`str`): - Required. Parent table that all the streams should - belong to, in the form of - ``projects/{project}/datasets/{dataset}/tables/{table}``. - - This corresponds to the ``parent`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsResponse: - Response message for BatchCommitWriteStreams. - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([parent]) - if request is not None and has_flattened_params: - raise ValueError("If the `request` argument is set, then none of " - "the individual field arguments should be set.") - - request = storage.BatchCommitWriteStreamsRequest(request) - - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if parent is not None: - request.parent = parent - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.batch_commit_write_streams, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("parent", request.parent), - )), - ) - - # Send the request. - response = await rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - async def flush_rows(self, - request: storage.FlushRowsRequest = None, - *, - write_stream: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> storage.FlushRowsResponse: - r"""Flushes rows to a BUFFERED stream. If users are appending rows - to BUFFERED stream, flush operation is required in order for the - rows to become available for reading. A Flush operation flushes - up to any previously flushed offset in a BUFFERED stream, to the - offset specified in the request. Flush is not supported on the - \_default stream, since it is not BUFFERED. - - Args: - request (:class:`google.cloud.bigquery_storage_v1beta2.types.FlushRowsRequest`): - The request object. Request message for `FlushRows`. - write_stream (:class:`str`): - Required. The stream that is the - target of the flush operation. - - This corresponds to the ``write_stream`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.FlushRowsResponse: - Respond message for FlushRows. - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([write_stream]) - if request is not None and has_flattened_params: - raise ValueError("If the `request` argument is set, then none of " - "the individual field arguments should be set.") - - request = storage.FlushRowsRequest(request) - - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if write_stream is not None: - request.write_stream = write_stream - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = gapic_v1.method_async.wrap_method( - self._client._transport.flush_rows, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=DEFAULT_CLIENT_INFO, - ) - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("write_stream", request.write_stream), - )), - ) - - # Send the request. - response = await rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - - - - -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - "google-cloud-bigquery-storage", - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() - - -__all__ = ( - "BigQueryWriteAsyncClient", -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py deleted file mode 100644 index 74329a11..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py +++ /dev/null @@ -1,831 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from collections import OrderedDict -from distutils import util -import os -import re -from typing import Callable, Dict, Optional, Iterable, Iterator, Sequence, Tuple, Type, Union -import pkg_resources - -from google.api_core import client_options as client_options_lib # type: ignore -from google.api_core import exceptions as core_exceptions # type: ignore -from google.api_core import gapic_v1 # type: ignore -from google.api_core import retry as retries # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.auth.transport import mtls # type: ignore -from google.auth.transport.grpc import SslCredentials # type: ignore -from google.auth.exceptions import MutualTLSChannelError # type: ignore -from google.oauth2 import service_account # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream -from google.cloud.bigquery_storage_v1beta2.types import table -from google.protobuf import timestamp_pb2 # type: ignore -from google.rpc import status_pb2 # type: ignore -from .transports.base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO -from .transports.grpc import BigQueryWriteGrpcTransport -from .transports.grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport - - -class BigQueryWriteClientMeta(type): - """Metaclass for the BigQueryWrite client. - - This provides class-level methods for building and retrieving - support objects (e.g. transport) without polluting the client instance - objects. - """ - _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryWriteTransport]] - _transport_registry["grpc"] = BigQueryWriteGrpcTransport - _transport_registry["grpc_asyncio"] = BigQueryWriteGrpcAsyncIOTransport - - def get_transport_class(cls, - label: str = None, - ) -> Type[BigQueryWriteTransport]: - """Returns an appropriate transport class. - - Args: - label: The name of the desired transport. If none is - provided, then the first transport in the registry is used. - - Returns: - The transport class to use. - """ - # If a specific transport is requested, return that one. - if label: - return cls._transport_registry[label] - - # No transport is requested; return the default (that is, the first one - # in the dictionary). - return next(iter(cls._transport_registry.values())) - - -class BigQueryWriteClient(metaclass=BigQueryWriteClientMeta): - """BigQuery Write API. - The Write API can be used to write data to BigQuery. - """ - - @staticmethod - def _get_default_mtls_endpoint(api_endpoint): - """Converts api endpoint to mTLS endpoint. - - Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to - "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. - Args: - api_endpoint (Optional[str]): the api endpoint to convert. - Returns: - str: converted mTLS api endpoint. - """ - if not api_endpoint: - return api_endpoint - - mtls_endpoint_re = re.compile( - r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" - ) - - m = mtls_endpoint_re.match(api_endpoint) - name, mtls, sandbox, googledomain = m.groups() - if mtls or not googledomain: - return api_endpoint - - if sandbox: - return api_endpoint.replace( - "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" - ) - - return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") - - DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" - DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore - DEFAULT_ENDPOINT - ) - - @classmethod - def from_service_account_info(cls, info: dict, *args, **kwargs): - """Creates an instance of this client using the provided credentials - info. - - Args: - info (dict): The service account private key info. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryWriteClient: The constructed client. - """ - credentials = service_account.Credentials.from_service_account_info(info) - kwargs["credentials"] = credentials - return cls(*args, **kwargs) - - @classmethod - def from_service_account_file(cls, filename: str, *args, **kwargs): - """Creates an instance of this client using the provided credentials - file. - - Args: - filename (str): The path to the service account private key json - file. - args: Additional arguments to pass to the constructor. - kwargs: Additional arguments to pass to the constructor. - - Returns: - BigQueryWriteClient: The constructed client. - """ - credentials = service_account.Credentials.from_service_account_file( - filename) - kwargs["credentials"] = credentials - return cls(*args, **kwargs) - - from_service_account_json = from_service_account_file - - @property - def transport(self) -> BigQueryWriteTransport: - """Returns the transport used by the client instance. - - Returns: - BigQueryWriteTransport: The transport used by the client - instance. - """ - return self._transport - - @staticmethod - def table_path(project: str,dataset: str,table: str,) -> str: - """Returns a fully-qualified table string.""" - return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) - - @staticmethod - def parse_table_path(path: str) -> Dict[str,str]: - """Parses a table path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def write_stream_path(project: str,dataset: str,table: str,stream: str,) -> str: - """Returns a fully-qualified write_stream string.""" - return "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}".format(project=project, dataset=dataset, table=table, stream=stream, ) - - @staticmethod - def parse_write_stream_path(path: str) -> Dict[str,str]: - """Parses a write_stream path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)/streams/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_billing_account_path(billing_account: str, ) -> str: - """Returns a fully-qualified billing_account string.""" - return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) - - @staticmethod - def parse_common_billing_account_path(path: str) -> Dict[str,str]: - """Parse a billing_account path into its component segments.""" - m = re.match(r"^billingAccounts/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_folder_path(folder: str, ) -> str: - """Returns a fully-qualified folder string.""" - return "folders/{folder}".format(folder=folder, ) - - @staticmethod - def parse_common_folder_path(path: str) -> Dict[str,str]: - """Parse a folder path into its component segments.""" - m = re.match(r"^folders/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_organization_path(organization: str, ) -> str: - """Returns a fully-qualified organization string.""" - return "organizations/{organization}".format(organization=organization, ) - - @staticmethod - def parse_common_organization_path(path: str) -> Dict[str,str]: - """Parse a organization path into its component segments.""" - m = re.match(r"^organizations/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_project_path(project: str, ) -> str: - """Returns a fully-qualified project string.""" - return "projects/{project}".format(project=project, ) - - @staticmethod - def parse_common_project_path(path: str) -> Dict[str,str]: - """Parse a project path into its component segments.""" - m = re.match(r"^projects/(?P.+?)$", path) - return m.groupdict() if m else {} - - @staticmethod - def common_location_path(project: str, location: str, ) -> str: - """Returns a fully-qualified location string.""" - return "projects/{project}/locations/{location}".format(project=project, location=location, ) - - @staticmethod - def parse_common_location_path(path: str) -> Dict[str,str]: - """Parse a location path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) - return m.groupdict() if m else {} - - def __init__(self, *, - credentials: Optional[ga_credentials.Credentials] = None, - transport: Union[str, BigQueryWriteTransport, None] = None, - client_options: Optional[client_options_lib.ClientOptions] = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - ) -> None: - """Instantiates the big query write client. - - Args: - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - transport (Union[str, BigQueryWriteTransport]): The - transport to use. If set to None, a transport is chosen - automatically. - client_options (google.api_core.client_options.ClientOptions): Custom options for the - client. It won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: - "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable - is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If - not provided, the default SSL client certificate will be used if - present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not - set, no client certificate will be used. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - - Raises: - google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport - creation failed for any reason. - """ - if isinstance(client_options, dict): - client_options = client_options_lib.from_dict(client_options) - if client_options is None: - client_options = client_options_lib.ClientOptions() - - # Create SSL credentials for mutual TLS if needed. - use_client_cert = bool(util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"))) - - client_cert_source_func = None - is_mtls = False - if use_client_cert: - if client_options.client_cert_source: - is_mtls = True - client_cert_source_func = client_options.client_cert_source - else: - is_mtls = mtls.has_default_client_cert_source() - if is_mtls: - client_cert_source_func = mtls.default_client_cert_source() - else: - client_cert_source_func = None - - # Figure out which api endpoint to use. - if client_options.api_endpoint is not None: - api_endpoint = client_options.api_endpoint - else: - use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") - if use_mtls_env == "never": - api_endpoint = self.DEFAULT_ENDPOINT - elif use_mtls_env == "always": - api_endpoint = self.DEFAULT_MTLS_ENDPOINT - elif use_mtls_env == "auto": - if is_mtls: - api_endpoint = self.DEFAULT_MTLS_ENDPOINT - else: - api_endpoint = self.DEFAULT_ENDPOINT - else: - raise MutualTLSChannelError( - "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " - "values: never, auto, always" - ) - - # Save or instantiate the transport. - # Ordinarily, we provide the transport, but allowing a custom transport - # instance provides an extensibility point for unusual situations. - if isinstance(transport, BigQueryWriteTransport): - # transport is a BigQueryWriteTransport instance. - if credentials or client_options.credentials_file: - raise ValueError("When providing a transport instance, " - "provide its credentials directly.") - if client_options.scopes: - raise ValueError( - "When providing a transport instance, provide its scopes " - "directly." - ) - self._transport = transport - else: - Transport = type(self).get_transport_class(transport) - self._transport = Transport( - credentials=credentials, - credentials_file=client_options.credentials_file, - host=api_endpoint, - scopes=client_options.scopes, - client_cert_source_for_mtls=client_cert_source_func, - quota_project_id=client_options.quota_project_id, - client_info=client_info, - always_use_jwt_access=( - Transport == type(self).get_transport_class("grpc") - or Transport == type(self).get_transport_class("grpc_asyncio") - ), - ) - - def create_write_stream(self, - request: storage.CreateWriteStreamRequest = None, - *, - parent: str = None, - write_stream: stream.WriteStream = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> stream.WriteStream: - r"""Creates a write stream to the given table. Additionally, every - table has a special COMMITTED stream named '_default' to which - data can be written. This stream doesn't need to be created - using CreateWriteStream. It is a stream that can be used - simultaneously by any number of clients. Data written to this - stream is considered committed as soon as an acknowledgement is - received. - - Args: - request (google.cloud.bigquery_storage_v1beta2.types.CreateWriteStreamRequest): - The request object. Request message for - `CreateWriteStream`. - parent (str): - Required. Reference to the table to which the stream - belongs, in the format of - ``projects/{project}/datasets/{dataset}/tables/{table}``. - - This corresponds to the ``parent`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - write_stream (google.cloud.bigquery_storage_v1beta2.types.WriteStream): - Required. Stream to be created. - This corresponds to the ``write_stream`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.WriteStream: - Information about a single stream - that gets data inside the storage - system. - - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([parent, write_stream]) - if request is not None and has_flattened_params: - raise ValueError('If the `request` argument is set, then none of ' - 'the individual field arguments should be set.') - - # Minor optimization to avoid making a copy if the user passes - # in a storage.CreateWriteStreamRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.CreateWriteStreamRequest): - request = storage.CreateWriteStreamRequest(request) - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if parent is not None: - request.parent = parent - if write_stream is not None: - request.write_stream = write_stream - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.create_write_stream] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("parent", request.parent), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def append_rows(self, - requests: Iterator[storage.AppendRowsRequest] = None, - *, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> Iterable[storage.AppendRowsResponse]: - r"""Appends data to the given stream. - - If ``offset`` is specified, the ``offset`` is checked against - the end of stream. The server returns ``OUT_OF_RANGE`` in - ``AppendRowsResponse`` if an attempt is made to append to an - offset beyond the current end of the stream or - ``ALREADY_EXISTS`` if user provids an ``offset`` that has - already been written to. User can retry with adjusted offset - within the same RPC stream. If ``offset`` is not specified, - append happens at the end of the stream. - - The response contains the offset at which the append happened. - Responses are received in the same order in which requests are - sent. There will be one response for each successful request. If - the ``offset`` is not set in response, it means append didn't - happen due to some errors. If one request fails, all the - subsequent requests will also fail until a success request is - made again. - - If the stream is of ``PENDING`` type, data will only be - available for read operations after the stream is committed. - - Args: - requests (Iterator[google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest]): - The request object iterator. Request message for `AppendRows`. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - Iterable[google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse]: - Response message for AppendRows. - """ - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.append_rows] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - )), - ) - - # Send the request. - response = rpc( - requests, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def get_write_stream(self, - request: storage.GetWriteStreamRequest = None, - *, - name: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> stream.WriteStream: - r"""Gets a write stream. - - Args: - request (google.cloud.bigquery_storage_v1beta2.types.GetWriteStreamRequest): - The request object. Request message for - `GetWriteStreamRequest`. - name (str): - Required. Name of the stream to get, in the form of - ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. - - This corresponds to the ``name`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.WriteStream: - Information about a single stream - that gets data inside the storage - system. - - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([name]) - if request is not None and has_flattened_params: - raise ValueError('If the `request` argument is set, then none of ' - 'the individual field arguments should be set.') - - # Minor optimization to avoid making a copy if the user passes - # in a storage.GetWriteStreamRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.GetWriteStreamRequest): - request = storage.GetWriteStreamRequest(request) - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if name is not None: - request.name = name - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.get_write_stream] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("name", request.name), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def finalize_write_stream(self, - request: storage.FinalizeWriteStreamRequest = None, - *, - name: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> storage.FinalizeWriteStreamResponse: - r"""Finalize a write stream so that no new data can be appended to - the stream. Finalize is not supported on the '_default' stream. - - Args: - request (google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamRequest): - The request object. Request message for invoking - `FinalizeWriteStream`. - name (str): - Required. Name of the stream to finalize, in the form of - ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. - - This corresponds to the ``name`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamResponse: - Response message for FinalizeWriteStream. - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([name]) - if request is not None and has_flattened_params: - raise ValueError('If the `request` argument is set, then none of ' - 'the individual field arguments should be set.') - - # Minor optimization to avoid making a copy if the user passes - # in a storage.FinalizeWriteStreamRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.FinalizeWriteStreamRequest): - request = storage.FinalizeWriteStreamRequest(request) - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if name is not None: - request.name = name - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.finalize_write_stream] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("name", request.name), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def batch_commit_write_streams(self, - request: storage.BatchCommitWriteStreamsRequest = None, - *, - parent: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> storage.BatchCommitWriteStreamsResponse: - r"""Atomically commits a group of ``PENDING`` streams that belong to - the same ``parent`` table. Streams must be finalized before - commit and cannot be committed multiple times. Once a stream is - committed, data in the stream becomes available for read - operations. - - Args: - request (google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsRequest): - The request object. Request message for - `BatchCommitWriteStreams`. - parent (str): - Required. Parent table that all the streams should - belong to, in the form of - ``projects/{project}/datasets/{dataset}/tables/{table}``. - - This corresponds to the ``parent`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsResponse: - Response message for BatchCommitWriteStreams. - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([parent]) - if request is not None and has_flattened_params: - raise ValueError('If the `request` argument is set, then none of ' - 'the individual field arguments should be set.') - - # Minor optimization to avoid making a copy if the user passes - # in a storage.BatchCommitWriteStreamsRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.BatchCommitWriteStreamsRequest): - request = storage.BatchCommitWriteStreamsRequest(request) - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if parent is not None: - request.parent = parent - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.batch_commit_write_streams] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("parent", request.parent), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - def flush_rows(self, - request: storage.FlushRowsRequest = None, - *, - write_stream: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, - timeout: float = None, - metadata: Sequence[Tuple[str, str]] = (), - ) -> storage.FlushRowsResponse: - r"""Flushes rows to a BUFFERED stream. If users are appending rows - to BUFFERED stream, flush operation is required in order for the - rows to become available for reading. A Flush operation flushes - up to any previously flushed offset in a BUFFERED stream, to the - offset specified in the request. Flush is not supported on the - \_default stream, since it is not BUFFERED. - - Args: - request (google.cloud.bigquery_storage_v1beta2.types.FlushRowsRequest): - The request object. Request message for `FlushRows`. - write_stream (str): - Required. The stream that is the - target of the flush operation. - - This corresponds to the ``write_stream`` field - on the ``request`` instance; if ``request`` is provided, this - should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, - should be retried. - timeout (float): The timeout for this request. - metadata (Sequence[Tuple[str, str]]): Strings which should be - sent along with the request as metadata. - - Returns: - google.cloud.bigquery_storage_v1beta2.types.FlushRowsResponse: - Respond message for FlushRows. - """ - # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have - # gotten any keyword arguments that map to the request. - has_flattened_params = any([write_stream]) - if request is not None and has_flattened_params: - raise ValueError('If the `request` argument is set, then none of ' - 'the individual field arguments should be set.') - - # Minor optimization to avoid making a copy if the user passes - # in a storage.FlushRowsRequest. - # There's no risk of modifying the input as we've already verified - # there are no flattened fields. - if not isinstance(request, storage.FlushRowsRequest): - request = storage.FlushRowsRequest(request) - # If we have keyword arguments corresponding to fields on the - # request, apply these. - if write_stream is not None: - request.write_stream = write_stream - - # Wrap the RPC method; this adds retry and timeout information, - # and friendly error handling. - rpc = self._transport._wrapped_methods[self._transport.flush_rows] - - # Certain fields should be provided within the metadata header; - # add these here. - metadata = tuple(metadata) + ( - gapic_v1.routing_header.to_grpc_metadata(( - ("write_stream", request.write_stream), - )), - ) - - # Send the request. - response = rpc( - request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - # Done; return the response. - return response - - - - - -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - "google-cloud-bigquery-storage", - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() - - -__all__ = ( - "BigQueryWriteClient", -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py deleted file mode 100644 index bf916a92..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from collections import OrderedDict -from typing import Dict, Type - -from .base import BigQueryWriteTransport -from .grpc import BigQueryWriteGrpcTransport -from .grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport - - -# Compile a registry of transports. -_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryWriteTransport]] -_transport_registry['grpc'] = BigQueryWriteGrpcTransport -_transport_registry['grpc_asyncio'] = BigQueryWriteGrpcAsyncIOTransport - -__all__ = ( - 'BigQueryWriteTransport', - 'BigQueryWriteGrpcTransport', - 'BigQueryWriteGrpcAsyncIOTransport', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py deleted file mode 100644 index cce7ef3e..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py +++ /dev/null @@ -1,284 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import abc -from typing import Awaitable, Callable, Dict, Optional, Sequence, Union -import packaging.version -import pkg_resources - -import google.auth # type: ignore -import google.api_core # type: ignore -from google.api_core import exceptions as core_exceptions # type: ignore -from google.api_core import gapic_v1 # type: ignore -from google.api_core import retry as retries # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.oauth2 import service_account # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream - -try: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( - gapic_version=pkg_resources.get_distribution( - 'google-cloud-bigquery-storage', - ).version, - ) -except pkg_resources.DistributionNotFound: - DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() - -try: - # google.auth.__version__ was added in 1.26.0 - _GOOGLE_AUTH_VERSION = google.auth.__version__ -except AttributeError: - try: # try pkg_resources if it is available - _GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version - except pkg_resources.DistributionNotFound: # pragma: NO COVER - _GOOGLE_AUTH_VERSION = None - - -class BigQueryWriteTransport(abc.ABC): - """Abstract transport class for BigQueryWrite.""" - - AUTH_SCOPES = ( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.insertdata', - 'https://www.googleapis.com/auth/cloud-platform', - ) - - DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' - def __init__( - self, *, - host: str = DEFAULT_HOST, - credentials: ga_credentials.Credentials = None, - credentials_file: Optional[str] = None, - scopes: Optional[Sequence[str]] = None, - quota_project_id: Optional[str] = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - always_use_jwt_access: Optional[bool] = False, - **kwargs, - ) -> None: - """Instantiate the transport. - - Args: - host (Optional[str]): - The hostname to connect to. - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is mutually exclusive with credentials. - scopes (Optional[Sequence[str]]): A list of scopes. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - always_use_jwt_access (Optional[bool]): Whether self signed JWT should - be used for service account credentials. - """ - # Save the hostname. Default to port 443 (HTTPS) if none is specified. - if ':' not in host: - host += ':443' - self._host = host - - scopes_kwargs = self._get_scopes_kwargs(self._host, scopes) - - # Save the scopes. - self._scopes = scopes - - # If no credentials are provided, then determine the appropriate - # defaults. - if credentials and credentials_file: - raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") - - if credentials_file is not None: - credentials, _ = google.auth.load_credentials_from_file( - credentials_file, - **scopes_kwargs, - quota_project_id=quota_project_id - ) - - elif credentials is None: - credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) - - # If the credentials is service account credentials, then always try to use self signed JWT. - if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): - credentials = credentials.with_always_use_jwt_access(True) - - # Save the credentials. - self._credentials = credentials - - # TODO(busunkim): This method is in the base transport - # to avoid duplicating code across the transport classes. These functions - # should be deleted once the minimum required versions of google-auth is increased. - - # TODO: Remove this function once google-auth >= 1.25.0 is required - @classmethod - def _get_scopes_kwargs(cls, host: str, scopes: Optional[Sequence[str]]) -> Dict[str, Optional[Sequence[str]]]: - """Returns scopes kwargs to pass to google-auth methods depending on the google-auth version""" - - scopes_kwargs = {} - - if _GOOGLE_AUTH_VERSION and ( - packaging.version.parse(_GOOGLE_AUTH_VERSION) - >= packaging.version.parse("1.25.0") - ): - scopes_kwargs = {"scopes": scopes, "default_scopes": cls.AUTH_SCOPES} - else: - scopes_kwargs = {"scopes": scopes or cls.AUTH_SCOPES} - - return scopes_kwargs - - def _prep_wrapped_messages(self, client_info): - # Precompute the wrapped methods. - self._wrapped_methods = { - self.create_write_stream: gapic_v1.method.wrap_method( - self.create_write_stream, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ResourceExhausted, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=client_info, - ), - self.append_rows: gapic_v1.method.wrap_method( - self.append_rows, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.ResourceExhausted, - core_exceptions.ServiceUnavailable, - ), - deadline=86400.0, - ), - default_timeout=86400.0, - client_info=client_info, - ), - self.get_write_stream: gapic_v1.method.wrap_method( - self.get_write_stream, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=client_info, - ), - self.finalize_write_stream: gapic_v1.method.wrap_method( - self.finalize_write_stream, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=client_info, - ), - self.batch_commit_write_streams: gapic_v1.method.wrap_method( - self.batch_commit_write_streams, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=client_info, - ), - self.flush_rows: gapic_v1.method.wrap_method( - self.flush_rows, - default_retry=retries.Retry( -initial=0.1,maximum=60.0,multiplier=1.3, predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=client_info, - ), - } - - @property - def create_write_stream(self) -> Callable[ - [storage.CreateWriteStreamRequest], - Union[ - stream.WriteStream, - Awaitable[stream.WriteStream] - ]]: - raise NotImplementedError() - - @property - def append_rows(self) -> Callable[ - [storage.AppendRowsRequest], - Union[ - storage.AppendRowsResponse, - Awaitable[storage.AppendRowsResponse] - ]]: - raise NotImplementedError() - - @property - def get_write_stream(self) -> Callable[ - [storage.GetWriteStreamRequest], - Union[ - stream.WriteStream, - Awaitable[stream.WriteStream] - ]]: - raise NotImplementedError() - - @property - def finalize_write_stream(self) -> Callable[ - [storage.FinalizeWriteStreamRequest], - Union[ - storage.FinalizeWriteStreamResponse, - Awaitable[storage.FinalizeWriteStreamResponse] - ]]: - raise NotImplementedError() - - @property - def batch_commit_write_streams(self) -> Callable[ - [storage.BatchCommitWriteStreamsRequest], - Union[ - storage.BatchCommitWriteStreamsResponse, - Awaitable[storage.BatchCommitWriteStreamsResponse] - ]]: - raise NotImplementedError() - - @property - def flush_rows(self) -> Callable[ - [storage.FlushRowsRequest], - Union[ - storage.FlushRowsResponse, - Awaitable[storage.FlushRowsResponse] - ]]: - raise NotImplementedError() - - -__all__ = ( - 'BigQueryWriteTransport', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py deleted file mode 100644 index 8fa44995..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py +++ /dev/null @@ -1,420 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import warnings -from typing import Callable, Dict, Optional, Sequence, Tuple, Union - -from google.api_core import grpc_helpers # type: ignore -from google.api_core import gapic_v1 # type: ignore -import google.auth # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.auth.transport.grpc import SslCredentials # type: ignore - -import grpc # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream -from .base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO - - -class BigQueryWriteGrpcTransport(BigQueryWriteTransport): - """gRPC backend transport for BigQueryWrite. - - BigQuery Write API. - The Write API can be used to write data to BigQuery. - - This class defines the same methods as the primary client, so the - primary client can load the underlying transport implementation - and call it. - - It sends protocol buffers over the wire using gRPC (which is built on - top of HTTP/2); the ``grpcio`` package must be installed. - """ - _stubs: Dict[str, Callable] - - def __init__(self, *, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: str = None, - scopes: Sequence[str] = None, - channel: grpc.Channel = None, - api_mtls_endpoint: str = None, - client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, - ssl_channel_credentials: grpc.ChannelCredentials = None, - client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, - quota_project_id: Optional[str] = None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - always_use_jwt_access: Optional[bool] = False, - ) -> None: - """Instantiate the transport. - - Args: - host (Optional[str]): - The hostname to connect to. - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - This argument is ignored if ``channel`` is provided. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. - scopes (Optional(Sequence[str])): A list of scopes. This argument is - ignored if ``channel`` is provided. - channel (Optional[grpc.Channel]): A ``Channel`` instance through - which to make calls. - api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. - If provided, it overrides the ``host`` argument and tries to create - a mutual TLS channel with client SSL credentials from - ``client_cert_source`` or applicatin default SSL credentials. - client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): - Deprecated. A callback to provide client SSL certificate bytes and - private key bytes, both in PEM format. It is ignored if - ``api_mtls_endpoint`` is None. - ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials - for grpc channel. It is ignored if ``channel`` is provided. - client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): - A callback to provide client certificate bytes and private key bytes, - both in PEM format. It is used to configure mutual TLS channel. It is - ignored if ``channel`` or ``ssl_channel_credentials`` is provided. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - always_use_jwt_access (Optional[bool]): Whether self signed JWT should - be used for service account credentials. - - Raises: - google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport - creation failed for any reason. - google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` - and ``credentials_file`` are passed. - """ - self._grpc_channel = None - self._ssl_channel_credentials = ssl_channel_credentials - self._stubs: Dict[str, Callable] = {} - - if api_mtls_endpoint: - warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) - if client_cert_source: - warnings.warn("client_cert_source is deprecated", DeprecationWarning) - - if channel: - # Ignore credentials if a channel was passed. - credentials = False - # If a channel was explicitly provided, set it. - self._grpc_channel = channel - self._ssl_channel_credentials = None - - else: - if api_mtls_endpoint: - host = api_mtls_endpoint - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - self._ssl_channel_credentials = SslCredentials().ssl_credentials - - else: - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - - # The base transport sets the host, credentials and scopes - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes, - quota_project_id=quota_project_id, - client_info=client_info, - always_use_jwt_access=always_use_jwt_access, - ) - - if not self._grpc_channel: - self._grpc_channel = type(self).create_channel( - self._host, - credentials=self._credentials, - credentials_file=credentials_file, - scopes=self._scopes, - ssl_credentials=self._ssl_channel_credentials, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - # Wrap messages. This must be done after self._grpc_channel exists - self._prep_wrapped_messages(client_info) - - @classmethod - def create_channel(cls, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: str = None, - scopes: Optional[Sequence[str]] = None, - quota_project_id: Optional[str] = None, - **kwargs) -> grpc.Channel: - """Create and return a gRPC channel object. - Args: - host (Optional[str]): The host for the channel to use. - credentials (Optional[~.Credentials]): The - authorization credentials to attach to requests. These - credentials identify this application to the service. If - none are specified, the client will attempt to ascertain - the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is mutually exclusive with credentials. - scopes (Optional[Sequence[str]]): A optional list of scopes needed for this - service. These are only used when credentials are not specified and - are passed to :func:`google.auth.default`. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - kwargs (Optional[dict]): Keyword arguments, which are passed to the - channel creation. - Returns: - grpc.Channel: A gRPC channel object. - - Raises: - google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` - and ``credentials_file`` are passed. - """ - - return grpc_helpers.create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - quota_project_id=quota_project_id, - default_scopes=cls.AUTH_SCOPES, - scopes=scopes, - default_host=cls.DEFAULT_HOST, - **kwargs - ) - - @property - def grpc_channel(self) -> grpc.Channel: - """Return the channel designed to connect to this service. - """ - return self._grpc_channel - - @property - def create_write_stream(self) -> Callable[ - [storage.CreateWriteStreamRequest], - stream.WriteStream]: - r"""Return a callable for the create write stream method over gRPC. - - Creates a write stream to the given table. Additionally, every - table has a special COMMITTED stream named '_default' to which - data can be written. This stream doesn't need to be created - using CreateWriteStream. It is a stream that can be used - simultaneously by any number of clients. Data written to this - stream is considered committed as soon as an acknowledgement is - received. - - Returns: - Callable[[~.CreateWriteStreamRequest], - ~.WriteStream]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'create_write_stream' not in self._stubs: - self._stubs['create_write_stream'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/CreateWriteStream', - request_serializer=storage.CreateWriteStreamRequest.serialize, - response_deserializer=stream.WriteStream.deserialize, - ) - return self._stubs['create_write_stream'] - - @property - def append_rows(self) -> Callable[ - [storage.AppendRowsRequest], - storage.AppendRowsResponse]: - r"""Return a callable for the append rows method over gRPC. - - Appends data to the given stream. - - If ``offset`` is specified, the ``offset`` is checked against - the end of stream. The server returns ``OUT_OF_RANGE`` in - ``AppendRowsResponse`` if an attempt is made to append to an - offset beyond the current end of the stream or - ``ALREADY_EXISTS`` if user provids an ``offset`` that has - already been written to. User can retry with adjusted offset - within the same RPC stream. If ``offset`` is not specified, - append happens at the end of the stream. - - The response contains the offset at which the append happened. - Responses are received in the same order in which requests are - sent. There will be one response for each successful request. If - the ``offset`` is not set in response, it means append didn't - happen due to some errors. If one request fails, all the - subsequent requests will also fail until a success request is - made again. - - If the stream is of ``PENDING`` type, data will only be - available for read operations after the stream is committed. - - Returns: - Callable[[~.AppendRowsRequest], - ~.AppendRowsResponse]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'append_rows' not in self._stubs: - self._stubs['append_rows'] = self.grpc_channel.stream_stream( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/AppendRows', - request_serializer=storage.AppendRowsRequest.serialize, - response_deserializer=storage.AppendRowsResponse.deserialize, - ) - return self._stubs['append_rows'] - - @property - def get_write_stream(self) -> Callable[ - [storage.GetWriteStreamRequest], - stream.WriteStream]: - r"""Return a callable for the get write stream method over gRPC. - - Gets a write stream. - - Returns: - Callable[[~.GetWriteStreamRequest], - ~.WriteStream]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'get_write_stream' not in self._stubs: - self._stubs['get_write_stream'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/GetWriteStream', - request_serializer=storage.GetWriteStreamRequest.serialize, - response_deserializer=stream.WriteStream.deserialize, - ) - return self._stubs['get_write_stream'] - - @property - def finalize_write_stream(self) -> Callable[ - [storage.FinalizeWriteStreamRequest], - storage.FinalizeWriteStreamResponse]: - r"""Return a callable for the finalize write stream method over gRPC. - - Finalize a write stream so that no new data can be appended to - the stream. Finalize is not supported on the '_default' stream. - - Returns: - Callable[[~.FinalizeWriteStreamRequest], - ~.FinalizeWriteStreamResponse]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'finalize_write_stream' not in self._stubs: - self._stubs['finalize_write_stream'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FinalizeWriteStream', - request_serializer=storage.FinalizeWriteStreamRequest.serialize, - response_deserializer=storage.FinalizeWriteStreamResponse.deserialize, - ) - return self._stubs['finalize_write_stream'] - - @property - def batch_commit_write_streams(self) -> Callable[ - [storage.BatchCommitWriteStreamsRequest], - storage.BatchCommitWriteStreamsResponse]: - r"""Return a callable for the batch commit write streams method over gRPC. - - Atomically commits a group of ``PENDING`` streams that belong to - the same ``parent`` table. Streams must be finalized before - commit and cannot be committed multiple times. Once a stream is - committed, data in the stream becomes available for read - operations. - - Returns: - Callable[[~.BatchCommitWriteStreamsRequest], - ~.BatchCommitWriteStreamsResponse]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'batch_commit_write_streams' not in self._stubs: - self._stubs['batch_commit_write_streams'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/BatchCommitWriteStreams', - request_serializer=storage.BatchCommitWriteStreamsRequest.serialize, - response_deserializer=storage.BatchCommitWriteStreamsResponse.deserialize, - ) - return self._stubs['batch_commit_write_streams'] - - @property - def flush_rows(self) -> Callable[ - [storage.FlushRowsRequest], - storage.FlushRowsResponse]: - r"""Return a callable for the flush rows method over gRPC. - - Flushes rows to a BUFFERED stream. If users are appending rows - to BUFFERED stream, flush operation is required in order for the - rows to become available for reading. A Flush operation flushes - up to any previously flushed offset in a BUFFERED stream, to the - offset specified in the request. Flush is not supported on the - \_default stream, since it is not BUFFERED. - - Returns: - Callable[[~.FlushRowsRequest], - ~.FlushRowsResponse]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'flush_rows' not in self._stubs: - self._stubs['flush_rows'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FlushRows', - request_serializer=storage.FlushRowsRequest.serialize, - response_deserializer=storage.FlushRowsResponse.deserialize, - ) - return self._stubs['flush_rows'] - - -__all__ = ( - 'BigQueryWriteGrpcTransport', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py deleted file mode 100644 index d5b164f0..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py +++ /dev/null @@ -1,424 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import warnings -from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union - -from google.api_core import gapic_v1 # type: ignore -from google.api_core import grpc_helpers_async # type: ignore -from google.auth import credentials as ga_credentials # type: ignore -from google.auth.transport.grpc import SslCredentials # type: ignore -import packaging.version - -import grpc # type: ignore -from grpc.experimental import aio # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream -from .base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO -from .grpc import BigQueryWriteGrpcTransport - - -class BigQueryWriteGrpcAsyncIOTransport(BigQueryWriteTransport): - """gRPC AsyncIO backend transport for BigQueryWrite. - - BigQuery Write API. - The Write API can be used to write data to BigQuery. - - This class defines the same methods as the primary client, so the - primary client can load the underlying transport implementation - and call it. - - It sends protocol buffers over the wire using gRPC (which is built on - top of HTTP/2); the ``grpcio`` package must be installed. - """ - - _grpc_channel: aio.Channel - _stubs: Dict[str, Callable] = {} - - @classmethod - def create_channel(cls, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: Optional[str] = None, - scopes: Optional[Sequence[str]] = None, - quota_project_id: Optional[str] = None, - **kwargs) -> aio.Channel: - """Create and return a gRPC AsyncIO channel object. - Args: - host (Optional[str]): The host for the channel to use. - credentials (Optional[~.Credentials]): The - authorization credentials to attach to requests. These - credentials identify this application to the service. If - none are specified, the client will attempt to ascertain - the credentials from the environment. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. - scopes (Optional[Sequence[str]]): A optional list of scopes needed for this - service. These are only used when credentials are not specified and - are passed to :func:`google.auth.default`. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - kwargs (Optional[dict]): Keyword arguments, which are passed to the - channel creation. - Returns: - aio.Channel: A gRPC AsyncIO channel object. - """ - - return grpc_helpers_async.create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - quota_project_id=quota_project_id, - default_scopes=cls.AUTH_SCOPES, - scopes=scopes, - default_host=cls.DEFAULT_HOST, - **kwargs - ) - - def __init__(self, *, - host: str = 'bigquerystorage.googleapis.com', - credentials: ga_credentials.Credentials = None, - credentials_file: Optional[str] = None, - scopes: Optional[Sequence[str]] = None, - channel: aio.Channel = None, - api_mtls_endpoint: str = None, - client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, - ssl_channel_credentials: grpc.ChannelCredentials = None, - client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, - quota_project_id=None, - client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, - always_use_jwt_access: Optional[bool] = False, - ) -> None: - """Instantiate the transport. - - Args: - host (Optional[str]): - The hostname to connect to. - credentials (Optional[google.auth.credentials.Credentials]): The - authorization credentials to attach to requests. These - credentials identify the application to the service; if none - are specified, the client will attempt to ascertain the - credentials from the environment. - This argument is ignored if ``channel`` is provided. - credentials_file (Optional[str]): A file with credentials that can - be loaded with :func:`google.auth.load_credentials_from_file`. - This argument is ignored if ``channel`` is provided. - scopes (Optional[Sequence[str]]): A optional list of scopes needed for this - service. These are only used when credentials are not specified and - are passed to :func:`google.auth.default`. - channel (Optional[aio.Channel]): A ``Channel`` instance through - which to make calls. - api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. - If provided, it overrides the ``host`` argument and tries to create - a mutual TLS channel with client SSL credentials from - ``client_cert_source`` or applicatin default SSL credentials. - client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): - Deprecated. A callback to provide client SSL certificate bytes and - private key bytes, both in PEM format. It is ignored if - ``api_mtls_endpoint`` is None. - ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials - for grpc channel. It is ignored if ``channel`` is provided. - client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): - A callback to provide client certificate bytes and private key bytes, - both in PEM format. It is used to configure mutual TLS channel. It is - ignored if ``channel`` or ``ssl_channel_credentials`` is provided. - quota_project_id (Optional[str]): An optional project to use for billing - and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing - your own client library. - always_use_jwt_access (Optional[bool]): Whether self signed JWT should - be used for service account credentials. - - Raises: - google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport - creation failed for any reason. - google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` - and ``credentials_file`` are passed. - """ - self._grpc_channel = None - self._ssl_channel_credentials = ssl_channel_credentials - self._stubs: Dict[str, Callable] = {} - - if api_mtls_endpoint: - warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) - if client_cert_source: - warnings.warn("client_cert_source is deprecated", DeprecationWarning) - - if channel: - # Ignore credentials if a channel was passed. - credentials = False - # If a channel was explicitly provided, set it. - self._grpc_channel = channel - self._ssl_channel_credentials = None - else: - if api_mtls_endpoint: - host = api_mtls_endpoint - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - self._ssl_channel_credentials = SslCredentials().ssl_credentials - - else: - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - - # The base transport sets the host, credentials and scopes - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes, - quota_project_id=quota_project_id, - client_info=client_info, - always_use_jwt_access=always_use_jwt_access, - ) - - if not self._grpc_channel: - self._grpc_channel = type(self).create_channel( - self._host, - credentials=self._credentials, - credentials_file=credentials_file, - scopes=self._scopes, - ssl_credentials=self._ssl_channel_credentials, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - # Wrap messages. This must be done after self._grpc_channel exists - self._prep_wrapped_messages(client_info) - - @property - def grpc_channel(self) -> aio.Channel: - """Create the channel designed to connect to this service. - - This property caches on the instance; repeated calls return - the same channel. - """ - # Return the channel from cache. - return self._grpc_channel - - @property - def create_write_stream(self) -> Callable[ - [storage.CreateWriteStreamRequest], - Awaitable[stream.WriteStream]]: - r"""Return a callable for the create write stream method over gRPC. - - Creates a write stream to the given table. Additionally, every - table has a special COMMITTED stream named '_default' to which - data can be written. This stream doesn't need to be created - using CreateWriteStream. It is a stream that can be used - simultaneously by any number of clients. Data written to this - stream is considered committed as soon as an acknowledgement is - received. - - Returns: - Callable[[~.CreateWriteStreamRequest], - Awaitable[~.WriteStream]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'create_write_stream' not in self._stubs: - self._stubs['create_write_stream'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/CreateWriteStream', - request_serializer=storage.CreateWriteStreamRequest.serialize, - response_deserializer=stream.WriteStream.deserialize, - ) - return self._stubs['create_write_stream'] - - @property - def append_rows(self) -> Callable[ - [storage.AppendRowsRequest], - Awaitable[storage.AppendRowsResponse]]: - r"""Return a callable for the append rows method over gRPC. - - Appends data to the given stream. - - If ``offset`` is specified, the ``offset`` is checked against - the end of stream. The server returns ``OUT_OF_RANGE`` in - ``AppendRowsResponse`` if an attempt is made to append to an - offset beyond the current end of the stream or - ``ALREADY_EXISTS`` if user provids an ``offset`` that has - already been written to. User can retry with adjusted offset - within the same RPC stream. If ``offset`` is not specified, - append happens at the end of the stream. - - The response contains the offset at which the append happened. - Responses are received in the same order in which requests are - sent. There will be one response for each successful request. If - the ``offset`` is not set in response, it means append didn't - happen due to some errors. If one request fails, all the - subsequent requests will also fail until a success request is - made again. - - If the stream is of ``PENDING`` type, data will only be - available for read operations after the stream is committed. - - Returns: - Callable[[~.AppendRowsRequest], - Awaitable[~.AppendRowsResponse]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'append_rows' not in self._stubs: - self._stubs['append_rows'] = self.grpc_channel.stream_stream( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/AppendRows', - request_serializer=storage.AppendRowsRequest.serialize, - response_deserializer=storage.AppendRowsResponse.deserialize, - ) - return self._stubs['append_rows'] - - @property - def get_write_stream(self) -> Callable[ - [storage.GetWriteStreamRequest], - Awaitable[stream.WriteStream]]: - r"""Return a callable for the get write stream method over gRPC. - - Gets a write stream. - - Returns: - Callable[[~.GetWriteStreamRequest], - Awaitable[~.WriteStream]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'get_write_stream' not in self._stubs: - self._stubs['get_write_stream'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/GetWriteStream', - request_serializer=storage.GetWriteStreamRequest.serialize, - response_deserializer=stream.WriteStream.deserialize, - ) - return self._stubs['get_write_stream'] - - @property - def finalize_write_stream(self) -> Callable[ - [storage.FinalizeWriteStreamRequest], - Awaitable[storage.FinalizeWriteStreamResponse]]: - r"""Return a callable for the finalize write stream method over gRPC. - - Finalize a write stream so that no new data can be appended to - the stream. Finalize is not supported on the '_default' stream. - - Returns: - Callable[[~.FinalizeWriteStreamRequest], - Awaitable[~.FinalizeWriteStreamResponse]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'finalize_write_stream' not in self._stubs: - self._stubs['finalize_write_stream'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FinalizeWriteStream', - request_serializer=storage.FinalizeWriteStreamRequest.serialize, - response_deserializer=storage.FinalizeWriteStreamResponse.deserialize, - ) - return self._stubs['finalize_write_stream'] - - @property - def batch_commit_write_streams(self) -> Callable[ - [storage.BatchCommitWriteStreamsRequest], - Awaitable[storage.BatchCommitWriteStreamsResponse]]: - r"""Return a callable for the batch commit write streams method over gRPC. - - Atomically commits a group of ``PENDING`` streams that belong to - the same ``parent`` table. Streams must be finalized before - commit and cannot be committed multiple times. Once a stream is - committed, data in the stream becomes available for read - operations. - - Returns: - Callable[[~.BatchCommitWriteStreamsRequest], - Awaitable[~.BatchCommitWriteStreamsResponse]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'batch_commit_write_streams' not in self._stubs: - self._stubs['batch_commit_write_streams'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/BatchCommitWriteStreams', - request_serializer=storage.BatchCommitWriteStreamsRequest.serialize, - response_deserializer=storage.BatchCommitWriteStreamsResponse.deserialize, - ) - return self._stubs['batch_commit_write_streams'] - - @property - def flush_rows(self) -> Callable[ - [storage.FlushRowsRequest], - Awaitable[storage.FlushRowsResponse]]: - r"""Return a callable for the flush rows method over gRPC. - - Flushes rows to a BUFFERED stream. If users are appending rows - to BUFFERED stream, flush operation is required in order for the - rows to become available for reading. A Flush operation flushes - up to any previously flushed offset in a BUFFERED stream, to the - offset specified in the request. Flush is not supported on the - \_default stream, since it is not BUFFERED. - - Returns: - Callable[[~.FlushRowsRequest], - Awaitable[~.FlushRowsResponse]]: - A function that, when called, will call the underlying RPC - on the server. - """ - # Generate a "stub function" on-the-fly which will actually make - # the request. - # gRPC handles serialization and deserialization, so we just need - # to pass in the functions for each. - if 'flush_rows' not in self._stubs: - self._stubs['flush_rows'] = self.grpc_channel.unary_unary( - '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FlushRows', - request_serializer=storage.FlushRowsRequest.serialize, - response_deserializer=storage.FlushRowsResponse.deserialize, - ) - return self._stubs['flush_rows'] - - -__all__ = ( - 'BigQueryWriteGrpcAsyncIOTransport', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py deleted file mode 100644 index b73a912f..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py +++ /dev/null @@ -1,92 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from .arrow import ( - ArrowRecordBatch, - ArrowSchema, - ArrowSerializationOptions, -) -from .avro import ( - AvroRows, - AvroSchema, -) -from .protobuf import ( - ProtoRows, - ProtoSchema, -) -from .storage import ( - AppendRowsRequest, - AppendRowsResponse, - BatchCommitWriteStreamsRequest, - BatchCommitWriteStreamsResponse, - CreateReadSessionRequest, - CreateWriteStreamRequest, - FinalizeWriteStreamRequest, - FinalizeWriteStreamResponse, - FlushRowsRequest, - FlushRowsResponse, - GetWriteStreamRequest, - ReadRowsRequest, - ReadRowsResponse, - SplitReadStreamRequest, - SplitReadStreamResponse, - StorageError, - StreamStats, - ThrottleState, -) -from .stream import ( - ReadSession, - ReadStream, - WriteStream, - DataFormat, -) -from .table import ( - TableFieldSchema, - TableSchema, -) - -__all__ = ( - 'ArrowRecordBatch', - 'ArrowSchema', - 'ArrowSerializationOptions', - 'AvroRows', - 'AvroSchema', - 'ProtoRows', - 'ProtoSchema', - 'AppendRowsRequest', - 'AppendRowsResponse', - 'BatchCommitWriteStreamsRequest', - 'BatchCommitWriteStreamsResponse', - 'CreateReadSessionRequest', - 'CreateWriteStreamRequest', - 'FinalizeWriteStreamRequest', - 'FinalizeWriteStreamResponse', - 'FlushRowsRequest', - 'FlushRowsResponse', - 'GetWriteStreamRequest', - 'ReadRowsRequest', - 'ReadRowsResponse', - 'SplitReadStreamRequest', - 'SplitReadStreamResponse', - 'StorageError', - 'StreamStats', - 'ThrottleState', - 'ReadSession', - 'ReadStream', - 'WriteStream', - 'DataFormat', - 'TableFieldSchema', - 'TableSchema', -) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py deleted file mode 100644 index 7bf84070..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package='google.cloud.bigquery.storage.v1beta2', - manifest={ - 'ArrowSchema', - 'ArrowRecordBatch', - 'ArrowSerializationOptions', - }, -) - - -class ArrowSchema(proto.Message): - r"""Arrow schema as specified in - https://arrow.apache.org/docs/python/api/datatypes.html and - serialized to bytes using IPC: - https://arrow.apache.org/docs/format/Columnar.html#serialization- - and-interprocess-communication-ipc - See code samples on how this message can be deserialized. - - Attributes: - serialized_schema (bytes): - IPC serialized Arrow schema. - """ - - serialized_schema = proto.Field( - proto.BYTES, - number=1, - ) - - -class ArrowRecordBatch(proto.Message): - r"""Arrow RecordBatch. - Attributes: - serialized_record_batch (bytes): - IPC-serialized Arrow RecordBatch. - """ - - serialized_record_batch = proto.Field( - proto.BYTES, - number=1, - ) - - -class ArrowSerializationOptions(proto.Message): - r"""Contains options specific to Arrow Serialization. - Attributes: - format_ (google.cloud.bigquery_storage_v1beta2.types.ArrowSerializationOptions.Format): - The Arrow IPC format to use. - """ - class Format(proto.Enum): - r"""The IPC format to use when serializing Arrow streams.""" - FORMAT_UNSPECIFIED = 0 - ARROW_0_14 = 1 - ARROW_0_15 = 2 - - format_ = proto.Field( - proto.ENUM, - number=1, - enum=Format, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py deleted file mode 100644 index e982615c..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package='google.cloud.bigquery.storage.v1beta2', - manifest={ - 'AvroSchema', - 'AvroRows', - }, -) - - -class AvroSchema(proto.Message): - r"""Avro schema. - Attributes: - schema (str): - Json serialized schema, as described at - https://avro.apache.org/docs/1.8.1/spec.html. - """ - - schema = proto.Field( - proto.STRING, - number=1, - ) - - -class AvroRows(proto.Message): - r"""Avro rows. - Attributes: - serialized_binary_rows (bytes): - Binary serialized rows in a block. - """ - - serialized_binary_rows = proto.Field( - proto.BYTES, - number=1, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py deleted file mode 100644 index 07175d85..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - -from google.protobuf import descriptor_pb2 # type: ignore - - -__protobuf__ = proto.module( - package='google.cloud.bigquery.storage.v1beta2', - manifest={ - 'ProtoSchema', - 'ProtoRows', - }, -) - - -class ProtoSchema(proto.Message): - r"""ProtoSchema describes the schema of the serialized protocol - buffer data rows. - - Attributes: - proto_descriptor (google.protobuf.descriptor_pb2.DescriptorProto): - Descriptor for input message. The descriptor - has to be self contained, including all the - nested types, excepted for proto buffer well - known types - (https://developers.google.com/protocol- - buffers/docs/reference/google.protobuf). - """ - - proto_descriptor = proto.Field( - proto.MESSAGE, - number=1, - message=descriptor_pb2.DescriptorProto, - ) - - -class ProtoRows(proto.Message): - r""" - Attributes: - serialized_rows (Sequence[bytes]): - A sequence of rows serialized as a Protocol - Buffer. - See https://developers.google.com/protocol- - buffers/docs/overview for more information on - deserializing this field. - """ - - serialized_rows = proto.RepeatedField( - proto.BYTES, - number=1, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py deleted file mode 100644 index b7d65dfc..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py +++ /dev/null @@ -1,618 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import arrow -from google.cloud.bigquery_storage_v1beta2.types import avro -from google.cloud.bigquery_storage_v1beta2.types import protobuf -from google.cloud.bigquery_storage_v1beta2.types import stream -from google.cloud.bigquery_storage_v1beta2.types import table -from google.protobuf import timestamp_pb2 # type: ignore -from google.protobuf import wrappers_pb2 # type: ignore -from google.rpc import status_pb2 # type: ignore - - -__protobuf__ = proto.module( - package='google.cloud.bigquery.storage.v1beta2', - manifest={ - 'CreateReadSessionRequest', - 'ReadRowsRequest', - 'ThrottleState', - 'StreamStats', - 'ReadRowsResponse', - 'SplitReadStreamRequest', - 'SplitReadStreamResponse', - 'CreateWriteStreamRequest', - 'AppendRowsRequest', - 'AppendRowsResponse', - 'GetWriteStreamRequest', - 'BatchCommitWriteStreamsRequest', - 'BatchCommitWriteStreamsResponse', - 'FinalizeWriteStreamRequest', - 'FinalizeWriteStreamResponse', - 'FlushRowsRequest', - 'FlushRowsResponse', - 'StorageError', - }, -) - - -class CreateReadSessionRequest(proto.Message): - r"""Request message for ``CreateReadSession``. - Attributes: - parent (str): - Required. The request project that owns the session, in the - form of ``projects/{project_id}``. - read_session (google.cloud.bigquery_storage_v1beta2.types.ReadSession): - Required. Session to be created. - max_stream_count (int): - Max initial number of streams. If unset or - zero, the server will provide a value of streams - so as to produce reasonable throughput. Must be - non-negative. The number of streams may be lower - than the requested number, depending on the - amount parallelism that is reasonable for the - table. Error will be returned if the max count - is greater than the current system max limit of - 1,000. - - Streams must be read starting from offset 0. - """ - - parent = proto.Field( - proto.STRING, - number=1, - ) - read_session = proto.Field( - proto.MESSAGE, - number=2, - message=stream.ReadSession, - ) - max_stream_count = proto.Field( - proto.INT32, - number=3, - ) - - -class ReadRowsRequest(proto.Message): - r"""Request message for ``ReadRows``. - Attributes: - read_stream (str): - Required. Stream to read rows from. - offset (int): - The offset requested must be less than the - last row read from Read. Requesting a larger - offset is undefined. If not specified, start - reading from offset zero. - """ - - read_stream = proto.Field( - proto.STRING, - number=1, - ) - offset = proto.Field( - proto.INT64, - number=2, - ) - - -class ThrottleState(proto.Message): - r"""Information on if the current connection is being throttled. - Attributes: - throttle_percent (int): - How much this connection is being throttled. - Zero means no throttling, 100 means fully - throttled. - """ - - throttle_percent = proto.Field( - proto.INT32, - number=1, - ) - - -class StreamStats(proto.Message): - r"""Estimated stream statistics for a given Stream. - Attributes: - progress (google.cloud.bigquery_storage_v1beta2.types.StreamStats.Progress): - Represents the progress of the current - stream. - """ - - class Progress(proto.Message): - r""" - Attributes: - at_response_start (float): - The fraction of rows assigned to the stream that have been - processed by the server so far, not including the rows in - the current response message. - - This value, along with ``at_response_end``, can be used to - interpolate the progress made as the rows in the message are - being processed using the following formula: - ``at_response_start + (at_response_end - at_response_start) * rows_processed_from_response / rows_in_response``. - - Note that if a filter is provided, the ``at_response_end`` - value of the previous response may not necessarily be equal - to the ``at_response_start`` value of the current response. - at_response_end (float): - Similar to ``at_response_start``, except that this value - includes the rows in the current response. - """ - - at_response_start = proto.Field( - proto.DOUBLE, - number=1, - ) - at_response_end = proto.Field( - proto.DOUBLE, - number=2, - ) - - progress = proto.Field( - proto.MESSAGE, - number=2, - message=Progress, - ) - - -class ReadRowsResponse(proto.Message): - r"""Response from calling ``ReadRows`` may include row data, progress - and throttling information. - - Attributes: - avro_rows (google.cloud.bigquery_storage_v1beta2.types.AvroRows): - Serialized row data in AVRO format. - arrow_record_batch (google.cloud.bigquery_storage_v1beta2.types.ArrowRecordBatch): - Serialized row data in Arrow RecordBatch - format. - row_count (int): - Number of serialized rows in the rows block. - stats (google.cloud.bigquery_storage_v1beta2.types.StreamStats): - Statistics for the stream. - throttle_state (google.cloud.bigquery_storage_v1beta2.types.ThrottleState): - Throttling state. If unset, the latest - response still describes the current throttling - status. - avro_schema (google.cloud.bigquery_storage_v1beta2.types.AvroSchema): - Output only. Avro schema. - arrow_schema (google.cloud.bigquery_storage_v1beta2.types.ArrowSchema): - Output only. Arrow schema. - """ - - avro_rows = proto.Field( - proto.MESSAGE, - number=3, - oneof='rows', - message=avro.AvroRows, - ) - arrow_record_batch = proto.Field( - proto.MESSAGE, - number=4, - oneof='rows', - message=arrow.ArrowRecordBatch, - ) - row_count = proto.Field( - proto.INT64, - number=6, - ) - stats = proto.Field( - proto.MESSAGE, - number=2, - message='StreamStats', - ) - throttle_state = proto.Field( - proto.MESSAGE, - number=5, - message='ThrottleState', - ) - avro_schema = proto.Field( - proto.MESSAGE, - number=7, - oneof='schema', - message=avro.AvroSchema, - ) - arrow_schema = proto.Field( - proto.MESSAGE, - number=8, - oneof='schema', - message=arrow.ArrowSchema, - ) - - -class SplitReadStreamRequest(proto.Message): - r"""Request message for ``SplitReadStream``. - Attributes: - name (str): - Required. Name of the stream to split. - fraction (float): - A value in the range (0.0, 1.0) that - specifies the fractional point at which the - original stream should be split. The actual - split point is evaluated on pre-filtered rows, - so if a filter is provided, then there is no - guarantee that the division of the rows between - the new child streams will be proportional to - this fractional value. Additionally, because the - server-side unit for assigning data is - collections of rows, this fraction will always - map to a data storage boundary on the server - side. - """ - - name = proto.Field( - proto.STRING, - number=1, - ) - fraction = proto.Field( - proto.DOUBLE, - number=2, - ) - - -class SplitReadStreamResponse(proto.Message): - r""" - Attributes: - primary_stream (google.cloud.bigquery_storage_v1beta2.types.ReadStream): - Primary stream, which contains the beginning portion of - \|original_stream|. An empty value indicates that the - original stream can no longer be split. - remainder_stream (google.cloud.bigquery_storage_v1beta2.types.ReadStream): - Remainder stream, which contains the tail of - \|original_stream|. An empty value indicates that the - original stream can no longer be split. - """ - - primary_stream = proto.Field( - proto.MESSAGE, - number=1, - message=stream.ReadStream, - ) - remainder_stream = proto.Field( - proto.MESSAGE, - number=2, - message=stream.ReadStream, - ) - - -class CreateWriteStreamRequest(proto.Message): - r"""Request message for ``CreateWriteStream``. - Attributes: - parent (str): - Required. Reference to the table to which the stream - belongs, in the format of - ``projects/{project}/datasets/{dataset}/tables/{table}``. - write_stream (google.cloud.bigquery_storage_v1beta2.types.WriteStream): - Required. Stream to be created. - """ - - parent = proto.Field( - proto.STRING, - number=1, - ) - write_stream = proto.Field( - proto.MESSAGE, - number=2, - message=stream.WriteStream, - ) - - -class AppendRowsRequest(proto.Message): - r"""Request message for ``AppendRows``. - Attributes: - write_stream (str): - Required. The stream that is the target of the append - operation. This value must be specified for the initial - request. If subsequent requests specify the stream name, it - must equal to the value provided in the first request. To - write to the \_default stream, populate this field with a - string in the format - ``projects/{project}/datasets/{dataset}/tables/{table}/_default``. - offset (google.protobuf.wrappers_pb2.Int64Value): - If present, the write is only performed if the next append - offset is same as the provided value. If not present, the - write is performed at the current end of stream. Specifying - a value for this field is not allowed when calling - AppendRows for the '_default' stream. - proto_rows (google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest.ProtoData): - Rows in proto format. - trace_id (str): - Id set by client to annotate its identity. - Only initial request setting is respected. - """ - - class ProtoData(proto.Message): - r"""Proto schema and data. - Attributes: - writer_schema (google.cloud.bigquery_storage_v1beta2.types.ProtoSchema): - Proto schema used to serialize the data. - rows (google.cloud.bigquery_storage_v1beta2.types.ProtoRows): - Serialized row data in protobuf message - format. - """ - - writer_schema = proto.Field( - proto.MESSAGE, - number=1, - message=protobuf.ProtoSchema, - ) - rows = proto.Field( - proto.MESSAGE, - number=2, - message=protobuf.ProtoRows, - ) - - write_stream = proto.Field( - proto.STRING, - number=1, - ) - offset = proto.Field( - proto.MESSAGE, - number=2, - message=wrappers_pb2.Int64Value, - ) - proto_rows = proto.Field( - proto.MESSAGE, - number=4, - oneof='rows', - message=ProtoData, - ) - trace_id = proto.Field( - proto.STRING, - number=6, - ) - - -class AppendRowsResponse(proto.Message): - r"""Response message for ``AppendRows``. - Attributes: - append_result (google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse.AppendResult): - Result if the append is successful. - error (google.rpc.status_pb2.Status): - Error returned when problems were encountered. If present, - it indicates rows were not accepted into the system. Users - can retry or continue with other append requests within the - same connection. - - Additional information about error signalling: - - ALREADY_EXISTS: Happens when an append specified an offset, - and the backend already has received data at this offset. - Typically encountered in retry scenarios, and can be - ignored. - - OUT_OF_RANGE: Returned when the specified offset in the - stream is beyond the current end of the stream. - - INVALID_ARGUMENT: Indicates a malformed request or data. - - ABORTED: Request processing is aborted because of prior - failures. The request can be retried if previous failure is - addressed. - - INTERNAL: Indicates server side error(s) that can be - retried. - updated_schema (google.cloud.bigquery_storage_v1beta2.types.TableSchema): - If backend detects a schema update, pass it - to user so that user can use it to input new - type of message. It will be empty when no schema - updates have occurred. - """ - - class AppendResult(proto.Message): - r"""AppendResult is returned for successful append requests. - Attributes: - offset (google.protobuf.wrappers_pb2.Int64Value): - The row offset at which the last append - occurred. The offset will not be set if - appending using default streams. - """ - - offset = proto.Field( - proto.MESSAGE, - number=1, - message=wrappers_pb2.Int64Value, - ) - - append_result = proto.Field( - proto.MESSAGE, - number=1, - oneof='response', - message=AppendResult, - ) - error = proto.Field( - proto.MESSAGE, - number=2, - oneof='response', - message=status_pb2.Status, - ) - updated_schema = proto.Field( - proto.MESSAGE, - number=3, - message=table.TableSchema, - ) - - -class GetWriteStreamRequest(proto.Message): - r"""Request message for ``GetWriteStreamRequest``. - Attributes: - name (str): - Required. Name of the stream to get, in the form of - ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. - """ - - name = proto.Field( - proto.STRING, - number=1, - ) - - -class BatchCommitWriteStreamsRequest(proto.Message): - r"""Request message for ``BatchCommitWriteStreams``. - Attributes: - parent (str): - Required. Parent table that all the streams should belong - to, in the form of - ``projects/{project}/datasets/{dataset}/tables/{table}``. - write_streams (Sequence[str]): - Required. The group of streams that will be - committed atomically. - """ - - parent = proto.Field( - proto.STRING, - number=1, - ) - write_streams = proto.RepeatedField( - proto.STRING, - number=2, - ) - - -class BatchCommitWriteStreamsResponse(proto.Message): - r"""Response message for ``BatchCommitWriteStreams``. - Attributes: - commit_time (google.protobuf.timestamp_pb2.Timestamp): - The time at which streams were committed in microseconds - granularity. This field will only exist when there are no - stream errors. **Note** if this field is not set, it means - the commit was not successful. - stream_errors (Sequence[google.cloud.bigquery_storage_v1beta2.types.StorageError]): - Stream level error if commit failed. Only - streams with error will be in the list. - If empty, there is no error and all streams are - committed successfully. If non empty, certain - streams have errors and ZERO stream is committed - due to atomicity guarantee. - """ - - commit_time = proto.Field( - proto.MESSAGE, - number=1, - message=timestamp_pb2.Timestamp, - ) - stream_errors = proto.RepeatedField( - proto.MESSAGE, - number=2, - message='StorageError', - ) - - -class FinalizeWriteStreamRequest(proto.Message): - r"""Request message for invoking ``FinalizeWriteStream``. - Attributes: - name (str): - Required. Name of the stream to finalize, in the form of - ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. - """ - - name = proto.Field( - proto.STRING, - number=1, - ) - - -class FinalizeWriteStreamResponse(proto.Message): - r"""Response message for ``FinalizeWriteStream``. - Attributes: - row_count (int): - Number of rows in the finalized stream. - """ - - row_count = proto.Field( - proto.INT64, - number=1, - ) - - -class FlushRowsRequest(proto.Message): - r"""Request message for ``FlushRows``. - Attributes: - write_stream (str): - Required. The stream that is the target of - the flush operation. - offset (google.protobuf.wrappers_pb2.Int64Value): - Ending offset of the flush operation. Rows - before this offset(including this offset) will - be flushed. - """ - - write_stream = proto.Field( - proto.STRING, - number=1, - ) - offset = proto.Field( - proto.MESSAGE, - number=2, - message=wrappers_pb2.Int64Value, - ) - - -class FlushRowsResponse(proto.Message): - r"""Respond message for ``FlushRows``. - Attributes: - offset (int): - The rows before this offset (including this - offset) are flushed. - """ - - offset = proto.Field( - proto.INT64, - number=1, - ) - - -class StorageError(proto.Message): - r"""Structured custom BigQuery Storage error message. The error - can be attached as error details in the returned rpc Status. In - particular, the use of error codes allows more structured error - handling, and reduces the need to evaluate unstructured error - text strings. - - Attributes: - code (google.cloud.bigquery_storage_v1beta2.types.StorageError.StorageErrorCode): - BigQuery Storage specific error code. - entity (str): - Name of the failed entity. - error_message (str): - Message that describes the error. - """ - class StorageErrorCode(proto.Enum): - r"""Error code for ``StorageError``.""" - STORAGE_ERROR_CODE_UNSPECIFIED = 0 - TABLE_NOT_FOUND = 1 - STREAM_ALREADY_COMMITTED = 2 - STREAM_NOT_FOUND = 3 - INVALID_STREAM_TYPE = 4 - INVALID_STREAM_STATE = 5 - STREAM_FINALIZED = 6 - - code = proto.Field( - proto.ENUM, - number=1, - enum=StorageErrorCode, - ) - entity = proto.Field( - proto.STRING, - number=2, - ) - error_message = proto.Field( - proto.STRING, - number=3, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py deleted file mode 100644 index 028c48ac..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py +++ /dev/null @@ -1,253 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - -from google.cloud.bigquery_storage_v1beta2.types import arrow -from google.cloud.bigquery_storage_v1beta2.types import avro -from google.cloud.bigquery_storage_v1beta2.types import table as gcbs_table -from google.protobuf import timestamp_pb2 # type: ignore - - -__protobuf__ = proto.module( - package='google.cloud.bigquery.storage.v1beta2', - manifest={ - 'DataFormat', - 'ReadSession', - 'ReadStream', - 'WriteStream', - }, -) - - -class DataFormat(proto.Enum): - r"""Data format for input or output data.""" - DATA_FORMAT_UNSPECIFIED = 0 - AVRO = 1 - ARROW = 2 - - -class ReadSession(proto.Message): - r"""Information about the ReadSession. - Attributes: - name (str): - Output only. Unique identifier for the session, in the form - ``projects/{project_id}/locations/{location}/sessions/{session_id}``. - expire_time (google.protobuf.timestamp_pb2.Timestamp): - Output only. Time at which the session becomes invalid. - After this time, subsequent requests to read this Session - will return errors. The expire_time is automatically - assigned and currently cannot be specified or updated. - data_format (google.cloud.bigquery_storage_v1beta2.types.DataFormat): - Immutable. Data format of the output data. - avro_schema (google.cloud.bigquery_storage_v1beta2.types.AvroSchema): - Output only. Avro schema. - arrow_schema (google.cloud.bigquery_storage_v1beta2.types.ArrowSchema): - Output only. Arrow schema. - table (str): - Immutable. Table that this ReadSession is reading from, in - the form - \`projects/{project_id}/datasets/{dataset_id}/tables/{table_id} - table_modifiers (google.cloud.bigquery_storage_v1beta2.types.ReadSession.TableModifiers): - Optional. Any modifiers which are applied - when reading from the specified table. - read_options (google.cloud.bigquery_storage_v1beta2.types.ReadSession.TableReadOptions): - Optional. Read options for this session (e.g. - column selection, filters). - streams (Sequence[google.cloud.bigquery_storage_v1beta2.types.ReadStream]): - Output only. A list of streams created with the session. - - At least one stream is created with the session. In the - future, larger request_stream_count values *may* result in - this list being unpopulated, in that case, the user will - need to use a List method to get the streams instead, which - is not yet available. - """ - - class TableModifiers(proto.Message): - r"""Additional attributes when reading a table. - Attributes: - snapshot_time (google.protobuf.timestamp_pb2.Timestamp): - The snapshot time of the table. If not set, - interpreted as now. - """ - - snapshot_time = proto.Field( - proto.MESSAGE, - number=1, - message=timestamp_pb2.Timestamp, - ) - - class TableReadOptions(proto.Message): - r"""Options dictating how we read a table. - Attributes: - selected_fields (Sequence[str]): - Names of the fields in the table that should be read. If - empty, all fields will be read. If the specified field is a - nested field, all the sub-fields in the field will be - selected. The output field order is unrelated to the order - of fields in selected_fields. - row_restriction (str): - SQL text filtering statement, similar to a WHERE clause in a - query. Aggregates are not supported. - - Examples: "int_field > 5" "date_field = CAST('2014-9-27' as - DATE)" "nullable_field is not NULL" "st_equals(geo_field, - st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 - AND 5.0" - - Restricted to a maximum length for 1 MB. - arrow_serialization_options (google.cloud.bigquery_storage_v1beta2.types.ArrowSerializationOptions): - Optional. Options specific to the Apache - Arrow output format. - """ - - selected_fields = proto.RepeatedField( - proto.STRING, - number=1, - ) - row_restriction = proto.Field( - proto.STRING, - number=2, - ) - arrow_serialization_options = proto.Field( - proto.MESSAGE, - number=3, - message=arrow.ArrowSerializationOptions, - ) - - name = proto.Field( - proto.STRING, - number=1, - ) - expire_time = proto.Field( - proto.MESSAGE, - number=2, - message=timestamp_pb2.Timestamp, - ) - data_format = proto.Field( - proto.ENUM, - number=3, - enum='DataFormat', - ) - avro_schema = proto.Field( - proto.MESSAGE, - number=4, - oneof='schema', - message=avro.AvroSchema, - ) - arrow_schema = proto.Field( - proto.MESSAGE, - number=5, - oneof='schema', - message=arrow.ArrowSchema, - ) - table = proto.Field( - proto.STRING, - number=6, - ) - table_modifiers = proto.Field( - proto.MESSAGE, - number=7, - message=TableModifiers, - ) - read_options = proto.Field( - proto.MESSAGE, - number=8, - message=TableReadOptions, - ) - streams = proto.RepeatedField( - proto.MESSAGE, - number=10, - message='ReadStream', - ) - - -class ReadStream(proto.Message): - r"""Information about a single stream that gets data out of the storage - system. Most of the information about ``ReadStream`` instances is - aggregated, making ``ReadStream`` lightweight. - - Attributes: - name (str): - Output only. Name of the stream, in the form - ``projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}``. - """ - - name = proto.Field( - proto.STRING, - number=1, - ) - - -class WriteStream(proto.Message): - r"""Information about a single stream that gets data inside the - storage system. - - Attributes: - name (str): - Output only. Name of the stream, in the form - ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. - type_ (google.cloud.bigquery_storage_v1beta2.types.WriteStream.Type): - Immutable. Type of the stream. - create_time (google.protobuf.timestamp_pb2.Timestamp): - Output only. Create time of the stream. For the \_default - stream, this is the creation_time of the table. - commit_time (google.protobuf.timestamp_pb2.Timestamp): - Output only. Commit time of the stream. If a stream is of - ``COMMITTED`` type, then it will have a commit_time same as - ``create_time``. If the stream is of ``PENDING`` type, - commit_time being empty means it is not committed. - table_schema (google.cloud.bigquery_storage_v1beta2.types.TableSchema): - Output only. The schema of the destination table. It is only - returned in ``CreateWriteStream`` response. Caller should - generate data that's compatible with this schema to send in - initial ``AppendRowsRequest``. The table schema could go out - of date during the life time of the stream. - """ - class Type(proto.Enum): - r"""Type enum of the stream.""" - TYPE_UNSPECIFIED = 0 - COMMITTED = 1 - PENDING = 2 - BUFFERED = 3 - - name = proto.Field( - proto.STRING, - number=1, - ) - type_ = proto.Field( - proto.ENUM, - number=2, - enum=Type, - ) - create_time = proto.Field( - proto.MESSAGE, - number=3, - message=timestamp_pb2.Timestamp, - ) - commit_time = proto.Field( - proto.MESSAGE, - number=4, - message=timestamp_pb2.Timestamp, - ) - table_schema = proto.Field( - proto.MESSAGE, - number=5, - message=gcbs_table.TableSchema, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py deleted file mode 100644 index 2c32504e..00000000 --- a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py +++ /dev/null @@ -1,113 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package='google.cloud.bigquery.storage.v1beta2', - manifest={ - 'TableSchema', - 'TableFieldSchema', - }, -) - - -class TableSchema(proto.Message): - r"""Schema of a table - Attributes: - fields (Sequence[google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema]): - Describes the fields in a table. - """ - - fields = proto.RepeatedField( - proto.MESSAGE, - number=1, - message='TableFieldSchema', - ) - - -class TableFieldSchema(proto.Message): - r"""A field in TableSchema - Attributes: - name (str): - Required. The field name. The name must contain only letters - (a-z, A-Z), numbers (0-9), or underscores (_), and must - start with a letter or underscore. The maximum length is 128 - characters. - type_ (google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema.Type): - Required. The field data type. - mode (google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema.Mode): - Optional. The field mode. The default value - is NULLABLE. - fields (Sequence[google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema]): - Optional. Describes the nested schema fields - if the type property is set to STRUCT. - description (str): - Optional. The field description. The maximum - length is 1,024 characters. - """ - class Type(proto.Enum): - r"""""" - TYPE_UNSPECIFIED = 0 - STRING = 1 - INT64 = 2 - DOUBLE = 3 - STRUCT = 4 - BYTES = 5 - BOOL = 6 - TIMESTAMP = 7 - DATE = 8 - TIME = 9 - DATETIME = 10 - GEOGRAPHY = 11 - NUMERIC = 12 - BIGNUMERIC = 13 - INTERVAL = 14 - JSON = 15 - - class Mode(proto.Enum): - r"""""" - MODE_UNSPECIFIED = 0 - NULLABLE = 1 - REQUIRED = 2 - REPEATED = 3 - - name = proto.Field( - proto.STRING, - number=1, - ) - type_ = proto.Field( - proto.ENUM, - number=2, - enum=Type, - ) - mode = proto.Field( - proto.ENUM, - number=3, - enum=Mode, - ) - fields = proto.RepeatedField( - proto.MESSAGE, - number=4, - message='TableFieldSchema', - ) - description = proto.Field( - proto.STRING, - number=6, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/mypy.ini b/owl-bot-staging/v1beta2/mypy.ini deleted file mode 100644 index 4505b485..00000000 --- a/owl-bot-staging/v1beta2/mypy.ini +++ /dev/null @@ -1,3 +0,0 @@ -[mypy] -python_version = 3.6 -namespace_packages = True diff --git a/owl-bot-staging/v1beta2/noxfile.py b/owl-bot-staging/v1beta2/noxfile.py deleted file mode 100644 index 4b78e039..00000000 --- a/owl-bot-staging/v1beta2/noxfile.py +++ /dev/null @@ -1,132 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import pathlib -import shutil -import subprocess -import sys - - -import nox # type: ignore - -CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() - -LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt" -PACKAGE_NAME = subprocess.check_output([sys.executable, "setup.py", "--name"], encoding="utf-8") - - -nox.sessions = [ - "unit", - "cover", - "mypy", - "check_lower_bounds" - # exclude update_lower_bounds from default - "docs", -] - -@nox.session(python=['3.6', '3.7', '3.8', '3.9']) -def unit(session): - """Run the unit test suite.""" - - session.install('coverage', 'pytest', 'pytest-cov', 'asyncmock', 'pytest-asyncio') - session.install('-e', '.') - - session.run( - 'py.test', - '--quiet', - '--cov=google/cloud/bigquery_storage_v1beta2/', - '--cov-config=.coveragerc', - '--cov-report=term', - '--cov-report=html', - os.path.join('tests', 'unit', ''.join(session.posargs)) - ) - - -@nox.session(python='3.7') -def cover(session): - """Run the final coverage report. - This outputs the coverage report aggregating coverage from the unit - test runs (not system test runs), and then erases coverage data. - """ - session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=100") - - session.run("coverage", "erase") - - -@nox.session(python=['3.6', '3.7']) -def mypy(session): - """Run the type checker.""" - session.install('mypy', 'types-pkg_resources') - session.install('.') - session.run( - 'mypy', - '--explicit-package-bases', - 'google', - ) - - -@nox.session -def update_lower_bounds(session): - """Update lower bounds in constraints.txt to match setup.py""" - session.install('google-cloud-testutils') - session.install('.') - - session.run( - 'lower-bound-checker', - 'update', - '--package-name', - PACKAGE_NAME, - '--constraints-file', - str(LOWER_BOUND_CONSTRAINTS_FILE), - ) - - -@nox.session -def check_lower_bounds(session): - """Check lower bounds in setup.py are reflected in constraints file""" - session.install('google-cloud-testutils') - session.install('.') - - session.run( - 'lower-bound-checker', - 'check', - '--package-name', - PACKAGE_NAME, - '--constraints-file', - str(LOWER_BOUND_CONSTRAINTS_FILE), - ) - -@nox.session(python='3.6') -def docs(session): - """Build the docs for this library.""" - - session.install("-e", ".") - session.install("sphinx<3.0.0", "alabaster", "recommonmark") - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run( - "sphinx-build", - "-W", # warnings as errors - "-T", # show full traceback on exception - "-N", # no colors - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) diff --git a/owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py b/owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py deleted file mode 100644 index 9ba710dc..00000000 --- a/owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py +++ /dev/null @@ -1,184 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import os -import libcst as cst -import pathlib -import sys -from typing import (Any, Callable, Dict, List, Sequence, Tuple) - - -def partition( - predicate: Callable[[Any], bool], - iterator: Sequence[Any] -) -> Tuple[List[Any], List[Any]]: - """A stable, out-of-place partition.""" - results = ([], []) - - for i in iterator: - results[int(predicate(i))].append(i) - - # Returns trueList, falseList - return results[1], results[0] - - -class bigquery_storageCallTransformer(cst.CSTTransformer): - CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') - METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { - 'append_rows': ('write_stream', 'offset', 'proto_rows', 'trace_id', ), - 'batch_commit_write_streams': ('parent', 'write_streams', ), - 'create_read_session': ('parent', 'read_session', 'max_stream_count', ), - 'create_write_stream': ('parent', 'write_stream', ), - 'finalize_write_stream': ('name', ), - 'flush_rows': ('write_stream', 'offset', ), - 'get_write_stream': ('name', ), - 'read_rows': ('read_stream', 'offset', ), - 'split_read_stream': ('name', 'fraction', ), - } - - def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: - try: - key = original.func.attr.value - kword_params = self.METHOD_TO_PARAMS[key] - except (AttributeError, KeyError): - # Either not a method from the API or too convoluted to be sure. - return updated - - # If the existing code is valid, keyword args come after positional args. - # Therefore, all positional args must map to the first parameters. - args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) - if any(k.keyword.value == "request" for k in kwargs): - # We've already fixed this file, don't fix it again. - return updated - - kwargs, ctrl_kwargs = partition( - lambda a: not a.keyword.value in self.CTRL_PARAMS, - kwargs - ) - - args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] - ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) - for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) - - request_arg = cst.Arg( - value=cst.Dict([ - cst.DictElement( - cst.SimpleString("'{}'".format(name)), -cst.Element(value=arg.value) - ) - # Note: the args + kwargs looks silly, but keep in mind that - # the control parameters had to be stripped out, and that - # those could have been passed positionally or by keyword. - for name, arg in zip(kword_params, args + kwargs)]), - keyword=cst.Name("request") - ) - - return updated.with_changes( - args=[request_arg] + ctrl_kwargs - ) - - -def fix_files( - in_dir: pathlib.Path, - out_dir: pathlib.Path, - *, - transformer=bigquery_storageCallTransformer(), -): - """Duplicate the input dir to the output dir, fixing file method calls. - - Preconditions: - * in_dir is a real directory - * out_dir is a real, empty directory - """ - pyfile_gen = ( - pathlib.Path(os.path.join(root, f)) - for root, _, files in os.walk(in_dir) - for f in files if os.path.splitext(f)[1] == ".py" - ) - - for fpath in pyfile_gen: - with open(fpath, 'r') as f: - src = f.read() - - # Parse the code and insert method call fixes. - tree = cst.parse_module(src) - updated = tree.visit(transformer) - - # Create the path and directory structure for the new file. - updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) - updated_path.parent.mkdir(parents=True, exist_ok=True) - - # Generate the updated source file at the corresponding path. - with open(updated_path, 'w') as f: - f.write(updated.code) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="""Fix up source that uses the bigquery_storage client library. - -The existing sources are NOT overwritten but are copied to output_dir with changes made. - -Note: This tool operates at a best-effort level at converting positional - parameters in client method calls to keyword based parameters. - Cases where it WILL FAIL include - A) * or ** expansion in a method call. - B) Calls via function or method alias (includes free function calls) - C) Indirect or dispatched calls (e.g. the method is looked up dynamically) - - These all constitute false negatives. The tool will also detect false - positives when an API method shares a name with another method. -""") - parser.add_argument( - '-d', - '--input-directory', - required=True, - dest='input_dir', - help='the input directory to walk for python files to fix up', - ) - parser.add_argument( - '-o', - '--output-directory', - required=True, - dest='output_dir', - help='the directory to output files fixed via un-flattening', - ) - args = parser.parse_args() - input_dir = pathlib.Path(args.input_dir) - output_dir = pathlib.Path(args.output_dir) - if not input_dir.is_dir(): - print( - f"input directory '{input_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if not output_dir.is_dir(): - print( - f"output directory '{output_dir}' does not exist or is not a directory", - file=sys.stderr, - ) - sys.exit(-1) - - if os.listdir(output_dir): - print( - f"output directory '{output_dir}' is not empty", - file=sys.stderr, - ) - sys.exit(-1) - - fix_files(input_dir, output_dir) diff --git a/owl-bot-staging/v1beta2/setup.py b/owl-bot-staging/v1beta2/setup.py deleted file mode 100644 index 591e1fba..00000000 --- a/owl-bot-staging/v1beta2/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import io -import os -import setuptools # type: ignore - -version = '0.1.0' - -package_root = os.path.abspath(os.path.dirname(__file__)) - -readme_filename = os.path.join(package_root, 'README.rst') -with io.open(readme_filename, encoding='utf-8') as readme_file: - readme = readme_file.read() - -setuptools.setup( - name='google-cloud-bigquery-storage', - version=version, - long_description=readme, - packages=setuptools.PEP420PackageFinder.find(), - namespace_packages=('google', 'google.cloud'), - platforms='Posix; MacOS X; Windows', - include_package_data=True, - install_requires=( - 'google-api-core[grpc] >= 1.27.0, < 3.0.0dev', - 'libcst >= 0.2.5', - 'proto-plus >= 1.15.0', - 'packaging >= 14.3', ), - python_requires='>=3.6', - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Topic :: Internet', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - zip_safe=False, -) diff --git a/owl-bot-staging/v1beta2/tests/__init__.py b/owl-bot-staging/v1beta2/tests/__init__.py deleted file mode 100644 index b54a5fcc..00000000 --- a/owl-bot-staging/v1beta2/tests/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ - -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/owl-bot-staging/v1beta2/tests/unit/__init__.py b/owl-bot-staging/v1beta2/tests/unit/__init__.py deleted file mode 100644 index b54a5fcc..00000000 --- a/owl-bot-staging/v1beta2/tests/unit/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ - -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py b/owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py deleted file mode 100644 index b54a5fcc..00000000 --- a/owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ - -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py deleted file mode 100644 index b54a5fcc..00000000 --- a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ - -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py deleted file mode 100644 index 1e3ad444..00000000 --- a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py +++ /dev/null @@ -1,1629 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import mock -import packaging.version - -import grpc -from grpc.experimental import aio -import math -import pytest -from proto.marshal.rules.dates import DurationRule, TimestampRule - - -from google.api_core import client_options -from google.api_core import exceptions as core_exceptions -from google.api_core import gapic_v1 -from google.api_core import grpc_helpers -from google.api_core import grpc_helpers_async -from google.auth import credentials as ga_credentials -from google.auth.exceptions import MutualTLSChannelError -from google.cloud.bigquery_storage_v1beta2.services.big_query_read import BigQueryReadAsyncClient -from google.cloud.bigquery_storage_v1beta2.services.big_query_read import BigQueryReadClient -from google.cloud.bigquery_storage_v1beta2.services.big_query_read import transports -from google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.base import _GOOGLE_AUTH_VERSION -from google.cloud.bigquery_storage_v1beta2.types import arrow -from google.cloud.bigquery_storage_v1beta2.types import avro -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream -from google.oauth2 import service_account -from google.protobuf import timestamp_pb2 # type: ignore -import google.auth - - -# TODO(busunkim): Once google-auth >= 1.25.0 is required transitively -# through google-api-core: -# - Delete the auth "less than" test cases -# - Delete these pytest markers (Make the "greater than or equal to" tests the default). -requires_google_auth_lt_1_25_0 = pytest.mark.skipif( - packaging.version.parse(_GOOGLE_AUTH_VERSION) >= packaging.version.parse("1.25.0"), - reason="This test requires google-auth < 1.25.0", -) -requires_google_auth_gte_1_25_0 = pytest.mark.skipif( - packaging.version.parse(_GOOGLE_AUTH_VERSION) < packaging.version.parse("1.25.0"), - reason="This test requires google-auth >= 1.25.0", -) - -def client_cert_source_callback(): - return b"cert bytes", b"key bytes" - - -# If default endpoint is localhost, then default mtls endpoint will be the same. -# This method modifies the default endpoint so the client can produce a different -# mtls endpoint for endpoint testing purposes. -def modify_default_endpoint(client): - return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT - - -def test__get_default_mtls_endpoint(): - api_endpoint = "example.googleapis.com" - api_mtls_endpoint = "example.mtls.googleapis.com" - sandbox_endpoint = "example.sandbox.googleapis.com" - sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" - non_googleapi = "api.example.com" - - assert BigQueryReadClient._get_default_mtls_endpoint(None) is None - assert BigQueryReadClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint - assert BigQueryReadClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint - assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint - assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint - assert BigQueryReadClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi - - -@pytest.mark.parametrize("client_class", [ - BigQueryReadClient, - BigQueryReadAsyncClient, -]) -def test_big_query_read_client_from_service_account_info(client_class): - creds = ga_credentials.AnonymousCredentials() - with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: - factory.return_value = creds - info = {"valid": True} - client = client_class.from_service_account_info(info) - assert client.transport._credentials == creds - assert isinstance(client, client_class) - - assert client.transport._host == 'bigquerystorage.googleapis.com:443' - - -@pytest.mark.parametrize("transport_class,transport_name", [ - (transports.BigQueryReadGrpcTransport, "grpc"), - (transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), -]) -def test_big_query_read_client_service_account_always_use_jwt(transport_class, transport_name): - with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: - creds = service_account.Credentials(None, None, None) - transport = transport_class(credentials=creds, always_use_jwt_access=True) - use_jwt.assert_called_once_with(True) - - with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: - creds = service_account.Credentials(None, None, None) - transport = transport_class(credentials=creds, always_use_jwt_access=False) - use_jwt.assert_not_called() - - -@pytest.mark.parametrize("client_class", [ - BigQueryReadClient, - BigQueryReadAsyncClient, -]) -def test_big_query_read_client_from_service_account_file(client_class): - creds = ga_credentials.AnonymousCredentials() - with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: - factory.return_value = creds - client = client_class.from_service_account_file("dummy/file/path.json") - assert client.transport._credentials == creds - assert isinstance(client, client_class) - - client = client_class.from_service_account_json("dummy/file/path.json") - assert client.transport._credentials == creds - assert isinstance(client, client_class) - - assert client.transport._host == 'bigquerystorage.googleapis.com:443' - - -def test_big_query_read_client_get_transport_class(): - transport = BigQueryReadClient.get_transport_class() - available_transports = [ - transports.BigQueryReadGrpcTransport, - ] - assert transport in available_transports - - transport = BigQueryReadClient.get_transport_class("grpc") - assert transport == transports.BigQueryReadGrpcTransport - - -@pytest.mark.parametrize("client_class,transport_class,transport_name", [ - (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), - (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), -]) -@mock.patch.object(BigQueryReadClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadClient)) -@mock.patch.object(BigQueryReadAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadAsyncClient)) -def test_big_query_read_client_client_options(client_class, transport_class, transport_name): - # Check that if channel is provided we won't create a new one. - with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: - transport = transport_class( - credentials=ga_credentials.AnonymousCredentials() - ) - client = client_class(transport=transport) - gtc.assert_not_called() - - # Check that if channel is provided via str we will create a new one. - with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: - client = client_class(transport=transport_name) - gtc.assert_called() - - # Check the case api_endpoint is provided. - options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host="squid.clam.whelk", - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is - # "never". - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is - # "always". - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_MTLS_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has - # unsupported value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): - with pytest.raises(MutualTLSChannelError): - client = client_class() - - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): - with pytest.raises(ValueError): - client = client_class() - - # Check the case quota_project_id is provided - options = client_options.ClientOptions(quota_project_id="octopus") - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id="octopus", - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - -@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ - (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "true"), - (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "true"), - (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "false"), - (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "false"), -]) -@mock.patch.object(BigQueryReadClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadClient)) -@mock.patch.object(BigQueryReadAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadAsyncClient)) -@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) -def test_big_query_read_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): - # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default - # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. - - # Check the case client_cert_source is provided. Whether client cert is used depends on - # GOOGLE_API_USE_CLIENT_CERTIFICATE value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): - options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - - if use_client_cert_env == "false": - expected_client_cert_source = None - expected_host = client.DEFAULT_ENDPOINT - else: - expected_client_cert_source = client_cert_source_callback - expected_host = client.DEFAULT_MTLS_ENDPOINT - - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=expected_host, - scopes=None, - client_cert_source_for_mtls=expected_client_cert_source, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case ADC client cert is provided. Whether client cert is used depends on - # GOOGLE_API_USE_CLIENT_CERTIFICATE value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): - with mock.patch.object(transport_class, '__init__') as patched: - with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): - with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): - if use_client_cert_env == "false": - expected_host = client.DEFAULT_ENDPOINT - expected_client_cert_source = None - else: - expected_host = client.DEFAULT_MTLS_ENDPOINT - expected_client_cert_source = client_cert_source_callback - - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=expected_host, - scopes=None, - client_cert_source_for_mtls=expected_client_cert_source, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case client_cert_source and ADC client cert are not provided. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): - with mock.patch.object(transport_class, '__init__') as patched: - with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - -@pytest.mark.parametrize("client_class,transport_class,transport_name", [ - (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), - (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), -]) -def test_big_query_read_client_client_options_scopes(client_class, transport_class, transport_name): - # Check the case scopes are provided. - options = client_options.ClientOptions( - scopes=["1", "2"], - ) - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=["1", "2"], - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - -@pytest.mark.parametrize("client_class,transport_class,transport_name", [ - (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), - (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), -]) -def test_big_query_read_client_client_options_credentials_file(client_class, transport_class, transport_name): - # Check the case credentials file is provided. - options = client_options.ClientOptions( - credentials_file="credentials.json" - ) - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file="credentials.json", - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - -def test_big_query_read_client_client_options_from_dict(): - with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadGrpcTransport.__init__') as grpc_transport: - grpc_transport.return_value = None - client = BigQueryReadClient( - client_options={'api_endpoint': 'squid.clam.whelk'} - ) - grpc_transport.assert_called_once_with( - credentials=None, - credentials_file=None, - host="squid.clam.whelk", - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - -def test_create_read_session(transport: str = 'grpc', request_type=storage.CreateReadSessionRequest): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.ReadSession( - name='name_value', - data_format=stream.DataFormat.AVRO, - table='table_value', - avro_schema=avro.AvroSchema(schema='schema_value'), - ) - response = client.create_read_session(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.CreateReadSessionRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, stream.ReadSession) - assert response.name == 'name_value' - assert response.data_format == stream.DataFormat.AVRO - assert response.table == 'table_value' - - -def test_create_read_session_from_dict(): - test_create_read_session(request_type=dict) - - -def test_create_read_session_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - client.create_read_session() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.CreateReadSessionRequest() - - -@pytest.mark.asyncio -async def test_create_read_session_async(transport: str = 'grpc_asyncio', request_type=storage.CreateReadSessionRequest): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession( - name='name_value', - data_format=stream.DataFormat.AVRO, - table='table_value', - )) - response = await client.create_read_session(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.CreateReadSessionRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, stream.ReadSession) - assert response.name == 'name_value' - assert response.data_format == stream.DataFormat.AVRO - assert response.table == 'table_value' - - -@pytest.mark.asyncio -async def test_create_read_session_async_from_dict(): - await test_create_read_session_async(request_type=dict) - - -def test_create_read_session_field_headers(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.CreateReadSessionRequest() - - request.read_session.table = 'read_session.table/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - call.return_value = stream.ReadSession() - client.create_read_session(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'read_session.table=read_session.table/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_create_read_session_field_headers_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.CreateReadSessionRequest() - - request.read_session.table = 'read_session.table/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) - await client.create_read_session(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'read_session.table=read_session.table/value', - ) in kw['metadata'] - - -def test_create_read_session_flattened(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.ReadSession() - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - client.create_read_session( - parent='parent_value', - read_session=stream.ReadSession(name='name_value'), - max_stream_count=1721, - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0].parent == 'parent_value' - assert args[0].read_session == stream.ReadSession(name='name_value') - assert args[0].max_stream_count == 1721 - - -def test_create_read_session_flattened_error(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - client.create_read_session( - storage.CreateReadSessionRequest(), - parent='parent_value', - read_session=stream.ReadSession(name='name_value'), - max_stream_count=1721, - ) - - -@pytest.mark.asyncio -async def test_create_read_session_flattened_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_read_session), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.ReadSession() - - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - response = await client.create_read_session( - parent='parent_value', - read_session=stream.ReadSession(name='name_value'), - max_stream_count=1721, - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0].parent == 'parent_value' - assert args[0].read_session == stream.ReadSession(name='name_value') - assert args[0].max_stream_count == 1721 - - -@pytest.mark.asyncio -async def test_create_read_session_flattened_error_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - await client.create_read_session( - storage.CreateReadSessionRequest(), - parent='parent_value', - read_session=stream.ReadSession(name='name_value'), - max_stream_count=1721, - ) - - -def test_read_rows(transport: str = 'grpc', request_type=storage.ReadRowsRequest): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = iter([storage.ReadRowsResponse()]) - response = client.read_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.ReadRowsRequest() - - # Establish that the response is the type that we expect. - for message in response: - assert isinstance(message, storage.ReadRowsResponse) - - -def test_read_rows_from_dict(): - test_read_rows(request_type=dict) - - -def test_read_rows_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - client.read_rows() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.ReadRowsRequest() - - -@pytest.mark.asyncio -async def test_read_rows_async(transport: str = 'grpc_asyncio', request_type=storage.ReadRowsRequest): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) - call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) - response = await client.read_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.ReadRowsRequest() - - # Establish that the response is the type that we expect. - message = await response.read() - assert isinstance(message, storage.ReadRowsResponse) - - -@pytest.mark.asyncio -async def test_read_rows_async_from_dict(): - await test_read_rows_async(request_type=dict) - - -def test_read_rows_field_headers(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.ReadRowsRequest() - - request.read_stream = 'read_stream/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - call.return_value = iter([storage.ReadRowsResponse()]) - client.read_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'read_stream=read_stream/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_read_rows_field_headers_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.ReadRowsRequest() - - request.read_stream = 'read_stream/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) - call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) - await client.read_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'read_stream=read_stream/value', - ) in kw['metadata'] - - -def test_read_rows_flattened(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = iter([storage.ReadRowsResponse()]) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - client.read_rows( - read_stream='read_stream_value', - offset=647, - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0].read_stream == 'read_stream_value' - assert args[0].offset == 647 - - -def test_read_rows_flattened_error(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - client.read_rows( - storage.ReadRowsRequest(), - read_stream='read_stream_value', - offset=647, - ) - - -@pytest.mark.asyncio -async def test_read_rows_flattened_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.read_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = iter([storage.ReadRowsResponse()]) - - call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - response = await client.read_rows( - read_stream='read_stream_value', - offset=647, - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0].read_stream == 'read_stream_value' - assert args[0].offset == 647 - - -@pytest.mark.asyncio -async def test_read_rows_flattened_error_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - await client.read_rows( - storage.ReadRowsRequest(), - read_stream='read_stream_value', - offset=647, - ) - - -def test_split_read_stream(transport: str = 'grpc', request_type=storage.SplitReadStreamRequest): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.split_read_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.SplitReadStreamResponse( - ) - response = client.split_read_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.SplitReadStreamRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, storage.SplitReadStreamResponse) - - -def test_split_read_stream_from_dict(): - test_split_read_stream(request_type=dict) - - -def test_split_read_stream_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.split_read_stream), - '__call__') as call: - client.split_read_stream() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.SplitReadStreamRequest() - - -@pytest.mark.asyncio -async def test_split_read_stream_async(transport: str = 'grpc_asyncio', request_type=storage.SplitReadStreamRequest): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.split_read_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse( - )) - response = await client.split_read_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.SplitReadStreamRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, storage.SplitReadStreamResponse) - - -@pytest.mark.asyncio -async def test_split_read_stream_async_from_dict(): - await test_split_read_stream_async(request_type=dict) - - -def test_split_read_stream_field_headers(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.SplitReadStreamRequest() - - request.name = 'name/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.split_read_stream), - '__call__') as call: - call.return_value = storage.SplitReadStreamResponse() - client.split_read_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'name=name/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_split_read_stream_field_headers_async(): - client = BigQueryReadAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.SplitReadStreamRequest() - - request.name = 'name/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.split_read_stream), - '__call__') as call: - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse()) - await client.split_read_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'name=name/value', - ) in kw['metadata'] - - -def test_credentials_transport_error(): - # It is an error to provide credentials and a transport instance. - transport = transports.BigQueryReadGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # It is an error to provide a credentials file and a transport instance. - transport = transports.BigQueryReadGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): - client = BigQueryReadClient( - client_options={"credentials_file": "credentials.json"}, - transport=transport, - ) - - # It is an error to provide scopes and a transport instance. - transport = transports.BigQueryReadGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): - client = BigQueryReadClient( - client_options={"scopes": ["1", "2"]}, - transport=transport, - ) - - -def test_transport_instance(): - # A client may be instantiated with a custom transport instance. - transport = transports.BigQueryReadGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - client = BigQueryReadClient(transport=transport) - assert client.transport is transport - -def test_transport_get_channel(): - # A client may be instantiated with a custom transport instance. - transport = transports.BigQueryReadGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - channel = transport.grpc_channel - assert channel - - transport = transports.BigQueryReadGrpcAsyncIOTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - channel = transport.grpc_channel - assert channel - -@pytest.mark.parametrize("transport_class", [ - transports.BigQueryReadGrpcTransport, - transports.BigQueryReadGrpcAsyncIOTransport, -]) -def test_transport_adc(transport_class): - # Test default credentials are used if not provided. - with mock.patch.object(google.auth, 'default') as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport_class() - adc.assert_called_once() - -def test_transport_grpc_default(): - # A client should use the gRPC transport by default. - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - assert isinstance( - client.transport, - transports.BigQueryReadGrpcTransport, - ) - -def test_big_query_read_base_transport_error(): - # Passing both a credentials object and credentials_file should raise an error - with pytest.raises(core_exceptions.DuplicateCredentialArgs): - transport = transports.BigQueryReadTransport( - credentials=ga_credentials.AnonymousCredentials(), - credentials_file="credentials.json" - ) - - -def test_big_query_read_base_transport(): - # Instantiate the base transport. - with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport.__init__') as Transport: - Transport.return_value = None - transport = transports.BigQueryReadTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Every method on the transport should just blindly - # raise NotImplementedError. - methods = ( - 'create_read_session', - 'read_rows', - 'split_read_stream', - ) - for method in methods: - with pytest.raises(NotImplementedError): - getattr(transport, method)(request=object()) - - -@requires_google_auth_gte_1_25_0 -def test_big_query_read_base_transport_with_credentials_file(): - # Instantiate the base transport with a credentials file - with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: - Transport.return_value = None - load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) - transport = transports.BigQueryReadTransport( - credentials_file="credentials.json", - quota_project_id="octopus", - ) - load_creds.assert_called_once_with("credentials.json", - scopes=None, - default_scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', -), - quota_project_id="octopus", - ) - - -@requires_google_auth_lt_1_25_0 -def test_big_query_read_base_transport_with_credentials_file_old_google_auth(): - # Instantiate the base transport with a credentials file - with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: - Transport.return_value = None - load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) - transport = transports.BigQueryReadTransport( - credentials_file="credentials.json", - quota_project_id="octopus", - ) - load_creds.assert_called_once_with("credentials.json", scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', - ), - quota_project_id="octopus", - ) - - -def test_big_query_read_base_transport_with_adc(): - # Test the default credentials are used if credentials and credentials_file are None. - with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: - Transport.return_value = None - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport = transports.BigQueryReadTransport() - adc.assert_called_once() - - -@requires_google_auth_gte_1_25_0 -def test_big_query_read_auth_adc(): - # If no credentials are provided, we should use ADC credentials. - with mock.patch.object(google.auth, 'default', autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - BigQueryReadClient() - adc.assert_called_once_with( - scopes=None, - default_scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', -), - quota_project_id=None, - ) - - -@requires_google_auth_lt_1_25_0 -def test_big_query_read_auth_adc_old_google_auth(): - # If no credentials are provided, we should use ADC credentials. - with mock.patch.object(google.auth, 'default', autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - BigQueryReadClient() - adc.assert_called_once_with( - scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.readonly', 'https://www.googleapis.com/auth/cloud-platform',), - quota_project_id=None, - ) - - -@pytest.mark.parametrize( - "transport_class", - [ - transports.BigQueryReadGrpcTransport, - transports.BigQueryReadGrpcAsyncIOTransport, - ], -) -@requires_google_auth_gte_1_25_0 -def test_big_query_read_transport_auth_adc(transport_class): - # If credentials and host are not provided, the transport class should use - # ADC credentials. - with mock.patch.object(google.auth, 'default', autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport_class(quota_project_id="octopus", scopes=["1", "2"]) - adc.assert_called_once_with( - scopes=["1", "2"], - default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.readonly', 'https://www.googleapis.com/auth/cloud-platform',), - quota_project_id="octopus", - ) - - -@pytest.mark.parametrize( - "transport_class", - [ - transports.BigQueryReadGrpcTransport, - transports.BigQueryReadGrpcAsyncIOTransport, - ], -) -@requires_google_auth_lt_1_25_0 -def test_big_query_read_transport_auth_adc_old_google_auth(transport_class): - # If credentials and host are not provided, the transport class should use - # ADC credentials. - with mock.patch.object(google.auth, "default", autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport_class(quota_project_id="octopus") - adc.assert_called_once_with(scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', -), - quota_project_id="octopus", - ) - - -@pytest.mark.parametrize( - "transport_class,grpc_helpers", - [ - (transports.BigQueryReadGrpcTransport, grpc_helpers), - (transports.BigQueryReadGrpcAsyncIOTransport, grpc_helpers_async) - ], -) -def test_big_query_read_transport_create_channel(transport_class, grpc_helpers): - # If credentials and host are not provided, the transport class should use - # ADC credentials. - with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( - grpc_helpers, "create_channel", autospec=True - ) as create_channel: - creds = ga_credentials.AnonymousCredentials() - adc.return_value = (creds, None) - transport_class( - quota_project_id="octopus", - scopes=["1", "2"] - ) - - create_channel.assert_called_with( - "bigquerystorage.googleapis.com:443", - credentials=creds, - credentials_file=None, - quota_project_id="octopus", - default_scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.readonly', - 'https://www.googleapis.com/auth/cloud-platform', -), - scopes=["1", "2"], - default_host="bigquerystorage.googleapis.com", - ssl_credentials=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - -@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) -def test_big_query_read_grpc_transport_client_cert_source_for_mtls( - transport_class -): - cred = ga_credentials.AnonymousCredentials() - - # Check ssl_channel_credentials is used if provided. - with mock.patch.object(transport_class, "create_channel") as mock_create_channel: - mock_ssl_channel_creds = mock.Mock() - transport_class( - host="squid.clam.whelk", - credentials=cred, - ssl_channel_credentials=mock_ssl_channel_creds - ) - mock_create_channel.assert_called_once_with( - "squid.clam.whelk:443", - credentials=cred, - credentials_file=None, - scopes=None, - ssl_credentials=mock_ssl_channel_creds, - quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls - # is used. - with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): - with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: - transport_class( - credentials=cred, - client_cert_source_for_mtls=client_cert_source_callback - ) - expected_cert, expected_key = client_cert_source_callback() - mock_ssl_cred.assert_called_once_with( - certificate_chain=expected_cert, - private_key=expected_key - ) - - -def test_big_query_read_host_no_port(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), - ) - assert client.transport._host == 'bigquerystorage.googleapis.com:443' - - -def test_big_query_read_host_with_port(): - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), - ) - assert client.transport._host == 'bigquerystorage.googleapis.com:8000' - -def test_big_query_read_grpc_transport_channel(): - channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) - - # Check that channel is used if provided. - transport = transports.BigQueryReadGrpcTransport( - host="squid.clam.whelk", - channel=channel, - ) - assert transport.grpc_channel == channel - assert transport._host == "squid.clam.whelk:443" - assert transport._ssl_channel_credentials == None - - -def test_big_query_read_grpc_asyncio_transport_channel(): - channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) - - # Check that channel is used if provided. - transport = transports.BigQueryReadGrpcAsyncIOTransport( - host="squid.clam.whelk", - channel=channel, - ) - assert transport.grpc_channel == channel - assert transport._host == "squid.clam.whelk:443" - assert transport._ssl_channel_credentials == None - - -# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are -# removed from grpc/grpc_asyncio transport constructor. -@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) -def test_big_query_read_transport_channel_mtls_with_client_cert_source( - transport_class -): - with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: - with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: - mock_ssl_cred = mock.Mock() - grpc_ssl_channel_cred.return_value = mock_ssl_cred - - mock_grpc_channel = mock.Mock() - grpc_create_channel.return_value = mock_grpc_channel - - cred = ga_credentials.AnonymousCredentials() - with pytest.warns(DeprecationWarning): - with mock.patch.object(google.auth, 'default') as adc: - adc.return_value = (cred, None) - transport = transport_class( - host="squid.clam.whelk", - api_mtls_endpoint="mtls.squid.clam.whelk", - client_cert_source=client_cert_source_callback, - ) - adc.assert_called_once() - - grpc_ssl_channel_cred.assert_called_once_with( - certificate_chain=b"cert bytes", private_key=b"key bytes" - ) - grpc_create_channel.assert_called_once_with( - "mtls.squid.clam.whelk:443", - credentials=cred, - credentials_file=None, - scopes=None, - ssl_credentials=mock_ssl_cred, - quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - assert transport.grpc_channel == mock_grpc_channel - assert transport._ssl_channel_credentials == mock_ssl_cred - - -# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are -# removed from grpc/grpc_asyncio transport constructor. -@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) -def test_big_query_read_transport_channel_mtls_with_adc( - transport_class -): - mock_ssl_cred = mock.Mock() - with mock.patch.multiple( - "google.auth.transport.grpc.SslCredentials", - __init__=mock.Mock(return_value=None), - ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), - ): - with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: - mock_grpc_channel = mock.Mock() - grpc_create_channel.return_value = mock_grpc_channel - mock_cred = mock.Mock() - - with pytest.warns(DeprecationWarning): - transport = transport_class( - host="squid.clam.whelk", - credentials=mock_cred, - api_mtls_endpoint="mtls.squid.clam.whelk", - client_cert_source=None, - ) - - grpc_create_channel.assert_called_once_with( - "mtls.squid.clam.whelk:443", - credentials=mock_cred, - credentials_file=None, - scopes=None, - ssl_credentials=mock_ssl_cred, - quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - assert transport.grpc_channel == mock_grpc_channel - - -def test_read_session_path(): - project = "squid" - location = "clam" - session = "whelk" - expected = "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) - actual = BigQueryReadClient.read_session_path(project, location, session) - assert expected == actual - - -def test_parse_read_session_path(): - expected = { - "project": "octopus", - "location": "oyster", - "session": "nudibranch", - } - path = BigQueryReadClient.read_session_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_read_session_path(path) - assert expected == actual - -def test_read_stream_path(): - project = "cuttlefish" - location = "mussel" - session = "winkle" - stream = "nautilus" - expected = "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) - actual = BigQueryReadClient.read_stream_path(project, location, session, stream) - assert expected == actual - - -def test_parse_read_stream_path(): - expected = { - "project": "scallop", - "location": "abalone", - "session": "squid", - "stream": "clam", - } - path = BigQueryReadClient.read_stream_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_read_stream_path(path) - assert expected == actual - -def test_table_path(): - project = "whelk" - dataset = "octopus" - table = "oyster" - expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) - actual = BigQueryReadClient.table_path(project, dataset, table) - assert expected == actual - - -def test_parse_table_path(): - expected = { - "project": "nudibranch", - "dataset": "cuttlefish", - "table": "mussel", - } - path = BigQueryReadClient.table_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_table_path(path) - assert expected == actual - -def test_common_billing_account_path(): - billing_account = "winkle" - expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) - actual = BigQueryReadClient.common_billing_account_path(billing_account) - assert expected == actual - - -def test_parse_common_billing_account_path(): - expected = { - "billing_account": "nautilus", - } - path = BigQueryReadClient.common_billing_account_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_common_billing_account_path(path) - assert expected == actual - -def test_common_folder_path(): - folder = "scallop" - expected = "folders/{folder}".format(folder=folder, ) - actual = BigQueryReadClient.common_folder_path(folder) - assert expected == actual - - -def test_parse_common_folder_path(): - expected = { - "folder": "abalone", - } - path = BigQueryReadClient.common_folder_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_common_folder_path(path) - assert expected == actual - -def test_common_organization_path(): - organization = "squid" - expected = "organizations/{organization}".format(organization=organization, ) - actual = BigQueryReadClient.common_organization_path(organization) - assert expected == actual - - -def test_parse_common_organization_path(): - expected = { - "organization": "clam", - } - path = BigQueryReadClient.common_organization_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_common_organization_path(path) - assert expected == actual - -def test_common_project_path(): - project = "whelk" - expected = "projects/{project}".format(project=project, ) - actual = BigQueryReadClient.common_project_path(project) - assert expected == actual - - -def test_parse_common_project_path(): - expected = { - "project": "octopus", - } - path = BigQueryReadClient.common_project_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_common_project_path(path) - assert expected == actual - -def test_common_location_path(): - project = "oyster" - location = "nudibranch" - expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) - actual = BigQueryReadClient.common_location_path(project, location) - assert expected == actual - - -def test_parse_common_location_path(): - expected = { - "project": "cuttlefish", - "location": "mussel", - } - path = BigQueryReadClient.common_location_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryReadClient.parse_common_location_path(path) - assert expected == actual - - -def test_client_withDEFAULT_CLIENT_INFO(): - client_info = gapic_v1.client_info.ClientInfo() - - with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: - client = BigQueryReadClient( - credentials=ga_credentials.AnonymousCredentials(), - client_info=client_info, - ) - prep.assert_called_once_with(client_info) - - with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: - transport_class = BigQueryReadClient.get_transport_class() - transport = transport_class( - credentials=ga_credentials.AnonymousCredentials(), - client_info=client_info, - ) - prep.assert_called_once_with(client_info) diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py deleted file mode 100644 index e49950ab..00000000 --- a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py +++ /dev/null @@ -1,2294 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import mock -import packaging.version - -import grpc -from grpc.experimental import aio -import math -import pytest -from proto.marshal.rules.dates import DurationRule, TimestampRule - - -from google.api_core import client_options -from google.api_core import exceptions as core_exceptions -from google.api_core import gapic_v1 -from google.api_core import grpc_helpers -from google.api_core import grpc_helpers_async -from google.auth import credentials as ga_credentials -from google.auth.exceptions import MutualTLSChannelError -from google.cloud.bigquery_storage_v1beta2.services.big_query_write import BigQueryWriteAsyncClient -from google.cloud.bigquery_storage_v1beta2.services.big_query_write import BigQueryWriteClient -from google.cloud.bigquery_storage_v1beta2.services.big_query_write import transports -from google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.base import _GOOGLE_AUTH_VERSION -from google.cloud.bigquery_storage_v1beta2.types import protobuf -from google.cloud.bigquery_storage_v1beta2.types import storage -from google.cloud.bigquery_storage_v1beta2.types import stream -from google.cloud.bigquery_storage_v1beta2.types import table -from google.oauth2 import service_account -from google.protobuf import descriptor_pb2 # type: ignore -from google.protobuf import timestamp_pb2 # type: ignore -from google.protobuf import wrappers_pb2 # type: ignore -from google.rpc import status_pb2 # type: ignore -import google.auth - - -# TODO(busunkim): Once google-auth >= 1.25.0 is required transitively -# through google-api-core: -# - Delete the auth "less than" test cases -# - Delete these pytest markers (Make the "greater than or equal to" tests the default). -requires_google_auth_lt_1_25_0 = pytest.mark.skipif( - packaging.version.parse(_GOOGLE_AUTH_VERSION) >= packaging.version.parse("1.25.0"), - reason="This test requires google-auth < 1.25.0", -) -requires_google_auth_gte_1_25_0 = pytest.mark.skipif( - packaging.version.parse(_GOOGLE_AUTH_VERSION) < packaging.version.parse("1.25.0"), - reason="This test requires google-auth >= 1.25.0", -) - -def client_cert_source_callback(): - return b"cert bytes", b"key bytes" - - -# If default endpoint is localhost, then default mtls endpoint will be the same. -# This method modifies the default endpoint so the client can produce a different -# mtls endpoint for endpoint testing purposes. -def modify_default_endpoint(client): - return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT - - -def test__get_default_mtls_endpoint(): - api_endpoint = "example.googleapis.com" - api_mtls_endpoint = "example.mtls.googleapis.com" - sandbox_endpoint = "example.sandbox.googleapis.com" - sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" - non_googleapi = "api.example.com" - - assert BigQueryWriteClient._get_default_mtls_endpoint(None) is None - assert BigQueryWriteClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint - assert BigQueryWriteClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint - assert BigQueryWriteClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint - assert BigQueryWriteClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint - assert BigQueryWriteClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi - - -@pytest.mark.parametrize("client_class", [ - BigQueryWriteClient, - BigQueryWriteAsyncClient, -]) -def test_big_query_write_client_from_service_account_info(client_class): - creds = ga_credentials.AnonymousCredentials() - with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: - factory.return_value = creds - info = {"valid": True} - client = client_class.from_service_account_info(info) - assert client.transport._credentials == creds - assert isinstance(client, client_class) - - assert client.transport._host == 'bigquerystorage.googleapis.com:443' - - -@pytest.mark.parametrize("transport_class,transport_name", [ - (transports.BigQueryWriteGrpcTransport, "grpc"), - (transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), -]) -def test_big_query_write_client_service_account_always_use_jwt(transport_class, transport_name): - with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: - creds = service_account.Credentials(None, None, None) - transport = transport_class(credentials=creds, always_use_jwt_access=True) - use_jwt.assert_called_once_with(True) - - with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: - creds = service_account.Credentials(None, None, None) - transport = transport_class(credentials=creds, always_use_jwt_access=False) - use_jwt.assert_not_called() - - -@pytest.mark.parametrize("client_class", [ - BigQueryWriteClient, - BigQueryWriteAsyncClient, -]) -def test_big_query_write_client_from_service_account_file(client_class): - creds = ga_credentials.AnonymousCredentials() - with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: - factory.return_value = creds - client = client_class.from_service_account_file("dummy/file/path.json") - assert client.transport._credentials == creds - assert isinstance(client, client_class) - - client = client_class.from_service_account_json("dummy/file/path.json") - assert client.transport._credentials == creds - assert isinstance(client, client_class) - - assert client.transport._host == 'bigquerystorage.googleapis.com:443' - - -def test_big_query_write_client_get_transport_class(): - transport = BigQueryWriteClient.get_transport_class() - available_transports = [ - transports.BigQueryWriteGrpcTransport, - ] - assert transport in available_transports - - transport = BigQueryWriteClient.get_transport_class("grpc") - assert transport == transports.BigQueryWriteGrpcTransport - - -@pytest.mark.parametrize("client_class,transport_class,transport_name", [ - (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), - (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), -]) -@mock.patch.object(BigQueryWriteClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteClient)) -@mock.patch.object(BigQueryWriteAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteAsyncClient)) -def test_big_query_write_client_client_options(client_class, transport_class, transport_name): - # Check that if channel is provided we won't create a new one. - with mock.patch.object(BigQueryWriteClient, 'get_transport_class') as gtc: - transport = transport_class( - credentials=ga_credentials.AnonymousCredentials() - ) - client = client_class(transport=transport) - gtc.assert_not_called() - - # Check that if channel is provided via str we will create a new one. - with mock.patch.object(BigQueryWriteClient, 'get_transport_class') as gtc: - client = client_class(transport=transport_name) - gtc.assert_called() - - # Check the case api_endpoint is provided. - options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host="squid.clam.whelk", - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is - # "never". - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is - # "always". - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_MTLS_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has - # unsupported value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): - with pytest.raises(MutualTLSChannelError): - client = client_class() - - # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): - with pytest.raises(ValueError): - client = client_class() - - # Check the case quota_project_id is provided - options = client_options.ClientOptions(quota_project_id="octopus") - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id="octopus", - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - -@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ - (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", "true"), - (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", "true"), - (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", "false"), - (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", "false"), -]) -@mock.patch.object(BigQueryWriteClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteClient)) -@mock.patch.object(BigQueryWriteAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteAsyncClient)) -@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) -def test_big_query_write_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): - # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default - # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. - - # Check the case client_cert_source is provided. Whether client cert is used depends on - # GOOGLE_API_USE_CLIENT_CERTIFICATE value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): - options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - - if use_client_cert_env == "false": - expected_client_cert_source = None - expected_host = client.DEFAULT_ENDPOINT - else: - expected_client_cert_source = client_cert_source_callback - expected_host = client.DEFAULT_MTLS_ENDPOINT - - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=expected_host, - scopes=None, - client_cert_source_for_mtls=expected_client_cert_source, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case ADC client cert is provided. Whether client cert is used depends on - # GOOGLE_API_USE_CLIENT_CERTIFICATE value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): - with mock.patch.object(transport_class, '__init__') as patched: - with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): - with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): - if use_client_cert_env == "false": - expected_host = client.DEFAULT_ENDPOINT - expected_client_cert_source = None - else: - expected_host = client.DEFAULT_MTLS_ENDPOINT - expected_client_cert_source = client_cert_source_callback - - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=expected_host, - scopes=None, - client_cert_source_for_mtls=expected_client_cert_source, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - # Check the case client_cert_source and ADC client cert are not provided. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): - with mock.patch.object(transport_class, '__init__') as patched: - with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - -@pytest.mark.parametrize("client_class,transport_class,transport_name", [ - (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), - (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), -]) -def test_big_query_write_client_client_options_scopes(client_class, transport_class, transport_name): - # Check the case scopes are provided. - options = client_options.ClientOptions( - scopes=["1", "2"], - ) - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=["1", "2"], - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - -@pytest.mark.parametrize("client_class,transport_class,transport_name", [ - (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), - (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), -]) -def test_big_query_write_client_client_options_credentials_file(client_class, transport_class, transport_name): - # Check the case credentials file is provided. - options = client_options.ClientOptions( - credentials_file="credentials.json" - ) - with mock.patch.object(transport_class, '__init__') as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file="credentials.json", - host=client.DEFAULT_ENDPOINT, - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - -def test_big_query_write_client_client_options_from_dict(): - with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteGrpcTransport.__init__') as grpc_transport: - grpc_transport.return_value = None - client = BigQueryWriteClient( - client_options={'api_endpoint': 'squid.clam.whelk'} - ) - grpc_transport.assert_called_once_with( - credentials=None, - credentials_file=None, - host="squid.clam.whelk", - scopes=None, - client_cert_source_for_mtls=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - always_use_jwt_access=True, - ) - - -def test_create_write_stream(transport: str = 'grpc', request_type=storage.CreateWriteStreamRequest): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.WriteStream( - name='name_value', - type_=stream.WriteStream.Type.COMMITTED, - ) - response = client.create_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.CreateWriteStreamRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, stream.WriteStream) - assert response.name == 'name_value' - assert response.type_ == stream.WriteStream.Type.COMMITTED - - -def test_create_write_stream_from_dict(): - test_create_write_stream(request_type=dict) - - -def test_create_write_stream_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_write_stream), - '__call__') as call: - client.create_write_stream() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.CreateWriteStreamRequest() - - -@pytest.mark.asyncio -async def test_create_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.CreateWriteStreamRequest): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( - name='name_value', - type_=stream.WriteStream.Type.COMMITTED, - )) - response = await client.create_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.CreateWriteStreamRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, stream.WriteStream) - assert response.name == 'name_value' - assert response.type_ == stream.WriteStream.Type.COMMITTED - - -@pytest.mark.asyncio -async def test_create_write_stream_async_from_dict(): - await test_create_write_stream_async(request_type=dict) - - -def test_create_write_stream_field_headers(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.CreateWriteStreamRequest() - - request.parent = 'parent/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_write_stream), - '__call__') as call: - call.return_value = stream.WriteStream() - client.create_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'parent=parent/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_create_write_stream_field_headers_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.CreateWriteStreamRequest() - - request.parent = 'parent/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_write_stream), - '__call__') as call: - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) - await client.create_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'parent=parent/value', - ) in kw['metadata'] - - -def test_create_write_stream_flattened(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.WriteStream() - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - client.create_write_stream( - parent='parent_value', - write_stream=stream.WriteStream(name='name_value'), - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0].parent == 'parent_value' - assert args[0].write_stream == stream.WriteStream(name='name_value') - - -def test_create_write_stream_flattened_error(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - client.create_write_stream( - storage.CreateWriteStreamRequest(), - parent='parent_value', - write_stream=stream.WriteStream(name='name_value'), - ) - - -@pytest.mark.asyncio -async def test_create_write_stream_flattened_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.create_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.WriteStream() - - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - response = await client.create_write_stream( - parent='parent_value', - write_stream=stream.WriteStream(name='name_value'), - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0].parent == 'parent_value' - assert args[0].write_stream == stream.WriteStream(name='name_value') - - -@pytest.mark.asyncio -async def test_create_write_stream_flattened_error_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - await client.create_write_stream( - storage.CreateWriteStreamRequest(), - parent='parent_value', - write_stream=stream.WriteStream(name='name_value'), - ) - - -def test_append_rows(transport: str = 'grpc', request_type=storage.AppendRowsRequest): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - requests = [request] - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.append_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = iter([storage.AppendRowsResponse()]) - response = client.append_rows(iter(requests)) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert next(args[0]) == request - - # Establish that the response is the type that we expect. - for message in response: - assert isinstance(message, storage.AppendRowsResponse) - - -def test_append_rows_from_dict(): - test_append_rows(request_type=dict) - - -@pytest.mark.asyncio -async def test_append_rows_async(transport: str = 'grpc_asyncio', request_type=storage.AppendRowsRequest): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - requests = [request] - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.append_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = mock.Mock(aio.StreamStreamCall, autospec=True) - call.return_value.read = mock.AsyncMock(side_effect=[storage.AppendRowsResponse()]) - response = await client.append_rows(iter(requests)) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert next(args[0]) == request - - # Establish that the response is the type that we expect. - message = await response.read() - assert isinstance(message, storage.AppendRowsResponse) - - -@pytest.mark.asyncio -async def test_append_rows_async_from_dict(): - await test_append_rows_async(request_type=dict) - - -def test_append_rows_flattened(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.append_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = iter([storage.AppendRowsResponse()]) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - client.append_rows( - write_stream='write_stream_value', - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0].write_stream == 'write_stream_value' - - -def test_append_rows_flattened_error(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - client.append_rows( - storage.AppendRowsRequest(), - write_stream='write_stream_value', - ) - - -@pytest.mark.asyncio -async def test_append_rows_flattened_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.append_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = iter([storage.AppendRowsResponse()]) - - call.return_value = mock.Mock(aio.StreamStreamCall, autospec=True) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - response = await client.append_rows( - write_stream='write_stream_value', - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0].write_stream == 'write_stream_value' - - -@pytest.mark.asyncio -async def test_append_rows_flattened_error_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - await client.append_rows( - storage.AppendRowsRequest(), - write_stream='write_stream_value', - ) - - -def test_get_write_stream(transport: str = 'grpc', request_type=storage.GetWriteStreamRequest): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.get_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.WriteStream( - name='name_value', - type_=stream.WriteStream.Type.COMMITTED, - ) - response = client.get_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.GetWriteStreamRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, stream.WriteStream) - assert response.name == 'name_value' - assert response.type_ == stream.WriteStream.Type.COMMITTED - - -def test_get_write_stream_from_dict(): - test_get_write_stream(request_type=dict) - - -def test_get_write_stream_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.get_write_stream), - '__call__') as call: - client.get_write_stream() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.GetWriteStreamRequest() - - -@pytest.mark.asyncio -async def test_get_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.GetWriteStreamRequest): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.get_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( - name='name_value', - type_=stream.WriteStream.Type.COMMITTED, - )) - response = await client.get_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.GetWriteStreamRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, stream.WriteStream) - assert response.name == 'name_value' - assert response.type_ == stream.WriteStream.Type.COMMITTED - - -@pytest.mark.asyncio -async def test_get_write_stream_async_from_dict(): - await test_get_write_stream_async(request_type=dict) - - -def test_get_write_stream_field_headers(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.GetWriteStreamRequest() - - request.name = 'name/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.get_write_stream), - '__call__') as call: - call.return_value = stream.WriteStream() - client.get_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'name=name/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_get_write_stream_field_headers_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.GetWriteStreamRequest() - - request.name = 'name/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.get_write_stream), - '__call__') as call: - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) - await client.get_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'name=name/value', - ) in kw['metadata'] - - -def test_get_write_stream_flattened(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.get_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.WriteStream() - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - client.get_write_stream( - name='name_value', - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0].name == 'name_value' - - -def test_get_write_stream_flattened_error(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - client.get_write_stream( - storage.GetWriteStreamRequest(), - name='name_value', - ) - - -@pytest.mark.asyncio -async def test_get_write_stream_flattened_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.get_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = stream.WriteStream() - - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - response = await client.get_write_stream( - name='name_value', - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0].name == 'name_value' - - -@pytest.mark.asyncio -async def test_get_write_stream_flattened_error_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - await client.get_write_stream( - storage.GetWriteStreamRequest(), - name='name_value', - ) - - -def test_finalize_write_stream(transport: str = 'grpc', request_type=storage.FinalizeWriteStreamRequest): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.finalize_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.FinalizeWriteStreamResponse( - row_count=992, - ) - response = client.finalize_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.FinalizeWriteStreamRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, storage.FinalizeWriteStreamResponse) - assert response.row_count == 992 - - -def test_finalize_write_stream_from_dict(): - test_finalize_write_stream(request_type=dict) - - -def test_finalize_write_stream_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.finalize_write_stream), - '__call__') as call: - client.finalize_write_stream() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.FinalizeWriteStreamRequest() - - -@pytest.mark.asyncio -async def test_finalize_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.FinalizeWriteStreamRequest): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.finalize_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse( - row_count=992, - )) - response = await client.finalize_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.FinalizeWriteStreamRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, storage.FinalizeWriteStreamResponse) - assert response.row_count == 992 - - -@pytest.mark.asyncio -async def test_finalize_write_stream_async_from_dict(): - await test_finalize_write_stream_async(request_type=dict) - - -def test_finalize_write_stream_field_headers(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.FinalizeWriteStreamRequest() - - request.name = 'name/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.finalize_write_stream), - '__call__') as call: - call.return_value = storage.FinalizeWriteStreamResponse() - client.finalize_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'name=name/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_finalize_write_stream_field_headers_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.FinalizeWriteStreamRequest() - - request.name = 'name/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.finalize_write_stream), - '__call__') as call: - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse()) - await client.finalize_write_stream(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'name=name/value', - ) in kw['metadata'] - - -def test_finalize_write_stream_flattened(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.finalize_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.FinalizeWriteStreamResponse() - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - client.finalize_write_stream( - name='name_value', - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0].name == 'name_value' - - -def test_finalize_write_stream_flattened_error(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - client.finalize_write_stream( - storage.FinalizeWriteStreamRequest(), - name='name_value', - ) - - -@pytest.mark.asyncio -async def test_finalize_write_stream_flattened_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.finalize_write_stream), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.FinalizeWriteStreamResponse() - - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse()) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - response = await client.finalize_write_stream( - name='name_value', - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0].name == 'name_value' - - -@pytest.mark.asyncio -async def test_finalize_write_stream_flattened_error_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - await client.finalize_write_stream( - storage.FinalizeWriteStreamRequest(), - name='name_value', - ) - - -def test_batch_commit_write_streams(transport: str = 'grpc', request_type=storage.BatchCommitWriteStreamsRequest): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.batch_commit_write_streams), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.BatchCommitWriteStreamsResponse( - ) - response = client.batch_commit_write_streams(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.BatchCommitWriteStreamsRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, storage.BatchCommitWriteStreamsResponse) - - -def test_batch_commit_write_streams_from_dict(): - test_batch_commit_write_streams(request_type=dict) - - -def test_batch_commit_write_streams_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.batch_commit_write_streams), - '__call__') as call: - client.batch_commit_write_streams() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.BatchCommitWriteStreamsRequest() - - -@pytest.mark.asyncio -async def test_batch_commit_write_streams_async(transport: str = 'grpc_asyncio', request_type=storage.BatchCommitWriteStreamsRequest): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.batch_commit_write_streams), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse( - )) - response = await client.batch_commit_write_streams(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.BatchCommitWriteStreamsRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, storage.BatchCommitWriteStreamsResponse) - - -@pytest.mark.asyncio -async def test_batch_commit_write_streams_async_from_dict(): - await test_batch_commit_write_streams_async(request_type=dict) - - -def test_batch_commit_write_streams_field_headers(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.BatchCommitWriteStreamsRequest() - - request.parent = 'parent/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.batch_commit_write_streams), - '__call__') as call: - call.return_value = storage.BatchCommitWriteStreamsResponse() - client.batch_commit_write_streams(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'parent=parent/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_batch_commit_write_streams_field_headers_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.BatchCommitWriteStreamsRequest() - - request.parent = 'parent/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.batch_commit_write_streams), - '__call__') as call: - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse()) - await client.batch_commit_write_streams(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'parent=parent/value', - ) in kw['metadata'] - - -def test_batch_commit_write_streams_flattened(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.batch_commit_write_streams), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.BatchCommitWriteStreamsResponse() - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - client.batch_commit_write_streams( - parent='parent_value', - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0].parent == 'parent_value' - - -def test_batch_commit_write_streams_flattened_error(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - client.batch_commit_write_streams( - storage.BatchCommitWriteStreamsRequest(), - parent='parent_value', - ) - - -@pytest.mark.asyncio -async def test_batch_commit_write_streams_flattened_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.batch_commit_write_streams), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.BatchCommitWriteStreamsResponse() - - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse()) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - response = await client.batch_commit_write_streams( - parent='parent_value', - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0].parent == 'parent_value' - - -@pytest.mark.asyncio -async def test_batch_commit_write_streams_flattened_error_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - await client.batch_commit_write_streams( - storage.BatchCommitWriteStreamsRequest(), - parent='parent_value', - ) - - -def test_flush_rows(transport: str = 'grpc', request_type=storage.FlushRowsRequest): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.flush_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.FlushRowsResponse( - offset=647, - ) - response = client.flush_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == storage.FlushRowsRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, storage.FlushRowsResponse) - assert response.offset == 647 - - -def test_flush_rows_from_dict(): - test_flush_rows(request_type=dict) - - -def test_flush_rows_empty_call(): - # This test is a coverage failsafe to make sure that totally empty calls, - # i.e. request == None and no flattened fields passed, work. - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport='grpc', - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.flush_rows), - '__call__') as call: - client.flush_rows() - call.assert_called() - _, args, _ = call.mock_calls[0] - assert args[0] == storage.FlushRowsRequest() - - -@pytest.mark.asyncio -async def test_flush_rows_async(transport: str = 'grpc_asyncio', request_type=storage.FlushRowsRequest): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # Everything is optional in proto3 as far as the runtime is concerned, - # and we are mocking out the actual API, so just send an empty request. - request = request_type() - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.flush_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse( - offset=647, - )) - response = await client.flush_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == storage.FlushRowsRequest() - - # Establish that the response is the type that we expect. - assert isinstance(response, storage.FlushRowsResponse) - assert response.offset == 647 - - -@pytest.mark.asyncio -async def test_flush_rows_async_from_dict(): - await test_flush_rows_async(request_type=dict) - - -def test_flush_rows_field_headers(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.FlushRowsRequest() - - request.write_stream = 'write_stream/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.flush_rows), - '__call__') as call: - call.return_value = storage.FlushRowsResponse() - client.flush_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'write_stream=write_stream/value', - ) in kw['metadata'] - - -@pytest.mark.asyncio -async def test_flush_rows_field_headers_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Any value that is part of the HTTP/1.1 URI should be sent as - # a field header. Set these to a non-empty value. - request = storage.FlushRowsRequest() - - request.write_stream = 'write_stream/value' - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.flush_rows), - '__call__') as call: - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse()) - await client.flush_rows(request) - - # Establish that the underlying gRPC stub method was called. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0] == request - - # Establish that the field header was sent. - _, _, kw = call.mock_calls[0] - assert ( - 'x-goog-request-params', - 'write_stream=write_stream/value', - ) in kw['metadata'] - - -def test_flush_rows_flattened(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.flush_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.FlushRowsResponse() - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - client.flush_rows( - write_stream='write_stream_value', - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) == 1 - _, args, _ = call.mock_calls[0] - assert args[0].write_stream == 'write_stream_value' - - -def test_flush_rows_flattened_error(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - client.flush_rows( - storage.FlushRowsRequest(), - write_stream='write_stream_value', - ) - - -@pytest.mark.asyncio -async def test_flush_rows_flattened_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Mock the actual call within the gRPC stub, and fake the request. - with mock.patch.object( - type(client.transport.flush_rows), - '__call__') as call: - # Designate an appropriate return value for the call. - call.return_value = storage.FlushRowsResponse() - - call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse()) - # Call the method with a truthy value for each flattened field, - # using the keyword arguments to the method. - response = await client.flush_rows( - write_stream='write_stream_value', - ) - - # Establish that the underlying call was made with the expected - # request object values. - assert len(call.mock_calls) - _, args, _ = call.mock_calls[0] - assert args[0].write_stream == 'write_stream_value' - - -@pytest.mark.asyncio -async def test_flush_rows_flattened_error_async(): - client = BigQueryWriteAsyncClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Attempting to call a method with both a request object and flattened - # fields is an error. - with pytest.raises(ValueError): - await client.flush_rows( - storage.FlushRowsRequest(), - write_stream='write_stream_value', - ) - - -def test_credentials_transport_error(): - # It is an error to provide credentials and a transport instance. - transport = transports.BigQueryWriteGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - transport=transport, - ) - - # It is an error to provide a credentials file and a transport instance. - transport = transports.BigQueryWriteGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): - client = BigQueryWriteClient( - client_options={"credentials_file": "credentials.json"}, - transport=transport, - ) - - # It is an error to provide scopes and a transport instance. - transport = transports.BigQueryWriteGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - with pytest.raises(ValueError): - client = BigQueryWriteClient( - client_options={"scopes": ["1", "2"]}, - transport=transport, - ) - - -def test_transport_instance(): - # A client may be instantiated with a custom transport instance. - transport = transports.BigQueryWriteGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - client = BigQueryWriteClient(transport=transport) - assert client.transport is transport - -def test_transport_get_channel(): - # A client may be instantiated with a custom transport instance. - transport = transports.BigQueryWriteGrpcTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - channel = transport.grpc_channel - assert channel - - transport = transports.BigQueryWriteGrpcAsyncIOTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - channel = transport.grpc_channel - assert channel - -@pytest.mark.parametrize("transport_class", [ - transports.BigQueryWriteGrpcTransport, - transports.BigQueryWriteGrpcAsyncIOTransport, -]) -def test_transport_adc(transport_class): - # Test default credentials are used if not provided. - with mock.patch.object(google.auth, 'default') as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport_class() - adc.assert_called_once() - -def test_transport_grpc_default(): - # A client should use the gRPC transport by default. - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - ) - assert isinstance( - client.transport, - transports.BigQueryWriteGrpcTransport, - ) - -def test_big_query_write_base_transport_error(): - # Passing both a credentials object and credentials_file should raise an error - with pytest.raises(core_exceptions.DuplicateCredentialArgs): - transport = transports.BigQueryWriteTransport( - credentials=ga_credentials.AnonymousCredentials(), - credentials_file="credentials.json" - ) - - -def test_big_query_write_base_transport(): - # Instantiate the base transport. - with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport.__init__') as Transport: - Transport.return_value = None - transport = transports.BigQueryWriteTransport( - credentials=ga_credentials.AnonymousCredentials(), - ) - - # Every method on the transport should just blindly - # raise NotImplementedError. - methods = ( - 'create_write_stream', - 'append_rows', - 'get_write_stream', - 'finalize_write_stream', - 'batch_commit_write_streams', - 'flush_rows', - ) - for method in methods: - with pytest.raises(NotImplementedError): - getattr(transport, method)(request=object()) - - -@requires_google_auth_gte_1_25_0 -def test_big_query_write_base_transport_with_credentials_file(): - # Instantiate the base transport with a credentials file - with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages') as Transport: - Transport.return_value = None - load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) - transport = transports.BigQueryWriteTransport( - credentials_file="credentials.json", - quota_project_id="octopus", - ) - load_creds.assert_called_once_with("credentials.json", - scopes=None, - default_scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.insertdata', - 'https://www.googleapis.com/auth/cloud-platform', -), - quota_project_id="octopus", - ) - - -@requires_google_auth_lt_1_25_0 -def test_big_query_write_base_transport_with_credentials_file_old_google_auth(): - # Instantiate the base transport with a credentials file - with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages') as Transport: - Transport.return_value = None - load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) - transport = transports.BigQueryWriteTransport( - credentials_file="credentials.json", - quota_project_id="octopus", - ) - load_creds.assert_called_once_with("credentials.json", scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.insertdata', - 'https://www.googleapis.com/auth/cloud-platform', - ), - quota_project_id="octopus", - ) - - -def test_big_query_write_base_transport_with_adc(): - # Test the default credentials are used if credentials and credentials_file are None. - with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages') as Transport: - Transport.return_value = None - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport = transports.BigQueryWriteTransport() - adc.assert_called_once() - - -@requires_google_auth_gte_1_25_0 -def test_big_query_write_auth_adc(): - # If no credentials are provided, we should use ADC credentials. - with mock.patch.object(google.auth, 'default', autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - BigQueryWriteClient() - adc.assert_called_once_with( - scopes=None, - default_scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.insertdata', - 'https://www.googleapis.com/auth/cloud-platform', -), - quota_project_id=None, - ) - - -@requires_google_auth_lt_1_25_0 -def test_big_query_write_auth_adc_old_google_auth(): - # If no credentials are provided, we should use ADC credentials. - with mock.patch.object(google.auth, 'default', autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - BigQueryWriteClient() - adc.assert_called_once_with( - scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.insertdata', 'https://www.googleapis.com/auth/cloud-platform',), - quota_project_id=None, - ) - - -@pytest.mark.parametrize( - "transport_class", - [ - transports.BigQueryWriteGrpcTransport, - transports.BigQueryWriteGrpcAsyncIOTransport, - ], -) -@requires_google_auth_gte_1_25_0 -def test_big_query_write_transport_auth_adc(transport_class): - # If credentials and host are not provided, the transport class should use - # ADC credentials. - with mock.patch.object(google.auth, 'default', autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport_class(quota_project_id="octopus", scopes=["1", "2"]) - adc.assert_called_once_with( - scopes=["1", "2"], - default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.insertdata', 'https://www.googleapis.com/auth/cloud-platform',), - quota_project_id="octopus", - ) - - -@pytest.mark.parametrize( - "transport_class", - [ - transports.BigQueryWriteGrpcTransport, - transports.BigQueryWriteGrpcAsyncIOTransport, - ], -) -@requires_google_auth_lt_1_25_0 -def test_big_query_write_transport_auth_adc_old_google_auth(transport_class): - # If credentials and host are not provided, the transport class should use - # ADC credentials. - with mock.patch.object(google.auth, "default", autospec=True) as adc: - adc.return_value = (ga_credentials.AnonymousCredentials(), None) - transport_class(quota_project_id="octopus") - adc.assert_called_once_with(scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.insertdata', - 'https://www.googleapis.com/auth/cloud-platform', -), - quota_project_id="octopus", - ) - - -@pytest.mark.parametrize( - "transport_class,grpc_helpers", - [ - (transports.BigQueryWriteGrpcTransport, grpc_helpers), - (transports.BigQueryWriteGrpcAsyncIOTransport, grpc_helpers_async) - ], -) -def test_big_query_write_transport_create_channel(transport_class, grpc_helpers): - # If credentials and host are not provided, the transport class should use - # ADC credentials. - with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( - grpc_helpers, "create_channel", autospec=True - ) as create_channel: - creds = ga_credentials.AnonymousCredentials() - adc.return_value = (creds, None) - transport_class( - quota_project_id="octopus", - scopes=["1", "2"] - ) - - create_channel.assert_called_with( - "bigquerystorage.googleapis.com:443", - credentials=creds, - credentials_file=None, - quota_project_id="octopus", - default_scopes=( - 'https://www.googleapis.com/auth/bigquery', - 'https://www.googleapis.com/auth/bigquery.insertdata', - 'https://www.googleapis.com/auth/cloud-platform', -), - scopes=["1", "2"], - default_host="bigquerystorage.googleapis.com", - ssl_credentials=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - -@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) -def test_big_query_write_grpc_transport_client_cert_source_for_mtls( - transport_class -): - cred = ga_credentials.AnonymousCredentials() - - # Check ssl_channel_credentials is used if provided. - with mock.patch.object(transport_class, "create_channel") as mock_create_channel: - mock_ssl_channel_creds = mock.Mock() - transport_class( - host="squid.clam.whelk", - credentials=cred, - ssl_channel_credentials=mock_ssl_channel_creds - ) - mock_create_channel.assert_called_once_with( - "squid.clam.whelk:443", - credentials=cred, - credentials_file=None, - scopes=None, - ssl_credentials=mock_ssl_channel_creds, - quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - - # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls - # is used. - with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): - with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: - transport_class( - credentials=cred, - client_cert_source_for_mtls=client_cert_source_callback - ) - expected_cert, expected_key = client_cert_source_callback() - mock_ssl_cred.assert_called_once_with( - certificate_chain=expected_cert, - private_key=expected_key - ) - - -def test_big_query_write_host_no_port(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), - ) - assert client.transport._host == 'bigquerystorage.googleapis.com:443' - - -def test_big_query_write_host_with_port(): - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), - ) - assert client.transport._host == 'bigquerystorage.googleapis.com:8000' - -def test_big_query_write_grpc_transport_channel(): - channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) - - # Check that channel is used if provided. - transport = transports.BigQueryWriteGrpcTransport( - host="squid.clam.whelk", - channel=channel, - ) - assert transport.grpc_channel == channel - assert transport._host == "squid.clam.whelk:443" - assert transport._ssl_channel_credentials == None - - -def test_big_query_write_grpc_asyncio_transport_channel(): - channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) - - # Check that channel is used if provided. - transport = transports.BigQueryWriteGrpcAsyncIOTransport( - host="squid.clam.whelk", - channel=channel, - ) - assert transport.grpc_channel == channel - assert transport._host == "squid.clam.whelk:443" - assert transport._ssl_channel_credentials == None - - -# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are -# removed from grpc/grpc_asyncio transport constructor. -@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) -def test_big_query_write_transport_channel_mtls_with_client_cert_source( - transport_class -): - with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: - with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: - mock_ssl_cred = mock.Mock() - grpc_ssl_channel_cred.return_value = mock_ssl_cred - - mock_grpc_channel = mock.Mock() - grpc_create_channel.return_value = mock_grpc_channel - - cred = ga_credentials.AnonymousCredentials() - with pytest.warns(DeprecationWarning): - with mock.patch.object(google.auth, 'default') as adc: - adc.return_value = (cred, None) - transport = transport_class( - host="squid.clam.whelk", - api_mtls_endpoint="mtls.squid.clam.whelk", - client_cert_source=client_cert_source_callback, - ) - adc.assert_called_once() - - grpc_ssl_channel_cred.assert_called_once_with( - certificate_chain=b"cert bytes", private_key=b"key bytes" - ) - grpc_create_channel.assert_called_once_with( - "mtls.squid.clam.whelk:443", - credentials=cred, - credentials_file=None, - scopes=None, - ssl_credentials=mock_ssl_cred, - quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - assert transport.grpc_channel == mock_grpc_channel - assert transport._ssl_channel_credentials == mock_ssl_cred - - -# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are -# removed from grpc/grpc_asyncio transport constructor. -@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) -def test_big_query_write_transport_channel_mtls_with_adc( - transport_class -): - mock_ssl_cred = mock.Mock() - with mock.patch.multiple( - "google.auth.transport.grpc.SslCredentials", - __init__=mock.Mock(return_value=None), - ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), - ): - with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: - mock_grpc_channel = mock.Mock() - grpc_create_channel.return_value = mock_grpc_channel - mock_cred = mock.Mock() - - with pytest.warns(DeprecationWarning): - transport = transport_class( - host="squid.clam.whelk", - credentials=mock_cred, - api_mtls_endpoint="mtls.squid.clam.whelk", - client_cert_source=None, - ) - - grpc_create_channel.assert_called_once_with( - "mtls.squid.clam.whelk:443", - credentials=mock_cred, - credentials_file=None, - scopes=None, - ssl_credentials=mock_ssl_cred, - quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - assert transport.grpc_channel == mock_grpc_channel - - -def test_table_path(): - project = "squid" - dataset = "clam" - table = "whelk" - expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) - actual = BigQueryWriteClient.table_path(project, dataset, table) - assert expected == actual - - -def test_parse_table_path(): - expected = { - "project": "octopus", - "dataset": "oyster", - "table": "nudibranch", - } - path = BigQueryWriteClient.table_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryWriteClient.parse_table_path(path) - assert expected == actual - -def test_write_stream_path(): - project = "cuttlefish" - dataset = "mussel" - table = "winkle" - stream = "nautilus" - expected = "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}".format(project=project, dataset=dataset, table=table, stream=stream, ) - actual = BigQueryWriteClient.write_stream_path(project, dataset, table, stream) - assert expected == actual - - -def test_parse_write_stream_path(): - expected = { - "project": "scallop", - "dataset": "abalone", - "table": "squid", - "stream": "clam", - } - path = BigQueryWriteClient.write_stream_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryWriteClient.parse_write_stream_path(path) - assert expected == actual - -def test_common_billing_account_path(): - billing_account = "whelk" - expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) - actual = BigQueryWriteClient.common_billing_account_path(billing_account) - assert expected == actual - - -def test_parse_common_billing_account_path(): - expected = { - "billing_account": "octopus", - } - path = BigQueryWriteClient.common_billing_account_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryWriteClient.parse_common_billing_account_path(path) - assert expected == actual - -def test_common_folder_path(): - folder = "oyster" - expected = "folders/{folder}".format(folder=folder, ) - actual = BigQueryWriteClient.common_folder_path(folder) - assert expected == actual - - -def test_parse_common_folder_path(): - expected = { - "folder": "nudibranch", - } - path = BigQueryWriteClient.common_folder_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryWriteClient.parse_common_folder_path(path) - assert expected == actual - -def test_common_organization_path(): - organization = "cuttlefish" - expected = "organizations/{organization}".format(organization=organization, ) - actual = BigQueryWriteClient.common_organization_path(organization) - assert expected == actual - - -def test_parse_common_organization_path(): - expected = { - "organization": "mussel", - } - path = BigQueryWriteClient.common_organization_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryWriteClient.parse_common_organization_path(path) - assert expected == actual - -def test_common_project_path(): - project = "winkle" - expected = "projects/{project}".format(project=project, ) - actual = BigQueryWriteClient.common_project_path(project) - assert expected == actual - - -def test_parse_common_project_path(): - expected = { - "project": "nautilus", - } - path = BigQueryWriteClient.common_project_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryWriteClient.parse_common_project_path(path) - assert expected == actual - -def test_common_location_path(): - project = "scallop" - location = "abalone" - expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) - actual = BigQueryWriteClient.common_location_path(project, location) - assert expected == actual - - -def test_parse_common_location_path(): - expected = { - "project": "squid", - "location": "clam", - } - path = BigQueryWriteClient.common_location_path(**expected) - - # Check that the path construction is reversible. - actual = BigQueryWriteClient.parse_common_location_path(path) - assert expected == actual - - -def test_client_withDEFAULT_CLIENT_INFO(): - client_info = gapic_v1.client_info.ClientInfo() - - with mock.patch.object(transports.BigQueryWriteTransport, '_prep_wrapped_messages') as prep: - client = BigQueryWriteClient( - credentials=ga_credentials.AnonymousCredentials(), - client_info=client_info, - ) - prep.assert_called_once_with(client_info) - - with mock.patch.object(transports.BigQueryWriteTransport, '_prep_wrapped_messages') as prep: - transport_class = BigQueryWriteClient.get_transport_class() - transport = transport_class( - credentials=ga_credentials.AnonymousCredentials(), - client_info=client_info, - ) - prep.assert_called_once_with(client_info)