From a351734ae16b4a689b89e6a42f63ea3ea5ad84ca Mon Sep 17 00:00:00 2001 From: MF2199 <38331387+mf2199@users.noreply.github.com> Date: Tue, 21 Jul 2020 14:04:02 -0400 Subject: [PATCH 1/2] feat(bigtable): Managed Backups wrappers (#57) * [new] managed backup wrappers + unit tests * feat: managed backups wrappers * fix: docstrings + blacken * fix: cleanup * refactor: ``backup``, ``list_backups`` and ``retore_table`` methods moved to the ``Table`` class. * feat: `reaload` and `is_ready` methods removed * refactor: `re` parser made local * feat: integration test * refactor: cleanup * fix: format * refactor: `name`, `cluster` property getters & `table_list_backups` feat: new `__eq__` and `__ne__` convenience methods * refactor: using `BigtableTableAdminClient.table_path` in lieu of `format` * fix: `from_pb2` method to include all `backup_pb` fields * refactor: cleanup * format: blacken * feat: reinstated `Backup.reload` + test method * fix: docstring typos * cleanup: minor cleanup * cleanup: minor cleanup * fix: ASCII encoding * fix: Python 2 compatibility issue * fix: SphinxWarning [possible cause] * fix: lint errors Co-authored-by: kolea2 <45548808+kolea2@users.noreply.github.com> --- google/cloud/bigtable/backup.py | 393 ++++++++++++++++ google/cloud/bigtable/instance.py | 5 +- google/cloud/bigtable/table.py | 180 +++++++- tests/system.py | 59 +++ tests/unit/test_backup.py | 725 ++++++++++++++++++++++++++++++ tests/unit/test_instance.py | 4 + tests/unit/test_table.py | 151 +++++++ 7 files changed, 1512 insertions(+), 5 deletions(-) create mode 100644 google/cloud/bigtable/backup.py create mode 100644 tests/unit/test_backup.py diff --git a/google/cloud/bigtable/backup.py b/google/cloud/bigtable/backup.py new file mode 100644 index 000000000..c6a2826dd --- /dev/null +++ b/google/cloud/bigtable/backup.py @@ -0,0 +1,393 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A user-friendly wrapper for a Google Cloud Bigtable Backup.""" + +import re + +from google.cloud._helpers import _datetime_to_pb_timestamp +from google.cloud.bigtable_admin_v2.gapic.bigtable_table_admin_client import ( + BigtableTableAdminClient, +) +from google.cloud.bigtable_admin_v2.types import table_pb2 +from google.cloud.exceptions import NotFound +from google.protobuf import field_mask_pb2 + +_BACKUP_NAME_RE = re.compile( + r"^projects/(?P[^/]+)/" + r"instances/(?P[a-z][-a-z0-9]*)/" + r"clusters/(?P[a-z][-a-z0-9]*)/" + r"backups/(?P[a-z][a-z0-9_\-]*[a-z0-9])$" +) + +_TABLE_NAME_RE = re.compile( + r"^projects/(?P[^/]+)/" + r"instances/(?P[a-z][-a-z0-9]*)/" + r"tables/(?P[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$" +) + + +class Backup(object): + """Representation of a Google Cloud Bigtable Backup. + + A :class: `Backup` can be used to: + + * :meth:`create` the backup + * :meth:`update` the backup + * :meth:`delete` the backup + + :type backup_id: str + :param backup_id: The ID of the backup. + + :type instance: :class:`~google.cloud.bigtable.instance.Instance` + :param instance: The Instance that owns this Backup. + + :type cluster_id: str + :param cluster_id: (Optional) The ID of the Cluster that contains this Backup. + Required for calling 'delete', 'exists' etc. methods. + + :type table_id: str + :param table_id: (Optional) The ID of the Table that the Backup is for. + Required if the 'create' method will be called. + + :type expire_time: :class:`datetime.datetime` + :param expire_time: (Optional) The expiration time after which the Backup + will be automatically deleted. Required if the `create` + method will be called. + """ + + def __init__( + self, backup_id, instance, cluster_id=None, table_id=None, expire_time=None + ): + self.backup_id = backup_id + self._instance = instance + self._cluster = cluster_id + self.table_id = table_id + self._expire_time = expire_time + + self._parent = None + self._source_table = None + self._start_time = None + self._end_time = None + self._size_bytes = None + self._state = None + + @property + def name(self): + """Backup name used in requests. + + The Backup name is of the form + + ``"projects/../instances/../clusters/../backups/{backup_id}"`` + + :rtype: str + :returns: The Backup name. + + :raises: ValueError: If the 'cluster' has not been set. + """ + if not self._cluster: + raise ValueError('"cluster" parameter must be set') + + return BigtableTableAdminClient.backup_path( + project=self._instance._client.project, + instance=self._instance.instance_id, + cluster=self._cluster, + backup=self.backup_id, + ) + + @property + def cluster(self): + """The ID of the [parent] cluster used in requests. + + :rtype: str + :returns: The ID of the cluster containing the Backup. + """ + return self._cluster + + @cluster.setter + def cluster(self, cluster_id): + self._cluster = cluster_id + + @property + def parent(self): + """Name of the parent cluster used in requests. + + .. note:: + This property will return None if ``cluster`` is not set. + + The parent name is of the form + + ``"projects/{project}/instances/{instance_id}/clusters/{cluster}"`` + + :rtype: str + :returns: A full path to the parent cluster. + """ + if not self._parent and self._cluster: + self._parent = BigtableTableAdminClient.cluster_path( + project=self._instance._client.project, + instance=self._instance.instance_id, + cluster=self._cluster, + ) + return self._parent + + @property + def source_table(self): + """The full name of the Table from which this Backup is created. + + .. note:: + This property will return None if ``table_id`` is not set. + + The table name is of the form + + ``"projects/../instances/../tables/{source_table}"`` + + :rtype: str + :returns: The Table name. + """ + if not self._source_table and self.table_id: + self._source_table = BigtableTableAdminClient.table_path( + project=self._instance._client.project, + instance=self._instance.instance_id, + table=self.table_id, + ) + return self._source_table + + @property + def expire_time(self): + """Expiration time used in the creation requests. + + :rtype: :class:`datetime.datetime` + :returns: A 'datetime' object representing the expiration time of + this Backup. + """ + return self._expire_time + + @expire_time.setter + def expire_time(self, new_expire_time): + self._expire_time = new_expire_time + + @property + def start_time(self): + """The time this Backup was started. + + :rtype: :class:`datetime.datetime` + :returns: A 'datetime' object representing the time when the creation + of this Backup had started. + """ + return self._start_time + + @property + def end_time(self): + """The time this Backup was finished. + + :rtype: :class:`datetime.datetime` + :returns: A 'datetime' object representing the time when the creation + of this Backup was finished. + """ + return self._end_time + + @property + def size_bytes(self): + """The size of this Backup, in bytes. + + :rtype: int + :returns: The size of this Backup, in bytes. + """ + return self._size_bytes + + @property + def state(self): + """ The current state of this Backup. + + :rtype: :class:`~google.cloud.bigtable_admin_v2.gapic.enums.Backup.State` + :returns: The current state of this Backup. + """ + return self._state + + @classmethod + def from_pb(cls, backup_pb, instance): + """Creates a Backup instance from a protobuf message. + + :type backup_pb: :class:`table_pb2.Backup` + :param backup_pb: A Backup protobuf object. + + :type instance: :class:`Instance ` + :param instance: The Instance that owns the Backup. + + :rtype: :class:`~google.cloud.bigtable.backup.Backup` + :returns: The backup parsed from the protobuf response. + :raises: ValueError: If the backup name does not match the expected + format or the parsed project ID does not match the + project ID on the Instance's client, or if the + parsed instance ID does not match the Instance ID. + """ + match = _BACKUP_NAME_RE.match(backup_pb.name) + if match is None: + raise ValueError( + "Backup protobuf name was not in the expected format.", backup_pb.name + ) + if match.group("project") != instance._client.project: + raise ValueError( + "Project ID of the Backup does not match the Project ID " + "of the instance's client" + ) + + instance_id = match.group("instance_id") + if instance_id != instance.instance_id: + raise ValueError( + "Instance ID of the Backup does not match the Instance ID " + "of the instance" + ) + backup_id = match.group("backup_id") + cluster_id = match.group("cluster_id") + + match = _TABLE_NAME_RE.match(backup_pb.source_table) + table_id = match.group("table_id") if match else None + + expire_time = backup_pb.expire_time + + backup = cls( + backup_id, + instance, + cluster_id=cluster_id, + table_id=table_id, + expire_time=expire_time, + ) + backup._start_time = backup_pb.start_time + backup._end_time = backup_pb.end_time + backup._size_bytes = backup_pb.size_bytes + backup._state = backup_pb.state + + return backup + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return other.backup_id == self.backup_id and other._instance == self._instance + + def __ne__(self, other): + return not self == other + + def create(self, cluster_id=None): + """Creates this backup within its instance. + + :type cluster_id: str + :param cluster_id: (Optional) The ID of the Cluster for the newly + created Backup. + + :rtype: :class:`~google.api_core.operation.Operation` + :returns: :class:`~google.cloud.bigtable_admin_v2.types._OperationFuture` + instance, to be used to poll the status of the 'create' request + :raises Conflict: if the Backup already exists + :raises NotFound: if the Instance owning the Backup does not exist + :raises BadRequest: if the `table` or `expire_time` values are invalid, + or `expire_time` is not set + """ + if not self._expire_time: + raise ValueError('"expire_time" parameter must be set') + # TODO: Consider implementing a method that sets a default value of + # `expire_time`, e.g. 1 week from the creation of the Backup. + if not self.table_id: + raise ValueError('"table" parameter must be set') + + if cluster_id: + self._cluster = cluster_id + + if not self._cluster: + raise ValueError('"cluster" parameter must be set') + + backup = table_pb2.Backup( + source_table=self.source_table, + expire_time=_datetime_to_pb_timestamp(self.expire_time), + ) + + api = self._instance._client.table_admin_client + return api.create_backup(self.parent, self.backup_id, backup) + + def get(self): + """Retrieves metadata of a pending or completed Backup. + + :returns: An instance of + :class:`~google.cloud.bigtable_admin_v2.types.Backup` + + :raises google.api_core.exceptions.GoogleAPICallError: If the request + failed for any reason. + :raises google.api_core.exceptions.RetryError: If the request failed + due to a retryable error and retry attempts failed. + :raises ValueError: If the parameters are invalid. + """ + api = self._instance._client.table_admin_client + try: + return api.get_backup(self.name) + except NotFound: + return None + + def reload(self): + """Refreshes the stored backup properties.""" + backup = self.get() + self._source_table = backup.source_table + self._expire_time = backup.expire_time + self._start_time = backup.start_time + self._end_time = backup.end_time + self._size_bytes = backup.size_bytes + self._state = backup.state + + def exists(self): + """Tests whether this Backup exists. + + :rtype: bool + :returns: True if the Backup exists, else False. + """ + return self.get() is not None + + def update_expire_time(self, new_expire_time): + """Update the expire time of this Backup. + + :type new_expire_time: :class:`datetime.datetime` + :param new_expire_time: the new expiration time timestamp + """ + backup_update = table_pb2.Backup( + name=self.name, expire_time=_datetime_to_pb_timestamp(new_expire_time), + ) + update_mask = field_mask_pb2.FieldMask(paths=["expire_time"]) + api = self._instance._client.table_admin_client + api.update_backup(backup_update, update_mask) + self._expire_time = new_expire_time + + def delete(self): + """Delete this Backup.""" + self._instance._client.table_admin_client.delete_backup(self.name) + + def restore(self, table_id): + """Creates a new Table by restoring from this Backup. The new Table + must be in the same Instance as the Instance containing the Backup. + The returned Table ``long-running operation`` can be used to track the + progress of the operation and to cancel it. The ``response`` type is + ``Table``, if successful. + + :param table_id: The ID of the Table to create and restore to. + This Table must not already exist. + :returns: An instance of + :class:`~google.cloud.bigtable_admin_v2.types._OperationFuture`. + + :raises: google.api_core.exceptions.AlreadyExists: If the table + already exists. + :raises: google.api_core.exceptions.GoogleAPICallError: If the request + failed for any reason. + :raises: google.api_core.exceptions.RetryError: If the request failed + due to a retryable error and retry attempts failed. + :raises: ValueError: If the parameters are invalid. + """ + api = self._instance._client.table_admin_client + return api.restore_table(self._instance.name, table_id, self.name) diff --git a/google/cloud/bigtable/instance.py b/google/cloud/bigtable/instance.py index e0a30590b..0c8b81fa3 100644 --- a/google/cloud/bigtable/instance.py +++ b/google/cloud/bigtable/instance.py @@ -14,12 +14,11 @@ """User-friendly container for Google Cloud Bigtable Instance.""" - import re -from google.cloud.bigtable.table import Table -from google.cloud.bigtable.cluster import Cluster from google.cloud.bigtable.app_profile import AppProfile +from google.cloud.bigtable.cluster import Cluster +from google.cloud.bigtable.table import Table from google.protobuf import field_mask_pb2 diff --git a/google/cloud/bigtable/table.py b/google/cloud/bigtable/table.py index 4852ff6e1..983bfcc14 100644 --- a/google/cloud/bigtable/table.py +++ b/google/cloud/bigtable/table.py @@ -14,7 +14,6 @@ """User-friendly container for Google Cloud Bigtable Table.""" - from grpc import StatusCode from google.api_core import timeout @@ -24,6 +23,7 @@ from google.api_core.retry import Retry from google.api_core.gapic_v1.method import wrap_method from google.cloud._helpers import _to_bytes +from google.cloud.bigtable.backup import Backup from google.cloud.bigtable.column_family import _gc_rule_from_pb from google.cloud.bigtable.column_family import ColumnFamily from google.cloud.bigtable.batcher import MutationsBatcher @@ -38,6 +38,9 @@ from google.cloud.bigtable.row_set import RowRange from google.cloud.bigtable import enums from google.cloud.bigtable_v2.proto import bigtable_pb2 as data_messages_v2_pb2 +from google.cloud.bigtable_admin_v2.gapic.bigtable_table_admin_client import ( + BigtableTableAdminClient, +) from google.cloud.bigtable_admin_v2.proto import table_pb2 as admin_messages_v2_pb2 from google.cloud.bigtable_admin_v2.proto import ( bigtable_table_admin_pb2 as table_admin_messages_v2_pb2, @@ -45,7 +48,6 @@ import warnings - # Maximum number of mutations in bulk (MutateRowsRequest message): # (https://cloud.google.com/bigtable/docs/reference/data/rpc/ # google.bigtable.v2#google.bigtable.v2.MutateRowRequest) @@ -782,6 +784,179 @@ def mutations_batcher(self, flush_count=FLUSH_COUNT, max_row_bytes=MAX_ROW_BYTES """ return MutationsBatcher(self, flush_count, max_row_bytes) + def backup(self, backup_id, cluster_id=None, expire_time=None): + """Factory to create a Backup linked to this Table. + + :type backup_id: str + :param backup_id: The ID of the Backup to be created. + + :type cluster_id: str + :param cluster_id: (Optional) The ID of the Cluster. Required for + calling 'delete', 'exists' etc. methods. + + :type expire_time: :class:`datetime.datetime` + :param expire_time: (Optional) The expiration time of this new Backup. + Required, if the `create` method needs to be called. + """ + return Backup( + backup_id, + self._instance, + cluster_id=cluster_id, + table_id=self.table_id, + expire_time=expire_time, + ) + + def list_backups(self, cluster_id=None, filter_=None, order_by=None, page_size=0): + """List Backups for this Table. + + :type cluster_id: str + :param cluster_id: (Optional) Specifies a single cluster to list + Backups from. If none is specified, the returned list + contains all the Backups in this Instance. + + :type filter_: str + :param filter_: (Optional) A filter expression that filters backups + listed in the response. The expression must specify + the field name, a comparison operator, and the value + that you want to use for filtering. The value must be + a string, a number, or a boolean. The comparison + operator must be <, >, <=, >=, !=, =, or :. Colon ':' + represents a HAS operator which is roughly synonymous + with equality. Filter rules are case insensitive. + + The fields eligible for filtering are: + + - ``name`` + - ``source_table`` + - ``state`` + - ``start_time`` (values of the format YYYY-MM-DDTHH:MM:SSZ) + - ``end_time`` (values of the format YYYY-MM-DDTHH:MM:SSZ) + - ``expire_time`` (values of the format YYYY-MM-DDTHH:MM:SSZ) + - ``size_bytes`` + + To filter on multiple expressions, provide each + separate expression within parentheses. By default, + each expression is an AND expression. However, you can + include AND, OR, and NOT expressions explicitly. + + Some examples of using filters are: + + - ``name:"exact"`` --> The Backup name is the string "exact". + - ``name:howl`` --> The Backup name contains the string "howl" + - ``source_table:prod`` --> The source table's name contains + the string "prod". + - ``state:CREATING`` --> The Backup is pending creation. + - ``state:READY`` --> The Backup is created and ready for use. + - ``(name:howl) AND (start_time < \"2020-05-28T14:50:00Z\")`` + --> The Backup name contains the string "howl" and + the Backup start time is before 2020-05-28T14:50:00Z. + - ``size_bytes > 10000000000`` --> The Backup size is greater + than 10GB + + :type order_by: str + :param order_by: (Optional) An expression for specifying the sort order + of the results of the request. The string value should + specify one or more fields in ``Backup``. The full + syntax is described at https://aip.dev/132#ordering. + + Fields supported are: \\* name \\* source_table \\* + expire_time \\* start_time \\* end_time \\* + size_bytes \\* state + + For example, "start_time". The default sorting order + is ascending. To specify descending order for the + field, a suffix " desc" should be appended to the + field name. For example, "start_time desc". Redundant + space characters in the syntax are insigificant. If + order_by is empty, results will be sorted by + ``start_time`` in descending order starting from + the most recently created backup. + + :type page_size: int + :param page_size: (Optional) The maximum number of resources contained + in the underlying API response. If page streaming is + performed per-resource, this parameter does not + affect the return value. If page streaming is + performed per-page, this determines the maximum + number of resources in a page. + + :rtype: :class:`~google.api_core.page_iterator.Iterator` + :returns: Iterator of :class:`~google.cloud.bigtable.backup.Backup` + resources within the current Instance. + :raises: :class:`ValueError ` if one of the + returned Backups' name is not of the expected format. + """ + cluster_id = cluster_id or "-" + + backups_filter = "source_table:{}".format(self.name) + if filter_: + backups_filter = "({}) AND ({})".format(backups_filter, filter_) + + parent = BigtableTableAdminClient.cluster_path( + project=self._instance._client.project, + instance=self._instance.instance_id, + cluster=cluster_id, + ) + client = self._instance._client.table_admin_client + backup_list_pb = client.list_backups( + parent=parent, + filter_=backups_filter, + order_by=order_by, + page_size=page_size, + ) + + result = [] + for backup_pb in backup_list_pb: + result.append(Backup.from_pb(backup_pb, self._instance)) + + return result + + def restore(self, new_table_id, cluster_id=None, backup_id=None, backup_name=None): + """Creates a new Table by restoring from the Backup specified by either + `backup_id` or `backup_name`. The returned ``long-running operation`` + can be used to track the progress of the operation and to cancel it. + The ``response`` type is ``Table``, if successful. + + :type new_table_id: str + :param new_table_id: The ID of the Table to create and restore to. + This Table must not already exist. + + :type cluster_id: str + :param cluster_id: The ID of the Cluster containing the Backup. + This parameter gets overriden by `backup_name`, if + the latter is provided. + + :type backup_id: str + :param backup_id: The ID of the Backup to restore the Table from. + This parameter gets overriden by `backup_name`, if + the latter is provided. + + :type backup_name: str + :param backup_name: (Optional) The full name of the Backup to restore + from. If specified, it overrides the `cluster_id` + and `backup_id` parameters even of such specified. + + :return: An instance of + :class:`~google.cloud.bigtable_admin_v2.types._OperationFuture`. + + :raises: google.api_core.exceptions.AlreadyExists: If the table + already exists. + :raises: google.api_core.exceptions.GoogleAPICallError: If the request + failed for any reason. + :raises: google.api_core.exceptions.RetryError: If the request failed + due to a retryable error and retry attempts failed. + :raises: ValueError: If the parameters are invalid. + """ + api = self._instance._client.table_admin_client + if not backup_name: + backup_name = BigtableTableAdminClient.backup_path( + project=self._instance._client.project, + instance=self._instance.instance_id, + cluster=cluster_id, + backup=backup_id, + ) + return api.restore_table(self._instance.name, new_table_id, backup_name) + class _RetryableMutateRowsWorker(object): """A callable worker that can retry to mutate rows with transient errors. @@ -797,6 +972,7 @@ class _RetryableMutateRowsWorker(object): StatusCode.ABORTED.value[0], StatusCode.UNAVAILABLE.value[0], ) + # pylint: enable=unsubscriptable-object def __init__(self, client, table_name, rows, app_profile_id=None, timeout=None): diff --git a/tests/system.py b/tests/system.py index dd77dd936..c41c90a6a 100644 --- a/tests/system.py +++ b/tests/system.py @@ -15,6 +15,7 @@ import datetime import operator import os +import time import unittest from google.api_core.exceptions import TooManyRequests @@ -652,10 +653,13 @@ def tearDownClass(cls): def setUp(self): self.tables_to_delete = [] + self.backups_to_delete = [] def tearDown(self): for table in self.tables_to_delete: table.delete() + for backup in self.backups_to_delete: + backup.delete() def _skip_if_emulated(self, message): # NOTE: This method is necessary because ``Config.IN_EMULATOR`` @@ -829,6 +833,61 @@ def test_delete_column_family(self): # Make sure we have successfully deleted it. self.assertEqual(temp_table.list_column_families(), {}) + def test_backup(self): + temp_table_id = "test-backup-table" + temp_table = Config.INSTANCE_DATA.table(temp_table_id) + temp_table.create() + self.tables_to_delete.append(temp_table) + + temp_backup_id = "test-backup" + + # TODO: consider using `datetime.datetime.now().timestamp()` + # when support for Python 2 is fully dropped + expire = int(time.mktime(datetime.datetime.now().timetuple())) + 604800 + + # Testing `Table.backup()` factory + temp_backup = temp_table.backup( + temp_backup_id, + cluster_id=CLUSTER_ID_DATA, + expire_time=datetime.datetime.utcfromtimestamp(expire), + ) + + # Sanity check for `Backup.exists()` method + self.assertFalse(temp_backup.exists()) + + # Testing `Backup.create()` method + temp_backup.create().result() + + # Implicit testing of `Backup.delete()` method + self.backups_to_delete.append(temp_backup) + + # Testing `Backup.exists()` method + self.assertTrue(temp_backup.exists()) + + # Testing `Table.list_backups()` method + temp_table_backup = temp_table.list_backups()[0] + self.assertEqual(temp_backup_id, temp_table_backup.backup_id) + self.assertEqual(CLUSTER_ID_DATA, temp_table_backup.cluster) + self.assertEqual(expire, temp_table_backup.expire_time.seconds) + + # Testing `Backup.update_expire_time()` method + expire += 3600 # A one-hour change in the `expire_time` parameter + temp_backup.update_expire_time(datetime.datetime.utcfromtimestamp(expire)) + + # Testing `Backup.get()` method + temp_table_backup = temp_backup.get() + self.assertEqual(expire, temp_table_backup.expire_time.seconds) + + # Testing `Table.restore()` and `Backup.retore()` methods + restored_table_id = "test-backup-table-restored" + restored_table = Config.INSTANCE_DATA.table(restored_table_id) + temp_table.restore( + restored_table_id, cluster_id=CLUSTER_ID_DATA, backup_id=temp_backup_id + ).result() + tables = Config.INSTANCE_DATA.list_tables() + self.assertIn(restored_table, tables) + restored_table.delete() + class TestDataAPI(unittest.TestCase): @classmethod diff --git a/tests/unit/test_backup.py b/tests/unit/test_backup.py new file mode 100644 index 000000000..587202a84 --- /dev/null +++ b/tests/unit/test_backup.py @@ -0,0 +1,725 @@ +# Copyright 2020 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import datetime +import mock +import unittest + +from ._testing import _make_credentials +from google.cloud._helpers import UTC + + +class TestBackup(unittest.TestCase): + PROJECT_ID = "project-id" + INSTANCE_ID = "instance-id" + INSTANCE_NAME = "projects/" + PROJECT_ID + "/instances/" + INSTANCE_ID + CLUSTER_ID = "cluster-id" + CLUSTER_NAME = INSTANCE_NAME + "/clusters/" + CLUSTER_ID + TABLE_ID = "table-id" + TABLE_NAME = INSTANCE_NAME + "/tables/" + TABLE_ID + BACKUP_ID = "backup-id" + BACKUP_NAME = CLUSTER_NAME + "/backups/" + BACKUP_ID + + @staticmethod + def _get_target_class(): + from google.cloud.bigtable.backup import Backup + + return Backup + + @staticmethod + def _make_table_admin_client(): + from google.cloud.bigtable_admin_v2 import BigtableTableAdminClient + + return mock.create_autospec(BigtableTableAdminClient, instance=True) + + def _make_one(self, *args, **kwargs): + return self._get_target_class()(*args, **kwargs) + + def _make_timestamp(self): + return datetime.datetime.utcnow().replace(tzinfo=UTC) + + def test_constructor_defaults(self): + instance = _Instance(self.INSTANCE_NAME) + backup = self._make_one(self.BACKUP_ID, instance) + + self.assertEqual(backup.backup_id, self.BACKUP_ID) + self.assertIs(backup._instance, instance) + self.assertIsNone(backup._cluster) + self.assertIsNone(backup.table_id) + self.assertIsNone(backup._expire_time) + + self.assertIsNone(backup._parent) + self.assertIsNone(backup._source_table) + self.assertIsNone(backup._start_time) + self.assertIsNone(backup._end_time) + self.assertIsNone(backup._size_bytes) + self.assertIsNone(backup._state) + + def test_constructor_non_defaults(self): + instance = _Instance(self.INSTANCE_NAME) + expire_time = self._make_timestamp() + + backup = self._make_one( + self.BACKUP_ID, + instance, + cluster_id=self.CLUSTER_ID, + table_id=self.TABLE_ID, + expire_time=expire_time, + ) + + self.assertEqual(backup.backup_id, self.BACKUP_ID) + self.assertIs(backup._instance, instance) + self.assertIs(backup._cluster, self.CLUSTER_ID) + self.assertEqual(backup.table_id, self.TABLE_ID) + self.assertEqual(backup._expire_time, expire_time) + + self.assertIsNone(backup._parent) + self.assertIsNone(backup._source_table) + self.assertIsNone(backup._start_time) + self.assertIsNone(backup._end_time) + self.assertIsNone(backup._size_bytes) + self.assertIsNone(backup._state) + + def test_from_pb_project_mismatch(self): + from google.cloud.bigtable_admin_v2.proto import table_pb2 + + alt_project_id = "alt-project-id" + client = _Client(project=alt_project_id) + instance = _Instance(self.INSTANCE_NAME, client) + backup_pb = table_pb2.Backup(name=self.BACKUP_NAME) + klasse = self._get_target_class() + + with self.assertRaises(ValueError): + klasse.from_pb(backup_pb, instance) + + def test_from_pb_instance_mismatch(self): + from google.cloud.bigtable_admin_v2.proto import table_pb2 + + alt_instance = "/projects/%s/instances/alt-instance" % self.PROJECT_ID + client = _Client() + instance = _Instance(alt_instance, client) + backup_pb = table_pb2.Backup(name=self.BACKUP_NAME) + klasse = self._get_target_class() + + with self.assertRaises(ValueError): + klasse.from_pb(backup_pb, instance) + + def test_from_pb_bad_name(self): + from google.cloud.bigtable_admin_v2.proto import table_pb2 + + client = _Client() + instance = _Instance(self.INSTANCE_NAME, client) + backup_pb = table_pb2.Backup(name="invalid_name") + klasse = self._get_target_class() + + with self.assertRaises(ValueError): + klasse.from_pb(backup_pb, instance) + + def test_from_pb_success(self): + from google.cloud.bigtable_admin_v2.gapic import enums + from google.cloud.bigtable_admin_v2.proto import table_pb2 + from google.cloud._helpers import _datetime_to_pb_timestamp + + client = _Client() + instance = _Instance(self.INSTANCE_NAME, client) + timestamp = _datetime_to_pb_timestamp(self._make_timestamp()) + size_bytes = 1234 + state = enums.Backup.State.READY + backup_pb = table_pb2.Backup( + name=self.BACKUP_NAME, + source_table=self.TABLE_NAME, + expire_time=timestamp, + start_time=timestamp, + end_time=timestamp, + size_bytes=size_bytes, + state=state, + ) + klasse = self._get_target_class() + + backup = klasse.from_pb(backup_pb, instance) + + self.assertTrue(isinstance(backup, klasse)) + self.assertEqual(backup._instance, instance) + self.assertEqual(backup.backup_id, self.BACKUP_ID) + self.assertEqual(backup.cluster, self.CLUSTER_ID) + self.assertEqual(backup.table_id, self.TABLE_ID) + self.assertEqual(backup._expire_time, timestamp) + self.assertEqual(backup._start_time, timestamp) + self.assertEqual(backup._end_time, timestamp) + self.assertEqual(backup._size_bytes, size_bytes) + self.assertEqual(backup._state, state) + + def test_property_name(self): + from google.cloud.bigtable.client import Client + from google.cloud.bigtable_admin_v2.gapic import bigtable_table_admin_client + + api = bigtable_table_admin_client.BigtableTableAdminClient(mock.Mock()) + credentials = _make_credentials() + client = Client(project=self.PROJECT_ID, credentials=credentials, admin=True) + client._table_admin_client = api + instance = _Instance(self.INSTANCE_NAME, client) + + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + self.assertEqual(backup.name, self.BACKUP_NAME) + + def test_property_cluster(self): + backup = self._make_one( + self.BACKUP_ID, _Instance(self.INSTANCE_NAME), cluster_id=self.CLUSTER_ID + ) + self.assertEqual(backup.cluster, self.CLUSTER_ID) + + def test_property_cluster_setter(self): + backup = self._make_one(self.BACKUP_ID, _Instance(self.INSTANCE_NAME)) + backup.cluster = self.CLUSTER_ID + self.assertEqual(backup.cluster, self.CLUSTER_ID) + + def test_property_parent_none(self): + backup = self._make_one(self.BACKUP_ID, _Instance(self.INSTANCE_NAME),) + self.assertIsNone(backup.parent) + + def test_property_parent_w_cluster(self): + from google.cloud.bigtable.client import Client + from google.cloud.bigtable_admin_v2.gapic import bigtable_table_admin_client + + api = bigtable_table_admin_client.BigtableTableAdminClient(mock.Mock()) + credentials = _make_credentials() + client = Client(project=self.PROJECT_ID, credentials=credentials, admin=True) + client._table_admin_client = api + instance = _Instance(self.INSTANCE_NAME, client) + + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + self.assertEqual(backup._cluster, self.CLUSTER_ID) + self.assertEqual(backup.parent, self.CLUSTER_NAME) + + def test_property_source_table_none(self): + from google.cloud.bigtable.client import Client + from google.cloud.bigtable_admin_v2.gapic import bigtable_table_admin_client + + api = bigtable_table_admin_client.BigtableTableAdminClient(mock.Mock()) + credentials = _make_credentials() + client = Client(project=self.PROJECT_ID, credentials=credentials, admin=True) + client._table_admin_client = api + instance = _Instance(self.INSTANCE_NAME, client) + + backup = self._make_one(self.BACKUP_ID, instance) + self.assertIsNone(backup.source_table) + + def test_property_source_table_valid(self): + from google.cloud.bigtable.client import Client + from google.cloud.bigtable_admin_v2.gapic import bigtable_table_admin_client + + api = bigtable_table_admin_client.BigtableTableAdminClient(mock.Mock()) + credentials = _make_credentials() + client = Client(project=self.PROJECT_ID, credentials=credentials, admin=True) + client._table_admin_client = api + instance = _Instance(self.INSTANCE_NAME, client) + + backup = self._make_one(self.BACKUP_ID, instance, table_id=self.TABLE_ID) + self.assertEqual(backup.source_table, self.TABLE_NAME) + + def test_property_expire_time(self): + instance = _Instance(self.INSTANCE_NAME) + expire_time = self._make_timestamp() + backup = self._make_one(self.BACKUP_ID, instance, expire_time=expire_time) + self.assertEqual(backup.expire_time, expire_time) + + def test_property_expire_time_setter(self): + instance = _Instance(self.INSTANCE_NAME) + expire_time = self._make_timestamp() + backup = self._make_one(self.BACKUP_ID, instance) + backup.expire_time = expire_time + self.assertEqual(backup.expire_time, expire_time) + + def test_property_start_time(self): + instance = _Instance(self.INSTANCE_NAME) + backup = self._make_one(self.BACKUP_ID, instance) + expected = backup._start_time = self._make_timestamp() + self.assertEqual(backup.start_time, expected) + + def test_property_end_time(self): + instance = _Instance(self.INSTANCE_NAME) + backup = self._make_one(self.BACKUP_ID, instance) + expected = backup._end_time = self._make_timestamp() + self.assertEqual(backup.end_time, expected) + + def test_property_size(self): + instance = _Instance(self.INSTANCE_NAME) + backup = self._make_one(self.BACKUP_ID, instance) + expected = backup._size_bytes = 10 + self.assertEqual(backup.size_bytes, expected) + + def test_property_state(self): + from google.cloud.bigtable_admin_v2.gapic import enums + + instance = _Instance(self.INSTANCE_NAME) + backup = self._make_one(self.BACKUP_ID, instance) + expected = backup._state = enums.Backup.State.READY + self.assertEqual(backup.state, expected) + + def test___eq__(self): + instance = object() + backup1 = self._make_one(self.BACKUP_ID, instance) + backup2 = self._make_one(self.BACKUP_ID, instance) + self.assertTrue(backup1 == backup2) + + def test___eq__different_types(self): + instance = object() + backup1 = self._make_one(self.BACKUP_ID, instance) + backup2 = object() + self.assertFalse(backup1 == backup2) + + def test___ne__same_value(self): + instance = object() + backup1 = self._make_one(self.BACKUP_ID, instance) + backup2 = self._make_one(self.BACKUP_ID, instance) + self.assertFalse(backup1 != backup2) + + def test___ne__(self): + backup1 = self._make_one("backup_1", "instance1") + backup2 = self._make_one("backup_2", "instance2") + self.assertTrue(backup1 != backup2) + + def test_create_grpc_error(self): + from google.api_core.exceptions import GoogleAPICallError + from google.api_core.exceptions import Unknown + from google.cloud._helpers import _datetime_to_pb_timestamp + from google.cloud.bigtable_admin_v2.types import table_pb2 + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.create_backup.side_effect = Unknown("testing") + + timestamp = self._make_timestamp() + backup = self._make_one( + self.BACKUP_ID, + _Instance(self.INSTANCE_NAME, client=client), + table_id=self.TABLE_ID, + expire_time=timestamp, + ) + + backup_pb = table_pb2.Backup( + source_table=self.TABLE_NAME, + expire_time=_datetime_to_pb_timestamp(timestamp), + ) + + with self.assertRaises(GoogleAPICallError): + backup.create(self.CLUSTER_ID) + + api.create_backup.assert_called_once_with( + parent=self.CLUSTER_NAME, backup_id=self.BACKUP_ID, backup=backup_pb, + ) + + def test_create_already_exists(self): + from google.cloud._helpers import _datetime_to_pb_timestamp + from google.cloud.bigtable_admin_v2.types import table_pb2 + from google.cloud.exceptions import Conflict + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.create_backup.side_effect = Conflict("testing") + + timestamp = self._make_timestamp() + backup = self._make_one( + self.BACKUP_ID, + _Instance(self.INSTANCE_NAME, client=client), + table_id=self.TABLE_ID, + expire_time=timestamp, + ) + + backup_pb = table_pb2.Backup( + source_table=self.TABLE_NAME, + expire_time=_datetime_to_pb_timestamp(timestamp), + ) + + with self.assertRaises(Conflict): + backup.create(self.CLUSTER_ID) + + api.create_backup.assert_called_once_with( + parent=self.CLUSTER_NAME, backup_id=self.BACKUP_ID, backup=backup_pb, + ) + + def test_create_instance_not_found(self): + from google.cloud._helpers import _datetime_to_pb_timestamp + from google.cloud.bigtable_admin_v2.types import table_pb2 + from google.cloud.exceptions import NotFound + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.create_backup.side_effect = NotFound("testing") + + timestamp = self._make_timestamp() + backup = self._make_one( + self.BACKUP_ID, + _Instance(self.INSTANCE_NAME, client=client), + table_id=self.TABLE_ID, + expire_time=timestamp, + ) + + backup_pb = table_pb2.Backup( + source_table=self.TABLE_NAME, + expire_time=_datetime_to_pb_timestamp(timestamp), + ) + + with self.assertRaises(NotFound): + backup.create(self.CLUSTER_ID) + + api.create_backup.assert_called_once_with( + parent=self.CLUSTER_NAME, backup_id=self.BACKUP_ID, backup=backup_pb, + ) + + def test_create_cluster_not_set(self): + backup = self._make_one( + self.BACKUP_ID, + _Instance(self.INSTANCE_NAME), + table_id=self.TABLE_ID, + expire_time=self._make_timestamp(), + ) + + with self.assertRaises(ValueError): + backup.create() + + def test_create_table_not_set(self): + backup = self._make_one( + self.BACKUP_ID, + _Instance(self.INSTANCE_NAME), + expire_time=self._make_timestamp(), + ) + + with self.assertRaises(ValueError): + backup.create(self.CLUSTER_ID) + + def test_create_expire_time_not_set(self): + backup = self._make_one( + self.BACKUP_ID, _Instance(self.INSTANCE_NAME), table_id=self.TABLE_ID, + ) + + with self.assertRaises(ValueError): + backup.create(self.CLUSTER_ID) + + def test_create_success(self): + from google.cloud._helpers import _datetime_to_pb_timestamp + from google.cloud.bigtable_admin_v2.types import table_pb2 + + op_future = object() + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.create_backup.return_value = op_future + + timestamp = self._make_timestamp() + backup = self._make_one( + self.BACKUP_ID, + _Instance(self.INSTANCE_NAME, client=client), + table_id=self.TABLE_ID, + expire_time=timestamp, + ) + + backup_pb = table_pb2.Backup( + source_table=self.TABLE_NAME, + expire_time=_datetime_to_pb_timestamp(timestamp), + ) + + future = backup.create(self.CLUSTER_ID) + self.assertEqual(backup._cluster, self.CLUSTER_ID) + self.assertIs(future, op_future) + + api.create_backup.assert_called_once_with( + parent=self.CLUSTER_NAME, backup_id=self.BACKUP_ID, backup=backup_pb, + ) + + def test_exists_grpc_error(self): + from google.api_core.exceptions import Unknown + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.get_backup.side_effect = Unknown("testing") + + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + + with self.assertRaises(Unknown): + backup.exists() + + api.get_backup.assert_called_once_with(self.BACKUP_NAME) + + def test_exists_not_found(self): + from google.api_core.exceptions import NotFound + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.get_backup.side_effect = NotFound("testing") + + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + + self.assertFalse(backup.exists()) + + api.get_backup.assert_called_once_with(self.BACKUP_NAME) + + def test_get(self): + from google.cloud.bigtable_admin_v2.gapic import enums + from google.cloud.bigtable_admin_v2.proto import table_pb2 + from google.cloud._helpers import _datetime_to_pb_timestamp + + timestamp = _datetime_to_pb_timestamp(self._make_timestamp()) + state = enums.Backup.State.READY + + client = _Client() + backup_pb = table_pb2.Backup( + name=self.BACKUP_NAME, + source_table=self.TABLE_NAME, + expire_time=timestamp, + start_time=timestamp, + end_time=timestamp, + size_bytes=0, + state=state, + ) + api = client.table_admin_client = self._make_table_admin_client() + api.get_backup.return_value = backup_pb + + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + + self.assertEqual(backup.get(), backup_pb) + + def test_reload(self): + from google.cloud.bigtable_admin_v2.gapic import enums + from google.cloud.bigtable_admin_v2.proto import table_pb2 + from google.cloud._helpers import _datetime_to_pb_timestamp + + timestamp = _datetime_to_pb_timestamp(self._make_timestamp()) + state = enums.Backup.State.READY + + client = _Client() + backup_pb = table_pb2.Backup( + name=self.BACKUP_NAME, + source_table=self.TABLE_NAME, + expire_time=timestamp, + start_time=timestamp, + end_time=timestamp, + size_bytes=0, + state=state, + ) + api = client.table_admin_client = self._make_table_admin_client() + api.get_backup.return_value = backup_pb + + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + + backup.reload() + self.assertEqual(backup._source_table, self.TABLE_NAME) + self.assertEqual(backup._expire_time, timestamp) + self.assertEqual(backup._start_time, timestamp) + self.assertEqual(backup._end_time, timestamp) + self.assertEqual(backup._size_bytes, 0) + self.assertEqual(backup._state, state) + + def test_exists_success(self): + from google.cloud.bigtable_admin_v2.proto import table_pb2 + + client = _Client() + backup_pb = table_pb2.Backup(name=self.BACKUP_NAME) + api = client.table_admin_client = self._make_table_admin_client() + api.get_backup.return_value = backup_pb + + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + + self.assertTrue(backup.exists()) + + api.get_backup.assert_called_once_with(self.BACKUP_NAME) + + def test_delete_grpc_error(self): + from google.api_core.exceptions import Unknown + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.delete_backup.side_effect = Unknown("testing") + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + + with self.assertRaises(Unknown): + backup.delete() + + api.delete_backup.assert_called_once_with(self.BACKUP_NAME) + + def test_delete_not_found(self): + from google.api_core.exceptions import NotFound + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.delete_backup.side_effect = NotFound("testing") + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + + with self.assertRaises(NotFound): + backup.delete() + + api.delete_backup.assert_called_once_with(self.BACKUP_NAME) + + def test_delete_success(self): + from google.protobuf.empty_pb2 import Empty + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.delete_backup.return_value = Empty() + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + + backup.delete() + + api.delete_backup.assert_called_once_with(self.BACKUP_NAME) + + def test_update_expire_time_grpc_error(self): + from google.api_core.exceptions import Unknown + from google.cloud._helpers import _datetime_to_pb_timestamp + from google.cloud.bigtable_admin_v2.types import table_pb2 + from google.protobuf import field_mask_pb2 + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.update_backup.side_effect = Unknown("testing") + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + expire_time = self._make_timestamp() + + with self.assertRaises(Unknown): + backup.update_expire_time(expire_time) + + backup_update = table_pb2.Backup( + name=self.BACKUP_NAME, expire_time=_datetime_to_pb_timestamp(expire_time), + ) + update_mask = field_mask_pb2.FieldMask(paths=["expire_time"]) + api.update_backup.assert_called_once_with( + backup_update, update_mask, + ) + + def test_update_expire_time_not_found(self): + from google.api_core.exceptions import NotFound + from google.cloud._helpers import _datetime_to_pb_timestamp + from google.cloud.bigtable_admin_v2.types import table_pb2 + from google.protobuf import field_mask_pb2 + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.update_backup.side_effect = NotFound("testing") + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + expire_time = self._make_timestamp() + + with self.assertRaises(NotFound): + backup.update_expire_time(expire_time) + + backup_update = table_pb2.Backup( + name=self.BACKUP_NAME, expire_time=_datetime_to_pb_timestamp(expire_time), + ) + update_mask = field_mask_pb2.FieldMask(paths=["expire_time"]) + api.update_backup.assert_called_once_with( + backup_update, update_mask, + ) + + def test_update_expire_time_success(self): + from google.cloud._helpers import _datetime_to_pb_timestamp + from google.cloud.bigtable_admin_v2.proto import table_pb2 + from google.protobuf import field_mask_pb2 + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.update_backup.return_type = table_pb2.Backup(name=self.BACKUP_NAME) + instance = _Instance(self.INSTANCE_NAME, client=client) + backup = self._make_one(self.BACKUP_ID, instance, cluster_id=self.CLUSTER_ID) + expire_time = self._make_timestamp() + + backup.update_expire_time(expire_time) + + backup_update = table_pb2.Backup( + name=self.BACKUP_NAME, expire_time=_datetime_to_pb_timestamp(expire_time), + ) + update_mask = field_mask_pb2.FieldMask(paths=["expire_time"]) + api.update_backup.assert_called_once_with( + backup_update, update_mask, + ) + + def test_restore_grpc_error(self): + from google.api_core.exceptions import GoogleAPICallError + from google.api_core.exceptions import Unknown + + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.restore_table.side_effect = Unknown("testing") + + timestamp = self._make_timestamp() + backup = self._make_one( + self.BACKUP_ID, + _Instance(self.INSTANCE_NAME, client=client), + cluster_id=self.CLUSTER_ID, + table_id=self.TABLE_NAME, + expire_time=timestamp, + ) + + with self.assertRaises(GoogleAPICallError): + backup.restore(self.TABLE_ID) + + api.restore_table.assert_called_once_with( + parent=self.INSTANCE_NAME, table_id=self.TABLE_ID, backup=self.BACKUP_NAME, + ) + + def test_restore_cluster_not_set(self): + client = _Client() + client.table_admin_client = self._make_table_admin_client() + backup = self._make_one( + self.BACKUP_ID, + _Instance(self.INSTANCE_NAME, client=client), + table_id=self.TABLE_ID, + expire_time=self._make_timestamp(), + ) + + with self.assertRaises(ValueError): + backup.restore(self.TABLE_ID) + + def test_restore_success(self): + op_future = object() + client = _Client() + api = client.table_admin_client = self._make_table_admin_client() + api.restore_table.return_value = op_future + + timestamp = self._make_timestamp() + backup = self._make_one( + self.BACKUP_ID, + _Instance(self.INSTANCE_NAME, client=client), + cluster_id=self.CLUSTER_ID, + table_id=self.TABLE_NAME, + expire_time=timestamp, + ) + + future = backup.restore(self.TABLE_ID) + self.assertEqual(backup._cluster, self.CLUSTER_ID) + self.assertIs(future, op_future) + + api.restore_table.assert_called_once_with( + parent=self.INSTANCE_NAME, table_id=self.TABLE_ID, backup=self.BACKUP_NAME, + ) + + +class _Client(object): + def __init__(self, project=TestBackup.PROJECT_ID): + self.project = project + self.project_name = "projects/" + self.project + + +class _Instance(object): + def __init__(self, name, client=None): + self.name = name + self.instance_id = name.rsplit("/", 1)[1] + self._client = client diff --git a/tests/unit/test_instance.py b/tests/unit/test_instance.py index b129d4edc..14dd0bf58 100644 --- a/tests/unit/test_instance.py +++ b/tests/unit/test_instance.py @@ -39,6 +39,10 @@ class TestInstance(unittest.TestCase): ) TABLE_ID = "table_id" TABLE_NAME = INSTANCE_NAME + "/tables/" + TABLE_ID + CLUSTER_ID = "cluster-id" + CLUSTER_NAME = INSTANCE_NAME + "/clusters/" + CLUSTER_ID + BACKUP_ID = "backup-id" + BACKUP_NAME = CLUSTER_NAME + "/backups/" + BACKUP_ID @staticmethod def _get_target_class(): diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index d4bb621c2..f7377bc76 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -128,8 +128,12 @@ class TestTable(unittest.TestCase): PROJECT_ID = "project-id" INSTANCE_ID = "instance-id" INSTANCE_NAME = "projects/" + PROJECT_ID + "/instances/" + INSTANCE_ID + CLUSTER_ID = "cluster-id" + CLUSTER_NAME = INSTANCE_NAME + "/clusters/" + CLUSTER_ID TABLE_ID = "table-id" TABLE_NAME = INSTANCE_NAME + "/tables/" + TABLE_ID + BACKUP_ID = "backup-id" + BACKUP_NAME = CLUSTER_NAME + "/backups/" + BACKUP_ID ROW_KEY = b"row-key" ROW_KEY_1 = b"row-key-1" ROW_KEY_2 = b"row-key-2" @@ -1153,6 +1157,153 @@ def test_test_iam_permissions(self): resource=table.name, permissions=permissions ) + def test_backup_factory_defaults(self): + from google.cloud.bigtable.backup import Backup + + instance = self._make_one(self.INSTANCE_ID, None) + table = self._make_one(self.TABLE_ID, instance) + backup = table.backup(self.BACKUP_ID) + + self.assertIsInstance(backup, Backup) + self.assertEqual(backup.backup_id, self.BACKUP_ID) + self.assertIs(backup._instance, instance) + self.assertIsNone(backup._cluster) + self.assertEqual(backup.table_id, self.TABLE_ID) + self.assertIsNone(backup._expire_time) + + self.assertIsNone(backup._parent) + self.assertIsNone(backup._source_table) + self.assertIsNone(backup._start_time) + self.assertIsNone(backup._end_time) + self.assertIsNone(backup._size_bytes) + self.assertIsNone(backup._state) + + def test_backup_factory_non_defaults(self): + import datetime + from google.cloud._helpers import UTC + from google.cloud.bigtable.backup import Backup + + instance = self._make_one(self.INSTANCE_ID, None) + table = self._make_one(self.TABLE_ID, instance) + timestamp = datetime.datetime.utcnow().replace(tzinfo=UTC) + backup = table.backup( + self.BACKUP_ID, cluster_id=self.CLUSTER_ID, expire_time=timestamp, + ) + + self.assertIsInstance(backup, Backup) + self.assertEqual(backup.backup_id, self.BACKUP_ID) + self.assertIs(backup._instance, instance) + + self.assertEqual(backup.backup_id, self.BACKUP_ID) + self.assertIs(backup._cluster, self.CLUSTER_ID) + self.assertEqual(backup.table_id, self.TABLE_ID) + self.assertEqual(backup._expire_time, timestamp) + self.assertIsNone(backup._start_time) + self.assertIsNone(backup._end_time) + self.assertIsNone(backup._size_bytes) + self.assertIsNone(backup._state) + + def _list_backups_helper(self, cluster_id=None, filter_=None, **kwargs): + from google.cloud.bigtable_admin_v2.gapic import ( + bigtable_instance_admin_client, + bigtable_table_admin_client, + ) + from google.cloud.bigtable_admin_v2.proto import ( + bigtable_table_admin_pb2, + table_pb2, + ) + from google.cloud.bigtable.backup import Backup + + instance_api = bigtable_instance_admin_client.BigtableInstanceAdminClient + table_api = bigtable_table_admin_client.BigtableTableAdminClient(mock.Mock()) + client = self._make_client( + project=self.PROJECT_ID, credentials=_make_credentials(), admin=True + ) + instance = client.instance(instance_id=self.INSTANCE_ID) + table = self._make_one(self.TABLE_ID, instance) + + client._instance_admin_client = instance_api + client._table_admin_client = table_api + + parent = self.INSTANCE_NAME + "/clusters/cluster" + backups_pb = bigtable_table_admin_pb2.ListBackupsResponse( + backups=[ + table_pb2.Backup(name=parent + "/backups/op1"), + table_pb2.Backup(name=parent + "/backups/op2"), + table_pb2.Backup(name=parent + "/backups/op3"), + ] + ) + + api = table_api._inner_api_calls["list_backups"] = mock.Mock( + return_value=backups_pb + ) + + backups_filter = "source_table:{}".format(self.TABLE_NAME) + if filter_: + backups_filter = "({}) AND ({})".format(backups_filter, filter_) + + backups = table.list_backups(cluster_id=cluster_id, filter_=filter_, **kwargs) + + for backup in backups: + self.assertIsInstance(backup, Backup) + + if not cluster_id: + cluster_id = "-" + parent = "{}/clusters/{}".format(self.INSTANCE_NAME, cluster_id) + + expected_metadata = [ + ("x-goog-request-params", "parent={}".format(parent)), + ] + api.assert_called_once_with( + bigtable_table_admin_pb2.ListBackupsRequest( + parent=parent, filter=backups_filter, **kwargs + ), + retry=mock.ANY, + timeout=mock.ANY, + metadata=expected_metadata, + ) + + def test_list_backups_defaults(self): + self._list_backups_helper() + + def test_list_backups_w_options(self): + self._list_backups_helper( + cluster_id="cluster", filter_="filter", order_by="order_by", page_size=10 + ) + + def _restore_helper(self, backup_name=None): + from google.cloud.bigtable_admin_v2 import BigtableTableAdminClient + from google.cloud.bigtable_admin_v2.gapic import bigtable_instance_admin_client + from google.cloud.bigtable.instance import Instance + + op_future = object() + instance_api = bigtable_instance_admin_client.BigtableInstanceAdminClient + + client = mock.Mock(project=self.PROJECT_ID, instance_admin_client=instance_api) + instance = Instance(self.INSTANCE_ID, client=client) + table = self._make_one(self.TABLE_ID, instance) + + api = client.table_admin_client = mock.create_autospec( + BigtableTableAdminClient, instance=True + ) + api.restore_table.return_value = op_future + + if backup_name: + future = table.restore(self.TABLE_ID, backup_name=self.BACKUP_NAME) + else: + future = table.restore(self.TABLE_ID, self.CLUSTER_ID, self.BACKUP_ID) + self.assertIs(future, op_future) + + api.restore_table.assert_called_once_with( + parent=self.INSTANCE_NAME, table_id=self.TABLE_ID, backup=self.BACKUP_NAME, + ) + + def test_restore_table_w_backup_id(self): + self._restore_helper() + + def test_restore_table_w_backup_name(self): + self._restore_helper(backup_name=self.BACKUP_NAME) + class Test__RetryableMutateRowsWorker(unittest.TestCase): from grpc import StatusCode From 3a092ded2fcf13ea87ad39f6e4d6917ca736dfdf Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2020 14:34:01 -0400 Subject: [PATCH 2/2] chore: release 1.4.0 (#65) * updated CHANGELOG.md [ci skip] * updated setup.cfg [ci skip] * updated setup.py Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eeb76023..56a43a742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-cloud-bigtable/#history +## [1.4.0](https://www.github.com/googleapis/python-bigtable/compare/v1.3.0...v1.4.0) (2020-07-21) + + +### Features + +* **bigtable:** Managed Backups wrappers ([#57](https://www.github.com/googleapis/python-bigtable/issues/57)) ([a351734](https://www.github.com/googleapis/python-bigtable/commit/a351734ae16b4a689b89e6a42f63ea3ea5ad84ca)) + ## [1.3.0](https://www.github.com/googleapis/python-bigtable/compare/v1.2.1...v1.3.0) (2020-07-16) diff --git a/setup.py b/setup.py index 73efe3540..a8f544560 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ name = 'google-cloud-bigtable' description = 'Google Cloud Bigtable API client library' -version = "1.3.0" +version = "1.4.0" # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta'