diff --git a/.kokoro/continuous/storage_perf_bench.cfg b/.kokoro/continuous/storage_perf_bench.cfg new file mode 100644 index 000000000..8f43917d9 --- /dev/null +++ b/.kokoro/continuous/storage_perf_bench.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 237a2362b..f1ab207f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,33 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +## [1.27.0](https://www.github.com/googleapis/python-storage/compare/v1.26.0...v1.27.0) (2020-04-01) + + +### Features + +* generate signed URLs for blobs/buckets using virtual hostname ([#58](https://www.github.com/googleapis/python-storage/issues/58)) ([23df542](https://www.github.com/googleapis/python-storage/commit/23df542d0669852b05139023d5ef1ae14a09f4c7)) +* **storage:** Add cname support for V4 signature ([#72](https://www.github.com/googleapis/python-storage/issues/72)) ([cc853af](https://www.github.com/googleapis/python-storage/commit/cc853af6bf8e44e5b16e8cdfb3a275629ffb1f27)) +* **storage:** add conformance tests for virtual hosted style signed URLs ([#83](https://www.github.com/googleapis/python-storage/issues/83)) ([5adc8b0](https://www.github.com/googleapis/python-storage/commit/5adc8b0e6ffe28185a4085cd1fc8c1b4998094aa)) +* **storage:** add get notification method ([#77](https://www.github.com/googleapis/python-storage/issues/77)) ([f602252](https://www.github.com/googleapis/python-storage/commit/f6022521bee0824e1b291211703afc5eae6c6891)) +* **storage:** improve v4 signature query parameters encoding ([#48](https://www.github.com/googleapis/python-storage/issues/48)) ([8df0b55](https://www.github.com/googleapis/python-storage/commit/8df0b554a1904787889309707f08c6b8683cad44)) + + +### Bug Fixes + +* **storage:** fix blob metadata to None regression ([#60](https://www.github.com/googleapis/python-storage/issues/60)) ([a834d1b](https://www.github.com/googleapis/python-storage/commit/a834d1b54aa96152ced4d841c4e0c241acd2d8d8)) +* add classifer for Python 3.8 ([#63](https://www.github.com/googleapis/python-storage/issues/63)) ([1b9b6bc](https://www.github.com/googleapis/python-storage/commit/1b9b6bc2601ee336a8399266852fb850e368b30a)) +* make v4 signing formatting consistent w/ spec ([#56](https://www.github.com/googleapis/python-storage/issues/56)) ([8712da8](https://www.github.com/googleapis/python-storage/commit/8712da84c93600a736e72a097c42a49b4724347d)) +* use correct IAM object admin role ([#71](https://www.github.com/googleapis/python-storage/issues/71)) ([2e27edd](https://www.github.com/googleapis/python-storage/commit/2e27edd3fe65cd5e17c12bf11f2b58f611937d61)) +* **storage:** remove docstring of retrun in reload method ([#78](https://www.github.com/googleapis/python-storage/issues/78)) ([4abeb1c](https://www.github.com/googleapis/python-storage/commit/4abeb1c0810c4e5d716758536da9fc204fa4c2a9)) +* **storage:** use OrderedDict while encoding POST policy ([#95](https://www.github.com/googleapis/python-storage/issues/95)) ([df560e1](https://www.github.com/googleapis/python-storage/commit/df560e178369a6d03140e412a25af6ec7444f5a1)) + ## [1.26.0](https://www.github.com/googleapis/python-storage/compare/v1.25.0...v1.26.0) (2020-02-12) ### Features -* **storage:** add support for signing URLs using token ([#9889](https://www.github.com/googleapis/python-storage/issues/9889)) ([ad280bf](https://www.github.com/googleapis/python-storage/commit/ad280bf506d3d7a37c402d06eac07422a5fe80af)) +* **storage:** add support for signing URLs using token ([#9889](https://www.github.com/googleapis/google-cloud-python/issues/9889)) ([ad280bf](https://www.github.com/googleapis/python-storage/commit/ad280bf506d3d7a37c402d06eac07422a5fe80af)) * add timeout parameter to public methods ([#44](https://www.github.com/googleapis/python-storage/issues/44)) ([63abf07](https://www.github.com/googleapis/python-storage/commit/63abf0778686df1caa001270dd22f9df0daf0c78)) diff --git a/docs/conf.py b/docs/conf.py index 4f6c1ec3a..9a042ba1e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -340,7 +340,7 @@ intersphinx_mapping = { "python": ("http://python.readthedocs.org/en/latest/", None), "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), - "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None), + "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), "grpc": ("https://grpc.io/grpc/python/", None), } diff --git a/docs/snippets.py b/docs/snippets.py index 8171d5cf8..16b324023 100644 --- a/docs/snippets.py +++ b/docs/snippets.py @@ -51,8 +51,7 @@ def storage_get_started(client, to_delete): @snippet def client_bucket_acl(client, to_delete): bucket_name = "system-test-bucket" - bucket = client.bucket(bucket_name) - bucket.create() + client.create_bucket(bucket_name) # [START client_bucket_acl] client = storage.Client() diff --git a/google/cloud/storage/_helpers.py b/google/cloud/storage/_helpers.py index ae53a7b65..1a1aca8ce 100644 --- a/google/cloud/storage/_helpers.py +++ b/google/cloud/storage/_helpers.py @@ -130,7 +130,7 @@ def reload(self, client=None, timeout=_DEFAULT_TIMEOUT): :param client: the client to use. If not passed, falls back to the ``client`` stored on the current object. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -191,7 +191,7 @@ def patch(self, client=None, timeout=_DEFAULT_TIMEOUT): :param client: the client to use. If not passed, falls back to the ``client`` stored on the current object. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -227,7 +227,7 @@ def update(self, client=None, timeout=_DEFAULT_TIMEOUT): :param client: the client to use. If not passed, falls back to the ``client`` stored on the current object. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). diff --git a/google/cloud/storage/_signing.py b/google/cloud/storage/_signing.py index e7c8e3328..38610b6ff 100644 --- a/google/cloud/storage/_signing.py +++ b/google/cloud/storage/_signing.py @@ -18,7 +18,6 @@ import collections import datetime import hashlib -import re import json import six @@ -31,8 +30,6 @@ NOW = datetime.datetime.utcnow # To be replaced by tests. -MULTIPLE_SPACES_RE = r"\s+" -MULTIPLE_SPACES = re.compile(MULTIPLE_SPACES_RE) SERVICE_ACCOUNT_URL = ( "https://googleapis.dev/python/google-api-core/latest/" @@ -192,7 +189,7 @@ def get_canonical_headers(headers): normalized = collections.defaultdict(list) for key, val in headers: key = key.lower().strip() - val = MULTIPLE_SPACES.sub(" ", val.strip()) + val = " ".join(val.split()) normalized[key].append(val) ordered_headers = sorted((key, ",".join(val)) for key, val in normalized.items()) @@ -206,8 +203,8 @@ def get_canonical_headers(headers): ) -def canonicalize(method, resource, query_parameters, headers): - """Canonicalize method, resource +def canonicalize_v2(method, resource, query_parameters, headers): + """Canonicalize method, resource per the V2 spec. :type method: str :param method: The HTTP verb that will be used when requesting the URL. @@ -223,7 +220,7 @@ def canonicalize(method, resource, query_parameters, headers): :type query_parameters: dict :param query_parameters: - (Optional) Additional query paramtersto be included as part of the + (Optional) Additional query parameters to be included as part of the signed URLs. See: https://cloud.google.com/storage/docs/xml-api/reference-headers#query @@ -301,12 +298,13 @@ def generate_signed_url_v2( :type resource: str :param resource: A pointer to a specific resource (typically, ``/bucket-name/path/to/blob.txt``). + Caller should have already URL-encoded the value. :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] :param expiration: Point in time when the signed URL should expire. :type api_access_endpoint: str - :param api_access_endpoint: Optional URI base. Defaults to empty string. + :param api_access_endpoint: (Optional) URI base. Defaults to empty string. :type method: str :param method: The HTTP verb that will be used when requesting the URL. @@ -354,7 +352,7 @@ def generate_signed_url_v2( :type query_parameters: dict :param query_parameters: - (Optional) Additional query paramtersto be included as part of the + (Optional) Additional query parameters to be included as part of the signed URLs. See: https://cloud.google.com/storage/docs/xml-api/reference-headers#query @@ -368,7 +366,7 @@ def generate_signed_url_v2( """ expiration_stamp = get_expiration_seconds_v2(expiration) - canonical = canonicalize(method, resource, query_parameters, headers) + canonical = canonicalize_v2(method, resource, query_parameters, headers) # Generate the string to sign. elements_to_sign = [ @@ -462,12 +460,13 @@ def generate_signed_url_v4( :type resource: str :param resource: A pointer to a specific resource (typically, ``/bucket-name/path/to/blob.txt``). + Caller should have already URL-encoded the value. :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] :param expiration: Point in time when the signed URL should expire. :type api_access_endpoint: str - :param api_access_endpoint: Optional URI base. Defaults to + :param api_access_endpoint: (Optional) URI base. Defaults to "https://storage.googleapis.com/" :type method: str @@ -510,7 +509,7 @@ def generate_signed_url_v4( :type query_parameters: dict :param query_parameters: - (Optional) Additional query paramtersto be included as part of the + (Optional) Additional query parameters to be included as part of the signed URLs. See: https://cloud.google.com/storage/docs/xml-api/reference-headers#query @@ -532,9 +531,7 @@ def generate_signed_url_v4( expiration_seconds = get_expiration_seconds_v4(expiration) if _request_timestamp is None: - now = NOW() - request_timestamp = now.strftime("%Y%m%dT%H%M%SZ") - datestamp = now.date().strftime("%Y%m%d") + request_timestamp, datestamp = get_v4_now_dtstamps() else: request_timestamp = _request_timestamp datestamp = _request_timestamp[:8] @@ -554,7 +551,7 @@ def generate_signed_url_v4( header_names = [key.lower() for key in headers] if "host" not in header_names: - headers["Host"] = "storage.googleapis.com" + headers["Host"] = six.moves.urllib.parse.urlparse(api_access_endpoint).netloc if method.upper() == "RESUMABLE": method = "POST" @@ -586,8 +583,14 @@ def generate_signed_url_v4( if generation is not None: query_parameters["generation"] = generation - ordered_query_parameters = sorted(query_parameters.items()) - canonical_query_string = six.moves.urllib.parse.urlencode(ordered_query_parameters) + canonical_query_string = _url_encode(query_parameters) + + lowercased_headers = dict(ordered_headers) + + if "x-goog-content-sha256" in lowercased_headers: + payload = lowercased_headers["x-goog-content-sha256"] + else: + payload = "UNSIGNED-PAYLOAD" canonical_elements = [ method, @@ -595,7 +598,7 @@ def generate_signed_url_v4( canonical_query_string, canonical_header_string, signed_headers, - "UNSIGNED-PAYLOAD", + payload, ] canonical_request = "\n".join(canonical_elements) @@ -624,6 +627,18 @@ def generate_signed_url_v4( ) +def get_v4_now_dtstamps(): + """Get current timestamp and datestamp in V4 valid format. + + :rtype: str, str + :returns: Current timestamp, datestamp. + """ + now = NOW() + timestamp = now.strftime("%Y%m%dT%H%M%SZ") + datestamp = now.date().strftime("%Y%m%d") + return timestamp, datestamp + + def _sign_message(message, access_token, service_account_email): """Signs a message. @@ -666,3 +681,34 @@ def _sign_message(message, access_token, service_account_email): data = json.loads(response.data.decode("utf-8")) return data["signature"] + + +def _url_encode(query_params): + """Encode query params into URL. + + :type query_params: dict + :param query_params: Query params to be encoded. + + :rtype: str + :returns: URL encoded query params. + """ + params = [ + "{}={}".format(_quote_param(name), _quote_param(value)) + for name, value in query_params.items() + ] + + return "&".join(sorted(params)) + + +def _quote_param(param): + """Quote query param. + + :type param: Any + :param param: Query param to be encoded. + + :rtype: str + :returns: URL encoded query param. + """ + if not isinstance(param, bytes): + param = str(param) + return six.moves.urllib.parse.quote(param, safe="~") diff --git a/google/cloud/storage/acl.py b/google/cloud/storage/acl.py index 7260c50b0..fb07faba9 100644 --- a/google/cloud/storage/acl.py +++ b/google/cloud/storage/acl.py @@ -92,8 +92,8 @@ class _ACLEntity(object): :param entity_type: The type of entity (ie, 'group' or 'user'). :type identifier: str - :param identifier: The ID or e-mail of the entity. For the special - entity types (like 'allUsers') this is optional. + :param identifier: (Optional) The ID or e-mail of the entity. For the special + entity types (like 'allUsers'). """ READER_ROLE = "READER" @@ -212,7 +212,7 @@ def _ensure_loaded(self, timeout=_DEFAULT_TIMEOUT): """Load if not already loaded. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -432,10 +432,10 @@ def reload(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the ACL's parent. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -465,16 +465,15 @@ def _save(self, acl, predefined, client, timeout=_DEFAULT_TIMEOUT): current entries. :type predefined: str - :param predefined: - (Optional) An identifier for a predefined ACL. Must be one of the + :param predefined: An identifier for a predefined ACL. Must be one of the keys in :attr:`PREDEFINED_JSON_ACLS` If passed, `acl` must be None. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the ACL's parent. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -514,10 +513,10 @@ def save(self, acl=None, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the ACL's parent. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -546,10 +545,10 @@ def save_predefined(self, predefined, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the ACL's parent. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -570,10 +569,10 @@ def clear(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the ACL's parent. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). diff --git a/google/cloud/storage/batch.py b/google/cloud/storage/batch.py index 89425f9b8..abfc88412 100644 --- a/google/cloud/storage/batch.py +++ b/google/cloud/storage/batch.py @@ -177,7 +177,7 @@ def _do_request( initialization of the object at a later time. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 93beced90..fb329d08d 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -123,20 +123,23 @@ class Blob(_PropertyMixin): :param bucket: The bucket to which this blob belongs. :type chunk_size: int - - :param chunk_size: The size of a chunk of data whenever iterating (in - bytes). This must be a multiple of 256 KB per the API - specification. + :param chunk_size: + (Optional) The size of a chunk of data whenever iterating (in bytes). + This must be a multiple of 256 KB per the API specification. :type encryption_key: bytes :param encryption_key: - Optional 32 byte encryption key for customer-supplied encryption. + (Optional) 32 byte encryption key for customer-supplied encryption. See https://cloud.google.com/storage/docs/encryption#customer-supplied. :type kms_key_name: str :param kms_key_name: - Optional resource name of Cloud KMS key used to encrypt the blob's + (Optional) Resource name of Cloud KMS key used to encrypt the blob's contents. + + :type generation: long + :param generation: (Optional) If present, selects a specific revision of + this object. """ _chunk_size = None # Default value for each instance. @@ -322,7 +325,7 @@ def from_string(cls, uri, client=None): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. + :param client: (Optional) The client to use. :rtype: :class:`google.cloud.storage.blob.Blob` :returns: The blob object created. @@ -361,6 +364,9 @@ def generate_signed_url( version=None, service_account_email=None, access_token=None, + virtual_hosted_style=False, + bucket_bound_hostname=None, + scheme="http", ): """Generates a signed URL for this blob. @@ -379,6 +385,21 @@ def generate_signed_url( amount of time, you can use this method to generate a URL that is only valid within a certain time period. + If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`, + ``https`` works only if using a ``CDN``. + + Example: + Generates a signed URL for this blob using bucket_bound_hostname and scheme. + + >>> from google.cloud import storage + >>> client = storage.Client() + >>> bucket = client.get_bucket('my-bucket-name') + >>> blob = client.get_blob('my-blob-name') + >>> url = blob.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld', + >>> version='v4') + >>> url = blob.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld', + >>> version='v4',scheme='https') # If using ``CDN`` + This is particularly useful if you don't want publicly accessible blobs, but don't want to require users to explicitly log in. @@ -387,7 +408,7 @@ def generate_signed_url( :param expiration: Point in time when the signed URL should expire. :type api_access_endpoint: str - :param api_access_endpoint: Optional URI base. + :param api_access_endpoint: (Optional) URI base. :type method: str :param method: The HTTP verb that will be used when requesting the URL. @@ -420,23 +441,22 @@ def generate_signed_url( :type headers: dict :param headers: (Optional) Additional HTTP headers to be included as part of the - signed URLs. See: + signed URLs. See: https://cloud.google.com/storage/docs/xml-api/reference-headers Requests using the signed URL *must* pass the specified header (name and value) with each request for the URL. :type query_parameters: dict :param query_parameters: - (Optional) Additional query paramtersto be included as part of the - signed URLs. See: + (Optional) Additional query parameters to be included as part of the + signed URLs. See: https://cloud.google.com/storage/docs/xml-api/reference-headers#query :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: (Optional) The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. - :type credentials: :class:`google.auth.credentials.Credentials` or :class:`NoneType` :param credentials: The authorization credentials to attach to requests. @@ -454,6 +474,23 @@ def generate_signed_url( :type access_token: str :param access_token: (Optional) Access token for a service account. + :type virtual_hosted_style: bool + :param virtual_hosted_style: + (Optional) If true, then construct the URL relative the bucket's + virtual hostname, e.g., '.storage.googleapis.com'. + + :type bucket_bound_hostname: str + :param bucket_bound_hostname: + (Optional) If passed, then construct the URL relative to the bucket-bound hostname. + Value can be a bare or with scheme, e.g., 'example.com' or 'http://example.com'. + See: https://cloud.google.com/storage/docs/request-endpoints#cname + + :type scheme: str + :param scheme: + (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use + this value as the scheme. ``https`` will work only when using a CDN. + Defaults to ``"http"``. + :raises: :exc:`ValueError` when version is invalid. :raises: :exc:`TypeError` when expiration is not a valid type. :raises: :exc:`AttributeError` if credentials is not an instance @@ -469,9 +506,25 @@ def generate_signed_url( raise ValueError("'version' must be either 'v2' or 'v4'") quoted_name = _quote(self.name, safe=b"/~") - resource = "/{bucket_name}/{quoted_name}".format( - bucket_name=self.bucket.name, quoted_name=quoted_name - ) + + if virtual_hosted_style: + api_access_endpoint = "https://{bucket_name}.storage.googleapis.com".format( + bucket_name=self.bucket.name + ) + elif bucket_bound_hostname: + if ":" in bucket_bound_hostname: + api_access_endpoint = bucket_bound_hostname + else: + api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( + scheme=scheme, bucket_bound_hostname=bucket_bound_hostname + ) + else: + resource = "/{bucket_name}/{quoted_name}".format( + bucket_name=self.bucket.name, quoted_name=quoted_name + ) + + if virtual_hosted_style or bucket_bound_hostname: + resource = "/{quoted_name}".format(quoted_name=quoted_name) if credentials is None: client = self._require_client(client) @@ -518,10 +571,10 @@ def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -561,10 +614,10 @@ def delete(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -643,17 +696,17 @@ def _do_download( :param download_url: The URL where the media can be accessed. :type headers: dict - :param headers: Optional headers to be sent with the request(s). + :param headers: Headers to be sent with the request(s). :type start: int - :param start: Optional, the first byte in a range to be downloaded. + :param start: (Optional) The first byte in a range to be downloaded. :type end: int - :param end: Optional, The last byte in a range to be downloaded. + :param end: (Optional) The last byte in a range to be downloaded. :type raw_download: bool :param raw_download: - Optional, If true, download the object without any expansion. + (Optional) If true, download the object without any expansion. """ if self.chunk_size is None: if raw_download: @@ -718,18 +771,18 @@ def download_to_file( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type start: int - :param start: Optional, the first byte in a range to be downloaded. + :param start: (Optional) The first byte in a range to be downloaded. :type end: int - :param end: Optional, The last byte in a range to be downloaded. + :param end: (Optional) The last byte in a range to be downloaded. :type raw_download: bool :param raw_download: - Optional, If true, download the object without any expansion. + (Optional) If true, download the object without any expansion. :raises: :class:`google.cloud.exceptions.NotFound` """ @@ -758,18 +811,18 @@ def download_to_filename( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type start: int - :param start: Optional, the first byte in a range to be downloaded. + :param start: (Optional) The first byte in a range to be downloaded. :type end: int - :param end: Optional, The last byte in a range to be downloaded. + :param end: (Optional) The last byte in a range to be downloaded. :type raw_download: bool :param raw_download: - Optional, If true, download the object without any expansion. + (Optional) If true, download the object without any expansion. :raises: :class:`google.cloud.exceptions.NotFound` """ @@ -800,18 +853,18 @@ def download_as_string(self, client=None, start=None, end=None, raw_download=Fal :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type start: int - :param start: Optional, the first byte in a range to be downloaded. + :param start: (Optional) The first byte in a range to be downloaded. :type end: int - :param end: Optional, The last byte in a range to be downloaded. + :param end: (Optional) The last byte in a range to be downloaded. :type raw_download: bool :param raw_download: - Optional, If true, download the object without any expansion. + (Optional) If true, download the object without any expansion. :rtype: bytes :returns: The data stored in this blob. @@ -837,7 +890,7 @@ def _get_content_type(self, content_type, filename=None): - The default value ('application/octet-stream') :type content_type: str - :param content_type: (Optional) type of content. + :param content_type: (Optional) Type of content. :type filename: str :param filename: (Optional) The name of the file where the content @@ -944,7 +997,7 @@ def _do_multipart_upload( argument will be removed in a future release.) :type predefined_acl: str - :param predefined_acl: (Optional) predefined access control list + :param predefined_acl: (Optional) Predefined access control list :rtype: :class:`~requests.Response` :returns: The "200 OK" response object returned after the multipart @@ -1024,7 +1077,7 @@ def _initiate_resumable_upload( concluded once ``stream`` is exhausted (or :data:`None`). :type predefined_acl: str - :param predefined_acl: (Optional) predefined access control list + :param predefined_acl: (Optional) Predefined access control list :type num_retries: int :param num_retries: Number of upload retries. (Deprecated: This @@ -1125,7 +1178,7 @@ def _do_resumable_upload( argument will be removed in a future release.) :type predefined_acl: str - :param predefined_acl: (Optional) predefined access control list + :param predefined_acl: (Optional) Predefined access control list :rtype: :class:`~requests.Response` :returns: The "200 OK" response object returned after the final chunk @@ -1181,7 +1234,7 @@ def _do_upload( argument will be removed in a future release.) :type predefined_acl: str - :param predefined_acl: (Optional) predefined access control list + :param predefined_acl: (Optional) Predefined access control list :rtype: dict :returns: The parsed JSON from the "200 OK" response. This will be the @@ -1256,7 +1309,7 @@ def upload_from_file( concluded once ``file_obj`` is exhausted. :type content_type: str - :param content_type: Optional type of content being uploaded. + :param content_type: (Optional) Type of content being uploaded. :type num_retries: int :param num_retries: Number of upload retries. (Deprecated: This @@ -1267,7 +1320,7 @@ def upload_from_file( to the ``client`` stored on the blob's bucket. :type predefined_acl: str - :param predefined_acl: (Optional) predefined access control list + :param predefined_acl: (Optional) Predefined access control list :raises: :class:`~google.cloud.exceptions.GoogleCloudError` if the upload response returns an error status. @@ -1321,14 +1374,14 @@ def upload_from_filename( :param filename: The path to the file. :type content_type: str - :param content_type: Optional type of content being uploaded. + :param content_type: (Optional) Type of content being uploaded. :type client: :class:`~google.cloud.storage.client.Client` :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type predefined_acl: str - :param predefined_acl: (Optional) predefined access control list + :param predefined_acl: (Optional) Predefined access control list """ content_type = self._get_content_type(content_type, filename=filename) @@ -1366,16 +1419,16 @@ def upload_from_string( text, it will be encoded as UTF-8. :type content_type: str - :param content_type: Optional type of content being uploaded. Defaults + :param content_type: (Optional) Type of content being uploaded. Defaults to ``'text/plain'``. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type predefined_acl: str - :param predefined_acl: (Optional) predefined access control list + :param predefined_acl: (Optional) Predefined access control list """ data = _to_bytes(data, encoding="utf-8") string_buffer = BytesIO(data) @@ -1430,7 +1483,7 @@ def create_resumable_upload_session( to that project. :type size: int - :param size: (Optional). The maximum number of bytes that can be + :param size: (Optional) The maximum number of bytes that can be uploaded using this session. If the size is not known when creating the session, this should be left blank. @@ -1498,11 +1551,11 @@ def get_iam_policy( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current object's bucket. :type requested_policy_version: int or ``NoneType`` - :param requested_policy_version: Optional. The version of IAM policies to request. + :param requested_policy_version: (Optional) The version of IAM policies to request. If a policy with a condition is requested without setting this, the server will return an error. This must be set to a value of 3 to retrieve IAM @@ -1513,7 +1566,7 @@ def get_iam_policy( than the one that was requested, based on the feature syntax in the policy fetched. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -1561,10 +1614,10 @@ def set_iam_policy(self, policy, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -1612,10 +1665,10 @@ def test_iam_permissions(self, permissions, client=None, timeout=_DEFAULT_TIMEOU :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -1643,7 +1696,7 @@ def make_public(self, client=None): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. """ self.acl.all().grant_read() @@ -1654,7 +1707,7 @@ def make_private(self, client=None): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. """ self.acl.all().revoke_read() @@ -1671,10 +1724,10 @@ def compose(self, sources, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -1710,17 +1763,17 @@ def rewrite(self, source, token=None, client=None, timeout=_DEFAULT_TIMEOUT): :param source: blob whose contents will be rewritten into this blob. :type token: str - :param token: Optional. Token returned from an earlier, not-completed + :param token: (Optional) Token returned from an earlier, not-completed call to rewrite the same source blob. If passed, result will include updated status, total bytes written. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -1794,7 +1847,7 @@ def update_storage_class(self, new_class, client=None): :attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`. :type client: :class:`~google.cloud.storage.client.Client` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. """ if new_class not in self.STORAGE_CLASSES: @@ -1861,6 +1914,9 @@ def update_storage_class(self, new_class, client=None): crc32c = _scalar_property("crc32c") """CRC32C checksum for this object. + This returns the blob's CRC32C checksum. To retrieve the value, first use a + reload method of the Blob class which loads the blob's properties from the server. + See `RFC 4960`_ and `API reference docs`_. If not set before upload, the server will compute the hash. @@ -1868,6 +1924,22 @@ def update_storage_class(self, new_class, client=None): :rtype: str or ``NoneType`` .. _RFC 4960: https://tools.ietf.org/html/rfc4960#appendix-B + + Example: + Retrieve the crc32c hash of blob. + + >>> from google.cloud import storage + >>> client = storage.Client() + >>> bucket = client.get_bucket("my-bucket-name") + >>> blob = bucket.blob('my-blob') + + >>> blob.crc32c # return None + >>> blob.reload() + >>> blob.crc32c # return crc32c hash + + >>> # Another approach + >>> blob = bucket.get_blob('my-blob') + >>> blob.crc32c # return crc32c hash """ @property @@ -1941,6 +2013,9 @@ def id(self): md5_hash = _scalar_property("md5Hash") """MD5 hash for this object. + This returns the blob's MD5 hash. To retrieve the value, first use a + reload method of the Blob class which loads the blob's properties from the server. + See `RFC 1321`_ and `API reference docs`_. If not set before upload, the server will compute the hash. @@ -1948,6 +2023,22 @@ def id(self): :rtype: str or ``NoneType`` .. _RFC 1321: https://tools.ietf.org/html/rfc1321 + + Example: + Retrieve the md5 hash of blob. + + >>> from google.cloud import storage + >>> client = storage.Client() + >>> bucket = client.get_bucket("my-bucket-name") + >>> blob = bucket.blob('my-blob') + + >>> blob.md5_hash # return None + >>> blob.reload() + >>> blob.md5_hash # return md5 hash + + >>> # Another approach + >>> blob = bucket.get_blob('my-blob') + >>> blob.md5_hash # return md5 hash """ @property @@ -1986,9 +2077,10 @@ def metadata(self, value): See https://cloud.google.com/storage/docs/json_api/v1/objects :type value: dict - :param value: (Optional) The blob metadata to set. + :param value: The blob metadata to set. """ - value = {k: str(v) for k, v in value.items()} + if value is not None: + value = {k: str(v) for k, v in value.items()} self._patch_property("metadata", value) @property diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 4914844dd..c2a909357 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -146,26 +146,26 @@ class LifecycleRuleConditions(dict): See: https://cloud.google.com/storage/docs/lifecycle :type age: int - :param age: (optional) apply rule action to items whos age, in days, + :param age: (Optional) Apply rule action to items whos age, in days, exceeds this value. :type created_before: datetime.date - :param created_before: (optional) apply rule action to items created + :param created_before: (Optional) Apply rule action to items created before this date. :type is_live: bool - :param is_live: (optional) if true, apply rule action to non-versioned + :param is_live: (Optional) If true, apply rule action to non-versioned items, or to items with no newer versions. If false, apply rule action to versioned items with at least one newer version. :type matches_storage_class: list(str), one or more of :attr:`Bucket.STORAGE_CLASSES`. - :param matches_storage_class: (optional) apply rule action to items which + :param matches_storage_class: (Optional) Apply rule action to items which whose storage class matches this value. :type number_of_newer_versions: int - :param number_of_newer_versions: (optional) apply rule action to versioned + :param number_of_newer_versions: (Optional) Apply rule action to versioned items having N newer versions. :raises ValueError: if no arguments are passed. @@ -316,11 +316,11 @@ class IAMConfiguration(dict): :type uniform_bucket_level_access_enabled: bool :params bucket_policy_only_enabled: - (optional) whether the IAM-only policy is enabled for the bucket. + (Optional) Whether the IAM-only policy is enabled for the bucket. :type uniform_bucket_level_locked_time: :class:`datetime.datetime` :params uniform_bucket_level_locked_time: - (optional) When the bucket's IAM-only policy was enabled. + (Optional) When the bucket's IAM-only policy was enabled. This value should normally only be set by the back-end API. :type bucket_policy_only_enabled: bool @@ -549,7 +549,7 @@ def from_string(cls, uri, client=None): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. + :param client: (Optional) The client to use. :rtype: :class:`google.cloud.storage.bucket.Bucket` :returns: The bucket object created. @@ -593,14 +593,14 @@ def blob( :type encryption_key: bytes :param encryption_key: - Optional 32 byte encryption key for customer-supplied encryption. + (Optional) 32 byte encryption key for customer-supplied encryption. :type kms_key_name: str :param kms_key_name: - Optional resource name of KMS key used to encrypt blob's content. + (Optional) Resource name of KMS key used to encrypt blob's content. :type generation: long - :param generation: Optional. If present, selects a specific revision of + :param generation: (Optional) If present, selects a specific revision of this object. :rtype: :class:`google.cloud.storage.blob.Blob` @@ -617,12 +617,13 @@ def blob( def notification( self, - topic_name, + topic_name=None, topic_project=None, custom_attributes=None, event_types=None, blob_name_prefix=None, payload_format=NONE_PAYLOAD_FORMAT, + notification_id=None, ): """Factory: create a notification resource for the bucket. @@ -632,12 +633,13 @@ def notification( """ return BucketNotification( self, - topic_name, + topic_name=topic_name, topic_project=topic_project, custom_attributes=custom_attributes, event_types=event_types, blob_name_prefix=blob_name_prefix, payload_format=payload_format, + notification_id=notification_id, ) def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): @@ -647,10 +649,10 @@ def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -693,7 +695,7 @@ def create( predefined_default_object_acl=None, timeout=_DEFAULT_TIMEOUT, ): - """Creates current bucket. + """DEPRECATED. Creates current bucket. If the bucket already exists, will raise :class:`google.cloud.exceptions.Conflict`. @@ -704,11 +706,11 @@ def create( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type project: str - :param project: Optional. The project under which the bucket is to + :param project: (Optional) The project under which the bucket is to be created. If not passed, uses the project set on the client. :raises ValueError: if :attr:`user_project` is set. @@ -716,65 +718,45 @@ def create( :attr:`project` is also None. :type location: str - :param location: Optional. The location of the bucket. If not passed, + :param location: (Optional) The location of the bucket. If not passed, the default location, US, will be used. See https://cloud.google.com/storage/docs/bucket-locations :type predefined_acl: str :param predefined_acl: - Optional. Name of predefined ACL to apply to bucket. See: + (Optional) Name of predefined ACL to apply to bucket. See: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl :type predefined_default_object_acl: str :param predefined_default_object_acl: - Optional. Name of predefined ACL to apply to bucket's objects. See: + (Optional) Name of predefined ACL to apply to bucket's objects. See: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. """ + warnings.warn( + "Bucket.create() is deprecated and will be removed in future." + "Use Client.create_bucket() instead.", + PendingDeprecationWarning, + stacklevel=1, + ) if self.user_project is not None: raise ValueError("Cannot create bucket with 'user_project' set.") client = self._require_client(client) - - if project is None: - project = client.project - - if project is None: - raise ValueError("Client project not set: pass an explicit project.") - - query_params = {"project": project} - - if predefined_acl is not None: - predefined_acl = BucketACL.validate_predefined(predefined_acl) - query_params["predefinedAcl"] = predefined_acl - - if predefined_default_object_acl is not None: - predefined_default_object_acl = DefaultObjectACL.validate_predefined( - predefined_default_object_acl - ) - query_params["predefinedDefaultObjectAcl"] = predefined_default_object_acl - - properties = {key: self._properties[key] for key in self._changes} - properties["name"] = self.name - - if location is not None: - properties["location"] = location - - api_response = client._connection.api_request( - method="POST", - path="/b", - query_params=query_params, - data=properties, - _target_object=self, + client.create_bucket( + bucket_or_name=self, + project=project, + location=location, + predefined_acl=predefined_acl, + predefined_default_object_acl=predefined_default_object_acl, timeout=timeout, ) - self._set_properties(api_response) def patch(self, client=None, timeout=_DEFAULT_TIMEOUT): """Sends all changed properties in a PATCH request. @@ -788,7 +770,7 @@ def patch(self, client=None, timeout=_DEFAULT_TIMEOUT): :param client: the client to use. If not passed, falls back to the ``client`` stored on the current object. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -859,21 +841,21 @@ def get_blob( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type encryption_key: bytes :param encryption_key: - Optional 32 byte encryption key for customer-supplied encryption. + (Optional) 32 byte encryption key for customer-supplied encryption. See https://cloud.google.com/storage/docs/encryption#customer-supplied. :type generation: long - :param generation: Optional. If present, selects a specific revision of + :param generation: (Optional) If present, selects a specific revision of this object. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -934,7 +916,7 @@ def list_blobs( token. :type prefix: str - :param prefix: (Optional) prefix used to filter blobs. + :param prefix: (Optional) Prefix used to filter blobs. :type delimiter: str :param delimiter: (Optional) Delimiter, used with ``prefix`` to @@ -963,7 +945,7 @@ def list_blobs( to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -1017,10 +999,10 @@ def list_notifications(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -1041,6 +1023,44 @@ def list_notifications(self, client=None, timeout=_DEFAULT_TIMEOUT): iterator.bucket = self return iterator + def get_notification(self, notification_id, client=None, timeout=_DEFAULT_TIMEOUT): + """Get Pub / Sub notification for this bucket. + + See: + https://cloud.google.com/storage/docs/json_api/v1/notifications/get + + If :attr:`user_project` is set, bills the API request to that project. + + :type notification_id: str + :param notification_id: The notification id to retrieve the notification configuration. + + :type client: :class:`~google.cloud.storage.client.Client` or + ``NoneType`` + :param client: (Optional) The client to use. If not passed, falls back + to the ``client`` stored on the current bucket. + :type timeout: float or tuple + :param timeout: (Optional) The amount of time, in seconds, to wait + for the server response. + + Can also be passed as a tuple (connect_timeout, read_timeout). + See :meth:`requests.Session.request` documentation for details. + + :rtype: :class:`.BucketNotification` + :returns: notification instance. + + Example: + Get notification using notification id. + + >>> from google.cloud import storage + >>> client = storage.Client() + >>> bucket = client.get_bucket('my-bucket-name') # API request. + >>> notification = bucket.get_notification(notification_id='id') # API request. + + """ + notification = self.notification(notification_id=notification_id) + notification.reload(client=client, timeout=timeout) + return notification + def delete(self, force=False, client=None, timeout=_DEFAULT_TIMEOUT): """Delete this bucket. @@ -1065,10 +1085,10 @@ def delete(self, force=False, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response on each request. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -1137,15 +1157,15 @@ def delete_blob( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type generation: long - :param generation: Optional. If present, permanently deletes a specific + :param generation: (Optional) If present, permanently deletes a specific revision of this object. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -1196,7 +1216,7 @@ def delete_blobs(self, blobs, on_error=None, client=None, timeout=_DEFAULT_TIMEO to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. The timeout applies to each individual blob delete request. @@ -1240,23 +1260,23 @@ def copy_blob( copied. :type new_name: str - :param new_name: (optional) the new name for the copied file. + :param new_name: (Optional) The new name for the copied file. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type preserve_acl: bool - :param preserve_acl: Optional. Copies ACL from old blob to new blob. + :param preserve_acl: (Optional) Copies ACL from old blob to new blob. Default: True. :type source_generation: long - :param source_generation: Optional. The generation of the blob to be + :param source_generation: (Optional) The generation of the blob to be copied. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -1315,11 +1335,11 @@ def rename_blob(self, blob, new_name, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. The timeout applies to each individual request. @@ -1908,7 +1928,7 @@ def requester_pays(self): def requester_pays(self, value): """Update whether requester pays for API requests for this bucket. - See https://cloud.google.com/storage/docs/ for + See https://cloud.google.com/storage/docs/using-requester-pays for details. :type value: convertible to boolean @@ -1974,11 +1994,11 @@ def get_iam_policy( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type requested_policy_version: int or ``NoneType`` - :param requested_policy_version: Optional. The version of IAM policies to request. + :param requested_policy_version: (Optional) The version of IAM policies to request. If a policy with a condition is requested without setting this, the server will return an error. This must be set to a value of 3 to retrieve IAM @@ -1990,7 +2010,7 @@ def get_iam_policy( feature syntax in the policy fetched. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -2055,11 +2075,11 @@ def set_iam_policy(self, policy, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -2100,11 +2120,11 @@ def test_iam_permissions(self, permissions, client=None, timeout=_DEFAULT_TIMEOU :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -2141,10 +2161,10 @@ def make_public( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. The timeout applies to each underlying request. @@ -2207,11 +2227,11 @@ def make_private( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. The timeout applies to each underlying request. @@ -2277,7 +2297,7 @@ def generate_upload_policy(self, conditions, expiration=None, client=None): /post-object#policydocument :type expiration: datetime - :param expiration: Optional expiration in UTC. If not specified, the + :param expiration: (Optional) Expiration in UTC. If not specified, the policy will expire in 1 hour. :type conditions: list @@ -2285,7 +2305,7 @@ def generate_upload_policy(self, conditions, expiration=None, client=None): `policy documents`_ documentation. :type client: :class:`~google.cloud.storage.client.Client` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :rtype: dict @@ -2325,7 +2345,7 @@ def lock_retention_policy(self, client=None, timeout=_DEFAULT_TIMEOUT): """Lock the bucket's retention policy. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -2374,6 +2394,9 @@ def generate_signed_url( client=None, credentials=None, version=None, + virtual_hosted_style=False, + bucket_bound_hostname=None, + scheme="http", ): """Generates a signed URL for this bucket. @@ -2392,6 +2415,20 @@ def generate_signed_url( amount of time, you can use this method to generate a URL that is only valid within a certain time period. + If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`, + ``https`` works only if using a ``CDN``. + + Example: + Generates a signed URL for this bucket using bucket_bound_hostname and scheme. + + >>> from google.cloud import storage + >>> client = storage.Client() + >>> bucket = client.get_bucket('my-bucket-name') + >>> url = bucket.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld', + >>> version='v4') + >>> url = bucket.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld', + >>> version='v4',scheme='https') # If using ``CDN`` + This is particularly useful if you don't want publicly accessible buckets, but don't want to require users to explicitly log in. @@ -2400,7 +2437,7 @@ def generate_signed_url( :param expiration: Point in time when the signed URL should expire. :type api_access_endpoint: str - :param api_access_endpoint: Optional URI base. + :param api_access_endpoint: (Optional) URI base. :type method: str :param method: The HTTP verb that will be used when requesting the URL. @@ -2415,7 +2452,7 @@ def generate_signed_url( :type query_parameters: dict :param query_parameters: - (Optional) Additional query paramtersto be included as part of the + (Optional) Additional query parameters to be included as part of the signed URLs. See: https://cloud.google.com/storage/docs/xml-api/reference-headers#query @@ -2436,6 +2473,23 @@ def generate_signed_url( :param version: (Optional) The version of signed credential to create. Must be one of 'v2' | 'v4'. + :type virtual_hosted_style: bool + :param virtual_hosted_style: + (Optional) If true, then construct the URL relative the bucket's + virtual hostname, e.g., '.storage.googleapis.com'. + + :type bucket_bound_hostname: str + :param bucket_bound_hostname: + (Optional) If pass, then construct the URL relative to the bucket-bound hostname. + Value cane be a bare or with scheme, e.g., 'example.com' or 'http://example.com'. + See: https://cloud.google.com/storage/docs/request-endpoints#cname + + :type scheme: str + :param scheme: + (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use + this value as the scheme. ``https`` will work only when using a CDN. + Defaults to ``"http"``. + :raises: :exc:`ValueError` when version is invalid. :raises: :exc:`TypeError` when expiration is not a valid type. :raises: :exc:`AttributeError` if credentials is not an instance @@ -2450,7 +2504,22 @@ def generate_signed_url( elif version not in ("v2", "v4"): raise ValueError("'version' must be either 'v2' or 'v4'") - resource = "/{bucket_name}".format(bucket_name=self.name) + if virtual_hosted_style: + api_access_endpoint = "https://{bucket_name}.storage.googleapis.com".format( + bucket_name=self.name + ) + elif bucket_bound_hostname: + if ":" in bucket_bound_hostname: + api_access_endpoint = bucket_bound_hostname + else: + api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( + scheme=scheme, bucket_bound_hostname=bucket_bound_hostname + ) + else: + resource = "/{bucket_name}".format(bucket_name=self.name) + + if virtual_hosted_style or bucket_bound_hostname: + resource = "/" if credentials is None: client = self._require_client(client) diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 41c123880..d72149fc7 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -14,18 +14,29 @@ """Client for interacting with the Google Cloud Storage API.""" +import base64 +import binascii +import collections +import datetime import functools - +import json +import warnings import google.api_core.client_options from google.auth.credentials import AnonymousCredentials from google.api_core import page_iterator -from google.cloud._helpers import _LocalStack +from google.cloud._helpers import _LocalStack, _NOW from google.cloud.client import ClientWithProject from google.cloud.exceptions import NotFound from google.cloud.storage._helpers import _get_storage_host from google.cloud.storage._http import Connection +from google.cloud.storage._signing import ( + get_expiration_seconds_v4, + get_v4_now_dtstamps, + ensure_signed_credentials, + _sign_message, +) from google.cloud.storage.batch import Batch from google.cloud.storage.bucket import Bucket from google.cloud.storage.blob import Blob @@ -222,7 +233,7 @@ def get_service_account_email(self, project=None, timeout=_DEFAULT_TIMEOUT): (Optional) Project ID to use for retreiving GCS service account email address. Defaults to the client's project. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -250,7 +261,7 @@ def bucket(self, bucket_name, user_project=None): :param bucket_name: The name of the bucket to be instantiated. :type user_project: str - :param user_project: (Optional) the project ID to be billed for API + :param user_project: (Optional) The project ID to be billed for API requests made via the bucket. :rtype: :class:`google.cloud.storage.bucket.Bucket` @@ -336,7 +347,7 @@ def lookup_bucket(self, bucket_name, timeout=_DEFAULT_TIMEOUT): :param bucket_name: The name of the bucket to get. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -373,23 +384,24 @@ def create_bucket( ]): The bucket resource to pass or name to create. requester_pays (bool): - Optional. Whether requester pays for API requests for this - bucket and its blobs. + DEPRECATED. Use Bucket().requester_pays instead. + (Optional) Whether requester pays for API requests for + this bucket and its blobs. project (str): - Optional. The project under which the bucket is to be created. + (Optional) The project under which the bucket is to be created. If not passed, uses the project set on the client. user_project (str): - Optional. The project ID to be billed for API requests + (Optional) The project ID to be billed for API requests made via created bucket. location (str): - Optional. The location of the bucket. If not passed, + (Optional) The location of the bucket. If not passed, the default location, US, will be used. See https://cloud.google.com/storage/docs/bucket-locations predefined_acl (str): - Optional. Name of predefined ACL to apply to bucket. See: + (Optional) Name of predefined ACL to apply to bucket. See: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl predefined_default_object_acl (str): - Optional. Name of predefined ACL to apply to bucket's objects. See: + (Optional) Name of predefined ACL to apply to bucket's objects. See: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl timeout (Optional[Union[float, Tuple[float, float]]]): The amount of time, in seconds, to wait for the server response. @@ -435,6 +447,11 @@ def create_bucket( raise ValueError("Client project not set: pass an explicit project.") if requester_pays is not None: + warnings.warn( + "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", + PendingDeprecationWarning, + stacklevel=1, + ) bucket.requester_pays = requester_pays query_params = {"project": project} @@ -482,9 +499,9 @@ def download_blob_to_file(self, blob_or_uri, file_obj, start=None, end=None): file_obj (file): A file handle to which to write the blob's data. start (int): - Optional. The first byte in a range to be downloaded. + (Optional) The first byte in a range to be downloaded. end (int): - Optional. The last byte in a range to be downloaded. + (Optional) The last byte in a range to be downloaded. Examples: Download a blob using using a blob resource. @@ -550,7 +567,7 @@ def list_blobs( token. prefix (str): - (Optional) prefix used to filter blobs. + (Optional) Prefix used to filter blobs. delimiter (str): (Optional) Delimiter, used with ``prefix`` to @@ -618,18 +635,18 @@ def list_buckets( This implements "storage.buckets.list". :type max_results: int - :param max_results: Optional. The maximum number of buckets to return. + :param max_results: (Optional) The maximum number of buckets to return. :type page_token: str :param page_token: - Optional. If present, return the next batch of buckets, using the + (Optional) If present, return the next batch of buckets, using the value, which must correspond to the ``nextPageToken`` value returned in the previous response. Deprecated: use the ``pages`` property of the returned iterator instead of manually passing the token. :type prefix: str - :param prefix: Optional. Filter results to buckets whose names begin + :param prefix: (Optional) Filter results to buckets whose names begin with this prefix. :type projection: str @@ -645,11 +662,11 @@ def list_buckets( bucket returned: 'items/id,nextPageToken' :type project: str - :param project: (Optional) the project whose buckets are to be listed. + :param project: (Optional) The project whose buckets are to be listed. If not passed, uses the project set on the client. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -702,14 +719,14 @@ def create_hmac_key( :param service_account_email: e-mail address of the service account :type project_id: str - :param project_id: (Optional) explicit project ID for the key. + :param project_id: (Optional) Explicit project ID for the key. Defaults to the client's project. :type user_project: str :param user_project: (Optional) This parameter is currently ignored. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -749,26 +766,26 @@ def list_hmac_keys( :type max_results: int :param max_results: - (Optional) max number of keys to return in a given page. + (Optional) Max number of keys to return in a given page. :type service_account_email: str :param service_account_email: - (Optional) limit keys to those created by the given service account. + (Optional) Limit keys to those created by the given service account. :type show_deleted_keys: bool :param show_deleted_keys: - (Optional) included deleted keys in the list. Default is to + (Optional) Included deleted keys in the list. Default is to exclude them. :type project_id: str - :param project_id: (Optional) explicit project ID for the key. + :param project_id: (Optional) Explicit project ID for the key. Defaults to the client's project. :type user_project: str :param user_project: (Optional) This parameter is currently ignored. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -813,11 +830,11 @@ def get_hmac_key_metadata( :param access_id: Unique ID of an existing key. :type project_id: str - :param project_id: (Optional) project ID of an existing key. + :param project_id: (Optional) Project ID of an existing key. Defaults to client's project. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -830,6 +847,181 @@ def get_hmac_key_metadata( metadata.reload(timeout=timeout) # raises NotFound for missing key return metadata + def generate_signed_post_policy_v4( + self, + bucket_name, + blob_name, + expiration, + conditions=None, + fields=None, + credentials=None, + virtual_hosted_style=False, + bucket_bound_hostname=None, + scheme=None, + service_account_email=None, + access_token=None, + ): + """Generate a V4 signed policy object. + + .. note:: + + Assumes ``credentials`` implements the + :class:`google.auth.credentials.Signing` interface. Also assumes + ``credentials`` has a ``service_account_email`` property which + identifies the credentials. + + Generated policy object allows user to upload objects with a POST request. + + :type bucket_name: str + :param bucket_name: Bucket name. + + :type blob_name: str + :param blob_name: Object name. + + :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] + :param expiration: Policy expiration time. + + :type conditions: list + :param conditions: (Optional) List of POST policy conditions, which are + used to restrict what is allowed in the request. + + :type fields: dict + :param fields: (Optional) Additional elements to include into request. + + :type credentials: :class:`google.auth.credentials.Signing` + :param credentials: (Optional) Credentials object with an associated private + key to sign text. + + :type virtual_hosted_style: bool + :param virtual_hosted_style: (Optional) If True, construct the URL relative to the bucket + virtual hostname, e.g., '.storage.googleapis.com'. + + :type bucket_bound_hostname: str + :param bucket_bound_hostname: + (Optional) If passed, construct the URL relative to the bucket-bound hostname. + Value can be bare or with a scheme, e.g., 'example.com' or 'http://example.com'. + See: https://cloud.google.com/storage/docs/request-endpoints#cname + + :type scheme: str + :param scheme: + (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use + this value as a scheme. ``https`` will work only when using a CDN. + Defaults to ``"http"``. + + :type service_account_email: str + :param service_account_email: (Optional) E-mail address of the service account. + + :type access_token: str + :param access_token: (Optional) Access token for a service account. + + :rtype: dict + :returns: Signed POST policy. + + Example: + Generate signed POST policy and upload a file. + + >>> from google.cloud import storage + >>> client = storage.Client() + >>> policy = client.generate_signed_post_policy_v4( + "bucket-name", + "blob-name", + expiration=datetime.datetime(2020, 3, 17), + conditions=[ + ["content-length-range", 0, 255] + ], + fields=[ + "x-goog-meta-hello" => "world" + ], + ) + >>> with open("bucket-name", "rb") as f: + files = {"file": ("bucket-name", f)} + requests.post(policy["url"], data=policy["fields"], files=files) + """ + credentials = self._credentials if credentials is None else credentials + ensure_signed_credentials(credentials) + + # prepare policy conditions and fields + timestamp, datestamp = get_v4_now_dtstamps() + + x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format( + email=credentials.signer_email, datestamp=datestamp + ) + required_conditions = [ + {"key": blob_name}, + {"x-goog-date": timestamp}, + {"x-goog-credential": x_goog_credential}, + {"x-goog-algorithm": "GOOG4-RSA-SHA256"}, + ] + + conditions = conditions or [] + policy_fields = {} + for key, value in sorted((fields or {}).items()): + if not key.startswith("x-ignore-"): + policy_fields[key] = value + conditions.append({key: value}) + + conditions += required_conditions + + # calculate policy expiration time + now = _NOW() + if expiration is None: + expiration = now + datetime.timedelta(hours=1) + + policy_expires = now + datetime.timedelta( + seconds=get_expiration_seconds_v4(expiration) + ) + + # encode policy for signing + policy = json.dumps( + collections.OrderedDict( + sorted( + { + "conditions": conditions, + "expiration": policy_expires.isoformat() + "Z", + }.items() + ) + ), + separators=(",", ":"), + ) + str_to_sign = base64.b64encode(policy.encode("utf-8")) + + # sign the policy and get its cryptographic signature + if access_token and service_account_email: + signature = _sign_message(str_to_sign, access_token, service_account_email) + signature_bytes = base64.b64decode(signature) + else: + signature_bytes = credentials.sign_bytes(str_to_sign) + + # get hexadecimal representation of the signature + signature = binascii.hexlify(signature_bytes).decode("utf-8") + + policy_fields.update( + { + "key": blob_name, + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": x_goog_credential, + "x-goog-date": timestamp, + "x-goog-signature": signature, + "policy": str_to_sign, + } + ) + # designate URL + if virtual_hosted_style: + url = "https://{}.storage.googleapis.com/".format(bucket_name) + + elif bucket_bound_hostname: + if ":" in bucket_bound_hostname: # URL includes scheme + url = bucket_bound_hostname + + else: # scheme is given separately + url = "{scheme}://{host}/".format( + scheme=scheme, host=bucket_bound_hostname + ) + else: + url = "https://storage.googleapis.com/{}/".format(bucket_name) + + return {"url": url, "fields": policy_fields} + def _item_to_bucket(iterator, item): """Convert a JSON bucket to the native object. diff --git a/google/cloud/storage/hmac_key.py b/google/cloud/storage/hmac_key.py index 296b38e92..d9c451c68 100644 --- a/google/cloud/storage/hmac_key.py +++ b/google/cloud/storage/hmac_key.py @@ -25,10 +25,10 @@ class HMACKeyMetadata(object): :param client: client associated with the key metadata. :type access_id: str - :param access_id: (Optional) unique ID of an existing key. + :param access_id: (Optional) Unique ID of an existing key. :type project_id: str - :param project_id: (Optional) project ID of an existing key. + :param project_id: (Optional) Project ID of an existing key. Defaults to client's project. :type user_project: str @@ -191,7 +191,7 @@ def exists(self, timeout=_DEFAULT_TIMEOUT): """Determine whether or not the key for this metadata exists. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -218,7 +218,7 @@ def reload(self, timeout=_DEFAULT_TIMEOUT): """Reload properties from Cloud Storage. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -240,7 +240,7 @@ def update(self, timeout=_DEFAULT_TIMEOUT): """Save writable properties to Cloud Storage. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -266,7 +266,7 @@ def delete(self, timeout=_DEFAULT_TIMEOUT): """Delete the key from Cloud Storage. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). diff --git a/google/cloud/storage/iam.py b/google/cloud/storage/iam.py index fb7e9e4ed..36c7412b8 100644 --- a/google/cloud/storage/iam.py +++ b/google/cloud/storage/iam.py @@ -25,7 +25,7 @@ STORAGE_OBJECT_VIEWER_ROLE = "roles/storage.objectViewer" """Role implying rights to view object properties, excluding ACLs.""" -STORAGE_OBJECT_ADMIN_ROLE = "roles/storage.objectViewer" +STORAGE_OBJECT_ADMIN_ROLE = "roles/storage.objectAdmin" """Role implying full control of objects.""" STORAGE_ADMIN_ROLE = "roles/storage.admin" diff --git a/google/cloud/storage/notification.py b/google/cloud/storage/notification.py index e9618f668..434a44dd1 100644 --- a/google/cloud/storage/notification.py +++ b/google/cloud/storage/notification.py @@ -50,40 +50,46 @@ class BucketNotification(object): :param bucket: Bucket to which the notification is bound. :type topic_name: str - :param topic_name: Topic name to which notifications are published. + :param topic_name: + (Optional) Topic name to which notifications are published. :type topic_project: str :param topic_project: - (Optional) project ID of topic to which notifications are published. + (Optional) Project ID of topic to which notifications are published. If not passed, uses the project ID of the bucket's client. :type custom_attributes: dict :param custom_attributes: - (Optional) additional attributes passed with notification events. + (Optional) Additional attributes passed with notification events. :type event_types: list(str) :param event_types: - (Optional) event types for which notificatin events are published. + (Optional) Event types for which notification events are published. :type blob_name_prefix: str :param blob_name_prefix: - (Optional) prefix of blob names for which notification events are - published.. + (Optional) Prefix of blob names for which notification events are + published. :type payload_format: str :param payload_format: - (Optional) format of payload for notification events. + (Optional) Format of payload for notification events. + + :type notification_id: str + :param notification_id: + (Optional) The ID of the notification. """ def __init__( self, bucket, - topic_name, + topic_name=None, topic_project=None, custom_attributes=None, event_types=None, blob_name_prefix=None, payload_format=NONE_PAYLOAD_FORMAT, + notification_id=None, ): self._bucket = bucket self._topic_name = topic_name @@ -107,6 +113,9 @@ def __init__( if blob_name_prefix is not None: self._properties["object_name_prefix"] = blob_name_prefix + if notification_id is not None: + self._properties["id"] = notification_id + self._properties["payload_format"] = payload_format @classmethod @@ -233,10 +242,10 @@ def create(self, client=None, timeout=_DEFAULT_TIMEOUT): to that project. :type client: :class:`~google.cloud.storage.client.Client` - :param client: (Optional) the client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the notification's bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -275,10 +284,10 @@ def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). @@ -317,17 +326,15 @@ def reload(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. - :rtype: bool - :returns: True, if the notification exists, else False. :raises ValueError: if the notification has no ID. """ if self.notification_id is None: @@ -355,10 +362,10 @@ def delete(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: Optional. The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type timeout: float or tuple - :param timeout: (optional) The amount of time, in seconds, to wait + :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). diff --git a/noxfile.py b/noxfile.py index 1b44b309f..058dcdd61 100644 --- a/noxfile.py +++ b/noxfile.py @@ -23,14 +23,14 @@ import nox -BLACK_VERSION = "black==19.3b0" +BLACK_VERSION = "black==19.10b0" BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] if os.path.exists("samples"): BLACK_PATHS.append("samples") -@nox.session(python="3.7") +@nox.session(python="3.8") def lint(session): """Run linters. @@ -56,7 +56,7 @@ def blacken(session): session.run("black", *BLACK_PATHS) -@nox.session(python="3.7") +@nox.session(python="3.8") def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" session.install("docutils", "pygments") @@ -89,7 +89,7 @@ def unit(session): default(session) -@nox.session(python=["2.7", "3.7"]) +@nox.session(python=["2.7", "3.8"]) def system(session): """Run the system test suite.""" system_test_path = os.path.join("tests", "system.py") @@ -124,7 +124,7 @@ def system(session): session.run("py.test", "--quiet", system_test_folder_path, *session.posargs) -@nox.session(python="3.7") +@nox.session(python="3.8") def cover(session): """Run the final coverage report. @@ -137,7 +137,7 @@ def cover(session): session.run("coverage", "erase") -@nox.session(python="3.7") +@nox.session(python="3.8") def docs(session): """Build the docs for this library.""" diff --git a/setup.py b/setup.py index f1a344032..c6bf342fe 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ name = "google-cloud-storage" description = "Google Cloud Storage API client library" -version = "1.26.0" +version = "1.27.0" # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' @@ -76,6 +76,7 @@ "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Operating System :: OS Independent", "Topic :: Internet", ], diff --git a/synth.metadata b/synth.metadata index cc98759ac..2ac902144 100644 --- a/synth.metadata +++ b/synth.metadata @@ -1,11 +1,25 @@ { - "updateTime": "2020-01-31T20:56:14.590164Z", + "updateTime": "2020-02-19T02:28:15.687395Z", "sources": [ + { + "git": { + "name": ".", + "remote": "https://github.com/googleapis/python-storage.git", + "sha": "6c9296ce6847e1ba35aec1a3cc22c020bedeb592" + } + }, + { + "git": { + "name": "synthtool", + "remote": "rpc://devrel/cloud/libraries/tools/autosynth", + "sha": "b4b7af4a16a07b40bfd8dcdda89f9f193ff4e2ed" + } + }, { "template": { "name": "python_split_library", "origin": "synthtool.gcp", - "version": "2019.10.17" + "version": "2020.2.4" } } ] diff --git a/tests/perf/benchwrapper.py b/tests/perf/benchwrapper.py index 9ebb3f455..c81d6bb20 100644 --- a/tests/perf/benchwrapper.py +++ b/tests/perf/benchwrapper.py @@ -2,7 +2,6 @@ import sys import time import grpc -import os from concurrent import futures import storage_pb2_grpc import storage_pb2 @@ -12,10 +11,10 @@ parser = argparse.ArgumentParser() -if os.environ.get("STORAGE_EMULATOR_HOST") is None: - sys.exit( - "This benchmarking server only works when connected to an emulator. Please set STORAGE_EMULATOR_HOST." - ) +# if os.environ.get("STORAGE_EMULATOR_HOST") is None: +# sys.exit( +# "This benchmarking server only works when connected to an emulator. Please set STORAGE_EMULATOR_HOST." +# ) parser.add_argument("--port", help="The port to run on.") @@ -24,7 +23,8 @@ if args.port is None: sys.exit("Usage: python3 main.py --port 8081") -client = storage.Client.create_anonymous_client() +# client = storage.Client.create_anonymous_client() +client = storage.Client() class StorageBenchWrapperServicer(storage_pb2_grpc.StorageBenchWrapperServicer): diff --git a/tests/system.py b/tests/system/test_system.py similarity index 94% rename from tests/system.py rename to tests/system/test_system.py index 995b984ed..2cb3dc0ab 100644 --- a/tests/system.py +++ b/tests/system/test_system.py @@ -40,6 +40,8 @@ USER_PROJECT = os.environ.get("GOOGLE_CLOUD_TESTS_USER_PROJECT") +DIRNAME = os.path.realpath(os.path.dirname(__file__)) +DATA_DIRNAME = os.path.abspath(os.path.join(DIRNAME, "..", "data")) def _bad_copy(bad_request): @@ -451,11 +453,10 @@ def test_bucket_get_blob_with_user_project(self): class TestStorageFiles(unittest.TestCase): - DIRNAME = os.path.realpath(os.path.dirname(__file__)) FILES = { - "logo": {"path": DIRNAME + "/data/CloudPlatform_128px_Retina.png"}, - "big": {"path": DIRNAME + "/data/five-point-one-mb-file.zip"}, - "simple": {"path": DIRNAME + "/data/simple.txt"}, + "logo": {"path": DATA_DIRNAME + "/CloudPlatform_128px_Retina.png"}, + "big": {"path": DATA_DIRNAME + "/five-point-one-mb-file.zip"}, + "simple": {"path": DATA_DIRNAME + "/simple.txt"}, } @classmethod @@ -1432,6 +1433,7 @@ def test_notification_minimal(self): bucket = retry_429_503(Config.CLIENT.create_bucket)(new_bucket_name) self.case_buckets_to_delete.append(new_bucket_name) self.assertEqual(list(bucket.list_notifications()), []) + notification = bucket.notification(self.TOPIC_NAME) retry_429_503(notification.create)() try: @@ -1448,7 +1450,7 @@ def test_notification_explicit(self): bucket = retry_429_503(Config.CLIENT.create_bucket)(new_bucket_name) self.case_buckets_to_delete.append(new_bucket_name) notification = bucket.notification( - self.TOPIC_NAME, + topic_name=self.TOPIC_NAME, custom_attributes=self.CUSTOM_ATTRIBUTES, event_types=self.event_types(), blob_name_prefix=self.BLOB_NAME_PREFIX, @@ -1462,6 +1464,7 @@ def test_notification_explicit(self): self.assertEqual(notification.event_types, self.event_types()) self.assertEqual(notification.blob_name_prefix, self.BLOB_NAME_PREFIX) self.assertEqual(notification.payload_format, self.payload_format()) + finally: notification.delete() @@ -1485,6 +1488,28 @@ def test_notification_w_user_project(self): finally: notification.delete() + def test_get_notification(self): + new_bucket_name = "get-notification" + unique_resource_id("-") + bucket = retry_429_503(Config.CLIENT.create_bucket)(new_bucket_name) + self.case_buckets_to_delete.append(new_bucket_name) + + notification = bucket.notification( + topic_name=self.TOPIC_NAME, + custom_attributes=self.CUSTOM_ATTRIBUTES, + payload_format=self.payload_format(), + ) + retry_429_503(notification.create)() + try: + self.assertTrue(notification.exists()) + self.assertIsNotNone(notification.notification_id) + notification_id = notification.notification_id + notification = bucket.get_notification(notification_id) + self.assertEqual(notification.notification_id, notification_id) + self.assertEqual(notification.custom_attributes, self.CUSTOM_ATTRIBUTES) + self.assertEqual(notification.payload_format, self.payload_format()) + finally: + notification.delete() + class TestAnonymousClient(unittest.TestCase): @@ -1494,7 +1519,7 @@ class TestAnonymousClient(unittest.TestCase): def test_access_to_public_bucket(self): anonymous = storage.Client.create_anonymous_client() bucket = anonymous.bucket(self.PUBLIC_BUCKET) - blob, = retry_429_503(bucket.list_blobs)(max_results=1) + (blob,) = retry_429_503(bucket.list_blobs)(max_results=1) with tempfile.TemporaryFile() as stream: retry_429_503(blob.download_to_file)(stream) @@ -1581,7 +1606,7 @@ def test_blob_w_explicit_kms_key_name(self): # We don't know the current version of the key. self.assertTrue(blob.kms_key_name.startswith(kms_key_name)) - listed, = list(self.bucket.list_blobs()) + (listed,) = list(self.bucket.list_blobs()) self.assertTrue(listed.kms_key_name.startswith(kms_key_name)) def test_bucket_w_default_kms_key_name(self): @@ -1930,3 +1955,67 @@ def test_ubla_set_unset_preserves_acls(self): self.assertEqual(bucket_acl_before, bucket_acl_after) self.assertEqual(blob_acl_before, blob_acl_after) + + +class TestV4POSTPolicies(unittest.TestCase): + def setUp(self): + self.case_buckets_to_delete = [] + + def tearDown(self): + for bucket_name in self.case_buckets_to_delete: + bucket = Config.CLIENT.bucket(bucket_name) + retry_429_harder(bucket.delete)(force=True) + + def test_get_signed_policy_v4(self): + bucket_name = "post_policy" + unique_resource_id("-") + self.assertRaises(exceptions.NotFound, Config.CLIENT.get_bucket, bucket_name) + retry_429_503(Config.CLIENT.create_bucket)(bucket_name) + self.case_buckets_to_delete.append(bucket_name) + + blob_name = "post_policy_obj.txt" + with open(blob_name, "w") as f: + f.write("DEADBEEF") + + policy = Config.CLIENT.generate_signed_post_policy_v4( + bucket_name, + blob_name, + conditions=[ + {"bucket": bucket_name}, + ["starts-with", "$Content-Type", "text/pla"], + ], + expiration=datetime.datetime.now() + datetime.timedelta(hours=1), + fields={"content-type": "text/plain"}, + ) + with open(blob_name, "r") as f: + files = {"file": (blob_name, f)} + response = requests.post(policy["url"], data=policy["fields"], files=files) + + os.remove(blob_name) + self.assertEqual(response.status_code, 204) + + def test_get_signed_policy_v4_invalid_field(self): + bucket_name = "post_policy" + unique_resource_id("-") + self.assertRaises(exceptions.NotFound, Config.CLIENT.get_bucket, bucket_name) + retry_429_503(Config.CLIENT.create_bucket)(bucket_name) + self.case_buckets_to_delete.append(bucket_name) + + blob_name = "post_policy_obj.txt" + with open(blob_name, "w") as f: + f.write("DEADBEEF") + + policy = Config.CLIENT.generate_signed_post_policy_v4( + bucket_name, + blob_name, + conditions=[ + {"bucket": bucket_name}, + ["starts-with", "$Content-Type", "text/pla"], + ], + expiration=datetime.datetime.now() + datetime.timedelta(hours=1), + fields={"x-goog-random": "invalid_field", "content-type": "text/plain"}, + ) + with open(blob_name, "r") as f: + files = {"file": (blob_name, f)} + response = requests.post(policy["url"], data=policy["fields"], files=files) + + os.remove(blob_name) + self.assertEqual(response.status_code, 400) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index df379f1e9..a864e9eae 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -11,3 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import io +import json +import os + + +def _read_local_json(json_file): + here = os.path.dirname(__file__) + json_path = os.path.abspath(os.path.join(here, json_file)) + with io.open(json_path, "r", encoding="utf-8-sig") as fileobj: + return json.load(fileobj) diff --git a/tests/unit/test__signing.py b/tests/unit/test__signing.py index ebd7f9c17..d1d2224e3 100644 --- a/tests/unit/test__signing.py +++ b/tests/unit/test__signing.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +# # Copyright 2017 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,9 +18,7 @@ import binascii import calendar import datetime -import io import json -import os import time import unittest @@ -27,12 +27,7 @@ import six from six.moves import urllib_parse - -def _read_local_json(json_file): - here = os.path.dirname(__file__) - json_path = os.path.abspath(os.path.join(here, json_file)) - with io.open(json_path, "r", encoding="utf-8-sig") as fileobj: - return json.load(fileobj) +from . import _read_local_json _SERVICE_ACCOUNT_JSON = _read_local_json("url_signer_v4_test_account.json") @@ -309,12 +304,12 @@ def test_w_embedded_ws(self): self.assertEqual(ordered, expected_ordered) -class Test_canonicalize(unittest.TestCase): +class Test_canonicalize_v2(unittest.TestCase): @staticmethod def _call_fut(*args, **kwargs): - from google.cloud.storage._signing import canonicalize + from google.cloud.storage._signing import canonicalize_v2 - return canonicalize(*args, **kwargs) + return canonicalize_v2(*args, **kwargs) def test_wo_headers_or_query_parameters(self): method = "GET" @@ -337,7 +332,7 @@ def test_w_headers_and_resumable(self): canonical.headers, ["x-goog-extension:foobar", "x-goog-resumable:start"] ) - def test_w_query_paramters(self): + def test_w_query_parameters(self): method = "GET" resource = "/bucket/blob" query_parameters = {"foo": "bar", "baz": "qux"} @@ -650,6 +645,9 @@ def test_w_custom_host_header(self): def test_w_custom_headers(self): self._generate_helper(headers={"x-goog-foo": "bar"}) + def test_w_custom_payload_hash_goog(self): + self._generate_helper(headers={"x-goog-content-sha256": "DEADBEEF"}) + def test_w_custom_query_parameters_w_string_value(self): self._generate_helper(query_parameters={"bar": "/"}) @@ -702,6 +700,77 @@ def test_sign_bytes_failure(self): ) +class TestCustomURLEncoding(unittest.TestCase): + def test_url_encode(self): + from google.cloud.storage._signing import _url_encode + + # param1 includes safe symbol ~ + # param# includes symbols, which must be encoded + query_params = {"param1": "value~1-2", "param#": "*value+value/"} + + self.assertEqual( + _url_encode(query_params), "param%23=%2Avalue%2Bvalue%2F¶m1=value~1-2" + ) + + +class TestQuoteParam(unittest.TestCase): + def test_ascii_symbols(self): + from google.cloud.storage._signing import _quote_param + + encoded_param = _quote_param("param") + self.assertIsInstance(encoded_param, str) + self.assertEqual(encoded_param, "param") + + def test_quoted_symbols(self): + from google.cloud.storage._signing import _quote_param + + encoded_param = _quote_param("!#$%&'()*+,/:;=?@[]") + self.assertIsInstance(encoded_param, str) + self.assertEqual( + encoded_param, "%21%23%24%25%26%27%28%29%2A%2B%2C%2F%3A%3B%3D%3F%40%5B%5D" + ) + + def test_unquoted_symbols(self): + from google.cloud.storage._signing import _quote_param + import string + + UNQUOTED = string.ascii_letters + string.digits + ".~_-" + + encoded_param = _quote_param(UNQUOTED) + self.assertIsInstance(encoded_param, str) + self.assertEqual(encoded_param, UNQUOTED) + + def test_unicode_symbols(self): + from google.cloud.storage._signing import _quote_param + + encoded_param = _quote_param("ЁЙЦЯЩЯЩ") + self.assertIsInstance(encoded_param, str) + self.assertEqual(encoded_param, "%D0%81%D0%99%D0%A6%D0%AF%D0%A9%D0%AF%D0%A9") + + def test_bytes(self): + from google.cloud.storage._signing import _quote_param + + encoded_param = _quote_param(b"bytes") + self.assertIsInstance(encoded_param, str) + self.assertEqual(encoded_param, "bytes") + + +class TestV4Stamps(unittest.TestCase): + def test_get_v4_now_dtstamps(self): + import datetime + from google.cloud.storage._signing import get_v4_now_dtstamps + + with mock.patch( + "google.cloud.storage._signing.NOW", + return_value=datetime.datetime(2020, 3, 12, 13, 14, 15), + ) as now_mock: + timestamp, datestamp = get_v4_now_dtstamps() + now_mock.assert_called_once() + + self.assertEqual(timestamp, "20200312T131415Z") + self.assertEqual(datestamp, "20200312") + + _DUMMY_SERVICE_ACCOUNT = None @@ -718,16 +787,22 @@ def dummy_service_account(): return _DUMMY_SERVICE_ACCOUNT -def _run_conformance_test(resource, test_data): - credentials = dummy_service_account() +_API_ACCESS_ENDPOINT = "https://storage.googleapis.com" + +def _run_conformance_test( + resource, test_data, api_access_endpoint=_API_ACCESS_ENDPOINT +): + credentials = dummy_service_account() url = Test_generate_signed_url_v4._call_fut( credentials, resource, expiration=test_data["expiration"], + api_access_endpoint=api_access_endpoint, method=test_data["method"], _request_timestamp=test_data["timestamp"], headers=test_data.get("headers"), + query_parameters=test_data.get("queryParameters"), ) assert url == test_data["expectedUrl"] @@ -741,14 +816,39 @@ def test_conformance_client(test_data): @pytest.mark.parametrize("test_data", _BUCKET_TESTS) def test_conformance_bucket(test_data): - resource = "/{}".format(test_data["bucket"]) - _run_conformance_test(resource, test_data) + global _API_ACCESS_ENDPOINT + if "urlStyle" in test_data and test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME": + _API_ACCESS_ENDPOINT = "{scheme}://{bucket_bound_hostname}".format( + scheme=test_data["scheme"], + bucket_bound_hostname=test_data["bucketBoundHostname"], + ) + resource = "/" + _run_conformance_test(resource, test_data, _API_ACCESS_ENDPOINT) + else: + resource = "/{}".format(test_data["bucket"]) + _run_conformance_test(resource, test_data) @pytest.mark.parametrize("test_data", _BLOB_TESTS) def test_conformance_blob(test_data): - resource = "/{}/{}".format(test_data["bucket"], test_data["object"]) - _run_conformance_test(resource, test_data) + global _API_ACCESS_ENDPOINT + if "urlStyle" in test_data: + if test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME": + _API_ACCESS_ENDPOINT = "{scheme}://{bucket_bound_hostname}".format( + scheme=test_data["scheme"], + bucket_bound_hostname=test_data["bucketBoundHostname"], + ) + + # For the VIRTUAL_HOSTED_STYLE + else: + _API_ACCESS_ENDPOINT = "{scheme}://{bucket_name}.storage.googleapis.com".format( + scheme=test_data["scheme"], bucket_name=test_data["bucket"] + ) + resource = "/{}".format(test_data["object"]) + _run_conformance_test(resource, test_data, _API_ACCESS_ENDPOINT) + else: + resource = "/{}/{}".format(test_data["bucket"], test_data["object"]) + _run_conformance_test(resource, test_data) def _make_credentials(signer_email=None): diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 1c2b1e90d..f656e6441 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -399,6 +399,9 @@ def _generate_signed_url_helper( encryption_key=None, access_token=None, service_account_email=None, + virtual_hosted_style=False, + bucket_bound_hostname=None, + scheme="http", ): from six.moves.urllib import parse from google.cloud._helpers import UTC @@ -442,6 +445,8 @@ def _generate_signed_url_helper( version=version, access_token=access_token, service_account_email=service_account_email, + virtual_hosted_style=virtual_hosted_style, + bucket_bound_hostname=bucket_bound_hostname, ) self.assertEqual(signed_uri, signer.return_value) @@ -452,7 +457,26 @@ def _generate_signed_url_helper( expected_creds = credentials encoded_name = blob_name.encode("utf-8") - expected_resource = "/name/{}".format(parse.quote(encoded_name, safe=b"/~")) + quoted_name = parse.quote(encoded_name, safe=b"/~") + + if virtual_hosted_style: + expected_api_access_endpoint = "https://{}.storage.googleapis.com".format( + bucket.name + ) + elif bucket_bound_hostname: + if ":" in bucket_bound_hostname: + expected_api_access_endpoint = bucket_bound_hostname + else: + expected_api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( + scheme=scheme, bucket_bound_hostname=bucket_bound_hostname + ) + else: + expected_api_access_endpoint = api_access_endpoint + expected_resource = "/{}/{}".format(bucket.name, quoted_name) + + if virtual_hosted_style or bucket_bound_hostname: + expected_resource = "/{}".format(quoted_name) + if encryption_key is not None: expected_headers = headers or {} if effective_version == "v2": @@ -465,7 +489,7 @@ def _generate_signed_url_helper( expected_kwargs = { "resource": expected_resource, "expiration": expiration, - "api_access_endpoint": api_access_endpoint, + "api_access_endpoint": expected_api_access_endpoint, "method": method.upper(), "content_md5": content_md5, "content_type": content_type, @@ -604,6 +628,17 @@ def test_generate_signed_url_v4_w_csek_and_headers(self): encryption_key=os.urandom(32), headers={"x-goog-foo": "bar"} ) + def test_generate_signed_url_v4_w_virtual_hostname(self): + self._generate_signed_url_v4_helper(virtual_hosted_style=True) + + def test_generate_signed_url_v4_w_bucket_bound_hostname_w_scheme(self): + self._generate_signed_url_v4_helper( + bucket_bound_hostname="http://cdn.example.com" + ) + + def test_generate_signed_url_v4_w_bucket_bound_hostname_w_bare_hostname(self): + self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com") + def test_generate_signed_url_v4_w_credentials(self): credentials = object() self._generate_signed_url_v4_helper(credentials=credentials) @@ -1215,6 +1250,16 @@ def test__get_writable_metadata_unwritable_field(self): expected = {"name": name} self.assertEqual(object_metadata, expected) + def test__set_metadata_to_none(self): + name = u"blob-name" + blob = self._make_one(name, bucket=None) + blob.storage_class = "NEARLINE" + blob.cache_control = "max-age=3600" + + with mock.patch("google.cloud.storage.blob.Blob._patch_property") as patch_prop: + blob.metadata = None + patch_prop.assert_called_once_with("metadata", None) + def test__get_upload_arguments(self): name = u"blob-name" key = b"[pXw@,p@@AfBfrR3x-2b2SCHR,.?YwRO" @@ -2426,7 +2471,7 @@ def test_rewrite_w_generations(self): self.assertEqual(rewritten, 33) self.assertEqual(size, 42) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "POST") self.assertEqual( kw["path"], diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 312fc0f65..365e1f0e1 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -615,223 +615,6 @@ def api_request(cls, *args, **kwargs): expected_cw = [((), expected_called_kwargs)] self.assertEqual(_FakeConnection._called_with, expected_cw) - def test_create_w_user_project(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - USER_PROJECT = "user-project-123" - - client = Client(project=PROJECT) - client._base_connection = _Connection() - - bucket = self._make_one(client, BUCKET_NAME, user_project=USER_PROJECT) - - with self.assertRaises(ValueError): - bucket.create() - - def test_create_w_missing_client_project(self): - from google.cloud.storage.client import Client - - BUCKET_NAME = "bucket-name" - - client = Client(project=None) - bucket = self._make_one(client, BUCKET_NAME) - - with self.assertRaises(ValueError): - bucket.create() - - def test_create_w_explicit_project(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - OTHER_PROJECT = "other-project-123" - DATA = {"name": BUCKET_NAME} - connection = _make_connection(DATA) - - client = Client(project=PROJECT) - client._base_connection = connection - - bucket = self._make_one(client, BUCKET_NAME) - bucket.create(project=OTHER_PROJECT) - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": OTHER_PROJECT}, - data=DATA, - _target_object=bucket, - timeout=self._get_default_timeout(), - ) - - def test_create_w_explicit_location(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - LOCATION = "us-central1" - DATA = {"location": LOCATION, "name": BUCKET_NAME} - - connection = _make_connection( - DATA, "{'location': 'us-central1', 'name': 'bucket-name'}" - ) - - client = Client(project=PROJECT) - client._base_connection = connection - - bucket = self._make_one(client, BUCKET_NAME) - bucket.create(location=LOCATION) - - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - data=DATA, - _target_object=bucket, - query_params={"project": "PROJECT"}, - timeout=self._get_default_timeout(), - ) - self.assertEqual(bucket.location, LOCATION) - - def test_create_hit(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _make_connection(DATA) - client = Client(project=PROJECT) - client._base_connection = connection - - bucket = self._make_one(client=client, name=BUCKET_NAME) - bucket.create(timeout=42) - - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": PROJECT}, - data=DATA, - _target_object=bucket, - timeout=42, - ) - - def test_create_w_extra_properties(self): - from google.cloud.storage.client import Client - - BUCKET_NAME = "bucket-name" - PROJECT = "PROJECT" - CORS = [ - { - "maxAgeSeconds": 60, - "methods": ["*"], - "origin": ["https://example.com/frontend"], - "responseHeader": ["X-Custom-Header"], - } - ] - LIFECYCLE_RULES = [{"action": {"type": "Delete"}, "condition": {"age": 365}}] - LOCATION = "eu" - LABELS = {"color": "red", "flavor": "cherry"} - STORAGE_CLASS = "NEARLINE" - DATA = { - "name": BUCKET_NAME, - "cors": CORS, - "lifecycle": {"rule": LIFECYCLE_RULES}, - "location": LOCATION, - "storageClass": STORAGE_CLASS, - "versioning": {"enabled": True}, - "billing": {"requesterPays": True}, - "labels": LABELS, - } - - connection = _make_connection(DATA) - client = Client(project=PROJECT) - client._base_connection = connection - - bucket = self._make_one(client=client, name=BUCKET_NAME) - bucket.cors = CORS - bucket.lifecycle_rules = LIFECYCLE_RULES - bucket.storage_class = STORAGE_CLASS - bucket.versioning_enabled = True - bucket.requester_pays = True - bucket.labels = LABELS - bucket.create(location=LOCATION) - - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": PROJECT}, - data=DATA, - _target_object=bucket, - timeout=self._get_default_timeout(), - ) - - def test_create_w_predefined_acl_invalid(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _Connection(DATA) - client = Client(project=PROJECT) - client._base_connection = connection - bucket = self._make_one(client=client, name=BUCKET_NAME) - - with self.assertRaises(ValueError): - bucket.create(predefined_acl="bogus") - - def test_create_w_predefined_acl_valid(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _Connection(DATA) - client = Client(project=PROJECT) - client._base_connection = connection - bucket = self._make_one(client=client, name=BUCKET_NAME) - bucket.create(predefined_acl="publicRead") - - kw, = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], "/b") - expected_qp = {"project": PROJECT, "predefinedAcl": "publicRead"} - self.assertEqual(kw["query_params"], expected_qp) - self.assertEqual(kw["data"], DATA) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - - def test_create_w_predefined_default_object_acl_invalid(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _Connection(DATA) - client = Client(project=PROJECT) - client._base_connection = connection - bucket = self._make_one(client=client, name=BUCKET_NAME) - - with self.assertRaises(ValueError): - bucket.create(predefined_default_object_acl="bogus") - - def test_create_w_predefined_default_object_acl_valid(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _Connection(DATA) - client = Client(project=PROJECT) - client._base_connection = connection - bucket = self._make_one(client=client, name=BUCKET_NAME) - bucket.create(predefined_default_object_acl="publicRead") - - kw, = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], "/b") - expected_qp = {"project": PROJECT, "predefinedDefaultObjectAcl": "publicRead"} - self.assertEqual(kw["query_params"], expected_qp) - self.assertEqual(kw["data"], DATA) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - def test_acl_property(self): from google.cloud.storage.acl import BucketACL @@ -865,7 +648,7 @@ def test_get_blob_miss(self): bucket = self._make_one(name=NAME) result = bucket.get_blob(NONESUCH, client=client, timeout=42) self.assertIsNone(result) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "GET") self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, NONESUCH)) self.assertEqual(kw["timeout"], 42) @@ -880,7 +663,7 @@ def test_get_blob_hit_w_user_project(self): blob = bucket.get_blob(BLOB_NAME, client=client) self.assertIs(blob.bucket, bucket) self.assertEqual(blob.name, BLOB_NAME) - kw, = connection._requested + (kw,) = connection._requested expected_qp = {"userProject": USER_PROJECT, "projection": "noAcl"} self.assertEqual(kw["method"], "GET") self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) @@ -898,7 +681,7 @@ def test_get_blob_hit_w_generation(self): self.assertIs(blob.bucket, bucket) self.assertEqual(blob.name, BLOB_NAME) self.assertEqual(blob.generation, GENERATION) - kw, = connection._requested + (kw,) = connection._requested expected_qp = {"generation": GENERATION, "projection": "noAcl"} self.assertEqual(kw["method"], "GET") self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) @@ -921,7 +704,7 @@ def test_get_blob_hit_with_kwargs(self): ) self.assertIs(blob.bucket, bucket) self.assertEqual(blob.name, BLOB_NAME) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "GET") self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) self.assertEqual(kw["headers"], _get_encryption_headers(KEY)) @@ -937,7 +720,7 @@ def test_list_blobs_defaults(self): iterator = bucket.list_blobs() blobs = list(iterator) self.assertEqual(blobs, []) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "GET") self.assertEqual(kw["path"], "/b/%s/o" % NAME) self.assertEqual(kw["query_params"], {"projection": "noAcl"}) @@ -979,7 +762,7 @@ def test_list_blobs_w_all_arguments_and_user_project(self): ) blobs = list(iterator) self.assertEqual(blobs, []) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "GET") self.assertEqual(kw["path"], "/b/%s/o" % NAME) self.assertEqual(kw["query_params"], EXPECTED) @@ -1043,6 +826,45 @@ def test_list_notifications(self): notification.payload_format, resource.get("payload_format") ) + def test_get_notification(self): + from google.cloud.storage.notification import _TOPIC_REF_FMT + from google.cloud.storage.notification import JSON_API_V1_PAYLOAD_FORMAT + + NAME = "name" + ETAG = "FACECABB" + NOTIFICATION_ID = "1" + SELF_LINK = "https://example.com/notification/1" + resources = { + "topic": _TOPIC_REF_FMT.format("my-project-123", "topic-1"), + "id": NOTIFICATION_ID, + "etag": ETAG, + "selfLink": SELF_LINK, + "payload_format": JSON_API_V1_PAYLOAD_FORMAT, + } + + connection = _make_connection(resources) + client = _Client(connection, project="my-project-123") + bucket = self._make_one(client=client, name=NAME) + notification = bucket.get_notification(notification_id=NOTIFICATION_ID) + + self.assertEqual(notification.notification_id, NOTIFICATION_ID) + self.assertEqual(notification.etag, ETAG) + self.assertEqual(notification.self_link, SELF_LINK) + self.assertIsNone(notification.custom_attributes) + self.assertIsNone(notification.event_types) + self.assertIsNone(notification.blob_name_prefix) + self.assertEqual(notification.payload_format, JSON_API_V1_PAYLOAD_FORMAT) + + def test_get_notification_miss(self): + from google.cloud.exceptions import NotFound + + response = NotFound("testing") + connection = _make_connection(response) + client = _Client(connection, project="my-project-123") + bucket = self._make_one(client=client, name="name") + with self.assertRaises(NotFound): + bucket.get_notification(notification_id="1") + def test_delete_miss(self): from google.cloud.exceptions import NotFound @@ -1152,7 +974,7 @@ def test_delete_blob_miss(self): client = _Client(connection) bucket = self._make_one(client=client, name=NAME) self.assertRaises(NotFound, bucket.delete_blob, NONESUCH) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "DELETE") self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, NONESUCH)) self.assertEqual(kw["query_params"], {}) @@ -1167,7 +989,7 @@ def test_delete_blob_hit_with_user_project(self): bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) result = bucket.delete_blob(BLOB_NAME, timeout=42) self.assertIsNone(result) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "DELETE") self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) self.assertEqual(kw["query_params"], {"userProject": USER_PROJECT}) @@ -1182,7 +1004,7 @@ def test_delete_blob_hit_with_generation(self): bucket = self._make_one(client=client, name=NAME) result = bucket.delete_blob(BLOB_NAME, generation=GENERATION) self.assertIsNone(result) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "DELETE") self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) self.assertEqual(kw["query_params"], {"generation": GENERATION}) @@ -1273,7 +1095,7 @@ def test_copy_blobs_wo_name(self): self.assertIs(new_blob.bucket, dest) self.assertEqual(new_blob.name, BLOB_NAME) - kw, = connection._requested + (kw,) = connection._requested COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( SOURCE, BLOB_NAME, DEST, BLOB_NAME ) @@ -1299,7 +1121,7 @@ def test_copy_blobs_source_generation(self): self.assertIs(new_blob.bucket, dest) self.assertEqual(new_blob.name, BLOB_NAME) - kw, = connection._requested + (kw,) = connection._requested COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( SOURCE, BLOB_NAME, DEST, BLOB_NAME ) @@ -1366,7 +1188,7 @@ def test_copy_blobs_w_name_and_user_project(self): COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( SOURCE, BLOB_NAME, DEST, NEW_NAME ) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "POST") self.assertEqual(kw["path"], COPY_PATH) self.assertEqual(kw["query_params"], {"userProject": USER_PROJECT}) @@ -1392,7 +1214,7 @@ def test_rename_blob(self): COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( BUCKET_NAME, BLOB_NAME, BUCKET_NAME, NEW_BLOB_NAME ) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "POST") self.assertEqual(kw["path"], COPY_PATH) self.assertEqual(kw["query_params"], {}) @@ -1417,7 +1239,7 @@ def test_rename_blob_to_itself(self): COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( BUCKET_NAME, BLOB_NAME, BUCKET_NAME, BLOB_NAME ) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "POST") self.assertEqual(kw["path"], COPY_PATH) self.assertEqual(kw["query_params"], {}) @@ -2013,6 +1835,51 @@ def test_versioning_enabled_getter(self): bucket = self._make_one(name=NAME, properties=before) self.assertEqual(bucket.versioning_enabled, True) + @mock.patch("warnings.warn") + def test_create_deprecated(self, mock_warn): + from google.cloud.storage.client import Client + + PROJECT = "PROJECT" + BUCKET_NAME = "bucket-name" + DATA = {"name": BUCKET_NAME} + connection = _make_connection(DATA) + client = Client(project=PROJECT) + client._base_connection = connection + + bucket = self._make_one(client=client, name=BUCKET_NAME) + bucket.create() + + connection.api_request.assert_called_once_with( + method="POST", + path="/b", + query_params={"project": PROJECT}, + data=DATA, + _target_object=bucket, + timeout=self._get_default_timeout(), + ) + + mock_warn.assert_called_with( + "Bucket.create() is deprecated and will be removed in future." + "Use Client.create_bucket() instead.", + PendingDeprecationWarning, + stacklevel=1, + ) + + def test_create_w_user_project(self): + from google.cloud.storage.client import Client + + PROJECT = "PROJECT" + BUCKET_NAME = "bucket-name" + DATA = {"name": BUCKET_NAME} + connection = _make_connection(DATA) + client = Client(project=PROJECT) + client._base_connection = connection + + bucket = self._make_one(client=client, name=BUCKET_NAME) + bucket._user_project = "USER_PROJECT" + with self.assertRaises(ValueError): + bucket.create() + def test_versioning_enabled_setter(self): NAME = "name" bucket = self._make_one(name=NAME) @@ -2848,7 +2715,7 @@ def test_lock_retention_policy_ok(self): bucket.lock_retention_policy(timeout=42) - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "POST") self.assertEqual(kw["path"], "/b/{}/lockRetentionPolicy".format(name)) self.assertEqual(kw["query_params"], {"ifMetagenerationMatch": 1234}) @@ -2879,7 +2746,7 @@ def test_lock_retention_policy_w_user_project(self): bucket.lock_retention_policy() - kw, = connection._requested + (kw,) = connection._requested self.assertEqual(kw["method"], "POST") self.assertEqual(kw["path"], "/b/{}/lockRetentionPolicy".format(name)) self.assertEqual( @@ -2911,6 +2778,9 @@ def _generate_signed_url_helper( query_parameters=None, credentials=None, expiration=None, + virtual_hosted_style=False, + bucket_bound_hostname=None, + scheme="http", ): from six.moves.urllib import parse from google.cloud._helpers import UTC @@ -2945,6 +2815,8 @@ def _generate_signed_url_helper( headers=headers, query_parameters=query_parameters, version=version, + virtual_hosted_style=virtual_hosted_style, + bucket_bound_hostname=bucket_bound_hostname, ) self.assertEqual(signed_uri, signer.return_value) @@ -2954,12 +2826,28 @@ def _generate_signed_url_helper( else: expected_creds = credentials - encoded_name = bucket_name.encode("utf-8") - expected_resource = "/{}".format(parse.quote(encoded_name)) + if virtual_hosted_style: + expected_api_access_endpoint = "https://{}.storage.googleapis.com".format( + bucket_name + ) + elif bucket_bound_hostname: + if ":" in bucket_bound_hostname: + expected_api_access_endpoint = bucket_bound_hostname + else: + expected_api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( + scheme=scheme, bucket_bound_hostname=bucket_bound_hostname + ) + else: + expected_api_access_endpoint = api_access_endpoint + expected_resource = "/{}".format(parse.quote(bucket_name)) + + if virtual_hosted_style or bucket_bound_hostname: + expected_resource = "/" + expected_kwargs = { "resource": expected_resource, "expiration": expiration, - "api_access_endpoint": api_access_endpoint, + "api_access_endpoint": expected_api_access_endpoint, "method": method.upper(), "headers": headers, "query_parameters": query_parameters, @@ -3088,6 +2976,17 @@ def test_generate_signed_url_v4_w_credentials(self): credentials = object() self._generate_signed_url_v4_helper(credentials=credentials) + def test_generate_signed_url_v4_w_virtual_hostname(self): + self._generate_signed_url_v4_helper(virtual_hosted_style=True) + + def test_generate_signed_url_v4_w_bucket_bound_hostname_w_scheme(self): + self._generate_signed_url_v4_helper( + bucket_bound_hostname="http://cdn.example.com" + ) + + def test_generate_signed_url_v4_w_bucket_bound_hostname_w_bare_hostname(self): + self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com") + class _Connection(object): _delete_bucket = False diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index b3e5874ef..8673bcfd0 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -12,15 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import base64 import io import json -import unittest - import mock import pytest +import re import requests +import unittest from six.moves import http_client +from google.oauth2.service_account import Credentials +from . import _read_local_json + +_SERVICE_ACCOUNT_JSON = _read_local_json("url_signer_v4_test_account.json") +_CONFORMANCE_TESTS = _read_local_json("url_signer_v4_test_data.json") +_POST_POLICY_TESTS = [test for test in _CONFORMANCE_TESTS if "policyInput" in test] +_DUMMY_CREDENTIALS = Credentials.from_service_account_info(_SERVICE_ACCOUNT_JSON) + def _make_credentials(): import google.auth.credentials @@ -28,6 +37,20 @@ def _make_credentials(): return mock.Mock(spec=google.auth.credentials.Credentials) +def _create_signing_credentials(): + import google.auth.credentials + + class _SigningCredentials( + google.auth.credentials.Credentials, google.auth.credentials.Signing + ): + pass + + credentials = mock.Mock(spec=_SigningCredentials) + credentials.sign_bytes = mock.Mock(return_value=b"Signature_bytes") + credentials.signer_email = "test@mail.com" + return credentials + + def _make_connection(*responses): import google.cloud.storage._http from google.cloud.exceptions import NotFound @@ -579,6 +602,44 @@ def test_create_bucket_w_conflict(self): timeout=self._get_default_timeout(), ) + @mock.patch("warnings.warn") + def test_create_requester_pays_deprecated(self, mock_warn): + from google.cloud.storage.bucket import Bucket + + project = "PROJECT" + credentials = _make_credentials() + client = self._make_one(project=project, credentials=credentials) + bucket_name = "bucket-name" + json_expected = {"name": bucket_name, "billing": {"requesterPays": True}} + http = _make_requests_session([_make_json_response(json_expected)]) + client._http_internal = http + + URI = "/".join( + [ + client._connection.API_BASE_URL, + "storage", + client._connection.API_VERSION, + "b?project=%s" % (project,), + ] + ) + + bucket = client.create_bucket(bucket_name, requester_pays=True) + + self.assertIsInstance(bucket, Bucket) + self.assertEqual(bucket.name, bucket_name) + self.assertTrue(bucket.requester_pays) + http.request.assert_called_once_with( + method="POST", url=URI, data=mock.ANY, headers=mock.ANY, timeout=mock.ANY + ) + json_sent = http.request.call_args_list[0][1]["data"] + self.assertEqual(json_expected, json.loads(json_sent)) + + mock_warn.assert_called_with( + "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", + PendingDeprecationWarning, + stacklevel=1, + ) + def test_create_bucket_w_predefined_acl_invalid(self): project = "PROJECT" bucket_name = "bucket-name" @@ -671,6 +732,100 @@ def test_create_bucket_w_explicit_location(self): ) self.assertEqual(bucket.location, location) + def test_create_bucket_w_explicit_project(self): + from google.cloud.storage.client import Client + + PROJECT = "PROJECT" + OTHER_PROJECT = "other-project-123" + BUCKET_NAME = "bucket-name" + DATA = {"name": BUCKET_NAME} + connection = _make_connection(DATA) + + client = Client(project=PROJECT) + client._base_connection = connection + + bucket = client.create_bucket(BUCKET_NAME, project=OTHER_PROJECT) + connection.api_request.assert_called_once_with( + method="POST", + path="/b", + query_params={"project": OTHER_PROJECT}, + data=DATA, + _target_object=bucket, + timeout=self._get_default_timeout(), + ) + + def test_create_w_extra_properties(self): + from google.cloud.storage.client import Client + from google.cloud.storage.bucket import Bucket + + BUCKET_NAME = "bucket-name" + PROJECT = "PROJECT" + CORS = [ + { + "maxAgeSeconds": 60, + "methods": ["*"], + "origin": ["https://example.com/frontend"], + "responseHeader": ["X-Custom-Header"], + } + ] + LIFECYCLE_RULES = [{"action": {"type": "Delete"}, "condition": {"age": 365}}] + LOCATION = "eu" + LABELS = {"color": "red", "flavor": "cherry"} + STORAGE_CLASS = "NEARLINE" + DATA = { + "name": BUCKET_NAME, + "cors": CORS, + "lifecycle": {"rule": LIFECYCLE_RULES}, + "location": LOCATION, + "storageClass": STORAGE_CLASS, + "versioning": {"enabled": True}, + "billing": {"requesterPays": True}, + "labels": LABELS, + } + + connection = _make_connection(DATA) + client = Client(project=PROJECT) + client._base_connection = connection + + bucket = Bucket(client=client, name=BUCKET_NAME) + bucket.cors = CORS + bucket.lifecycle_rules = LIFECYCLE_RULES + bucket.storage_class = STORAGE_CLASS + bucket.versioning_enabled = True + bucket.requester_pays = True + bucket.labels = LABELS + client.create_bucket(bucket, location=LOCATION) + + connection.api_request.assert_called_once_with( + method="POST", + path="/b", + query_params={"project": PROJECT}, + data=DATA, + _target_object=bucket, + timeout=self._get_default_timeout(), + ) + + def test_create_hit(self): + from google.cloud.storage.client import Client + + PROJECT = "PROJECT" + BUCKET_NAME = "bucket-name" + DATA = {"name": BUCKET_NAME} + connection = _make_connection(DATA) + client = Client(project=PROJECT) + client._base_connection = connection + + bucket = client.create_bucket(BUCKET_NAME) + + connection.api_request.assert_called_once_with( + method="POST", + path="/b", + query_params={"project": PROJECT}, + data=DATA, + _target_object=bucket, + timeout=self._get_default_timeout(), + ) + def test_create_bucket_w_string_success(self): from google.cloud.storage.bucket import Bucket @@ -687,16 +842,15 @@ def test_create_bucket_w_string_success(self): "b?project=%s" % (project,), ] ) - json_expected = {"name": bucket_name, "billing": {"requesterPays": True}} + json_expected = {"name": bucket_name} data = json_expected http = _make_requests_session([_make_json_response(data)]) client._http_internal = http - bucket = client.create_bucket(bucket_name, requester_pays=True) + bucket = client.create_bucket(bucket_name) self.assertIsInstance(bucket, Bucket) self.assertEqual(bucket.name, bucket_name) - self.assertTrue(bucket.requester_pays) http.request.assert_called_once_with( method="POST", url=URI, data=mock.ANY, headers=mock.ANY, timeout=mock.ANY ) @@ -1340,3 +1494,323 @@ def test_get_hmac_key_metadata_w_project(self): headers=mock.ANY, timeout=self._get_default_timeout(), ) + + def test_get_signed_policy_v4(self): + import datetime + + BUCKET_NAME = "bucket-name" + BLOB_NAME = "object-name" + EXPECTED_SIGN = "5369676e61747572655f6279746573" + EXPECTED_POLICY = b"eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsia2V5Ijoib2JqZWN0LW5hbWUifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAzMTJUMTE0NzE2WiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0QG1haWwuY29tLzIwMjAwMzEyL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDMtMjZUMDA6MDA6MTBaIn0=" + + client = self._make_one(project="PROJECT") + + dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() + with dtstamps_patch, now_patch, expire_secs_patch: + policy = client.generate_signed_post_policy_v4( + BUCKET_NAME, + BLOB_NAME, + expiration=datetime.datetime(2020, 3, 12), + conditions=[ + {"bucket": BUCKET_NAME}, + {"acl": "private"}, + ["starts-with", "$Content-Type", "text/plain"], + ], + credentials=_create_signing_credentials(), + ) + self.assertEqual( + policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/" + ) + fields = policy["fields"] + + self.assertEqual(fields["key"], BLOB_NAME) + self.assertEqual(fields["x-goog-algorithm"], "GOOG4-RSA-SHA256") + self.assertEqual(fields["x-goog-date"], "20200312T114716Z") + self.assertEqual( + fields["x-goog-credential"], + "test@mail.com/20200312/auto/storage/goog4_request", + ) + self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) + self.assertEqual(fields["policy"], EXPECTED_POLICY) + + def test_get_signed_policy_v4_without_credentials(self): + import datetime + + BUCKET_NAME = "bucket-name" + BLOB_NAME = "object-name" + EXPECTED_SIGN = "5369676e61747572655f6279746573" + EXPECTED_POLICY = b"eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsia2V5Ijoib2JqZWN0LW5hbWUifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAzMTJUMTE0NzE2WiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0QG1haWwuY29tLzIwMjAwMzEyL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDMtMjZUMDA6MDA6MTBaIn0=" + + client = self._make_one( + project="PROJECT", credentials=_create_signing_credentials() + ) + + dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() + with dtstamps_patch, now_patch, expire_secs_patch: + policy = client.generate_signed_post_policy_v4( + BUCKET_NAME, + BLOB_NAME, + expiration=datetime.datetime(2020, 3, 12), + conditions=[ + {"bucket": BUCKET_NAME}, + {"acl": "private"}, + ["starts-with", "$Content-Type", "text/plain"], + ], + ) + self.assertEqual( + policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/" + ) + fields = policy["fields"] + + self.assertEqual(fields["key"], BLOB_NAME) + self.assertEqual(fields["x-goog-algorithm"], "GOOG4-RSA-SHA256") + self.assertEqual(fields["x-goog-date"], "20200312T114716Z") + self.assertEqual( + fields["x-goog-credential"], + "test@mail.com/20200312/auto/storage/goog4_request", + ) + self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) + self.assertEqual(fields["policy"], EXPECTED_POLICY) + + def test_get_signed_policy_v4_with_fields(self): + import datetime + + BUCKET_NAME = "bucket-name" + BLOB_NAME = "object-name" + FIELD1_VALUE = "Value1" + EXPECTED_SIGN = "5369676e61747572655f6279746573" + EXPECTED_POLICY = b"eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsiZmllbGQxIjoiVmFsdWUxIn0seyJrZXkiOiJvYmplY3QtbmFtZSJ9LHsieC1nb29nLWRhdGUiOiIyMDIwMDMxMlQxMTQ3MTZaIn0seyJ4LWdvb2ctY3JlZGVudGlhbCI6InRlc3RAbWFpbC5jb20vMjAyMDAzMTIvYXV0by9zdG9yYWdlL2dvb2c0X3JlcXVlc3QifSx7IngtZ29vZy1hbGdvcml0aG0iOiJHT09HNC1SU0EtU0hBMjU2In1dLCJleHBpcmF0aW9uIjoiMjAyMC0wMy0yNlQwMDowMDoxMFoifQ==" + + client = self._make_one(project="PROJECT") + + dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() + with dtstamps_patch, now_patch, expire_secs_patch: + policy = client.generate_signed_post_policy_v4( + BUCKET_NAME, + BLOB_NAME, + expiration=datetime.datetime(2020, 3, 12), + conditions=[ + {"bucket": BUCKET_NAME}, + {"acl": "private"}, + ["starts-with", "$Content-Type", "text/plain"], + ], + fields={"field1": FIELD1_VALUE, "x-ignore-field": "Ignored_value"}, + credentials=_create_signing_credentials(), + ) + self.assertEqual( + policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/" + ) + fields = policy["fields"] + + self.assertEqual(fields["key"], BLOB_NAME) + self.assertEqual(fields["x-goog-algorithm"], "GOOG4-RSA-SHA256") + self.assertEqual(fields["x-goog-date"], "20200312T114716Z") + self.assertEqual(fields["field1"], FIELD1_VALUE) + self.assertNotIn("x-ignore-field", fields.keys()) + self.assertEqual( + fields["x-goog-credential"], + "test@mail.com/20200312/auto/storage/goog4_request", + ) + self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) + self.assertEqual(fields["policy"], EXPECTED_POLICY) + + def test_get_signed_policy_v4_virtual_hosted_style(self): + import datetime + + BUCKET_NAME = "bucket-name" + + client = self._make_one(project="PROJECT") + + dtstamps_patch, _, _ = _time_functions_patches() + with dtstamps_patch: + policy = client.generate_signed_post_policy_v4( + BUCKET_NAME, + "object-name", + expiration=datetime.datetime(2020, 3, 12), + virtual_hosted_style=True, + credentials=_create_signing_credentials(), + ) + self.assertEqual( + policy["url"], "https://{}.storage.googleapis.com/".format(BUCKET_NAME) + ) + + def test_get_signed_policy_v4_bucket_bound_hostname(self): + import datetime + + client = self._make_one(project="PROJECT") + + dtstamps_patch, _, _ = _time_functions_patches() + with dtstamps_patch: + policy = client.generate_signed_post_policy_v4( + "bucket-name", + "object-name", + expiration=datetime.datetime(2020, 3, 12), + bucket_bound_hostname="https://bucket.bound_hostname", + credentials=_create_signing_credentials(), + ) + self.assertEqual(policy["url"], "https://bucket.bound_hostname") + + def test_get_signed_policy_v4_bucket_bound_hostname_with_scheme(self): + import datetime + + client = self._make_one(project="PROJECT") + + dtstamps_patch, _, _ = _time_functions_patches() + with dtstamps_patch: + policy = client.generate_signed_post_policy_v4( + "bucket-name", + "object-name", + expiration=datetime.datetime(2020, 3, 12), + bucket_bound_hostname="bucket.bound_hostname", + scheme="http", + credentials=_create_signing_credentials(), + ) + self.assertEqual(policy["url"], "http://bucket.bound_hostname/") + + def test_get_signed_policy_v4_no_expiration(self): + BUCKET_NAME = "bucket-name" + EXPECTED_POLICY = b"eyJjb25kaXRpb25zIjpbeyJrZXkiOiJvYmplY3QtbmFtZSJ9LHsieC1nb29nLWRhdGUiOiIyMDIwMDMxMlQxMTQ3MTZaIn0seyJ4LWdvb2ctY3JlZGVudGlhbCI6InRlc3RAbWFpbC5jb20vMjAyMDAzMTIvYXV0by9zdG9yYWdlL2dvb2c0X3JlcXVlc3QifSx7IngtZ29vZy1hbGdvcml0aG0iOiJHT09HNC1SU0EtU0hBMjU2In1dLCJleHBpcmF0aW9uIjoiMjAyMC0wMy0yNlQwMDowMDoxMFoifQ==" + + client = self._make_one(project="PROJECT") + + dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() + with dtstamps_patch, now_patch, expire_secs_patch: + policy = client.generate_signed_post_policy_v4( + BUCKET_NAME, + "object-name", + expiration=None, + credentials=_create_signing_credentials(), + ) + + self.assertEqual( + policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/" + ) + self.assertEqual(policy["fields"]["policy"], EXPECTED_POLICY) + + def test_get_signed_policy_v4_with_access_token(self): + import datetime + + BUCKET_NAME = "bucket-name" + BLOB_NAME = "object-name" + EXPECTED_SIGN = "0c4003044105" + EXPECTED_POLICY = b"eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsia2V5Ijoib2JqZWN0LW5hbWUifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAzMTJUMTE0NzE2WiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0QG1haWwuY29tLzIwMjAwMzEyL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDMtMjZUMDA6MDA6MTBaIn0=" + + client = self._make_one(project="PROJECT") + + dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches() + with dtstamps_patch, now_patch, expire_secs_patch: + with mock.patch( + "google.cloud.storage.client._sign_message", return_value=b"DEADBEEF" + ): + policy = client.generate_signed_post_policy_v4( + BUCKET_NAME, + BLOB_NAME, + expiration=datetime.datetime(2020, 3, 12), + conditions=[ + {"bucket": BUCKET_NAME}, + {"acl": "private"}, + ["starts-with", "$Content-Type", "text/plain"], + ], + credentials=_create_signing_credentials(), + service_account_email="test@mail.com", + access_token="token", + ) + self.assertEqual( + policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/" + ) + fields = policy["fields"] + + self.assertEqual(fields["key"], BLOB_NAME) + self.assertEqual(fields["x-goog-algorithm"], "GOOG4-RSA-SHA256") + self.assertEqual(fields["x-goog-date"], "20200312T114716Z") + self.assertEqual( + fields["x-goog-credential"], + "test@mail.com/20200312/auto/storage/goog4_request", + ) + self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN) + self.assertEqual(fields["policy"], EXPECTED_POLICY) + + +@pytest.mark.parametrize("test_data", _POST_POLICY_TESTS) +def test_conformance_post_policy(test_data): + import datetime + from google.cloud.storage.client import Client + + in_data = test_data["policyInput"] + timestamp = datetime.datetime.strptime(in_data["timestamp"], "%Y-%m-%dT%H:%M:%SZ") + + client = Client(credentials=_DUMMY_CREDENTIALS) + + # mocking time functions + with mock.patch("google.cloud.storage._signing.NOW", return_value=timestamp): + with mock.patch( + "google.cloud.storage.client.get_expiration_seconds_v4", + return_value=in_data["expiration"], + ): + with mock.patch("google.cloud.storage.client._NOW", return_value=timestamp): + + policy = client.generate_signed_post_policy_v4( + bucket_name=in_data["bucket"], + blob_name=in_data["object"], + conditions=_prepare_conditions(in_data), + fields=in_data.get("fields"), + credentials=_DUMMY_CREDENTIALS, + expiration=in_data["expiration"], + virtual_hosted_style=in_data.get("urlStyle") + == "VIRTUAL_HOSTED_STYLE", + bucket_bound_hostname=in_data.get("bucketBoundHostname"), + scheme=in_data.get("scheme"), + ) + fields = policy["fields"] + out_data = test_data["policyOutput"] + + decoded_policy = base64.b64decode(fields["policy"]).decode("unicode_escape") + assert decoded_policy == out_data["expectedDecodedPolicy"] + + for field in ( + "x-goog-algorithm", + "x-goog-credential", + "x-goog-date", + "x-goog-signature", + ): + assert fields[field] == test_data["policyOutput"]["fields"][field] + + assert policy["url"] == out_data["url"] + + +def _prepare_conditions(in_data): + """Helper for V4 POST policy generation conformance tests. + + Convert conformance test data conditions dict into list. + + Args: + in_data (dict): conditions arg from conformance test data. + + Returns: + list: conditions arg to pass into generate_signed_post_policy_v4(). + """ + if "conditions" in in_data: + conditions = [] + for key, value in in_data["conditions"].items(): + # camel case to snake case with "-" separator + field = re.sub(r"(?