diff --git a/.coveragerc b/.coveragerc index dffe22a6f4..9d801989cb 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.flake8 b/.flake8 index 87f6e408c4..32986c7928 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 81f87c5691..91d742b5b9 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:5a4c19d17e597b92d786e569be101e636c9c2817731f80a5adec56b2aa8fe070 -# created: 2024-04-12T11:35:58.922854369Z + digest: sha256:d3de8a02819f65001effcbd3ea76ce97e9bcff035c7a89457f40f892c87c5b32 +# created: 2024-07-03T17:43:00.77142528Z diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml index 8b37ee8971..21786a4eb0 100644 --- a/.github/auto-label.yaml +++ b/.github/auto-label.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile index bdaf39fe22..a26ce61930 100644 --- a/.kokoro/docker/docs/Dockerfile +++ b/.kokoro/docker/docs/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh index 6f3972140e..c435402f47 100755 --- a/.kokoro/populate-secrets.sh +++ b/.kokoro/populate-secrets.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC. +# Copyright 2024 Google LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh index 7700c90ee9..da9ce803dd 100755 --- a/.kokoro/publish-docs.sh +++ b/.kokoro/publish-docs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 320ac51271..21a9b558c5 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 51f92b8e12..35ece0e4d2 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -4,21 +4,25 @@ # # pip-compile --allow-unsafe --generate-hashes requirements.in # -argcomplete==3.1.4 \ - --hash=sha256:72558ba729e4c468572609817226fb0a6e7e9a0a7d477b882be168c0b4a62b94 \ - --hash=sha256:fbe56f8cda08aa9a04b307d8482ea703e96a6a801611acb4be9bf3942017989f +argcomplete==3.4.0 \ + --hash=sha256:69a79e083a716173e5532e0fa3bef45f793f4e61096cf52b5a42c0211c8b8aa5 \ + --hash=sha256:c2abcdfe1be8ace47ba777d4fce319eb13bf8ad9dace8d085dcad6eded88057f # via nox -attrs==23.1.0 \ - --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ - --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 +attrs==23.2.0 \ + --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ + --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 # via gcp-releasetool -cachetools==5.3.2 \ - --hash=sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2 \ - --hash=sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1 +backports-tarfile==1.2.0 \ + --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ + --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 + # via jaraco-context +cachetools==5.3.3 \ + --hash=sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945 \ + --hash=sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105 # via google-auth -certifi==2023.7.22 \ - --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ - --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 +certifi==2024.6.2 \ + --hash=sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516 \ + --hash=sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56 # via requests cffi==1.16.0 \ --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ @@ -87,90 +91,90 @@ click==8.0.4 \ # -r requirements.in # gcp-docuploader # gcp-releasetool -colorlog==6.7.0 \ - --hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \ - --hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5 +colorlog==6.8.2 \ + --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ + --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33 # via # gcp-docuploader # nox -cryptography==42.0.5 \ - --hash=sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee \ - --hash=sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576 \ - --hash=sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d \ - --hash=sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30 \ - --hash=sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413 \ - --hash=sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb \ - --hash=sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da \ - --hash=sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4 \ - --hash=sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd \ - --hash=sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc \ - --hash=sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8 \ - --hash=sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1 \ - --hash=sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc \ - --hash=sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e \ - --hash=sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8 \ - --hash=sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940 \ - --hash=sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400 \ - --hash=sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7 \ - --hash=sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16 \ - --hash=sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278 \ - --hash=sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74 \ - --hash=sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec \ - --hash=sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1 \ - --hash=sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2 \ - --hash=sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c \ - --hash=sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922 \ - --hash=sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a \ - --hash=sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6 \ - --hash=sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1 \ - --hash=sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e \ - --hash=sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac \ - --hash=sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7 +cryptography==42.0.8 \ + --hash=sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad \ + --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ + --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ + --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ + --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ + --hash=sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648 \ + --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ + --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ + --hash=sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c \ + --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ + --hash=sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d \ + --hash=sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c \ + --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ + --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ + --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ + --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ + --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ + --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ + --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ + --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ + --hash=sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe \ + --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ + --hash=sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71 \ + --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ + --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ + --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ + --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ + --hash=sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842 \ + --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ + --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ + --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ + --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e # via # -r requirements.in # gcp-releasetool # secretstorage -distlib==0.3.7 \ - --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ - --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 +distlib==0.3.8 \ + --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ + --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 # via virtualenv -docutils==0.20.1 \ - --hash=sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6 \ - --hash=sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b +docutils==0.21.2 \ + --hash=sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f \ + --hash=sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2 # via readme-renderer -filelock==3.13.1 \ - --hash=sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e \ - --hash=sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c +filelock==3.15.4 \ + --hash=sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb \ + --hash=sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7 # via virtualenv gcp-docuploader==0.6.5 \ --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea # via -r requirements.in -gcp-releasetool==2.0.0 \ - --hash=sha256:3d73480b50ba243f22d7c7ec08b115a30e1c7817c4899781840c26f9c55b8277 \ - --hash=sha256:7aa9fd935ec61e581eb8458ad00823786d91756c25e492f372b2b30962f3c28f +gcp-releasetool==2.0.1 \ + --hash=sha256:34314a910c08e8911d9c965bd44f8f2185c4f556e737d719c33a41f6a610de96 \ + --hash=sha256:b0d5863c6a070702b10883d37c4bdfd74bf930fe417f36c0c965d3b7c779ae62 # via -r requirements.in -google-api-core==2.12.0 \ - --hash=sha256:c22e01b1e3c4dcd90998494879612c38d0a3411d1f7b679eb89e2abe3ce1f553 \ - --hash=sha256:ec6054f7d64ad13b41e43d96f735acbd763b0f3b695dabaa2d579673f6a6e160 +google-api-core==2.19.1 \ + --hash=sha256:f12a9b8309b5e21d92483bbd47ce2c445861ec7d269ef6784ecc0ea8c1fa6125 \ + --hash=sha256:f4695f1e3650b316a795108a76a1c416e6afb036199d1c1f1f110916df479ffd # via # google-cloud-core # google-cloud-storage -google-auth==2.23.4 \ - --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ - --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 +google-auth==2.31.0 \ + --hash=sha256:042c4702efa9f7d3c48d3a69341c209381b125faa6dbf3ebe56bc7e40ae05c23 \ + --hash=sha256:87805c36970047247c8afe614d4e3af8eceafc1ebba0c679fe75ddd1d575e871 # via # gcp-releasetool # google-api-core # google-cloud-core # google-cloud-storage -google-cloud-core==2.3.3 \ - --hash=sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb \ - --hash=sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863 +google-cloud-core==2.4.1 \ + --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ + --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 # via google-cloud-storage -google-cloud-storage==2.13.0 \ - --hash=sha256:ab0bf2e1780a1b74cf17fccb13788070b729f50c252f0c94ada2aae0ca95437d \ - --hash=sha256:f62dc4c7b6cd4360d072e3deb28035fbdad491ac3d9b0b1815a12daea10f37c7 +google-cloud-storage==2.17.0 \ + --hash=sha256:49378abff54ef656b52dca5ef0f2eba9aa83dc2b2c72c78714b03a1a95fe9388 \ + --hash=sha256:5b393bc766b7a3bc6f5407b9e665b2450d36282614b7945e570b3480a456d1e1 # via gcp-docuploader google-crc32c==1.5.0 \ --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ @@ -244,28 +248,36 @@ google-crc32c==1.5.0 \ # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.6.0 \ - --hash=sha256:972852f6c65f933e15a4a210c2b96930763b47197cdf4aa5f5bea435efb626e7 \ - --hash=sha256:fc03d344381970f79eebb632a3c18bb1828593a2dc5572b5f90115ef7d11e81b +google-resumable-media==2.7.1 \ + --hash=sha256:103ebc4ba331ab1bfdac0250f8033627a2cd7cde09e7ccff9181e31ba4315b2c \ + --hash=sha256:eae451a7b2e2cdbaaa0fd2eb00cc8a1ee5e95e16b55597359cbc3d27d7d90e33 # via google-cloud-storage -googleapis-common-protos==1.61.0 \ - --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ - --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b +googleapis-common-protos==1.63.2 \ + --hash=sha256:27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945 \ + --hash=sha256:27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87 # via google-api-core idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via requests -importlib-metadata==6.8.0 \ - --hash=sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb \ - --hash=sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743 +importlib-metadata==8.0.0 \ + --hash=sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f \ + --hash=sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812 # via # -r requirements.in # keyring # twine -jaraco-classes==3.3.0 \ - --hash=sha256:10afa92b6743f25c0cf5f37c6bb6e18e2c5bb84a16527ccfc0040ea377e7aaeb \ - --hash=sha256:c063dd08e89217cee02c8d5e5ec560f2c8ce6cdc2fcdc2e68f7b2e5547ed3621 +jaraco-classes==3.4.0 \ + --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ + --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 + # via keyring +jaraco-context==5.3.0 \ + --hash=sha256:3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266 \ + --hash=sha256:c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2 + # via keyring +jaraco-functools==4.0.1 \ + --hash=sha256:3b24ccb921d6b593bdceb56ce14799204f473976e2a9d4b15b04d0f2c2326664 \ + --hash=sha256:d33fa765374c0611b52f8b3a795f8900869aa88c84769d4d1746cd68fb28c3e8 # via keyring jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ @@ -273,13 +285,13 @@ jeepney==0.8.0 \ # via # keyring # secretstorage -jinja2==3.1.3 \ - --hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \ - --hash=sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90 +jinja2==3.1.4 \ + --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ + --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d # via gcp-releasetool -keyring==24.2.0 \ - --hash=sha256:4901caaf597bfd3bbd78c9a0c7c4c29fcd8310dab2cffefe749e916b6527acd6 \ - --hash=sha256:ca0746a19ec421219f4d713f848fa297a661a8a8c1504867e55bfb5e09091509 +keyring==25.2.1 \ + --hash=sha256:2458681cdefc0dbc0b7eb6cf75d0b98e59f9ad9b2d4edd319d18f68bdca95e50 \ + --hash=sha256:daaffd42dbda25ddafb1ad5fec4024e5bbcfe424597ca1ca452b299861e49f1b # via # gcp-releasetool # twine @@ -287,146 +299,153 @@ markdown-it-py==3.0.0 \ --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb # via rich -markupsafe==2.1.3 \ - --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ - --hash=sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e \ - --hash=sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431 \ - --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ - --hash=sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c \ - --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ - --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ - --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ - --hash=sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939 \ - --hash=sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c \ - --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ - --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ - --hash=sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9 \ - --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ - --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ - --hash=sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d \ - --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ - --hash=sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3 \ - --hash=sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00 \ - --hash=sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155 \ - --hash=sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac \ - --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ - --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ - --hash=sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8 \ - --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ - --hash=sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007 \ - --hash=sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24 \ - --hash=sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea \ - --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ - --hash=sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0 \ - --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ - --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ - --hash=sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2 \ - --hash=sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1 \ - --hash=sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707 \ - --hash=sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6 \ - --hash=sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c \ - --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ - --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ - --hash=sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779 \ - --hash=sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636 \ - --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ - --hash=sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad \ - --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ - --hash=sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc \ - --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ - --hash=sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48 \ - --hash=sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7 \ - --hash=sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e \ - --hash=sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b \ - --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ - --hash=sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5 \ - --hash=sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e \ - --hash=sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb \ - --hash=sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9 \ - --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ - --hash=sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc \ - --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ - --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ - --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 +markupsafe==2.1.5 \ + --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ + --hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \ + --hash=sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f \ + --hash=sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3 \ + --hash=sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532 \ + --hash=sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f \ + --hash=sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617 \ + --hash=sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df \ + --hash=sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4 \ + --hash=sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906 \ + --hash=sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f \ + --hash=sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4 \ + --hash=sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8 \ + --hash=sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371 \ + --hash=sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2 \ + --hash=sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465 \ + --hash=sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52 \ + --hash=sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6 \ + --hash=sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169 \ + --hash=sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad \ + --hash=sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2 \ + --hash=sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0 \ + --hash=sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029 \ + --hash=sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f \ + --hash=sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a \ + --hash=sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced \ + --hash=sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5 \ + --hash=sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c \ + --hash=sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf \ + --hash=sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9 \ + --hash=sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb \ + --hash=sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad \ + --hash=sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3 \ + --hash=sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1 \ + --hash=sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46 \ + --hash=sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc \ + --hash=sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a \ + --hash=sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee \ + --hash=sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900 \ + --hash=sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5 \ + --hash=sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea \ + --hash=sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f \ + --hash=sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5 \ + --hash=sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e \ + --hash=sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a \ + --hash=sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f \ + --hash=sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50 \ + --hash=sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a \ + --hash=sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b \ + --hash=sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4 \ + --hash=sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff \ + --hash=sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2 \ + --hash=sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46 \ + --hash=sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b \ + --hash=sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf \ + --hash=sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5 \ + --hash=sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5 \ + --hash=sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab \ + --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \ + --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68 # via jinja2 mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via markdown-it-py -more-itertools==10.1.0 \ - --hash=sha256:626c369fa0eb37bac0291bce8259b332fd59ac792fa5497b59837309cd5b114a \ - --hash=sha256:64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6 - # via jaraco-classes -nh3==0.2.14 \ - --hash=sha256:116c9515937f94f0057ef50ebcbcc10600860065953ba56f14473ff706371873 \ - --hash=sha256:18415df36db9b001f71a42a3a5395db79cf23d556996090d293764436e98e8ad \ - --hash=sha256:203cac86e313cf6486704d0ec620a992c8bc164c86d3a4fd3d761dd552d839b5 \ - --hash=sha256:2b0be5c792bd43d0abef8ca39dd8acb3c0611052ce466d0401d51ea0d9aa7525 \ - --hash=sha256:377aaf6a9e7c63962f367158d808c6a1344e2b4f83d071c43fbd631b75c4f0b2 \ - --hash=sha256:525846c56c2bcd376f5eaee76063ebf33cf1e620c1498b2a40107f60cfc6054e \ - --hash=sha256:5529a3bf99402c34056576d80ae5547123f1078da76aa99e8ed79e44fa67282d \ - --hash=sha256:7771d43222b639a4cd9e341f870cee336b9d886de1ad9bec8dddab22fe1de450 \ - --hash=sha256:88c753efbcdfc2644a5012938c6b9753f1c64a5723a67f0301ca43e7b85dcf0e \ - --hash=sha256:93a943cfd3e33bd03f77b97baa11990148687877b74193bf777956b67054dcc6 \ - --hash=sha256:9be2f68fb9a40d8440cbf34cbf40758aa7f6093160bfc7fb018cce8e424f0c3a \ - --hash=sha256:a0c509894fd4dccdff557068e5074999ae3b75f4c5a2d6fb5415e782e25679c4 \ - --hash=sha256:ac8056e937f264995a82bf0053ca898a1cb1c9efc7cd68fa07fe0060734df7e4 \ - --hash=sha256:aed56a86daa43966dd790ba86d4b810b219f75b4bb737461b6886ce2bde38fd6 \ - --hash=sha256:e8986f1dd3221d1e741fda0a12eaa4a273f1d80a35e31a1ffe579e7c621d069e \ - --hash=sha256:f99212a81c62b5f22f9e7c3e347aa00491114a5647e1f13bbebd79c3e5f08d75 +more-itertools==10.3.0 \ + --hash=sha256:e5d93ef411224fbcef366a6e8ddc4c5781bc6359d43412a65dd5964e46111463 \ + --hash=sha256:ea6a02e24a9161e51faad17a8782b92a0df82c12c1c8886fec7f0c3fa1a1b320 + # via + # jaraco-classes + # jaraco-functools +nh3==0.2.17 \ + --hash=sha256:0316c25b76289cf23be6b66c77d3608a4fdf537b35426280032f432f14291b9a \ + --hash=sha256:1a814dd7bba1cb0aba5bcb9bebcc88fd801b63e21e2450ae6c52d3b3336bc911 \ + --hash=sha256:1aa52a7def528297f256de0844e8dd680ee279e79583c76d6fa73a978186ddfb \ + --hash=sha256:22c26e20acbb253a5bdd33d432a326d18508a910e4dcf9a3316179860d53345a \ + --hash=sha256:40015514022af31975c0b3bca4014634fa13cb5dc4dbcbc00570acc781316dcc \ + --hash=sha256:40d0741a19c3d645e54efba71cb0d8c475b59135c1e3c580f879ad5514cbf028 \ + --hash=sha256:551672fd71d06cd828e282abdb810d1be24e1abb7ae2543a8fa36a71c1006fe9 \ + --hash=sha256:66f17d78826096291bd264f260213d2b3905e3c7fae6dfc5337d49429f1dc9f3 \ + --hash=sha256:85cdbcca8ef10733bd31f931956f7fbb85145a4d11ab9e6742bbf44d88b7e351 \ + --hash=sha256:a3f55fabe29164ba6026b5ad5c3151c314d136fd67415a17660b4aaddacf1b10 \ + --hash=sha256:b4427ef0d2dfdec10b641ed0bdaf17957eb625b2ec0ea9329b3d28806c153d71 \ + --hash=sha256:ba73a2f8d3a1b966e9cdba7b211779ad8a2561d2dba9674b8a19ed817923f65f \ + --hash=sha256:c21bac1a7245cbd88c0b0e4a420221b7bfa838a2814ee5bb924e9c2f10a1120b \ + --hash=sha256:c551eb2a3876e8ff2ac63dff1585236ed5dfec5ffd82216a7a174f7c5082a78a \ + --hash=sha256:c790769152308421283679a142dbdb3d1c46c79c823008ecea8e8141db1a2062 \ + --hash=sha256:d7a25fd8c86657f5d9d576268e3b3767c5cd4f42867c9383618be8517f0f022a # via readme-renderer -nox==2023.4.22 \ - --hash=sha256:0b1adc619c58ab4fa57d6ab2e7823fe47a32e70202f287d78474adcc7bda1891 \ - --hash=sha256:46c0560b0dc609d7d967dc99e22cb463d3c4caf54a5fda735d6c11b5177e3a9f +nox==2024.4.15 \ + --hash=sha256:6492236efa15a460ecb98e7b67562a28b70da006ab0be164e8821177577c0565 \ + --hash=sha256:ecf6700199cdfa9e5ea0a41ff5e6ef4641d09508eda6edb89d9987864115817f # via -r requirements.in -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 +packaging==24.1 \ + --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ + --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 # via # gcp-releasetool # nox -pkginfo==1.9.6 \ - --hash=sha256:4b7a555a6d5a22169fcc9cf7bfd78d296b0361adad412a346c1226849af5e546 \ - --hash=sha256:8fd5896e8718a4372f0ea9cc9d96f6417c9b986e23a4d116dda26b62cc29d046 +pkginfo==1.10.0 \ + --hash=sha256:5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297 \ + --hash=sha256:889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097 # via twine -platformdirs==3.11.0 \ - --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ - --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e +platformdirs==4.2.2 \ + --hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \ + --hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3 # via virtualenv -protobuf==4.25.3 \ - --hash=sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4 \ - --hash=sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8 \ - --hash=sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c \ - --hash=sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d \ - --hash=sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4 \ - --hash=sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa \ - --hash=sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c \ - --hash=sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019 \ - --hash=sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9 \ - --hash=sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c \ - --hash=sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2 +proto-plus==1.24.0 \ + --hash=sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445 \ + --hash=sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12 + # via google-api-core +protobuf==5.27.2 \ + --hash=sha256:0e341109c609749d501986b835f667c6e1e24531096cff9d34ae411595e26505 \ + --hash=sha256:176c12b1f1c880bf7a76d9f7c75822b6a2bc3db2d28baa4d300e8ce4cde7409b \ + --hash=sha256:354d84fac2b0d76062e9b3221f4abbbacdfd2a4d8af36bab0474f3a0bb30ab38 \ + --hash=sha256:4fadd8d83e1992eed0248bc50a4a6361dc31bcccc84388c54c86e530b7f58863 \ + --hash=sha256:54330f07e4949d09614707c48b06d1a22f8ffb5763c159efd5c0928326a91470 \ + --hash=sha256:610e700f02469c4a997e58e328cac6f305f649826853813177e6290416e846c6 \ + --hash=sha256:7fc3add9e6003e026da5fc9e59b131b8f22b428b991ccd53e2af8071687b4fce \ + --hash=sha256:9e8f199bf7f97bd7ecebffcae45ebf9527603549b2b562df0fbc6d4d688f14ca \ + --hash=sha256:a109916aaac42bff84702fb5187f3edadbc7c97fc2c99c5ff81dd15dcce0d1e5 \ + --hash=sha256:b848dbe1d57ed7c191dfc4ea64b8b004a3f9ece4bf4d0d80a367b76df20bf36e \ + --hash=sha256:f3ecdef226b9af856075f28227ff2c90ce3a594d092c39bee5513573f25e2714 # via # gcp-docuploader # gcp-releasetool # google-api-core # googleapis-common-protos -pyasn1==0.5.0 \ - --hash=sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57 \ - --hash=sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde + # proto-plus +pyasn1==0.6.0 \ + --hash=sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c \ + --hash=sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 \ - --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ - --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d +pyasn1-modules==0.4.0 \ + --hash=sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6 \ + --hash=sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b # via google-auth -pycparser==2.21 \ - --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ - --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 +pycparser==2.22 \ + --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ + --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc # via cffi -pygments==2.16.1 \ - --hash=sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692 \ - --hash=sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29 +pygments==2.18.0 \ + --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ + --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via # readme-renderer # rich @@ -434,20 +453,20 @@ pyjwt==2.8.0 \ --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 # via gcp-releasetool -pyperclip==1.8.2 \ - --hash=sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57 +pyperclip==1.9.0 \ + --hash=sha256:b7de0142ddc81bfc5c7507eea19da920b92252b548b96186caf94a5e2527d310 # via gcp-releasetool -python-dateutil==2.8.2 \ - --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ - --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 # via gcp-releasetool -readme-renderer==42.0 \ - --hash=sha256:13d039515c1f24de668e2c93f2e877b9dbe6c6c32328b90a40a49d8b2b85f36d \ - --hash=sha256:2d55489f83be4992fe4454939d1a051c33edbab778e82761d060c9fc6b308cd1 +readme-renderer==43.0 \ + --hash=sha256:1818dd28140813509eeed8d62687f7cd4f7bad90d4db586001c5dc09d4fde311 \ + --hash=sha256:19db308d86ecd60e5affa3b2a98f017af384678c63c88e5d4556a380e674f3f9 # via twine -requests==2.31.0 \ - --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ - --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via # gcp-releasetool # google-api-core @@ -462,9 +481,9 @@ rfc3986==2.0.0 \ --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c # via twine -rich==13.6.0 \ - --hash=sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245 \ - --hash=sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef +rich==13.7.1 \ + --hash=sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222 \ + --hash=sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432 # via twine rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ @@ -480,35 +499,39 @@ six==1.16.0 \ # via # gcp-docuploader # python-dateutil -twine==4.0.2 \ - --hash=sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8 \ - --hash=sha256:9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8 +tomli==2.0.1 \ + --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ + --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f + # via nox +twine==5.1.1 \ + --hash=sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997 \ + --hash=sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db # via -r requirements.in -typing-extensions==4.8.0 \ - --hash=sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0 \ - --hash=sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ + --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via -r requirements.in -urllib3==2.0.7 \ - --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ - --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e +urllib3==2.2.2 \ + --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \ + --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168 # via # requests # twine -virtualenv==20.24.6 \ - --hash=sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af \ - --hash=sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381 +virtualenv==20.26.3 \ + --hash=sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a \ + --hash=sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589 # via nox -wheel==0.41.3 \ - --hash=sha256:488609bc63a29322326e05560731bf7bfea8e48ad646e1f5e40d366607de0942 \ - --hash=sha256:4d4987ce51a49370ea65c0bfd2234e8ce80a12780820d9dc462597a6e60d0841 +wheel==0.43.0 \ + --hash=sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85 \ + --hash=sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81 # via -r requirements.in -zipp==3.17.0 \ - --hash=sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31 \ - --hash=sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0 +zipp==3.19.2 \ + --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ + --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==69.2.0 \ - --hash=sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e \ - --hash=sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c +setuptools==70.2.0 \ + --hash=sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05 \ + --hash=sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1 # via -r requirements.in diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh index 63ac41dfae..e9d8bd79a6 100755 --- a/.kokoro/test-samples-against-head.sh +++ b/.kokoro/test-samples-against-head.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 5a0f5fab6a..55910c8ba1 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh index 50b35a48c1..7933d82014 100755 --- a/.kokoro/test-samples.sh +++ b/.kokoro/test-samples.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh index d85b1f2676..48f7969970 100755 --- a/.kokoro/trampoline.sh +++ b/.kokoro/trampoline.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh index 59a7cf3a93..35fa529231 100755 --- a/.kokoro/trampoline_v2.sh +++ b/.kokoro/trampoline_v2.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.trampolinerc b/.trampolinerc index a7dfeb42c6..0080152373 100644 --- a/.trampolinerc +++ b/.trampolinerc @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/CHANGELOG.md b/CHANGELOG.md index b27da8d301..8249515719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://pypi.org/project/bigframes/#history +## [1.11.1](https://github.com/googleapis/python-bigquery-dataframes/compare/v1.11.0...v1.11.1) (2024-07-08) + + +### Documentation + +* Remove session and connection in llm notebook ([#821](https://github.com/googleapis/python-bigquery-dataframes/issues/821)) ([74170da](https://github.com/googleapis/python-bigquery-dataframes/commit/74170dabd323f1b08ad76241e37ff9f2a5b67ab5)) +* Remove the experimental flask icon from the public docs ([#820](https://github.com/googleapis/python-bigquery-dataframes/issues/820)) ([067ff17](https://github.com/googleapis/python-bigquery-dataframes/commit/067ff173f0abfcf5bf06d3fbdb6d12e0fa5283c3)) + ## [1.11.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v1.10.0...v1.11.0) (2024-07-01) diff --git a/MANIFEST.in b/MANIFEST.in index 02b1f4ba4b..16a933a629 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/bigframes/core/__init__.py b/bigframes/core/__init__.py index 00a36b9c05..9b858046bc 100644 --- a/bigframes/core/__init__.py +++ b/bigframes/core/__init__.py @@ -78,7 +78,7 @@ def from_cached( cls, original: ArrayValue, table: google.cloud.bigquery.Table, - ordering: orderings.ExpressionOrdering, + ordering: orderings.TotalOrdering, ): node = nodes.CachedTableNode( original_node=original.node, @@ -147,7 +147,7 @@ def _compiled_schema(self) -> schemata.ArraySchema: def as_cached( self: ArrayValue, cache_table: google.cloud.bigquery.Table, - ordering: Optional[orderings.ExpressionOrdering], + ordering: Optional[orderings.TotalOrdering], ) -> ArrayValue: """ Replace the node with an equivalent one that references a tabel where the value has been materialized to. diff --git a/bigframes/core/blocks.py b/bigframes/core/blocks.py index 379c661179..fef91f88dc 100644 --- a/bigframes/core/blocks.py +++ b/bigframes/core/blocks.py @@ -438,33 +438,28 @@ def reorder_levels(self, ids: typing.Sequence[str]): def _to_dataframe(self, result) -> pd.DataFrame: """Convert BigQuery data to pandas DataFrame with specific dtypes.""" - dtypes = dict(zip(self.index_columns, self.index.dtypes)) - dtypes.update(zip(self.value_columns, self.dtypes)) - result_dataframe = self.session._rows_to_dataframe(result, dtypes) + result_dataframe = self.session._rows_to_dataframe(result) # Runs strict validations to ensure internal type predictions and ibis are completely in sync # Do not execute these validations outside of testing suite. if "PYTEST_CURRENT_TEST" in os.environ: - self._validate_result_schema(result_dataframe) + self._validate_result_schema(result.schema) return result_dataframe - def _validate_result_schema(self, result_df: pd.DataFrame): + def _validate_result_schema( + self, bq_result_schema: list[bigquery.schema.SchemaField] + ): + actual_schema = tuple(bq_result_schema) ibis_schema = self.expr._compiled_schema internal_schema = self.expr.node.schema - actual_schema = bf_schema.ArraySchema( - tuple( - bf_schema.SchemaItem(name, dtype) # type: ignore - for name, dtype in result_df.dtypes.items() - ) - ) if not bigframes.features.PANDAS_VERSIONS.is_arrow_list_dtype_usable: return - if internal_schema != actual_schema: + if internal_schema.to_bigquery() != actual_schema: raise ValueError( - f"This error should only occur while testing. BigFrames internal schema: {internal_schema} does not match actual schema: {actual_schema}" + f"This error should only occur while testing. BigFrames internal schema: {internal_schema.to_bigquery()} does not match actual schema: {actual_schema}" ) - if ibis_schema != actual_schema: + if ibis_schema.to_bigquery() != actual_schema: raise ValueError( - f"This error should only occur while testing. Ibis schema: {ibis_schema} does not match actual schema: {actual_schema}" + f"This error should only occur while testing. Ibis schema: {ibis_schema.to_bigquery()} does not match actual schema: {actual_schema}" ) def to_arrow( @@ -2585,7 +2580,6 @@ def to_pandas(self, *, ordered: Optional[bool] = None) -> pd.Index: ) # Project down to only the index column. So the query can be cached to visualize other data. index_columns = list(self._block.index_columns) - dtypes = dict(zip(index_columns, self.dtypes)) expr = self._expr.select_columns(index_columns) results, _ = self.session._execute( expr, @@ -2593,7 +2587,7 @@ def to_pandas(self, *, ordered: Optional[bool] = None) -> pd.Index: if (ordered is not None) else self.session._strictly_ordered, ) - df = expr.session._rows_to_dataframe(results, dtypes) + df = expr.session._rows_to_dataframe(results) df = df.set_index(index_columns) index = df.index index.names = list(self._block._index_labels) # type:ignore diff --git a/bigframes/core/compile/api.py b/bigframes/core/compile/api.py index 9fba3081ca..1f7d0a4507 100644 --- a/bigframes/core/compile/api.py +++ b/bigframes/core/compile/api.py @@ -46,7 +46,7 @@ def compile_ordered( def compile_raw( node: bigframes.core.nodes.BigFrameNode, -) -> Tuple[str, bigframes.core.ordering.ExpressionOrdering]: +) -> Tuple[str, bigframes.core.ordering.TotalOrdering]: """Compile node into sql that exposes all columns, including hidden ordering-only columns.""" ir = compiler.compile_ordered_ir(node) sql = ir.raw_sql() diff --git a/bigframes/core/compile/compiled.py b/bigframes/core/compile/compiled.py index 907c918efd..cc601744c1 100644 --- a/bigframes/core/compile/compiled.py +++ b/bigframes/core/compile/compiled.py @@ -36,9 +36,9 @@ from bigframes.core.ordering import ( ascending_over, encode_order_string, - ExpressionOrdering, IntegerEncoding, OrderingExpression, + TotalOrdering, ) import bigframes.core.schema as schemata import bigframes.core.sql @@ -194,7 +194,7 @@ def _aggregate_base( if by_column_ids: result = table.group_by(by_column_ids).aggregate(**stats) # Must have deterministic ordering, so order by the unique "by" column - ordering = ExpressionOrdering( + ordering = TotalOrdering( tuple([ascending_over(column_id) for column_id in by_column_ids]), total_ordering_columns=frozenset(by_column_ids), ) @@ -210,7 +210,7 @@ def _aggregate_base( # Ordering is irrelevant for single-row output, but set ordering id regardless # as other ops(join etc.) expect it. # TODO: Maybe can make completely empty - ordering = ExpressionOrdering( + ordering = TotalOrdering( ordering_value_columns=tuple([]), total_ordering_columns=frozenset([]), ) @@ -279,7 +279,7 @@ def row_count(self) -> OrderedIR: return OrderedIR( ibis_table, (ibis_table["count"],), - ordering=ExpressionOrdering( + ordering=TotalOrdering( ordering_value_columns=(ascending_over("count"),), total_ordering_columns=frozenset(["count"]), ), @@ -519,7 +519,7 @@ def __init__( table: ibis_types.Table, columns: Sequence[ibis_types.Value], hidden_ordering_columns: Optional[Sequence[ibis_types.Value]] = None, - ordering: ExpressionOrdering = ExpressionOrdering(), + ordering: TotalOrdering = TotalOrdering(), predicates: Optional[Collection[ibis_types.BooleanValue]] = None, ): super().__init__(table, columns, predicates) @@ -598,10 +598,7 @@ def from_pandas( return cls( keys_memtable, columns=[keys_memtable[column].name(column) for column in pd_df.columns], - ordering=ExpressionOrdering( - ordering_value_columns=tuple([ascending_over(ORDER_ID_COLUMN)]), - total_ordering_columns=frozenset([ORDER_ID_COLUMN]), - ), + ordering=TotalOrdering.from_offset_col(ORDER_ID_COLUMN), hidden_ordering_columns=(keys_memtable[ORDER_ID_COLUMN],), ) @@ -760,7 +757,7 @@ def explode(self, column_ids: typing.Sequence[str]) -> OrderedIR: ], table_w_unnest[unnest_offset_id], ] - ordering = ExpressionOrdering( + ordering = TotalOrdering( ordering_value_columns=tuple( [ *self._ordering.ordering_value_columns, @@ -1153,7 +1150,7 @@ def _bake_ordering(self) -> OrderedIR: self._ibis_bindings[expr.scalar_expression.id] ) - new_ordering = ExpressionOrdering( + new_ordering = TotalOrdering( tuple(new_exprs), self._ordering.integer_encoding, self._ordering.string_encoding, @@ -1176,7 +1173,7 @@ def _project_offsets(self) -> OrderedIR: ordering_mode="offset_col", order_col_name=ORDER_ID_COLUMN ) columns = [table[column_name] for column_name in self._column_names] - ordering = ExpressionOrdering( + ordering = TotalOrdering( ordering_value_columns=tuple([ascending_over(ORDER_ID_COLUMN)]), total_ordering_columns=frozenset([ORDER_ID_COLUMN]), integer_encoding=IntegerEncoding(True, is_sequential=True), @@ -1300,7 +1297,7 @@ class Builder: def __init__( self, table: ibis_types.Table, - ordering: ExpressionOrdering, + ordering: TotalOrdering, columns: Collection[ibis_types.Value] = (), hidden_ordering_columns: Collection[ibis_types.Value] = (), predicates: Optional[Collection[ibis_types.BooleanValue]] = None, diff --git a/bigframes/core/compile/compiler.py b/bigframes/core/compile/compiler.py index 9272b8ad1b..ca9c479fff 100644 --- a/bigframes/core/compile/compiler.py +++ b/bigframes/core/compile/compiler.py @@ -195,7 +195,7 @@ def compile_read_table_ordered(node: nodes.ReadTableNode): ) else: integer_encoding = bf_ordering.IntegerEncoding() - ordering = bf_ordering.ExpressionOrdering( + ordering = bf_ordering.TotalOrdering( ordering_value_columns, integer_encoding=integer_encoding, total_ordering_columns=frozenset(node.total_order_cols), diff --git a/bigframes/core/compile/concat.py b/bigframes/core/compile/concat.py index 41a8f97821..35cf9ec5b4 100644 --- a/bigframes/core/compile/concat.py +++ b/bigframes/core/compile/concat.py @@ -21,9 +21,9 @@ import bigframes.core.compile.compiled as compiled from bigframes.core.ordering import ( ascending_over, - ExpressionOrdering, reencode_order_string, StringEncoding, + TotalOrdering, ) ORDER_ID_COLUMN = "bigframes_ordering_id" @@ -83,7 +83,7 @@ def concat_ordered( ) tables.append(table) combined_table = ibis.union(*tables) - ordering = ExpressionOrdering( + ordering = TotalOrdering( ordering_value_columns=tuple([ascending_over(ORDER_ID_COLUMN)]), total_ordering_columns=frozenset([ORDER_ID_COLUMN]), string_encoding=StringEncoding(True, prefix_size + max_encoding_size), diff --git a/bigframes/core/compile/default_ordering.py b/bigframes/core/compile/default_ordering.py index d8bdc0546b..7d7a41f742 100644 --- a/bigframes/core/compile/default_ordering.py +++ b/bigframes/core/compile/default_ordering.py @@ -82,7 +82,7 @@ def gen_default_ordering(table: ibis.table, use_double_hash: bool = True): itertools.chain(original_column_ids, order_values) ) - ordering = order.ExpressionOrdering( + ordering = order.TotalOrdering( ordering_value_columns=tuple( order.ascending_over(col.get_name()) for col in order_values ), diff --git a/bigframes/core/compile/single_column.py b/bigframes/core/compile/single_column.py index dbf25891bf..9b621c9c79 100644 --- a/bigframes/core/compile/single_column.py +++ b/bigframes/core/compile/single_column.py @@ -16,8 +16,6 @@ from __future__ import annotations -from typing import Mapping - import ibis import ibis.expr.datatypes as ibis_dtypes import ibis.expr.types as ibis_types @@ -84,7 +82,7 @@ def join_by_column_ordered( ) # Preserve ordering accross joins. - ordering = join_orderings( + ordering = orderings.join_orderings( left._ordering, right._ordering, l_mapping, @@ -173,33 +171,3 @@ def value_to_join_key(value: ibis_types.Value): if not value.type().is_string(): value = value.cast(ibis_dtypes.str) return value.fillna(ibis_types.literal("$NULL_SENTINEL$")) - - -def join_orderings( - left: orderings.ExpressionOrdering, - right: orderings.ExpressionOrdering, - left_id_mapping: Mapping[str, str], - right_id_mapping: Mapping[str, str], - left_order_dominates: bool = True, -) -> orderings.ExpressionOrdering: - left_ordering_refs = [ - ref.remap_names(left_id_mapping) for ref in left.all_ordering_columns - ] - right_ordering_refs = [ - ref.remap_names(right_id_mapping) for ref in right.all_ordering_columns - ] - if left_order_dominates: - joined_refs = [*left_ordering_refs, *right_ordering_refs] - else: - joined_refs = [*right_ordering_refs, *left_ordering_refs] - - left_total_order_cols = frozenset( - [left_id_mapping[id] for id in left.total_ordering_columns] - ) - right_total_order_cols = frozenset( - [right_id_mapping[id] for id in right.total_ordering_columns] - ) - return orderings.ExpressionOrdering( - ordering_value_columns=tuple(joined_refs), - total_ordering_columns=left_total_order_cols | right_total_order_cols, - ) diff --git a/bigframes/core/nodes.py b/bigframes/core/nodes.py index 65a1bd8084..a703cf1969 100644 --- a/bigframes/core/nodes.py +++ b/bigframes/core/nodes.py @@ -373,7 +373,7 @@ class CachedTableNode(BigFrameNode): table_id: str = field() physical_schema: Tuple[bq.SchemaField, ...] = field() - ordering: typing.Optional[orderings.ExpressionOrdering] = field() + ordering: typing.Optional[orderings.TotalOrdering] = field() def __post_init__(self): # enforce invariants diff --git a/bigframes/core/ordering.py b/bigframes/core/ordering.py index 1562592720..406ca52731 100644 --- a/bigframes/core/ordering.py +++ b/bigframes/core/ordering.py @@ -94,10 +94,42 @@ class IntegerEncoding: @dataclass(frozen=True) -class ExpressionOrdering: - """Immutable object that holds information about the ordering of rows in a ArrayValue object.""" +class RowOrdering: + """Immutable object that holds information about the ordering of rows in a ArrayValue object. May not be unambiguous.""" ordering_value_columns: typing.Tuple[OrderingExpression, ...] = () + + @property + def all_ordering_columns(self) -> Sequence[OrderingExpression]: + return list(self.ordering_value_columns) + + @property + def referenced_columns(self) -> Set[str]: + return set( + col + for part in self.ordering_value_columns + for col in part.scalar_expression.unbound_variables + ) + + def with_reverse(self) -> RowOrdering: + """Reverses the ordering.""" + return RowOrdering( + tuple([col.with_reverse() for col in self.ordering_value_columns]), + ) + + def with_column_remap(self, mapping: typing.Mapping[str, str]) -> RowOrdering: + new_value_columns = [ + col.remap_names(mapping) for col in self.all_ordering_columns + ] + return TotalOrdering( + tuple(new_value_columns), + ) + + +@dataclass(frozen=True) +class TotalOrdering(RowOrdering): + """Immutable object that holds information about the ordering of rows in a ArrayValue object. Guaranteed to be unambiguous.""" + integer_encoding: IntegerEncoding = IntegerEncoding(False) string_encoding: StringEncoding = StringEncoding(False) # A table has a total ordering defined by the identities of a set of 1 or more columns. @@ -106,8 +138,8 @@ class ExpressionOrdering: total_ordering_columns: frozenset[str] = field(default_factory=frozenset) @classmethod - def from_offset_col(cls, col: str) -> ExpressionOrdering: - return ExpressionOrdering( + def from_offset_col(cls, col: str) -> TotalOrdering: + return TotalOrdering( (ascending_over(col),), integer_encoding=IntegerEncoding(True, is_sequential=True), total_ordering_columns=frozenset({col}), @@ -119,7 +151,7 @@ def with_non_sequential(self): This is useful when filtering, but not sorting, an expression. """ if self.integer_encoding.is_sequential: - return ExpressionOrdering( + return TotalOrdering( self.ordering_value_columns, integer_encoding=IntegerEncoding( self.integer_encoding.is_encoded, is_sequential=False @@ -132,7 +164,7 @@ def with_non_sequential(self): def with_ordering_columns( self, ordering_value_columns: Sequence[OrderingExpression] = (), - ) -> ExpressionOrdering: + ) -> TotalOrdering: """Creates a new ordering that reorders by the given columns. Args: @@ -147,7 +179,7 @@ def with_ordering_columns( new_ordering = self._truncate_ordering( (*ordering_value_columns, *self.ordering_value_columns) ) - return ExpressionOrdering( + return TotalOrdering( new_ordering, total_ordering_columns=self.total_ordering_columns, ) @@ -173,7 +205,7 @@ def _truncate_ordering( def with_reverse(self): """Reverses the ordering.""" - return ExpressionOrdering( + return TotalOrdering( tuple([col.with_reverse() for col in self.ordering_value_columns]), total_ordering_columns=self.total_ordering_columns, ) @@ -185,7 +217,7 @@ def with_column_remap(self, mapping: typing.Mapping[str, str]): new_total_order = frozenset( mapping.get(col_id, col_id) for col_id in self.total_ordering_columns ) - return ExpressionOrdering( + return TotalOrdering( tuple(new_value_columns), integer_encoding=self.integer_encoding, string_encoding=self.string_encoding, @@ -211,18 +243,6 @@ def is_string_encoded(self) -> bool: def is_sequential(self) -> bool: return self.integer_encoding.is_encoded and self.integer_encoding.is_sequential - @property - def all_ordering_columns(self) -> Sequence[OrderingExpression]: - return list(self.ordering_value_columns) - - @property - def referenced_columns(self) -> Set[str]: - return set( - col - for part in self.ordering_value_columns - for col in part.scalar_expression.unbound_variables - ) - def encode_order_string( order_id: ibis_types.IntegerColumn, length: int = DEFAULT_ORDERING_ID_LENGTH @@ -257,3 +277,58 @@ def descending_over(id: str, nulls_last: bool = True) -> OrderingExpression: return OrderingExpression( expression.free_var(id), direction=OrderingDirection.DESC, na_last=nulls_last ) + + +@typing.overload +def join_orderings( + left: TotalOrdering, + right: TotalOrdering, + left_id_mapping: Mapping[str, str], + right_id_mapping: Mapping[str, str], + left_order_dominates: bool = True, +) -> TotalOrdering: + ... + + +@typing.overload +def join_orderings( + left: RowOrdering, + right: RowOrdering, + left_id_mapping: Mapping[str, str], + right_id_mapping: Mapping[str, str], + left_order_dominates: bool = True, +) -> RowOrdering: + ... + + +def join_orderings( + left: RowOrdering, + right: RowOrdering, + left_id_mapping: Mapping[str, str], + right_id_mapping: Mapping[str, str], + left_order_dominates: bool = True, +) -> RowOrdering: + left_ordering_refs = [ + ref.remap_names(left_id_mapping) for ref in left.all_ordering_columns + ] + right_ordering_refs = [ + ref.remap_names(right_id_mapping) for ref in right.all_ordering_columns + ] + if left_order_dominates: + joined_refs = [*left_ordering_refs, *right_ordering_refs] + else: + joined_refs = [*right_ordering_refs, *left_ordering_refs] + + if isinstance(left, TotalOrdering) and isinstance(right, TotalOrdering): + left_total_order_cols = frozenset( + [left_id_mapping[id] for id in left.total_ordering_columns] + ) + right_total_order_cols = frozenset( + [right_id_mapping[id] for id in right.total_ordering_columns] + ) + return TotalOrdering( + ordering_value_columns=tuple(joined_refs), + total_ordering_columns=left_total_order_cols | right_total_order_cols, + ) + else: + return RowOrdering(tuple(joined_refs)) diff --git a/bigframes/core/schema.py b/bigframes/core/schema.py index 30a2a6593a..ee27c6ff30 100644 --- a/bigframes/core/schema.py +++ b/bigframes/core/schema.py @@ -58,6 +58,12 @@ def dtypes(self) -> typing.Tuple[bigframes.dtypes.Dtype, ...]: def _mapping(self) -> typing.Dict[ColumnIdentifierType, bigframes.dtypes.Dtype]: return {item.column: item.dtype for item in self.items} + def to_bigquery(self) -> typing.Tuple[google.cloud.bigquery.SchemaField, ...]: + return tuple( + bigframes.dtypes.convert_to_schema_field(item.column, item.dtype) + for item in self.items + ) + def drop(self, columns: typing.Iterable[str]) -> ArraySchema: return ArraySchema( tuple(item for item in self.items if item.column not in columns) diff --git a/bigframes/dtypes.py b/bigframes/dtypes.py index 160802ded9..563904fbb6 100644 --- a/bigframes/dtypes.py +++ b/bigframes/dtypes.py @@ -70,7 +70,9 @@ class SimpleDtypeInfo: dtype: Dtype arrow_dtype: typing.Optional[pa.DataType] - type_kind: typing.Tuple[str, ...] # Should all correspond to the same db type + type_kind: typing.Tuple[ + str, ... + ] # Should all correspond to the same db type. Put preferred canonical sql type name first logical_bytes: int = ( 8 # this is approximate only, some types are variably sized, also, compression ) @@ -84,20 +86,23 @@ class SimpleDtypeInfo: SimpleDtypeInfo( dtype=INT_DTYPE, arrow_dtype=pa.int64(), - type_kind=("INT64", "INTEGER"), + type_kind=("INTEGER", "INT64"), orderable=True, clusterable=True, ), SimpleDtypeInfo( dtype=FLOAT_DTYPE, arrow_dtype=pa.float64(), - type_kind=("FLOAT64", "FLOAT"), + type_kind=("FLOAT", "FLOAT64"), orderable=True, ), SimpleDtypeInfo( dtype=BOOL_DTYPE, arrow_dtype=pa.bool_(), - type_kind=("BOOL", "BOOLEAN"), + type_kind=( + "BOOLEAN", + "BOOL", + ), logical_bytes=1, orderable=True, clusterable=True, @@ -143,7 +148,7 @@ class SimpleDtypeInfo: SimpleDtypeInfo( dtype=NUMERIC_DTYPE, arrow_dtype=pa.decimal128(38, 9), - type_kind=("NUMERIC",), + type_kind=("NUMERIC", "DECIMAL"), logical_bytes=16, orderable=True, clusterable=True, @@ -151,7 +156,7 @@ class SimpleDtypeInfo: SimpleDtypeInfo( dtype=BIGNUMERIC_DTYPE, arrow_dtype=pa.decimal256(76, 38), - type_kind=("BIGNUMERIC",), + type_kind=("BIGNUMERIC", "BIGDECIMAL"), logical_bytes=32, orderable=True, clusterable=True, @@ -417,6 +422,7 @@ def infer_literal_arrow_type(literal) -> typing.Optional[pa.DataType]: for mapping in SIMPLE_TYPES for type_kind in mapping.type_kind } +_BIGFRAMES_TO_TK = {mapping.dtype: mapping.type_kind[0] for mapping in SIMPLE_TYPES} def convert_schema_field( @@ -440,12 +446,44 @@ def convert_schema_field( if is_repeated: pa_type = pa.list_(bigframes_dtype_to_arrow_dtype(singular_type)) return field.name, pd.ArrowDtype(pa_type) - else: - return field.name, singular_type + return field.name, singular_type else: raise ValueError(f"Cannot handle type: {field.field_type}") +def convert_to_schema_field( + name: str, + bigframes_dtype: Dtype, +) -> google.cloud.bigquery.SchemaField: + if bigframes_dtype in _BIGFRAMES_TO_TK: + return google.cloud.bigquery.SchemaField( + name, _BIGFRAMES_TO_TK[bigframes_dtype] + ) + if isinstance(bigframes_dtype, pd.ArrowDtype): + if pa.types.is_list(bigframes_dtype.pyarrow_dtype): + inner_type = arrow_dtype_to_bigframes_dtype( + bigframes_dtype.pyarrow_dtype.value_type + ) + inner_field = convert_to_schema_field(name, inner_type) + return google.cloud.bigquery.SchemaField( + name, inner_field.field_type, mode="REPEATED", fields=inner_field.fields + ) + if pa.types.is_struct(bigframes_dtype.pyarrow_dtype): + inner_fields: list[pa.Field] = [] + struct_type = typing.cast(pa.StructType, bigframes_dtype.pyarrow_dtype) + for i in range(struct_type.num_fields): + field = struct_type.field(i) + inner_bf_type = arrow_dtype_to_bigframes_dtype(field.type) + inner_fields.append(convert_to_schema_field(field.name, inner_bf_type)) + + return google.cloud.bigquery.SchemaField( + name, "RECORD", fields=inner_fields + ) + raise ValueError( + f"No arrow conversion for {bigframes_dtype}. {constants.FEEDBACK_LINK}" + ) + + def bf_type_from_type_kind( bq_schema: list[google.cloud.bigquery.SchemaField], ) -> typing.Dict[str, Dtype]: diff --git a/bigframes/session/__init__.py b/bigframes/session/__init__.py index 867bdedf1c..0f7953d3d4 100644 --- a/bigframes/session/__init__.py +++ b/bigframes/session/__init__.py @@ -1910,7 +1910,7 @@ def _cache_with_offsets(self, array_value: core.ArrayValue): ) cached_replacement = array_value.as_cached( cache_table=self.bqclient.get_table(tmp_table), - ordering=order.ExpressionOrdering.from_offset_col(offset_column), + ordering=order.TotalOrdering.from_offset_col(offset_column), ).node self._cached_executions[array_value.node] = cached_replacement @@ -2038,7 +2038,7 @@ def _get_table_size(self, destination_table): return table.num_bytes def _rows_to_dataframe( - self, row_iterator: bigquery.table.RowIterator, dtypes: Dict + self, row_iterator: bigquery.table.RowIterator ) -> pandas.DataFrame: # Can ignore inferred datatype until dtype emulation breaks 1:1 mapping between BQ types and bigframes types dtypes_from_bq = bigframes.dtypes.bf_type_from_type_kind(row_iterator.schema) diff --git a/bigframes/version.py b/bigframes/version.py index 75f3ffb361..1186811c97 100644 --- a/bigframes/version.py +++ b/bigframes/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.11.0" +__version__ = "1.11.1" diff --git a/docs/conf.py b/docs/conf.py index af8c5efda8..23ec7a6b36 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docs/templates/toc.yml b/docs/templates/toc.yml index aee3058ecc..95bded9a60 100644 --- a/docs/templates/toc.yml +++ b/docs/templates/toc.yml @@ -204,4 +204,3 @@ uid: bigframes.bigquery name: bigframes.bigquery name: BigQuery DataFrames - status: beta diff --git a/notebooks/generative_ai/large_language_models.ipynb b/notebooks/generative_ai/large_language_models.ipynb index 08ef52b544..744706cab8 100644 --- a/notebooks/generative_ai/large_language_models.ipynb +++ b/notebooks/generative_ai/large_language_models.ipynb @@ -16,8 +16,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Prerequisites\n", - "Create session and define a BQ connection which we already created and allowlisted. " + "## Define the model" ] }, { @@ -29,33 +28,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/usr/local/google/home/garrettwu/src/bigframes/bigframes/session/__init__.py:1762: UserWarning: No explicit location is set, so using location US for the session.\n", - " return Session(context)\n" + "/usr/local/google/home/garrettwu/src/bigframes/bigframes/ml/llm.py:589: DefaultLocationWarning: No explicit location is set, so using location US for the session.\n", + " self.session = session or bpd.get_global_session()\n" ] - } - ], - "source": [ - "session = bigframes.pandas.get_global_session()\n", - "connection = f\"{session.bqclient.project}.us.bigframes-default-connection\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define the model" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ + }, { "data": { "text/html": [ - "Query job 12bcd690-ca99-4001-bf26-032f50e77d62 is DONE. 0 Bytes processed. Open Job" + "Query job 675a6c8a-213b-496c-9f77-b87bf7cfa5e0 is DONE. 0 Bytes processed. Open Job" ], "text/plain": [ "" @@ -66,7 +46,7 @@ } ], "source": [ - "model = GeminiTextGenerator(session=session, connection_name=connection)" + "model = GeminiTextGenerator()" ] }, { @@ -81,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -102,13 +82,13 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "Query job f8fe31c6-7d8a-4919-9492-8304a0083cca is DONE. 0 Bytes processed. Open Job" + "Query job 7967df2b-9f0f-45c8-a363-15f65891c3bf is DONE. 0 Bytes processed. Open Job" ], "text/plain": [ "" @@ -118,21 +98,17 @@ "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job 28bab71f-e218-4d92-9a50-dab41bb0c71f is DONE. 24 Bytes processed. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/garrettwu/src/bigframes/bigframes/core/__init__.py:108: PreviewWarning: Interpreting JSON column(s) as StringDtype. This behavior may change in future versions.\n", + " warnings.warn(\n" + ] }, { "data": { "text/html": [ - "Query job 01d66b61-459f-474e-9f66-d519f9c2f23d is DONE. 6 Bytes processed. Open Job" + "Query job 9a1f57cd-98e1-4eac-a1b3-8f88d61971cd is DONE. 6 Bytes processed. Open Job" ], "text/plain": [ "" @@ -144,7 +120,7 @@ { "data": { "text/html": [ - "Query job af606ca7-4bcf-4bd1-95fd-c516542b5a4f is DONE. 5.3 kB processed. Open Job" + "Query job 2a94a2cf-7d4c-4009-a798-d7a5d6d4049d is DONE. 8.5 kB processed. Open Job" ], "text/plain": [ "" @@ -183,28 +159,28 @@ " \n", " \n", " 0\n", - " **BigQuery**\n", + " ## BigQuery: A Serverless Data Warehouse\n", "\n", - "**Definition:**\n", - "\n", - "BigQuery is a s...\n", - " null\n", + "BigQ...\n", + " [{\"category\":1,\"probability\":1,\"probability_sc...\n", " \n", " What is BigQuery?\n", " \n", " \n", " 1\n", - " **BigQuery Machine Learning (BQML)**\n", + " ## BigQuery Machine Learning (BQML)\n", "\n", - "BQML is ...\n", - " null\n", + "BQML is a...\n", + " [{\"category\":1,\"probability\":1,\"probability_sc...\n", " \n", " What is BQML?\n", " \n", " \n", " 2\n", - " BigQuery DataFrame is a Python DataFrame imple...\n", - " null\n", + " ## What is BigQuery DataFrame?\n", + "\n", + "**BigQuery Dat...\n", + " [{\"category\":1,\"probability\":1,\"probability_sc...\n", " \n", " What is BigQuery DataFrame?\n", " \n", @@ -214,20 +190,20 @@ ], "text/plain": [ " ml_generate_text_llm_result \\\n", - "0 **BigQuery**\n", + "0 ## BigQuery: A Serverless Data Warehouse\n", "\n", - "**Definition:**\n", + "BigQ... \n", + "1 ## BigQuery Machine Learning (BQML)\n", "\n", - "BigQuery is a s... \n", - "1 **BigQuery Machine Learning (BQML)**\n", + "BQML is a... \n", + "2 ## What is BigQuery DataFrame?\n", "\n", - "BQML is ... \n", - "2 BigQuery DataFrame is a Python DataFrame imple... \n", + "**BigQuery Dat... \n", "\n", - " ml_generate_text_rai_result ml_generate_text_status \\\n", - "0 null \n", - "1 null \n", - "2 null \n", + " ml_generate_text_rai_result ml_generate_text_status \\\n", + "0 [{\"category\":1,\"probability\":1,\"probability_sc... \n", + "1 [{\"category\":1,\"probability\":1,\"probability_sc... \n", + "2 [{\"category\":1,\"probability\":1,\"probability_sc... \n", "\n", " prompt \n", "0 What is BigQuery? \n", @@ -235,7 +211,7 @@ "2 What is BigQuery DataFrame? " ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -255,16 +231,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'**BigQuery**\\n\\n**Definition:**\\n\\nBigQuery is a serverless, highly scalable, cloud-based data warehouse and analytics platform offered by Google Cloud.\\n\\n**Key Features:**\\n\\n* **Massive Scalability:** Can handle large datasets (petabytes or more) with fast query execution.\\n* **Elastic:** Automatically scales compute resources based on workload requirements.\\n* **Serverless:** Users do not need to manage infrastructure or provision resources.\\n* **Flexible Data Loading:** Supports a wide range of data sources, including files, databases, and streaming data.\\n* **SQL-Based Querying:** Uses standard SQL syntax for querying and analyzing data.\\n* **Machine Learning Integration:** Provides built-in machine learning capabilities for predictive analytics and data exploration.\\n* **Real-Time Analysis:** Supports streaming data analysis and interactive dashboards.\\n* **Collaboration and Sharing:** Allows multiple users to access and analyze data in a collaborative environment.\\n* **Cost-Effective:** Pay-as-you-go pricing based on data scanned and compute resources used.\\n\\n**Applications:**\\n\\n* Data warehousing and analytics\\n* Business intelligence and reporting\\n* Data science and machine learning\\n* Data exploration and visualization\\n* Marketing analytics\\n* Fraud detection and risk management\\n\\n**Benefits:**\\n\\n* Rapid data analysis on large datasets\\n* Reduced infrastructure management overhead\\n* Increased agility and flexibility\\n* Enhanced collaboration and data sharing\\n* Cost-effective data storage and analytics'" + "\"## BigQuery: A Serverless Data Warehouse\\n\\nBigQuery is a serverless, cloud-based data warehouse that enables scalable analysis of large datasets. It's a popular choice for businesses of all sizes due to its ability to handle petabytes of data and run complex queries quickly and efficiently. Let's delve into its key features:\\n\\n**Serverless Architecture:** BigQuery eliminates the need for server management, allowing you to focus on analyzing data. Google manages the infrastructure, scaling resources up or down automatically based on your needs.\\n\\n**Scalability:** BigQuery can handle massive datasets, scaling seamlessly as your data volume grows. It automatically distributes queries across its infrastructure, ensuring fast and efficient processing.\\n\\n**SQL-like Querying:** BigQuery uses a familiar SQL-like syntax, making it easy for data analysts and developers to learn and use. This allows them to leverage their existing SQL knowledge for data exploration and analysis.\\n\\n**Cost-Effectiveness:** BigQuery offers a pay-as-you-go pricing model, meaning you only pay for the resources you use. This makes it a cost-effective solution for businesses with varying data processing needs.\\n\\n**Integration with Google Cloud:** BigQuery integrates seamlessly with other Google Cloud services like Cloud Storage, Dataflow, and Machine Learning, enabling a comprehensive data processing and analysis workflow within the Google Cloud ecosystem.\\n\\n**Security and Reliability:** BigQuery offers robust security features and high availability, ensuring data protection and reliable access.\\n\\n**Use Cases:** BigQuery finds applications in various scenarios, including:\\n\\n* **Data Warehousing:** Store and analyze large amounts of structured and semi-structured data.\\n* **Business Intelligence:** Generate insights from data for informed decision-making.\\n* **Data Analytics:** Perform complex data analysis and extract valuable patterns.\\n* **Machine Learning:** Train and deploy machine learning models on large datasets.\\n\\n**Getting Started:** To get started with BigQuery, you can create a free trial account on Google Cloud Platform and explore its features. Numerous tutorials and documentation are available to help you learn and use BigQuery effectively.\\n\\n## Additional Resources:\\n\\n* **BigQuery Documentation:** https://cloud.google.com/bigquery/docs/\\n* **BigQuery Quickstart:** https://cloud.google.com/bigquery/docs/quickstarts/quickstart-console\\n* **BigQuery Pricing:** https://cloud.google.com/bigquery/pricing\\n\\nFeel free to ask if you have any further questions about BigQuery!\"" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } diff --git a/notebooks/remote_functions/remote_function_usecases.ipynb b/notebooks/remote_functions/remote_function_usecases.ipynb new file mode 100644 index 0000000000..3d7ae3e8c7 --- /dev/null +++ b/notebooks/remote_functions/remote_function_usecases.ipynb @@ -0,0 +1,1408 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "# Copyright 2023 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "id": "Y6QAttCqqMM0" + }, + "outputs": [], + "source": [ + "import bigframes.pandas as bpd" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 296 + }, + "id": "xraJ9RRzsvel", + "outputId": "6e3308cf-8de0-4b89-9128-4c6ddf3598c0" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shobs/code/bigframes/venv/lib/python3.10/site-packages/IPython/core/interactiveshell.py:3550: UserWarning: Reading cached table from 2024-06-28 02:49:31.716256+00:00 to avoid incompatibilies with previous reads of this table. To read the latest version, set `use_cache=False` or close the current session with Session.close() or bigframes.pandas.close_session().\n", + " exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + }, + { + "data": { + "text/html": [ + "Query job f72cda67-2a96-4cd2-a624-591c0d540fc9 is DONE. 582.8 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 65cf6ca3-73f0-49e6-84a8-1ff79af6ec75 is DONE. 82.0 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutes
50RaysRangers181
72PhilliesPirates192
89MarinersBlue Jays183
351AstrosAngels212
382RoyalsYankees259
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes\n", + "50 Rays Rangers 181\n", + "72 Phillies Pirates 192\n", + "89 Mariners Blue Jays 183\n", + "351 Astros Angels 212\n", + "382 Royals Yankees 259" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = bpd.read_gbq(\"bigquery-public-data.baseball.schedules\")[[\"homeTeamName\", \"awayTeamName\", \"duration_minutes\"]]\n", + "df.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Notes\n", + "\n", + "* The API reference documentation for the `remote_function` can be found at\n", + " https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.session.Session#bigframes_session_Session_remote_function\n", + "\n", + "* More code samples for `remote_function` can be found in the BigQuery\n", + " DataFrames API reference documentation, e.g.\n", + " * https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.series.Series#bigframes_series_Series_apply\n", + " * https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.dataframe.DataFrame#bigframes_dataframe_DataFrame_map\n", + " * https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.dataframe.DataFrame#bigframes_dataframe_DataFrame_apply\n", + "\n", + "* The following examples are only for the purpose of demonstrating\n", + "`remote_function` usage. They are not necessarily the best way to achieve the\n", + "end result.\n", + "\n", + "* In the examples in this notebook we are using `reuse=False` just as a caution\n", + " to avoid concurrent runs of this notebook in the same google cloud project\n", + " stepping over each other's remote function deployment. It may not be neccesary\n", + " in a simple use case." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Pt4mWYE1p5o8" + }, + "source": [ + "# Self-contained function\n", + "\n", + "Let's consider a scenario where we want to categorize the matches as short,\n", + "medium or long duration based on the `duration_minutes` column." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "VoCPBJ-ZpyeG", + "outputId": "19351206-116e-4da2-8ff0-f288b7745b27" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job f039d478-8dc4-4b60-8eda-179955e06586 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created cloud function 'projects/bigframes-dev/locations/us-central1/functions/bigframes-862150459da5240a6df1ce01c59b32d8-em4ibov0' and BQ remote function 'bigframes-dev._1b6c31ff1bcd5d2f6d86833cf8268317f1b12d57.bigframes_862150459da5240a6df1ce01c59b32d8_em4ibov0'.\n" + ] + } + ], + "source": [ + "@bpd.remote_function(reuse=False)\n", + "def duration_category(duration_minutes: int) -> str:\n", + " if duration_minutes < 90:\n", + " return \"short\"\n", + " elif duration_minutes < 180:\n", + " return \"medium\"\n", + " else:\n", + " return \"long\"\n", + "\n", + "print(f\"Created cloud function '{duration_category.bigframes_cloud_function}' and BQ remote function '{duration_category.bigframes_remote_function}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "oXgDB70Lp5cG", + "outputId": "c08aade0-8b03-425b-fc26-deafd89275a4" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 23e95831-d913-4d2b-97f6-588fc7967455 is DONE. 58.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job bb8b3d13-a521-4d45-b4c8-5686c944a9f2 is DONE. 157.2 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 2a4653f5-cc6b-4279-a45e-40f0f97090a7 is DONE. 98.8 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutesduration_cat
1911DodgersAngels132medium
2365AthleticsAngels134medium
1977AthleticsAngels139medium
554CubsAngels142medium
654AstrosAngels143medium
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes duration_cat\n", + "1911 Dodgers Angels 132 medium\n", + "2365 Athletics Angels 134 medium\n", + "1977 Athletics Angels 139 medium\n", + "554 Cubs Angels 142 medium\n", + "654 Astros Angels 143 medium" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(duration_cat=df[\"duration_minutes\"].apply(duration_category))\n", + "df1.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zTaNSVmuzEkc" + }, + "source": [ + "# Function referring to variables outside the function body\n", + "\n", + "Let's consider a slight variation of the earlier example where the labels for\n", + "the short, medium and long duration matches are defined outside the function\n", + "body. They would be captured at the time of `remote_function` deployment and\n", + "any change in their values in the notebook after the deployment will not\n", + "automatically propagate to the `remote_function`." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "id": "2UEmTbu4znyS" + }, + "outputs": [], + "source": [ + "DURATION_CATEGORY_SHORT = \"S\"\n", + "DURATION_CATEGORY_MEDIUM = \"M\"\n", + "DURATION_CATEGORY_LONG = \"L\"" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "G-73kpmrznHn", + "outputId": "b5923b7c-d412-43bf-9a20-3946154df81a" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 5d914fde-81ec-46eb-9219-9822f77dd9a2 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created cloud function 'projects/bigframes-dev/locations/us-central1/functions/bigframes-f3231b74ec807496f4894218d5d40ed5-688mx7hi' and BQ remote function 'bigframes-dev._1b6c31ff1bcd5d2f6d86833cf8268317f1b12d57.bigframes_f3231b74ec807496f4894218d5d40ed5_688mx7hi'.\n" + ] + } + ], + "source": [ + "@bpd.remote_function(reuse=False)\n", + "def duration_category(duration_minutes: int) -> str:\n", + " if duration_minutes < 90:\n", + " return DURATION_CATEGORY_SHORT\n", + " elif duration_minutes < 180:\n", + " return DURATION_CATEGORY_MEDIUM\n", + " else:\n", + " return DURATION_CATEGORY_LONG\n", + "\n", + "print(f\"Created cloud function '{duration_category.bigframes_cloud_function}' and BQ remote function '{duration_category.bigframes_remote_function}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "DWHKsfF-z7rL", + "outputId": "c736b57f-1fcb-464a-f725-eb203265ddc2" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job b0b39944-1e69-4185-97ba-985178ee241f is DONE. 58.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 90d99515-eb5e-4bcd-bce5-292eea09770e is DONE. 147.7 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job eb31d033-c871-49c5-a75e-4427e376516f is DONE. 89.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutesduration_cat
1911DodgersAngels132M
2365AthleticsAngels134M
1977AthleticsAngels139M
554CubsAngels142M
654AstrosAngels143M
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes duration_cat\n", + "1911 Dodgers Angels 132 M\n", + "2365 Athletics Angels 134 M\n", + "1977 Athletics Angels 139 M\n", + "554 Cubs Angels 142 M\n", + "654 Astros Angels 143 M" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(duration_cat=df[\"duration_minutes\"].apply(duration_category))\n", + "df1.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "J-1BIasNzKil" + }, + "source": [ + "# Function referring to imports (built-in) outside the function body\n", + "\n", + "Let's consider a scenario in which we want to categorize the matches in terms of\n", + "hour buckets. E.g. a match finishing in 0-60 minutes would be in 1h category,\n", + "61-120 minutes in 2h category and so on. The function itself makes use of the\n", + "`math` module (a built-in module in a standard python installation) which\n", + "happens to be imported outside the function body, let's say in one of the\n", + "previous cells. For the demo purpose we have aliased the import to `mymath`, but\n", + "it is not necessary.\n", + "\n", + "Later in the notebook we will see another example with a third-party module." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "id": "zlQfhcW41uzM" + }, + "outputs": [], + "source": [ + "import math as mymath" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "ktADchck2mh4", + "outputId": "9aed6aea-b361-4414-a0f6-8873e8291090" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 2895676f-d15c-40fd-8cf2-3a0436291e6b is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created cloud function 'projects/bigframes-dev/locations/us-central1/functions/bigframes-9b20b0257558a42da610d8998022c25e-7k62x9l6' and BQ remote function 'bigframes-dev._1b6c31ff1bcd5d2f6d86833cf8268317f1b12d57.bigframes_9b20b0257558a42da610d8998022c25e_7k62x9l6'.\n" + ] + } + ], + "source": [ + "@bpd.remote_function(reuse=False)\n", + "def duration_category(duration_minutes: int) -> str:\n", + " duration_hours = mymath.ceil(duration_minutes / 60)\n", + " return f\"{duration_hours}h\"\n", + "\n", + "print(f\"Created cloud function '{duration_category.bigframes_cloud_function}' and BQ remote function '{duration_category.bigframes_remote_function}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "ywAtZlJU3GoB", + "outputId": "d3c93a31-3367-4ccf-bdf7-62d5bbff4461" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 4efda755-2f54-4477-b48a-4a424c888559 is DONE. 58.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job a8992776-c2e8-4c3e-ab75-dfc01c5de89f is DONE. 150.1 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 3ea299b0-27ad-432b-8dbf-81da3aae884f is DONE. 91.7 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutesduration_cat
1911DodgersAngels1323h
2365AthleticsAngels1343h
1977AthleticsAngels1393h
554CubsAngels1423h
654AstrosAngels1433h
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes duration_cat\n", + "1911 Dodgers Angels 132 3h\n", + "2365 Athletics Angels 134 3h\n", + "1977 Athletics Angels 139 3h\n", + "554 Cubs Angels 142 3h\n", + "654 Astros Angels 143 3h" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(duration_cat=df[\"duration_minutes\"].apply(duration_category))\n", + "df1.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WO0FH7Bm3OxR" + }, + "source": [ + "# Function referring to another function outside the function body\n", + "\n", + "In this example let's create a `remote_function` from a function\n", + "`duration_category` which depends upon another function `get_hour_ceiling`,\n", + "which further depends on another function `get_minutes_in_hour`. This dependency\n", + "chain could be even longer in a real world example. The behaviors of the\n", + "dependencies would be captured at the time of the remote function\n", + "deployment.\n", + "\n", + "Please ntoe that any changes in those functions in the notebook after the\n", + "deployment would not automatically propagate to the remote function." + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "id": "0G91fWiF3pKg" + }, + "outputs": [], + "source": [ + "import math\n", + "\n", + "def get_minutes_in_hour():\n", + " return 60\n", + "\n", + "def get_hour_ceiling(minutes):\n", + " return math.ceil(minutes / get_minutes_in_hour())" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "lQrC8T2031EJ", + "outputId": "420e7c3d-54cb-4814-f973-c7678be61caa" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 411853db-bf83-4df8-af78-55b1ceb39cb1 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created cloud function 'projects/bigframes-dev/locations/us-central1/functions/bigframes-b54aa0aa752af6a3bd6d9d529dac373b-h4lgpy4y' and BQ remote function 'bigframes-dev._1b6c31ff1bcd5d2f6d86833cf8268317f1b12d57.bigframes_b54aa0aa752af6a3bd6d9d529dac373b_h4lgpy4y'.\n" + ] + } + ], + "source": [ + "@bpd.remote_function(reuse=False)\n", + "def duration_category(duration_minutes: int) -> str:\n", + " duration_hours = get_hour_ceiling(duration_minutes)\n", + " return f\"{duration_hours} hrs\"\n", + "\n", + "print(f\"Created cloud function '{duration_category.bigframes_cloud_function}' and BQ remote function '{duration_category.bigframes_remote_function}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "GVyrihii4EFG", + "outputId": "e979b649-4ed4-4b82-e814-54180420e3fc" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job d04abfa5-e2f2-4936-a708-ed97ef429df3 is DONE. 58.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 2fc4edf0-7a86-4532-b8fb-bd3f5d153dcb is DONE. 157.4 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job f7e6e18c-70d7-4b4e-926a-03b3a1abd1fe is DONE. 99.0 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutesduration_cat
1911DodgersAngels1323 hrs
2365AthleticsAngels1343 hrs
1977AthleticsAngels1393 hrs
554CubsAngels1423 hrs
654AstrosAngels1433 hrs
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes duration_cat\n", + "1911 Dodgers Angels 132 3 hrs\n", + "2365 Athletics Angels 134 3 hrs\n", + "1977 Athletics Angels 139 3 hrs\n", + "554 Cubs Angels 142 3 hrs\n", + "654 Astros Angels 143 3 hrs" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(duration_cat=df[\"duration_minutes\"].apply(duration_category))\n", + "df1.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Uu7SOoT94vSP" + }, + "source": [ + "# Function requiring external packages\n", + "\n", + "In this example let's say we want to redact the `homeTeamName` values, and we\n", + "choose to use a third party library `cryptography`. Any third party dependencies\n", + "can be specified in [pip format](https://pip.pypa.io/en/stable/reference/requirements-file-format/)\n", + "(with or without version number) as a list via the `packages` parameter." + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "3EUEyNcW41_l", + "outputId": "2d09d60f-da1a-4eab-86d3-0e62390a360c" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job c674e7b7-2349-4317-8f08-8bfd9aa99785 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "@bpd.remote_function(reuse=False, packages=[\"cryptography\"])\n", + "def get_hash(input: str) -> str:\n", + " from cryptography.fernet import Fernet\n", + "\n", + " # handle missing value\n", + " if input is None:\n", + " input = \"\"\n", + "\n", + " key = Fernet.generate_key()\n", + " f = Fernet(key)\n", + " return f.encrypt(input.encode()).decode()" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "OX1Hl7bR5uyd", + "outputId": "8ac3bf28-d16d-438b-b636-74ef2371715f" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job eb9384c9-de7d-4232-bdca-94b61b50ff89 is DONE. 60.5 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 11a736a5-96d1-4e62-90e2-576156131a94 is DONE. 388.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job c66a9ad1-60f7-4af1-ad7c-65e4eecbb035 is DONE. 330.0 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minuteshomeTeamNameRedacted
719AstrosAngels180gAAAAABmflbKCFygsmoTzFkUCObFSBJG29Ksk8HEtk82ib...
2295AstrosAngels204gAAAAABmflbKv-XzIxcNS92RO4fXYIAwA0kGWsAy-tI5fm...
1126AstrosAngels176gAAAAABmflbJdjgpqnfvmklU7Zg3NJUqlTMYMs44dLEkwg...
294AstrosAngels189gAAAAABmflbKmfBh4P3FnwyiIpVFek9TzF4GzwP_5rQmkv...
351AstrosAngels212gAAAAABmflbJ_mzqao9i7BtoYlMpb6y3bV3x7-cYuWGxsT...
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes \\\n", + "719 Astros Angels 180 \n", + "2295 Astros Angels 204 \n", + "1126 Astros Angels 176 \n", + "294 Astros Angels 189 \n", + "351 Astros Angels 212 \n", + "\n", + " homeTeamNameRedacted \n", + "719 gAAAAABmflbKCFygsmoTzFkUCObFSBJG29Ksk8HEtk82ib... \n", + "2295 gAAAAABmflbKv-XzIxcNS92RO4fXYIAwA0kGWsAy-tI5fm... \n", + "1126 gAAAAABmflbJdjgpqnfvmklU7Zg3NJUqlTMYMs44dLEkwg... \n", + "294 gAAAAABmflbKmfBh4P3FnwyiIpVFek9TzF4GzwP_5rQmkv... \n", + "351 gAAAAABmflbJ_mzqao9i7BtoYlMpb6y3bV3x7-cYuWGxsT... " + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(homeTeamNameRedacted=df[\"homeTeamName\"].apply(get_hash))\n", + "df1.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Function referring to imports (third-party) outside the function body\n", + "\n", + "In this scenario the function depends on a third party library and the module\n", + "from the third party library used in the function is imported outside the\n", + "function body in a previous cell. Below is such an example where the third-party\n", + "dependency is `humanize` and its module of the same name is imported outside the\n", + "function body." + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "import datetime as dt\n", + "import humanize" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 21b054a9-8fb2-418f-a17b-effdf5aba9b5 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created cloud function 'projects/bigframes-dev/locations/us-central1/functions/bigframes-0879f72acd9b8ede460b69c5a8cc0dcb-edxlst27' and BQ remote function 'bigframes-dev._1b6c31ff1bcd5d2f6d86833cf8268317f1b12d57.bigframes_0879f72acd9b8ede460b69c5a8cc0dcb_edxlst27'.\n" + ] + } + ], + "source": [ + "@bpd.remote_function(reuse=False, packages=[\"humanize\"])\n", + "def duration_category(duration_minutes: int) -> str:\n", + " timedelta = dt.timedelta(minutes=duration_minutes)\n", + " return humanize.naturaldelta(timedelta)\n", + "\n", + "print(f\"Created cloud function '{duration_category.bigframes_cloud_function}' and BQ remote function '{duration_category.bigframes_remote_function}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Query job d67b7cb9-9813-4863-99d1-01cf45ab4949 is DONE. 58.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 579ba853-a7b8-49df-9539-bf22f08d2370 is DONE. 162.2 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 72f9eb5d-1c1a-4ce8-8f2f-1f5a8f7cec99 is DONE. 103.9 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutesduration_cat
1911DodgersAngels1322 hours
2365AthleticsAngels1342 hours
1977AthleticsAngels1392 hours
554CubsAngels1422 hours
654AstrosAngels1432 hours
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes duration_cat\n", + "1911 Dodgers Angels 132 2 hours\n", + "2365 Athletics Angels 134 2 hours\n", + "1977 Athletics Angels 139 2 hours\n", + "554 Cubs Angels 142 2 hours\n", + "654 Astros Angels 143 2 hours" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(duration_cat=df[\"duration_minutes\"].apply(duration_category))\n", + "df1.peek()" + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/scripts/decrypt-secrets.sh b/scripts/decrypt-secrets.sh index 0018b421dd..120b0ddc43 100755 --- a/scripts/decrypt-secrets.sh +++ b/scripts/decrypt-secrets.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2023 Google LLC All rights reserved. +# Copyright 2024 Google LLC All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py index 1acc119835..8f5e248a0d 100644 --- a/scripts/readme-gen/readme_gen.py +++ b/scripts/readme-gen/readme_gen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.