fix: update docs pipeline (#109)

* fix: update docs pipeline

* move install

* fix dockerfile

* debug

* fix install

* Update Dockerfile

* Update publish-docs.sh

* fix buckets

* update bucket

* debug

* Update publish-docs.sh

* Update common.cfg

Author: averikitsch

.kokoro/docker/docs/Dockerfile

@@ -60,11 +60,6 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/* \
   && rm -f /var/cache/apt/archives/*.deb
 
-ENV VERSION v4.2.0
-ENV BINARY yq_linux_amd64
-RUN wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && \
-    chmod +x /usr/bin/yq
-
 ###################### Install python 3.9.13
 
 # Download python 3.9.13
@@ -85,4 +80,8 @@ RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
 # Test pip
 RUN python3 -m pip
 
+# Install build requirements
+COPY requirements.txt /requirements.txt
+RUN python3 -m pip install --require-hashes -r requirements.txt
+
 CMD ["python3.8"]

.kokoro/docker/docs/requirements.in

@@ -0,0 +1 @@
+nox

.kokoro/docker/docs/requirements.txt

@@ -0,0 +1,38 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile --allow-unsafe --generate-hashes requirements.in
+#
+argcomplete==3.2.3 \
+    --hash=sha256:bf7900329262e481be5a15f56f19736b376df6f82ed27576fa893652c5de6c23 \
+    --hash=sha256:c12355e0494c76a2a7b73e3a59b09024ca0ba1e279fb9ed6c1b82d5b74b6a70c
+    # via nox
+colorlog==6.8.2 \
+    --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \
+    --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33
+    # via nox
+distlib==0.3.8 \
+    --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \
+    --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64
+    # via virtualenv
+filelock==3.13.1 \
+    --hash=sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e \
+    --hash=sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c
+    # via virtualenv
+nox==2024.3.2 \
+    --hash=sha256:e53514173ac0b98dd47585096a55572fe504fecede58ced708979184d05440be \
+    --hash=sha256:f521ae08a15adbf5e11f16cb34e8d0e6ea521e0b92868f684e91677deb974553
+    # via -r requirements.in
+packaging==24.0 \
+    --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \
+    --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9
+    # via nox
+platformdirs==4.2.0 \
+    --hash=sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068 \
+    --hash=sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768
+    # via virtualenv
+virtualenv==20.25.1 \
+    --hash=sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a \
+    --hash=sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197
+    # via nox

.kokoro/docs/common.cfg

@@ -25,14 +25,14 @@ env_vars: {
 
 env_vars: {
     key: "STAGING_BUCKET"
-    value: "docs-staging"
+    value: "docs-staging"  # Used for googleapis.dev
 }
 
 env_vars: {
     key: "V2_STAGING_BUCKET"
     # Push non-cloud library docs to `docs-staging-v2-staging` instead of the
     # Cloud RAD bucket `docs-staging-v2`
-    value: "docs-staging-v2-staging"
+    value: "docs-staging-v2"
 }
 
 # It will upload the docker image after successful builds.
@@ -64,4 +64,4 @@ before_action {
       keyname: "docuploader_service_account"
     }
   }
-}
+}

.kokoro/publish-docs.sh

@@ -35,26 +35,27 @@ python3 -m docuploader create-metadata \
   --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
   --version=$VERSION \
   --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
-  --distribution-name=$(yq -oy '.project.name'  pyproject.toml) \
+  --distribution-name=$(jq --raw-output '.distribution_name // empty' .repo-metadata.json) \
   --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
   --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
   --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
 
 cat docs.metadata
 
 # upload docs
+echo "Uploading docs to ${STAGING_BUCKET}"
 python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"
 
 
-# docfx yaml files
+# docfx yaml files for CGC
 nox -s docfx
 
 # create metadata.
 python3 -m docuploader create-metadata \
   --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
   --version=$VERSION \
   --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
-  --distribution-name=$(yq -oy '.project.name'  pyproject.toml) \
+  --distribution-name=$(jq --raw-output '.distribution_name // empty' .repo-metadata.json) \
   --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
   --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
   --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)

DEVELOPER.md

@@ -89,9 +89,17 @@ Currently, manual PRs are made to the [Langchain GitHub repo](https://github.com
 
 ### API Reference
 
-API docs are templated in the `docs/` directory. See `.kokoro/` for the release pipeline.
+#### Build the documentation
+API docs are templated in the `docs/` directory.
 
-To test locally, run: `nox -s`
+To test locally, run: `nox -s docs`
+
+The nox session, `docs`, is used to create HTML to publish to googleapis.dev
+The nox session, `docfx`, is used to create YAML to publish to CGC.
+
+#### Publish the documentation
+
+The kokoro docs pipeline runs when a new release is created. See `.kokoro/` for the release pipeline.
 
 
 [provider]: https://python.langchain.com/docs/integrations/platforms/google

README.rst

@@ -58,7 +58,7 @@ Mac/Linux
     pip install virtualenv
     virtualenv <your-env>
     source <your-env>/bin/activate
-    <your-env>/bin/pip install google-cloud-bigquery
+    <your-env>/bin/pip install langchain-google-cloud-sql-pg
 
 
 Windows
@@ -69,7 +69,7 @@ Windows
     pip install virtualenv
     virtualenv <your-env>
     <your-env>\Scripts\activate
-    <your-env>\Scripts\pip.exe install google-cloud-bigquery
+    <your-env>\Scripts\pip.exe install langchain-google-cloud-sql-pg
 
 
 Example Usage