opencost 2024-01-22T05:39:29.463176966Z ???

Log level set to info

opencost 2024-01-22T05:39:29.464282851Z INF Starting cost-model version 1.106.7
(1c27486)
opencost 2024-01-22T05:39:29.464525144Z INF Prometheus/Thanos Client Max
Concurrency set to 5
opencost-ui running with BASE_URL=/model
opencost-ui /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt
to perform configuration
opencost-ui /docker-entrypoint.sh: Looking for shell scripts in /docker-
entrypoint.d/
opencost-ui /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-
ipv6-by-default.sh
opencost-ui 10-listen-on-ipv6-by-default.sh: info: /etc/nginx/conf.d/default.conf
is not a file or does not exist
opencost-ui /docker-entrypoint.sh: Sourcing /docker-entrypoint.d/15-local-
resolvers.envsh
opencost-ui /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-
templates.sh
opencost-ui /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-
processes.sh
opencost-ui /docker-entrypoint.sh: Configuration complete; ready for start up
opencost-ui 2024/01/22 05:39:33 [notice] 1#1: using the "epoll" event method
opencost-ui 2024/01/22 05:39:33 [notice] 1#1: nginx/1.25.3
opencost-ui 2024/01/22 05:39:33 [notice] 1#1: built by gcc 12.2.1 20220924 (Alpine
12.2.1_git20220924-r10)
opencost-ui 2024/01/22 05:39:33 [notice] 1#1: OS: Linux 5.15.120+
opencost-ui 2024/01/22 05:39:33 [notice] 1#1: getrlimit(RLIMIT_NOFILE):
1048576:1048576
opencost-ui 2024/01/22 05:39:33 [notice] 1#1: start worker processes
opencost-ui 2024/01/22 05:39:33 [notice] 1#1: start worker process 21
opencost-ui 2024/01/22 05:39:33 [notice] 1#1: start worker process 22
opencost-ui 127.0.0.1 - - [22/Jan/2024:05:40:30 +0000] "GET / HTTP/1.1" 200 422 "-"
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like
Gecko) Chrome/120.0.0.0 Safari/537.36" "-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:05:40:30 +0000] "GET /index.89ba2f7f.css
HTTP/1.1" 304 0 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS X
10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" "-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:05:40:30 +0000] "GET /index.a13ddd66.js
HTTP/1.1" 200 341398 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS
X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
"-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:05:40:31 +0000] "GET /logo.9a2faf1d.png
HTTP/1.1" 200 10934 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS
X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
"-"
opencost-ui 2024/01/22 05:40:31 [error] 22#22: *2 connect() failed (111: Connection
refused) while connecting to upstream, client: 127.0.0.1, server: _, request:
"GET /model/allocation/compute?
window=6d&aggregate=namespace&step=1d&accumulate=false HTTP/1.1", upstream:
"http://0.0.0.0:9003/allocation/compute?
window=6d&aggregate=namespace&step=1d&accumulate=false", host: "localhost:9090",
referrer: "http://localhost:9090/"
opencost-ui 127.0.0.1 - - [22/Jan/2024:05:40:31 +0000] "GET
/model/allocation/compute?window=6d&aggregate=namespace&step=1d&accumulate=false
HTTP/1.1" 502 559 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS X
10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" "-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:05:40:32 +0000] "GET /favicon.7eff484d.ico
HTTP/1.1" 200 25134 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS
X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
"-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:05:40:56 +0000] "GET / HTTP/1.1" 200 422 "-"
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like
Gecko) Chrome/120.0.0.0 Safari/537.36" "-"
opencost-ui 2024/01/22 05:40:56 [error] 22#22: *2 connect() failed (111: Connection
refused) while connecting to upstream, client: 127.0.0.1, server: _, request:
"GET /model/allocation/compute?
window=6d&aggregate=namespace&step=1d&accumulate=false HTTP/1.1", upstream:
"http://0.0.0.0:9003/allocation/compute?
window=6d&aggregate=namespace&step=1d&accumulate=false", host: "localhost:9090",
referrer: "http://localhost:9090/"
opencost-ui 127.0.0.1 - - [22/Jan/2024:05:40:56 +0000] "GET
/model/allocation/compute?window=6d&aggregate=namespace&step=1d&accumulate=false
HTTP/1.1" 502 559 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS X
10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" "-"

export PROJECT_ID=$(gcloud config get-value project)

gcloud iam service-accounts create compute-viewer-opencost --display-name "Compute
Read Only Account Created For Opencost" --format json
gcloud projects add-iam-policy-binding $PROJECT_ID --member serviceAccount:compute-
viewer-opencost@$PROJECT_ID.iam.gserviceaccount.com --role roles/compute.viewer
gcloud projects add-iam-policy-binding $PROJECT_ID --member serviceAccount:compute-
viewer-opencost@$PROJECT_ID.iam.gserviceaccount.com --role roles/bigquery.user
gcloud projects add-iam-policy-binding $PROJECT_ID --member serviceAccount:compute-
viewer-opencost@$PROJECT_ID.iam.gserviceaccount.com --role
roles/bigquery.dataViewer
gcloud projects add-iam-policy-binding $PROJECT_ID --member serviceAccount:compute-
viewer-opencost@$PROJECT_ID.iam.gserviceaccount.com --role roles/bigquery.jobUser
gcloud iam service-accounts keys create ./compute-viewer-opencost-key.json --iam-
account compute-viewer-opencost@$PROJECT_ID.iam.gserviceaccount.com

2024-01-22T06:46:12.464395832Z ??? Log level set to info

2024-01-22T06:46:12.464570419Z INF Starting cost-model version 1.108.0 (25d0064)
2024-01-22T06:46:12.464862327Z INF Prometheus/Thanos Client Max Concurrency set to
5
2024-01-22T06:47:22.500026171Z ERR Failed to query prometheus at http://my-
prometheus-server.prometheus.svc.cluster.local:80. Error: query error: 'Post
"http://my-prometheus-server.prometheus.svc.cluster.local:80/api/v1/query?
query=up&time=1705905972": dial tcp: lookup my-prometheus-
server.prometheus.svc.cluster.local on 172.16.0.10:53: read udp
172.16.69.210:51379->172.16.0.10:53: i/o timeout' fetching query 'up' .
Troubleshooting help available at: http://docs.kubecost.com/custom-
prom#troubleshoot
2024-01-22T06:48:32.518030824Z INF No valid prometheus config file at http://my-
prometheus-server.prometheus.svc.cluster.local:80. Error: Get "http://my-
prometheus-server.prometheus.svc.cluster.local:80/api/v1/status/config": dial tcp:
lookup my-prometheus-server.prometheus.svc.cluster.local on 172.16.0.10:53: read
udp 172.16.69.210:47318->172.16.0.10:53: i/o timeout . Troubleshooting help
available at: http://docs.kubecost.com/custom-prom#troubleshoot. Ignore if using
cortex/mimir/thanos here.
Stream closed EOF for opencost/opencost-85f49d47d6-85777 (opencost)

opencost 2024-01-22T07:48:43.330614317Z ??? Log level set to info

opencost 2024-01-22T07:48:43.3363519Z INF Starting cost-model version 1.108.0
(25d0064)
opencost 2024-01-22T07:48:43.336594933Z INF Prometheus/Thanos Client Max
Concurrency set to 5
opencost-ui running with BASE_URL=/model
opencost-ui /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt
to perform configuration
opencost-ui /docker-entrypoint.sh: Looking for shell scripts in /docker-
entrypoint.d/
opencost-ui /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-
ipv6-by-default.sh
opencost-ui 10-listen-on-ipv6-by-default.sh: info: /etc/nginx/conf.d/default.conf
is not a file or does not exist
opencost-ui /docker-entrypoint.sh: Sourcing /docker-entrypoint.d/15-local-
resolvers.envsh
opencost-ui /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-
templates.sh
opencost-ui /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-
processes.sh
opencost-ui /docker-entrypoint.sh: Configuration complete; ready for start up
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: using the "epoll" event method
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: nginx/1.25.3
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: built by gcc 12.2.1 20220924 (Alpine
12.2.1_git20220924-r10)
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: OS: Linux 5.15.120+
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: getrlimit(RLIMIT_NOFILE):
1048576:1048576
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: start worker processes
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: start worker process 22
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: start worker process 23
opencost 2024-01-22T07:49:53.385790464Z ERR Failed to query prometheus at
http://my-prometheus-server.prometheus.svc.cluster.local:80. Error: query error:
'Post "http://my-prometheus-server.prometheus.svc.cluster.local:80/api/v1/query?
query=up&time=1705909723": dial tcp: lookup my-prometheus-
server.prometheus.svc.cluster.local on 172.16.0.10:53: read udp
172.16.71.168:45236->172.16.0.10:53: i/o timeout' fetching query 'up' .
Troubleshooting help available at: http://docs.kubecost.com/custom-
prom#troubleshoot

(⎈|dscp-vitae:production)➜ ~ helm install my-prometheus --repo https://prometheus-

community.github.io/helm-charts prometheus \
--namespace prometheus --create-namespace \
--set prometheus-pushgateway.enabled=false \
--set alertmanager.enabled=false \
-f https://raw.githubusercontent.com/opencost/opencost/develop/kubernetes/
prometheus/extraScrapeConfigs.yaml
NAME: my-prometheus
LAST DEPLOYED: Mon Jan 22 15:14:04 2024
NAMESPACE: prometheus
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
The Prometheus server can be accessed via port 80 on the following DNS name from
within your cluster:
my-prometheus-server.prometheus.svc.cluster.local

Get the Prometheus server URL by running these commands in the same shell:
export POD_NAME=$(kubectl get pods --namespace prometheus -l
"app.kubernetes.io/name=prometheus,app.kubernetes.io/instance=my-prometheus" -o
jsonpath="{.items[0].metadata.name}")
kubectl --namespace prometheus port-forward $POD_NAME 9090

#################################################################################
###### WARNING: Pod Security Policy has been disabled by default since #####
###### it deprecated after k8s 1.25+. use #####
###### (index .Values "prometheus-node-exporter" "rbac" #####
###### . "pspEnabled") with (index .Values #####
###### "prometheus-node-exporter" "rbac" "pspAnnotations") #####
###### in case you still need it. #####
#################################################################################

For more information on running Prometheus, visit:

https://prometheus.io/

opencost 2024-01-22T07:48:43.330614317Z ??? Log level set to info

opencost 2024-01-22T07:48:43.3363519Z INF Starting cost-model version 1.108.0
(25d0064)
opencost 2024-01-22T07:48:43.336594933Z INF Prometheus/Thanos Client Max
Concurrency set to 5
opencost-ui running with BASE_URL=/model
opencost-ui /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt
to perform configuration
opencost-ui /docker-entrypoint.sh: Looking for shell scripts in /docker-
entrypoint.d/
opencost-ui /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-
ipv6-by-default.sh
opencost-ui 10-listen-on-ipv6-by-default.sh: info: /etc/nginx/conf.d/default.conf
is not a file or does not exist
opencost-ui /docker-entrypoint.sh: Sourcing /docker-entrypoint.d/15-local-
resolvers.envsh
opencost-ui /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-
templates.sh
opencost-ui /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-
processes.sh
opencost-ui /docker-entrypoint.sh: Configuration complete; ready for start up
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: using the "epoll" event method
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: nginx/1.25.3
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: built by gcc 12.2.1 20220924 (Alpine
12.2.1_git20220924-r10)
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: OS: Linux 5.15.120+
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: getrlimit(RLIMIT_NOFILE):
1048576:1048576
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: start worker processes
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: start worker process 22
opencost-ui 2024/01/22 07:48:43 [notice] 1#1: start worker process 23
opencost 2024-01-22T07:49:53.385790464Z ERR Failed to query prometheus at
http://my-prometheus-server.prometheus.svc.cluster.local:80. Error: query error:
'Post "http://my-prometheus-server.prometheus.svc.cluster.local:80/api/v1/query?
query=up&time=1705909723": dial tcp: lookup my-prometheus-
server.prometheus.svc.cluster.local on 172.16.0.10:53: read udp
172.16.71.168:45236->172.16.0.10:53: i/o timeout' fetching query 'up' .
Troubleshooting help available at: http://docs.kubecost.com/custom-
prom#troubleshoot
opencost 2024-01-22T07:51:03.423985612Z INF No valid prometheus config file at
http://my-prometheus-server.prometheus.svc.cluster.local:80. Error: Get "http://my-
prometheus-server.prometheus.svc.cluster.local:80/api/v1/status/config": dial tcp:
lookup my-prometheus-server.prometheus.svc.cluster.local on 172.16.0.10:53: read
udp 172.16.71.168:47123->172.16.0.10:53: i/o timeout . Troubleshooting help
available at: http://docs.kubecost.com/custom-prom#troubleshoot. Ignore if using
cortex/mimir/thanos here.
Stream closed EOF for opencost/opencost-6cf4f65bbf-rdjpw (opencost)
opencost-ui 127.0.0.1 - - [22/Jan/2024:07:54:29 +0000] "GET / HTTP/1.1" 200 548 "-"
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like
Gecko) Chrome/120.0.0.0 Safari/537.36" "-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:07:54:29 +0000] "GET
/index.runtime.b478a79e.js HTTP/1.1" 200 982 "http://localhost:9090/" "Mozilla/5.0
(Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/120.0.0.0 Safari/537.36" "-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:07:54:29 +0000] "GET /index.3406705b.css
HTTP/1.1" 304 0 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS X
10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" "-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:07:54:30 +0000] "GET /index.9d46bb91.js
HTTP/1.1" 200 538332 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS
X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
"-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:07:54:31 +0000] "GET /logo.b9464e00.png
HTTP/1.1" 200 10967 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS
X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
"-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:07:54:31 +0000] "GET
/model/allocation/compute?window=6d&aggregate=namespace&step=1d&accumulate=false
HTTP/1.1" 502 559 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS X
10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" "-"
opencost-ui 2024/01/22 07:54:31 [error] 22#22: *64 connect() failed (111:
Connection refused) while connecting to upstream, client: 127.0.0.1, server: _,
request: "GET /model/allocation/compute?
window=6d&aggregate=namespace&step=1d&accumulate=false HTTP/1.1", upstream:
"http://0.0.0.0:9003/allocation/compute?
window=6d&aggregate=namespace&step=1d&accumulate=false", host: "localhost:9090",
referrer: "http://localhost:9090/"
opencost-ui 2024/01/22 07:54:31 [error] 22#22: *64 connect() failed (111:
Connection refused) while connecting to upstream, client: 127.0.0.1, server: _,
request: "GET /model/allocation/compute?
window=6d&aggregate=namespace&step=1d&accumulate=false HTTP/1.1", upstream:
"http://0.0.0.0:9003/allocation/compute?
window=6d&aggregate=namespace&step=1d&accumulate=false", host: "localhost:9090",
referrer: "http://localhost:9090/"
opencost-ui 127.0.0.1 - - [22/Jan/2024:07:54:31 +0000] "GET
/model/allocation/compute?window=6d&aggregate=namespace&step=1d&accumulate=false
HTTP/1.1" 502 559 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS X
10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" "-"
opencost-ui 127.0.0.1 - - [22/Jan/2024:07:54:31 +0000] "GET /favicon.7eff484d.ico
HTTP/1.1" 200 25134 "http://localhost:9090/" "Mozilla/5.0 (Macintosh; Intel Mac OS
X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
"-"

helm repo add prometheus-community https://prometheus-community.github.io/helm-

charts

(⎈|dscp-vitae:production)➜ ~ helm install ksm prometheus-community/kube-state-

metrics -n prometheus
NAME: ksm
LAST DEPLOYED: Mon Jan 22 18:07:12 2024
NAMESPACE: prometheus
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
kube-state-metrics is a simple service that listens to the Kubernetes API server
and generates metrics about the state of the objects.
The exposed metrics can be found here:
https://github.com/kubernetes/kube-state-metrics/blob/master/docs/
README.md#exposed-metrics

The metrics are exported on the HTTP endpoint /metrics on the listening port.
In your case, ksm-kube-state-metrics.prometheus.svc.cluster.local:8080/metrics

They are served either as plaintext or protobuf depending on the Accept header.
They are designed to be consumed either by Prometheus itself or by a scraper that
is compatible with scraping a Prometheus client endpoint.

(⎈|dscp-vitae:production)➜ ~ helm install ne prometheus-community/prometheus-node-

exporter -n prometheus
NAME: ne
LAST DEPLOYED: Mon Jan 22 18:09:16 2024
NAMESPACE: prometheus
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
1. Get the application URL by running these commands:
export POD_NAME=$(kubectl get pods --namespace prometheus -l
"app.kubernetes.io/name=prometheus-node-exporter,app.kubernetes.io/instance=ne" -o
jsonpath="{.items[0].metadata.name}")
echo "Visit http://127.0.0.1:9100 to use your application"
kubectl port-forward --namespace prometheus $POD_NAME 9100

2024-01-23T02:34:00.874360039Z ??? Log level set to info

2024-01-23T02:34:00.874493127Z INF Starting cost-model version 1.108.0 (25d0064)
2024-01-23T02:34:00.874807607Z INF Prometheus/Thanos Client Max Concurrency set to
5
2024-01-23T02:35:10.900625325Z ERR Failed to query prometheus at http://my-
prometheus-server.prometheus.svc.cluster.local:80. Error: query error: 'Post
"http://my-prometheus-server.prometheus.svc.cluster.local:80/api/v1/query?
query=up&time=1705977240": dial tcp: lookup my-prometheus-
server.prometheus.svc.cluster.local on 172.16.0.10:53: read udp 172.16.71.9:35422-
>172.16.0.10:53: i/o timeout' fetching query 'up' . Troubleshooting help available
at: http://docs.kubecost.com/custom-prom#troubleshoot

helm install my-prometheus --repo https://prometheus-community.github.io/helm-

charts prometheus \
--namespace prometheus --create-namespace \
--set prometheus-pushgateway.enabled=false \
--set alertmanager.enabled=false \
-f https://raw.githubusercontent.com/opencost/opencost/develop/kubernetes/
prometheus/extraScrapeConfigs.yaml

** Please be patient while the chart is being deployed **

Watch the Prometheus Operator Deployment status using the command:

 kubectl get deploy -w --namespace prometheus -l app.kubernetes.io/name=kube-
prometheus-operator,app.kubernetes.io/instance=kube-prometheus

Watch the Prometheus StatefulSet status using the command:

kubectl get sts -w --namespace prometheus -l app.kubernetes.io/name=kube-

prometheus-prometheus,app.kubernetes.io/instance=kube-prometheus

Prometheus can be accessed via port "9090" on the following DNS name from within
your cluster:

kube-prometheus-prometheus.prometheus.svc.cluster.local
kube-prometheus-prometheus.prometheus.svc.cluster.local

To access Prometheus from outside the cluster execute the following commands:

echo "Prometheus URL: http://127.0.0.1:9090/"

kubectl port-forward --namespace prometheus svc/kube-prometheus-prometheus
9090:9090

2024-01-23T05:05:52.648407883Z INF ComputeAggregateCostModel: missed cache:

1d:1m:1.000000h:false (found false, disableAggregateCostModelCache true, noCache
false)
2024-01-23T05:05:52.649232885Z INF EXPORT_CSV_FILE is not set, CSV export is
disabled
2024-01-23T05:05:52.692559852Z INF Persistent Volume pricing not found for
standard-rwo: europe-west1,pd-balanced
2024-01-23T05:05:52.708407607Z INF Kubecost prometheus metrics not currently
available. Ingest this server's /metrics endpoint to get that data.
2024-01-23T05:05:52.708513477Z INF Error building cache [2024-01-22T05:04:52+0000,
2024-01-23T05:04:52+0000): empty data for range: [2024-01-22T06:04:52+0000, 2024-
01-23T05:04:52+0000)
2024-01-23T05:05:52.722059673Z INF Persistent Volume pricing not found for
standard-rwo: europe-west1,pd-balanced
2024-01-23T05:05:52.73142048Z INF Persistent Volume pricing not found for standard-
rwo: europe-west1,pd-balanced
2024-01-23T05:05:52.747694124Z WRN Cluster cost data count not found for cluster
default-cluster
2024-01-23T05:05:52.747767828Z INF caching 1d cluster costs for 11m0s
2024-01-23T05:05:52.748381293Z INF aggregation: warm cache: 1d
2024-01-23T05:05:54.89873508Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
}|...
2024-01-23T05:06:04.901827872Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
 }
}
]
}
}|...
2024-01-23T05:06:14.902761479Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
}|...
2024-01-23T05:06:24.903794026Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
}|...
2024-01-23T05:06:34.904570751Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
}|...
2024-01-23T05:06:44.905209503Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
}|...
2024-01-23T05:06:52.754509696Z INF Persistent Volume pricing not found for
standard-rwo: europe-west1,pd-balanced
2024-01-23T05:06:52.759082825Z INF Persistent Volume pricing not found for
standard-rwo: europe-west1,pd-balanced
2024-01-23T05:06:54.906068777Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
 }
}
]
}
}|...
2024-01-23T05:07:04.906738121Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
}|...
2024-01-23T05:07:14.907729697Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
}|...
2024-01-23T05:07:24.908368072Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
}|...
2024-01-23T05:07:34.90919605Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
}|...
2024-01-23T05:07:44.910742786Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
 }|...
2024-01-23T05:07:52.806201014Z INF Persistent Volume pricing not found for
standard-rwo: europe-west1,pd-balanced
2024-01-23T05:07:52.810319973Z INF Persistent Volume pricing not found for
standard-rwo: europe-west1,pd-balanced
2024-01-23T05:07:54.914178979Z ERR MultiCloudWatcher: Error getting file
'/var/configs/cloud-integration/cloud-integration.json': loadFile: Error reading
json: unmarshalerDecoder: BigQueryConfiguration: FromInterface: FromInterface:
missing 'projectID' property, error found in #10 byte of ...|
}
}|..., bigger context ...| }
}
}
]
}
}|...