Submit startup time request asynchronously via task queue

Author: yifanyang

ci/fireci/fireci/uploader.py

@@ -18,11 +18,13 @@
 import requests
 import subprocess
 
+from google.cloud import tasks_v2
+from google.protobuf import duration_pb2
 
 _logger = logging.getLogger('fireci.uploader')
 
 
-def post_report(test_report, metrics_service_url, access_token, metric_type):
+def post_report(test_report, metrics_service_url, access_token, metric_type, asynchronous=False):
   """Post a report to the metrics service backend."""
 
   endpoint = ''
@@ -39,9 +41,25 @@ def post_report(test_report, metrics_service_url, access_token, metric_type):
   _logger.info(f'Request data: {data}')
 
   request_url = f'{metrics_service_url}{endpoint}'
-  result = requests.post(request_url, data=data, headers=headers)
-
-  _logger.info(f'Response: {result.text}')
+  if not asynchronous:
+    result = requests.post(request_url, data=data, headers=headers)
+    _logger.info(f'Response: {result.text}')
+  else:
+    client = tasks_v2.CloudTasksClient()
+    parent = client.queue_path('fireescape-c4819', 'us-central1', 'task-queue')
+    duration = duration_pb2.Duration()
+    duration.FromSeconds(1200)
+    task = {
+      'http_request': {
+        'http_method': tasks_v2.HttpMethod.POST,
+        'url': request_url,
+        'body': data.encode(),
+        'headers': headers,
+      },
+      'dispatch_deadline': duration
+    }
+    response = client.create_task({'parent': parent, 'task': task})
+    _logger.info(f'Created task: {response.name}')
 
 
 def _construct_request_endpoint_for_github_actions(metric_type):

ci/fireci/fireciplugins/macrobenchmark/analyze/utils.py

@@ -18,7 +18,7 @@
 import tempfile
 
 from click import ClickException
-from google.cloud import storage
+from google.cloud.storage import Client
 from pathlib import Path
 from typing import List, TypedDict
 
@@ -41,7 +41,7 @@ def collect_data_points(ftl_results_dir: List[str], local_reports_dir: Path) ->
 
 def _download(ftl_results_dirs: List[str]) -> Path:
   ftl_results_bucket = 'fireescape-benchmark-results'
-  gcs = storage.Client()
+  gcs = Client()
 
   temp_dir = tempfile.mkdtemp(prefix='ftl-results-')
   for ftl_results_dir in ftl_results_dirs:

ci/fireci/fireciplugins/macrobenchmark/commands.py

@@ -163,7 +163,13 @@ def ci(pull_request, repeat):
     if ftl_results:
       metric_service_url = 'https://api.firebase-sdk-health-metrics.com'
       access_token = ci_utils.gcloud_identity_token()
-      uploader.post_report(startup_time_data, metric_service_url, access_token, 'startup-time')
+      uploader.post_report(
+        test_report=startup_time_data,
+        metrics_service_url=metric_service_url,
+        access_token=access_token,
+        metric_type='startup-time',
+        asynchronous=True
+      )
 
   if exception:
     raise exception

ci/fireci/setup.cfg

@@ -4,9 +4,10 @@ version = 0.1
 
 [options]
 install_requires =
-        protobuf==3.19
+        protobuf==3.19.6
         click==8.1.3
         google-cloud-storage==2.5.0
+        google-cloud-tasks==2.10.4
         mypy==0.991
         numpy==1.23.1
         pandas==1.5.1
@@ -26,7 +27,9 @@ console_scripts =
 
 [mypy]
 strict_optional = False
-[mypy-google.cloud]
+[mypy-google.cloud.storage]
+ignore_missing_imports = True
+[mypy-google.protobuf]
 ignore_missing_imports = True
 [mypy-matplotlib]
 ignore_missing_imports = True