Revert "Download newest orderfile from GS bucket for ChromeOS."
This reverts commit 6cfc16304f8f22ddfcc23a74a41951f4c52ba90b.
Reason for revert: Intend to cherry-pick to M77 branch. Will abandon the revert on master after that.
M77 branch is cut after the original CL went in. We fear that it might cause other toolings fail,
like the one mentioned in crbug/990747. The original CL doesn't have any functional impact for
Chrome until we actually flip the bits on (which we haven't done on master yet). So it should be
safe to revert the CL only for the branch.
Bug: chromium:990747
Original change's description:
> Download newest orderfile from GS bucket for ChromeOS.
>
> This patch gives us the ability to download the most recent
> orderfile (with name specified in a file) so that ChromeOS can
> use it for linking Chrome in the future.
>
> This patch generalizes the script used for downloading
> newest AFDO profiles for Android/Linux to be able to download
> any artifacts generated on Chrome OS. The script now takes
> arguments that can describe any profiles/orderfiles.
> Also move the script to a non Android-specific location.
>
> Bug: chromium:950627
> Test: The script is working locally for both android and chromeos
>
> Change-Id: I9eedbd6f39664013c5f61c7e64cbe0652bb13b7c
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1704936
> Commit-Queue: Tiancong Wang <[email protected]>
> Reviewed-by: Steven Bennetts <[email protected]>
> Reviewed-by: Andrew Grieve <[email protected]>
> Reviewed-by: Nico Weber <[email protected]>
> Reviewed-by: George Burgess <[email protected]>
> Reviewed-by: Dirk Pranke <[email protected]>
> Auto-Submit: Tiancong Wang <[email protected]>
> Cr-Commit-Position: refs/heads/master@{#681012}
[email protected],[email protected],[email protected],[email protected],[email protected],[email protected]
# Not skipping CQ checks because original CL landed > 1 day ago.
Change-Id: Iecee91f88ae0d0785f317bf49eb84cca32fcbc9b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1738345
Reviewed-by: George Burgess <[email protected]>
Cr-Commit-Position: refs/branch-heads/3865@{#275}
Cr-Branched-From: 0cdcc6158160790658d1f033d3db873603250124-refs/heads/master@{#681094}
diff --git a/.gitignore b/.gitignore
index 9752fac9..2dc13af 100644
--- a/.gitignore
+++ b/.gitignore
@@ -163,8 +163,6 @@
/chrome/web_ui_mojo_bindings.xml
/chromecast/internal
/chromeos/assistant/internal
-/chromeos/profiles/chromeos.orderfile.txt
-/chromeos/profiles/orderfile.local.txt
/cipd_cache/
/clank
/cloud_print/cloud_print_version_resources.xml
diff --git a/DEPS b/DEPS
index 02f9f69..7d7165a 100644
--- a/DEPS
+++ b/DEPS
@@ -2763,13 +2763,7 @@
'name': 'Fetch Android AFDO profile',
'pattern': '.',
'condition': 'checkout_android or checkout_linux',
- 'action': [ 'vpython',
- 'src/tools/download_cros_provided_profile.py',
- '--newest_state=src/chrome/android/profiles/newest.txt',
- '--local_state=src/chrome/android/profiles/local.txt',
- '--output_name=src/chrome/android/profiles/afdo.prof',
- '--gs_url_base=chromeos-prebuilt/afdo-job/llvm',
- ],
+ 'action': ['vpython', 'src/chrome/android/profiles/update_afdo_profile.py'],
},
{
'name': 'gvr_static_shim_android_arm_1',
@@ -2906,18 +2900,6 @@
],
},
{
- 'name': 'Fetch ChromeOS-specific orderfile for Chrome',
- 'pattern': '.',
- 'condition': 'checkout_chromeos or checkout_simplechrome',
- 'action': [ 'vpython',
- 'src/tools/download_cros_provided_profile.py',
- '--newest_state=src/chromeos/profiles/orderfile.newest.txt',
- '--local_state=src/chromeos/profiles/orderfile.local.txt',
- '--output_name=src/chromeos/profiles/chromeos.orderfile.txt',
- '--gs_url_base=chromeos-prebuilt/afdo-job/orderfiles/vetted',
- ],
- },
- {
# Pull doclava binaries if building for Android.
'name': 'doclava',
'pattern': '.',
diff --git a/chrome/android/profiles/update_afdo_profile.py b/chrome/android/profiles/update_afdo_profile.py
new file mode 100755
index 0000000..690521c
--- /dev/null
+++ b/chrome/android/profiles/update_afdo_profile.py
@@ -0,0 +1,143 @@
+#!/usr/bin/python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This script is used to update our local AFDO profiles.
+
+This uses profiles of Chrome provided by our friends from Chrome OS. Though the
+profiles are available externally, the bucket they sit in is otherwise
+unreadable by non-Googlers. Gsutil usage with this bucket is therefore quite
+awkward: you can't do anything but `cp` certain files with an external account,
+and you can't even do that if you're not yet authenticated.
+
+No authentication is necessary if you pull these profiles directly over
+https."""
+
+import argparse
+import contextlib
+import os
+import subprocess
+import sys
+import urllib2
+
+GS_HTTP_URL = 'https://storage.googleapis.com'
+GS_BASE_URL = GS_HTTP_URL + '/chromeos-prebuilt/afdo-job/llvm'
+PROFILE_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
+LOCAL_PROFILE_PATH = os.path.join(PROFILE_DIRECTORY, 'afdo.prof')
+
+# We use these to track the local profile; newest.txt is owned by git and tracks
+# the name of the newest profile we should pull, and local.txt is the most
+# recent profile we've successfully pulled.
+NEWEST_PROFILE_NAME_PATH = os.path.join(PROFILE_DIRECTORY, 'newest.txt')
+LOCAL_PROFILE_NAME_PATH = os.path.join(PROFILE_DIRECTORY, 'local.txt')
+
+
+def ReadUpToDateProfileName():
+ with open(NEWEST_PROFILE_NAME_PATH) as f:
+ return f.read().strip()
+
+
+def ReadLocalProfileName():
+ try:
+ with open(LOCAL_PROFILE_NAME_PATH) as f:
+ return f.read().strip()
+ except IOError:
+ # Assume it either didn't exist, or we couldn't read it. In either case, we
+ # should probably grab a new profile (and, in doing so, make this file sane
+ # again)
+ return None
+
+
+def WriteLocalProfileName(name):
+ with open(LOCAL_PROFILE_NAME_PATH, 'w') as f:
+ f.write(name)
+
+
+def CheckCallOrExit(cmd):
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = proc.communicate()
+ exit_code = proc.wait()
+ if not exit_code:
+ return
+
+ complaint_lines = [
+ '## %s failed with exit code %d' % (cmd[0], exit_code),
+ '## Full command: %s' % cmd,
+ '## Stdout:\n' + stdout,
+ '## Stderr:\n' + stderr,
+ ]
+ print >>sys.stderr, '\n'.join(complaint_lines)
+ sys.exit(1)
+
+
+def RetrieveProfile(desired_profile_name, out_path):
+ # vpython is > python 2.7.9, so we can expect urllib to validate HTTPS certs
+ # properly.
+ ext = os.path.splitext(desired_profile_name)[1]
+ compressed_path = out_path + ext
+ gs_prefix = 'gs://'
+ if not desired_profile_name.startswith(gs_prefix):
+ gs_url = GS_BASE_URL + '/' + desired_profile_name
+ else:
+ gs_url = GS_HTTP_URL + '/' + desired_profile_name[len(gs_prefix):]
+
+ with contextlib.closing(urllib2.urlopen(gs_url)) as u:
+ with open(compressed_path, 'wb') as f:
+ while True:
+ buf = u.read(4096)
+ if not buf:
+ break
+ f.write(buf)
+
+ if ext == '.bz2':
+ # NOTE: we can't use Python's bzip module, since it doesn't support
+ # multi-stream bzip files. It will silently succeed and give us a garbage
+ # profile.
+ # bzip2 removes the compressed file on success.
+ CheckCallOrExit(['bzip2', '-d', compressed_path])
+ elif ext == '.xz':
+ # ...And we can't use the `lzma` module, since it was introduced in python3.
+ # xz removes the compressed file on success.
+ CheckCallOrExit(['xz', '-d', compressed_path])
+ else:
+ # Wait until after downloading the file to check the file extension, so the
+ # user has something usable locally if the file extension is unrecognized.
+ raise ValueError(
+ 'Only bz2 and xz extensions are supported; "%s" is not' % ext)
+
+
+def CleanProfilesDirectory():
+ # Start with a clean slate, removing old profiles/downloads/etc.
+ old_artifacts = (p for p in os.listdir(PROFILE_DIRECTORY) if
+ p.startswith('chromeos-chrome-'))
+ for artifact in old_artifacts:
+ os.remove(os.path.join(PROFILE_DIRECTORY, artifact))
+
+
+def main():
+ parser = argparse.ArgumentParser('Downloads profiles provided by Chrome OS')
+ parser.add_argument('-f', '--force', action='store_true',
+ help='Fetch a profile even if the local one is current')
+ args = parser.parse_args()
+
+ up_to_date_profile = ReadUpToDateProfileName()
+ if not args.force:
+ local_profile_name = ReadLocalProfileName()
+ # In a perfect world, the local profile should always exist if we
+ # successfully read local_profile_name. If it's gone, though, the user
+ # probably removed it as a way to get us to download it again.
+ if local_profile_name == up_to_date_profile \
+ and os.path.exists(LOCAL_PROFILE_PATH):
+ return 0
+
+ CleanProfilesDirectory()
+
+ new_tmpfile = LOCAL_PROFILE_PATH + '.new'
+ RetrieveProfile(up_to_date_profile, new_tmpfile)
+ os.rename(new_tmpfile, LOCAL_PROFILE_PATH)
+ WriteLocalProfileName(up_to_date_profile)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/chromeos/profiles/OWNERS b/chromeos/profiles/OWNERS
deleted file mode 100644
index 0d73e4e..0000000
--- a/chromeos/profiles/OWNERS
+++ /dev/null
@@ -1 +0,0 @@
-per-file orderfile.newest.txt=*
diff --git a/chromeos/profiles/orderfile.newest.txt b/chromeos/profiles/orderfile.newest.txt
deleted file mode 100644
index 9e996d0..0000000
--- a/chromeos/profiles/orderfile.newest.txt
+++ /dev/null
@@ -1 +0,0 @@
-chromeos-chrome-orderfile-77.0.3849.0_rc-r1.orderfile.xz
diff --git a/tools/OWNERS b/tools/OWNERS
index 2f045af..d771315 100644
--- a/tools/OWNERS
+++ b/tools/OWNERS
@@ -54,6 +54,3 @@
per-file [email protected]
per-file [email protected]
per-file [email protected]
-
-per-file [email protected]
-per-file [email protected]
diff --git a/tools/download_cros_provided_profile.py b/tools/download_cros_provided_profile.py
deleted file mode 100755
index 45755d2..0000000
--- a/tools/download_cros_provided_profile.py
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/usr/bin/python
-# Copyright 2018 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""This script is used to update our local profiles (AFDO or orderfiles)
-
-This uses profiles of Chrome, or orderfiles for linking, provided by our
-friends from Chrome OS. Though the profiles are available externally,
-the bucket they sit in is otherwise unreadable by non-Googlers. Gsutil
-usage with this bucket is therefore quite awkward: you can't do anything
-but `cp` certain files with an external account, and you can't even do
-that if you're not yet authenticated.
-
-No authentication is necessary if you pull these profiles directly over
-https.
-"""
-
-import argparse
-import contextlib
-import os
-import subprocess
-import sys
-import urllib2
-
-GS_HTTP_URL = 'https://storage.googleapis.com'
-
-
-def ReadUpToDateProfileName(newest_profile_name_path):
- with open(newest_profile_name_path) as f:
- return f.read().strip()
-
-
-def ReadLocalProfileName(local_profile_name_path):
- try:
- with open(local_profile_name_path) as f:
- return f.read().strip()
- except IOError:
- # Assume it either didn't exist, or we couldn't read it. In either case, we
- # should probably grab a new profile (and, in doing so, make this file sane
- # again)
- return None
-
-
-def WriteLocalProfileName(name, local_profile_name_path):
- with open(local_profile_name_path, 'w') as f:
- f.write(name)
-
-
-def CheckCallOrExit(cmd):
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = proc.communicate()
- exit_code = proc.wait()
- if not exit_code:
- return
-
- complaint_lines = [
- '## %s failed with exit code %d' % (cmd[0], exit_code),
- '## Full command: %s' % cmd,
- '## Stdout:\n' + stdout,
- '## Stderr:\n' + stderr,
- ]
- print >>sys.stderr, '\n'.join(complaint_lines)
- sys.exit(1)
-
-
-def RetrieveProfile(desired_profile_name, out_path, gs_url_base):
- # vpython is > python 2.7.9, so we can expect urllib to validate HTTPS certs
- # properly.
- ext = os.path.splitext(desired_profile_name)[1]
- compressed_path = out_path + ext
- gs_prefix = 'gs://'
- if not desired_profile_name.startswith(gs_prefix):
- gs_url = os.path.join(GS_HTTP_URL, gs_url_base, desired_profile_name)
- else:
- gs_url = os.path.join(GS_HTTP_URL, desired_profile_name[len(gs_prefix):])
-
- with contextlib.closing(urllib2.urlopen(gs_url)) as u:
- with open(compressed_path, 'wb') as f:
- while True:
- buf = u.read(4096)
- if not buf:
- break
- f.write(buf)
-
- if ext == '.bz2':
- # NOTE: we can't use Python's bzip module, since it doesn't support
- # multi-stream bzip files. It will silently succeed and give us a garbage
- # profile.
- # bzip2 removes the compressed file on success.
- CheckCallOrExit(['bzip2', '-d', compressed_path])
- elif ext == '.xz':
- # ...And we can't use the `lzma` module, since it was introduced in python3.
- # xz removes the compressed file on success.
- CheckCallOrExit(['xz', '-d', compressed_path])
- else:
- # Wait until after downloading the file to check the file extension, so the
- # user has something usable locally if the file extension is unrecognized.
- raise ValueError(
- 'Only bz2 and xz extensions are supported; "%s" is not' % ext)
-
-
-def main():
- parser = argparse.ArgumentParser(
- 'Downloads profile/orderfile provided by Chrome OS')
-
- parser.add_argument(
- '--newest_state',
- required=True,
- help='Path to the file with name of the newest profile. '
- 'We use this file to track the name of the newest profile '
- 'we should pull'
- )
- parser.add_argument(
- '--local_state',
- required=True,
- help='Path of the file storing name of the local profile. '
- 'We use this file to track the most recent profile we\'ve '
- 'successfully pulled.'
- )
- parser.add_argument(
- '--gs_url_base',
- required=True,
- help='The base GS URL to search for the profile.'
- )
- parser.add_argument(
- '--output_name',
- required=True,
- help='Output name of the downloaded and uncompressed profile.'
- )
- parser.add_argument(
- '-f', '--force',
- action='store_true',
- help='Fetch a profile even if the local one is current'
- )
- args = parser.parse_args()
-
- up_to_date_profile = ReadUpToDateProfileName(args.newest_state)
- if not args.force:
- local_profile_name = ReadLocalProfileName(args.local_state)
- # In a perfect world, the local profile should always exist if we
- # successfully read local_profile_name. If it's gone, though, the user
- # probably removed it as a way to get us to download it again.
- if local_profile_name == up_to_date_profile \
- and os.path.exists(args.output_name):
- return 0
-
- new_tmpfile = args.output_name + '.new'
- RetrieveProfile(up_to_date_profile, new_tmpfile, args.gs_url_base)
- os.rename(new_tmpfile, args.output_name)
- WriteLocalProfileName(up_to_date_profile, args.local_state)
-
-
-if __name__ == '__main__':
- sys.exit(main())
