# Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import argparse import errno import hashlib import json import os import sys import tempfile import time import traceback import urllib import urllib2 # Default package repository URL. CIPD_BACKEND_URL = 'https://chrome-infra-packages.appspot.com' # ./cipd resolve \ # infra/tools/cipd/ \ # -version=git_revision:768586a9aa72fe6a41a8a0205ed56ceb1495625c CLIENT_VERSIONS = { 'linux-386': '3f5d6d3906c3b2c6a057d1ab1634ac06aa418708', 'linux-amd64': '1ea0b6b254ad3f546c826dd3e437798ace2c2480', 'mac-amd64': 'c111be343c692e5285113a6b1c999887adbb268e', 'windows-386': 'dc3d1bd5b4b93945640bac4bb047c333a8fa12fd', 'windows-amd64': 'bccdb9a605037e3dd2a8a64e79e08f691a6f159d', } class CipdBootstrapError(Exception): """Raised by install_cipd_client on fatal error.""" def install_cipd_client(path, package, version): """Installs CIPD client to /cipd. Args: path: root directory to install CIPD client into. package: cipd client package name, e.g. infra/tools/cipd/linux-amd64. version: version of the package to install. Returns: Absolute path to CIPD executable. """ print 'Ensuring CIPD client is up-to-date' version_file = os.path.join(path, 'VERSION') bin_file = os.path.join(path, 'cipd') # Resolve version to concrete instance ID, e.g "live" -> "abcdef0123....". instance_id = call_cipd_api( 'repo/v1/instance/resolve', {'package_name': package, 'version': version})['instance_id'] print 'CIPD client %s => %s' % (version, instance_id) # Already installed? installed_instance_id = (read_file(version_file) or '').strip() if installed_instance_id == instance_id and os.path.exists(bin_file): return bin_file, instance_id # Resolve instance ID to an URL to fetch client binary from. client_info = call_cipd_api( 'repo/v1/client', {'package_name': package, 'instance_id': instance_id}) print 'CIPD client binary info:\n%s' % dump_json(client_info) # Fetch the client. It is ~10 MB, so don't bother and fetch it into memory. status, raw_client_bin = fetch_url(client_info['client_binary']['fetch_url']) if status != 200: print 'Failed to fetch client binary, HTTP %d' % status raise CipdBootstrapError('Failed to fetch client binary, HTTP %d' % status) digest = hashlib.sha1(raw_client_bin).hexdigest() if digest != client_info['client_binary']['sha1']: raise CipdBootstrapError('Client SHA1 mismatch') # Success. print 'Fetched CIPD client %s:%s at %s' % (package, instance_id, bin_file) write_file(bin_file, raw_client_bin) os.chmod(bin_file, 0755) write_file(version_file, instance_id + '\n') return bin_file, instance_id def call_cipd_api(endpoint, query): """Sends GET request to CIPD backend, parses JSON response.""" url = '%s/_ah/api/%s' % (CIPD_BACKEND_URL, endpoint) if query: url += '?' + urllib.urlencode(query) status, body = fetch_url(url) if status != 200: raise CipdBootstrapError('Server replied with HTTP %d' % status) try: body = json.loads(body) except ValueError: raise CipdBootstrapError('Server returned invalid JSON') status = body.get('status') if status != 'SUCCESS': m = body.get('error_message') or '' raise CipdBootstrapError('Server replied with error %s: %s' % (status, m)) return body def fetch_url(url, headers=None): """Sends GET request (with retries). Args: url: URL to fetch. headers: dict with request headers. Returns: (200, reply body) on success. (HTTP code, None) on HTTP 401, 403, or 404 reply. Raises: Whatever urllib2 raises. """ req = urllib2.Request(url) req.add_header('User-Agent', 'cipd recipe bootstrap.py') for k, v in (headers or {}).iteritems(): req.add_header(str(k), str(v)) i = 0 while True: i += 1 try: print 'GET %s' % url return 200, urllib2.urlopen(req, timeout=60).read() except Exception as e: if isinstance(e, urllib2.HTTPError): print 'Failed to fetch %s, server returned HTTP %d' % (url, e.code) if e.code in (401, 403, 404): return e.code, None else: print 'Failed to fetch %s' % url if i == 20: raise print 'Retrying in %d sec.' % i time.sleep(i) def ensure_directory(path): """Creates a directory.""" # Handle a case where a file is being converted into a directory. chunks = path.split(os.sep) for i in xrange(len(chunks)): p = os.sep.join(chunks[:i+1]) if os.path.exists(p) and not os.path.isdir(p): os.remove(p) break try: os.makedirs(path) except OSError as e: if e.errno != errno.EEXIST: raise def read_file(path): """Returns contents of a file or None if missing.""" try: with open(path, 'r') as f: return f.read() except IOError as e: if e.errno == errno.ENOENT: return None raise def write_file(path, data): """Puts a file on disk, atomically.""" ensure_directory(os.path.dirname(path)) fd, temp_file = tempfile.mkstemp(dir=os.path.dirname(path)) with os.fdopen(fd, 'w') as f: f.write(data) if not sys.platform in ('linux2', 'darwin'): # On windows we should remove destination file if it exists. if os.path.exists(path): os.remove(path) # At this point we may crash, and it's OK, as next time we'll just # re-install CIPD from scratch. os.rename(temp_file, path) def dump_json(obj): """Pretty-formats object to JSON.""" return json.dumps(obj, indent=2, sort_keys=True, separators=(',',':')) def main(args): parser = argparse.ArgumentParser('bootstrap cipd') parser.add_argument('--json-output', default=None) parser.add_argument('--version', default=None) parser.add_argument('--platform', required=True) parser.add_argument('--dest-directory', required=True) opts = parser.parse_args(args) package = "infra/tools/cipd/%s" % opts.platform version = opts.version or CLIENT_VERSIONS[opts.platform] try: exe_path, instance_id = install_cipd_client(opts.dest_directory, package, version) result = { 'executable': exe_path, 'instance_id': instance_id } if opts.json_output: with open(opts.json_output, 'w') as f: json.dump(result, f) except Exception as e: print 'Exception installing cipd: %s' % e _exc_type, _exc_value, exc_traceback = sys.exc_info() traceback.print_tb(exc_traceback) return 1 return 0 if __name__ == '__main__': sys.exit(main(sys.argv[1:]))