131 lines
5.5 KiB
Python
131 lines
5.5 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
import logging
|
|
import json
|
|
import os
|
|
import urllib2
|
|
|
|
from . import base
|
|
from taskgraph.util.docker import (
|
|
docker_image,
|
|
generate_context_hash,
|
|
INDEX_PREFIX,
|
|
)
|
|
from taskgraph.util.templates import Templates
|
|
|
|
logger = logging.getLogger(__name__)
|
|
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
|
|
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
|
|
INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
|
|
|
|
|
|
class DockerImageTask(base.Task):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.index_paths = kwargs.pop('index_paths')
|
|
super(DockerImageTask, self).__init__(*args, **kwargs)
|
|
|
|
def __eq__(self, other):
|
|
return super(DockerImageTask, self).__eq__(other) and \
|
|
self.index_paths == other.index_paths
|
|
|
|
@classmethod
|
|
def load_tasks(cls, kind, path, config, params, loaded_tasks):
|
|
parameters = {
|
|
'pushlog_id': params.get('pushlog_id', 0),
|
|
'pushdate': params['moz_build_date'],
|
|
'pushtime': params['moz_build_date'][8:],
|
|
'year': params['moz_build_date'][0:4],
|
|
'month': params['moz_build_date'][4:6],
|
|
'day': params['moz_build_date'][6:8],
|
|
'project': params['project'],
|
|
'docker_image': docker_image,
|
|
'base_repository': params['base_repository'] or params['head_repository'],
|
|
'head_repository': params['head_repository'],
|
|
'head_ref': params['head_ref'] or params['head_rev'],
|
|
'head_rev': params['head_rev'],
|
|
'owner': params['owner'],
|
|
'level': params['level'],
|
|
'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
|
|
.format(repo=params['head_repository'], rev=params['head_rev']),
|
|
'index_image_prefix': INDEX_PREFIX,
|
|
'artifact_path': 'public/image.tar.zst',
|
|
}
|
|
|
|
tasks = []
|
|
templates = Templates(path)
|
|
for image_name, image_symbol in config['images'].iteritems():
|
|
context_path = os.path.join('testing', 'docker', image_name)
|
|
context_hash = generate_context_hash(GECKO, context_path, image_name)
|
|
|
|
image_parameters = dict(parameters)
|
|
image_parameters['image_name'] = image_name
|
|
image_parameters['context_hash'] = context_hash
|
|
|
|
image_task = templates.load('image.yml', image_parameters)
|
|
attributes = {'image_name': image_name}
|
|
|
|
# unique symbol for different docker image
|
|
if 'extra' in image_task['task']:
|
|
image_task['task']['extra']['treeherder']['symbol'] = image_symbol
|
|
|
|
# As an optimization, if the context hash exists for a high level, that image
|
|
# task ID will be used. The reasoning behind this is that eventually everything ends
|
|
# up on level 3 at some point if most tasks use this as a common image
|
|
# for a given context hash, a worker within Taskcluster does not need to contain
|
|
# the same image per branch.
|
|
index_paths = ['{}.level-{}.{}.hash.{}'.format(
|
|
INDEX_PREFIX, level, image_name, context_hash)
|
|
for level in range(int(params['level']), 4)]
|
|
|
|
tasks.append(cls(kind, 'build-docker-image-' + image_name,
|
|
task=image_task['task'], attributes=attributes,
|
|
index_paths=index_paths))
|
|
|
|
return tasks
|
|
|
|
def get_dependencies(self, taskgraph):
|
|
return []
|
|
|
|
def optimize(self, params):
|
|
for index_path in self.index_paths:
|
|
try:
|
|
url = INDEX_URL.format(index_path)
|
|
existing_task = json.load(urllib2.urlopen(url))
|
|
# Only return the task ID if the artifact exists for the indexed
|
|
# task. Otherwise, continue on looking at each of the branches. Method
|
|
# continues trying other branches in case mozilla-central has an expired
|
|
# artifact, but 'project' might not. Only return no task ID if all
|
|
# branches have been tried
|
|
request = urllib2.Request(
|
|
ARTIFACT_URL.format(existing_task['taskId'], 'public/image.tar.zst'))
|
|
request.get_method = lambda: 'HEAD'
|
|
urllib2.urlopen(request)
|
|
|
|
# HEAD success on the artifact is enough
|
|
return True, existing_task['taskId']
|
|
except urllib2.HTTPError:
|
|
pass
|
|
|
|
return False, None
|
|
|
|
@classmethod
|
|
def from_json(cls, task_dict):
|
|
# Generating index_paths for optimization
|
|
imgMeta = task_dict['task']['extra']['imageMeta']
|
|
image_name = imgMeta['imageName']
|
|
context_hash = imgMeta['contextHash']
|
|
index_paths = ['{}.level-{}.{}.hash.{}'.format(
|
|
INDEX_PREFIX, level, image_name, context_hash)
|
|
for level in range(int(imgMeta['level']), 4)]
|
|
docker_image_task = cls(kind='docker-image',
|
|
label=task_dict['label'],
|
|
attributes=task_dict['attributes'],
|
|
task=task_dict['task'],
|
|
index_paths=index_paths)
|
|
return docker_image_task
|