86 lines
3.6 KiB
Python
86 lines
3.6 KiB
Python
import json
|
|
import logging
|
|
import requests
|
|
from redo import retry
|
|
from requests import exceptions
|
|
|
|
logger = logging.getLogger(__name__)
|
|
headers = {
|
|
'User-Agent': 'TaskCluster'
|
|
}
|
|
|
|
# It's a list of project name which SETA is useful on
|
|
SETA_PROJECTS = ['mozilla-inbound', 'autoland']
|
|
SETA_ENDPOINT = "https://seta.herokuapp.com/data/setadetails/?branch=%s"
|
|
|
|
|
|
class SETA(object):
|
|
"""
|
|
Interface to the SETA service, which defines low-value tasks that can be optimized out
|
|
of the taskgraph.
|
|
"""
|
|
def __init__(self):
|
|
# cached low value tasks, by project
|
|
self.low_value_tasks = {}
|
|
|
|
def query_low_value_tasks(self, project):
|
|
# Request the set of low value tasks from the SETA service. Low value tasks will be
|
|
# optimized out of the task graph.
|
|
if project not in SETA_PROJECTS:
|
|
logger.debug("SETA is not enabled for project `{}`".format(project))
|
|
return []
|
|
|
|
logger.debug("Querying SETA service for low-value tasks on {}".format(project))
|
|
low_value_tasks = []
|
|
|
|
url = SETA_ENDPOINT % project
|
|
# Try to fetch the SETA data twice, falling back to an empty list of low value tasks.
|
|
# There are 10 seconds between each try.
|
|
try:
|
|
logger.debug("Retrieving low-value jobs list from SETA")
|
|
response = retry(requests.get, attempts=2, sleeptime=10,
|
|
args=(url, ),
|
|
kwargs={'timeout': 5, 'headers': headers})
|
|
task_list = json.loads(response.content).get('jobtypes', '')
|
|
if len(task_list) > 0:
|
|
low_value_tasks = task_list.values()[0]
|
|
|
|
# Bug 1315145, disable SETA for tier-1 platforms until backfill is implemented.
|
|
low_value_tasks = [x for x in low_value_tasks if x.find('debug') == -1]
|
|
low_value_tasks = [x for x in low_value_tasks if x.find('asan') == -1]
|
|
|
|
# In the event of request times out, requests will raise a TimeoutError.
|
|
except exceptions.Timeout:
|
|
logger.warning("SETA server is timeout, we will treat all test tasks as high value.")
|
|
|
|
# In the event of a network problem (e.g. DNS failure, refused connection, etc),
|
|
# requests will raise a ConnectionError.
|
|
except exceptions.ConnectionError:
|
|
logger.warning("SETA server is timeout, we will treat all test tasks as high value.")
|
|
|
|
# In the event of the rare invalid HTTP response(e.g 404, 401),
|
|
# requests will raise an HTTPError exception
|
|
except exceptions.HTTPError:
|
|
logger.warning("We got bad Http response from ouija,"
|
|
" we will treat all test tasks as high value.")
|
|
|
|
# We just print the error out as a debug message if we failed to catch the exception above
|
|
except exceptions.RequestException as error:
|
|
logger.warning(error)
|
|
|
|
# When we get invalid JSON (i.e. 500 error), it results in a ValueError (bug 1313426)
|
|
except ValueError as error:
|
|
logger.warning("Invalid JSON, possible server error: {}".format(error))
|
|
|
|
return low_value_tasks
|
|
|
|
def is_low_value_task(self, label, project):
|
|
# cache the low value tasks per project to avoid repeated SETA server queries
|
|
if project not in self.low_value_tasks:
|
|
self.low_value_tasks[project] = self.query_low_value_tasks(project)
|
|
return label in self.low_value_tasks[project]
|
|
|
|
# create a single instance of this class, and expose its `is_low_value_task`
|
|
# bound method as a module-level function
|
|
is_low_value_task = SETA().is_low_value_task
|