make repo public

handler.py

@@ -0,0 +1,131 @@
+import os
+from bs4 import BeautifulSoup
+import requests
+import json
+import boto3
+import botocore
+
+CRAN_SRC_R_URL = 'https://cran.rstudio.com/src/base/R-3/'
+batch_client = boto3.client('batch', region_name='us-east-1')
+
+
+def _to_list(input):
+    """Normalize a list (list or string) to list."""
+    if isinstance(input, list):
+        return input
+    return input.split(',')
+
+
+def _persist_r_versions(data):
+    """Ship current list to s3."""
+    s3 = boto3.resource('s3')
+    obj = s3.Object(os.environ['S3_BUCKET'], 'r/versions.json')
+    obj.put(Body=json.dumps(data), ContentType='application/json')
+
+
+def _cran_r_versions():
+    """Perform a lookup of CRAN-known R version."""
+    r = requests.get(CRAN_SRC_R_URL)
+    soup = BeautifulSoup(r.text, 'html.parser')
+    r_versions = []
+    for link in soup.find_all('a'):
+        href = link.get('href')
+        if 'R-' in href:
+            v = href.replace('.tar.gz', '').replace('R-', '')
+            if '-revised' not in v:  # reject 3.2.4-revised
+                r_versions.append(v)
+    return {'r_versions': r_versions}
+
+
+def _known_r_versions():
+    """Ship current list to s3."""
+    try:
+        s3 = boto3.resource('s3')
+        obj = s3.Object(os.environ['S3_BUCKET'], 'r/versions.json')
+        str = obj.get()['Body'].read().decode('utf-8')
+    except botocore.exceptions.ClientError:
+        print('Key not found, using empty list')
+        str = '{"r_versions":[]}'
+    return json.loads(str)
+
+
+def _compare_versions(fresh, known):
+    """Compare fresh cran list to new list and return unknown versions."""
+    new = set(fresh) - set(known)
+    return list(new)
+
+
+def _container_overrides(version):
+    """Generate container override parameter for jobs."""
+    overrides = {}
+    overrides['environment'] = [
+        {'name': 'R_VERSION', 'value': version},
+        {'name': 'S3_BUCKET', 'value': os.environ['S3_BUCKET']}
+    ]
+    return overrides
+
+
+def _submit_job(version, platform):
+    """Submit an R build job to AWS Batch."""
+    job_name = '-'.join(['R', version, platform])
+    job_name = job_name.replace('.', '_')
+    job_definition_arn = 'JOB_DEFINITION_ARN_{}'.format(platform.replace('-','_'))
+    args = {
+        'jobName': job_name,
+        'jobQueue': os.environ['JOB_QUEUE_ARN'],
+        'jobDefinition': os.environ[job_definition_arn],
+        'containerOverrides': _container_overrides(version)
+    }
+    if os.environ.get('DRYRUN'):
+        print('DRYRUN: would have queued {}'.format(job_name))
+        return 'dryrun-no-job-{}'.format(job_name)
+    else:
+        response = batch_client.submit_job(**args)
+        print("Started job for R:{},Platform:{},id:{}".format(version, platform, response['jobId']))
+        return response['jobId']
+
+
+def _versions_to_build(force):
+    cran_versions = _cran_r_versions()['r_versions']
+    known_versions = _known_r_versions()['r_versions']
+    new_versions = _compare_versions(cran_versions, known_versions)
+
+    if len(new_versions) > 0:
+        print('New R Versions found: %s' % new_versions)
+        _persist_r_versions(_cran_r_versions())
+
+    if force in [True, 'True', 'true']:
+        return cran_versions
+    else:
+        return new_versions
+
+
+def _check_for_job_status(jobs, status):
+    """Return a subset of job ids which match a given status."""
+    r = batch_client.list_jobs(jobQueue=os.environ['JOB_QUEUE_ARN'], jobStatus=status)
+    return [i['jobId'] for i in r['jobSummaryList'] if i['jobId'] in jobs]
+
+
+def queue_builds(event, context):
+    """Queue some builds."""
+    event['versions_to_build'] =  _versions_to_build(event.get('force', False))
+    event['supported_platforms'] = _to_list(os.environ.get('SUPPORTED_PLATFORMS', 'ubuntu-1604'))
+    job_ids = []
+    for version in event['versions_to_build']:
+        for platform in event['supported_platforms']:
+            # skip bionic builds with a configure script bug
+            if (platform == 'ubuntu-1804' or platform == 'opensuse-15') and version in ['3.3.0', '3.3.1', '3.3.2']:
+                continue
+            job_ids.append(_submit_job(version, platform))
+    event['jobIds'] = job_ids
+    return event
+
+
+def poll_running_jobs(event, context):
+    """Query job queue for current queue depth."""
+    event['failedJobIds'] = _check_for_job_status(event['jobIds'], 'FAILED')
+    event['succeededJobIds'] = _check_for_job_status(event['jobIds'], 'SUCCEEDED')
+    event['failedJobCount'] = len(event['failedJobIds'])
+    event['finishedJobCount'] = len(event['failedJobIds']) + len(event['succeededJobIds'])
+    event['unfinishedJobCount'] = len(event['jobIds']) - event['finishedJobCount']
+    return event