Auto merge of #16565 - aneeshusa:move-upload-nightly-script-to-python, r=metajack

Convert nightly upload script to Python

Now that MinGW and MSYS have been removed from the Windows builders,
bash is not available to run the previous upload_nightlies.sh script.
Convert the script to Python 2 for cross-platform support.
Additionally, switch to the `boto3` library for easy uploading
without needing to install `s3cmd`,
and move the code into mach for easy `boto3` installation
as the new `./mach upload-nightly` command.

Also, hard-code the paths to the packages instead of using
globs to look for them, as the paths are static.
(The paths used to contain timestamps,
but we now insert timestamps when uploading to S3
to improve reproducibility.)

<!-- Please describe your changes on the following line: -->

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [ ] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] These changes fix #16560 (github issue number if applicable).

<!-- Either: -->
- [ ] There are tests for these changes OR
- [x] These changes do not require tests because Buildbot will test them.

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/16565)
<!-- Reviewable:end -->
This commit is contained in:
bors-servo 2017-05-22 18:11:19 -05:00 committed by GitHub
commit 9468fae0d4
4 changed files with 129 additions and 119 deletions

View file

@ -24,4 +24,7 @@ ply == 3.8
# For Cross-platform colored terminal text
colorama == 0.3.7
# For package uploading
boto3 == 1.4.4
-e python/tidy

View file

@ -7,13 +7,17 @@
# option. This file may not be copied, modified, or distributed
# except according to those terms.
from __future__ import print_function, unicode_literals
from __future__ import absolute_import, print_function, unicode_literals
from datetime import datetime
import hashlib
import json
import os
import os.path as path
import shutil
import subprocess
import sys
import tempfile
from mach.decorators import (
CommandArgument,
@ -35,6 +39,26 @@ from servo.command_base import (
from servo.util import delete
PACKAGES = {
'android': [
'target/arm-linux-androideabi/release/servo.apk',
],
'linux': [
'target/release/servo-tech-demo.tar.gz',
],
'mac': [
'target/release/servo-tech-demo.dmg',
],
'macbrew': [
'target/release/brew/servo.tar.gz',
],
'windows-msvc': [
r'target\release\msi\Servo.msi',
r'target\release\msi\Servo.zip',
],
}
def otool(s):
o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE)
for l in o.stdout:
@ -374,3 +398,99 @@ class PackageCommands(CommandBase):
print(" ".join(exec_command))
return subprocess.call(exec_command, env=self.build_env())
@Command('upload-nightly',
description='Upload Servo nightly to S3',
category='package')
@CommandArgument('platform',
choices=PACKAGES.keys(),
help='Package platform type to upload')
def upload_nightly(self, platform):
import boto3
def nightly_filename(package, timestamp):
return '{}-{}'.format(
timestamp.isoformat() + 'Z', # The `Z` denotes UTC
path.basename(package)
)
def upload_to_s3(platform, package, timestamp):
s3 = boto3.client('s3')
BUCKET = 'servo-builds'
nightly_dir = 'nightly/{}'.format(platform)
filename = nightly_filename(package, timestamp)
package_upload_key = '{}/{}'.format(nightly_dir, filename)
extension = path.basename(package).partition('.')[2]
latest_upload_key = '{}/servo-latest.{}'.format(nightly_dir, extension)
s3.upload_file(package, BUCKET, package_upload_key)
copy_source = {
'Bucket': BUCKET,
'Key': package_upload_key,
}
s3.copy(copy_source, BUCKET, latest_upload_key)
def update_brew(package, timestamp):
print("Updating brew formula")
package_url = 'https://download.servo.org/nightly/macbrew/{}'.format(
nightly_filename(package, timestamp)
)
with open(package) as p:
digest = hashlib.sha256(p.read()).hexdigest()
brew_version = timestamp.strftime('%Y.%m.%d')
with tempfile.TemporaryDirectory(prefix='homebrew-servo') as tmp_dir:
def call_git(cmd, **kwargs):
subprocess.check_call(
['git', '-C', tmp_dir] + cmd,
**kwargs
)
call_git([
'clone',
'https://github.com/servo/homebrew-servo.git',
'.',
])
script_dir = path.dirname(path.realpath(__file__))
with open(path.join(script_dir, 'servo-binary-formula.rb.in')) as f:
formula = f.read()
formula = formula.replace('PACKAGEURL', package_url)
formula = formula.replace('SHA', digest)
formula = formula.replace('VERSION', brew_version)
with open(path.join(tmp_dir, 'Formula', 'servo-bin.rb')) as f:
f.write(formula)
call_git(['add', path.join('.', 'Formula', 'servo-bin.rb')])
call_git([
'-c', 'user.name=Tom Servo',
'-c', 'user.email=servo@servo.org',
'commit',
'--message=Version Bump: {}'.format(brew_version),
])
token = os.environ['GITHUB_HOMEBREW_TOKEN']
call_git([
'push',
'-qf',
'https://{}@github.com/servo/homebrew-servo.git'.format(token),
'master',
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
timestamp = datetime.utcnow().replace(microsecond=0)
for package in PACKAGES[platform]:
if not path.isfile(package):
print("Could not find package for {} at {}".format(
platform,
package
), file=sys.stderr)
return 1
upload_to_s3(platform, package, timestamp)
if platform == 'macbrew':
update_brew(package, timestamp)
return 0