From 5126beacfc52fd85284f9e6039c6b0c3cdfdeed2 Mon Sep 17 00:00:00 2001 From: dsc Date: Tue, 8 May 2012 17:46:50 -0700 Subject: [PATCH] Fabfile now bundles. --- Cokefile | 5 +- deploy.sh | 4 +- fabfile/__init__.py | 191 +++++++++++++++++++++++++++++++++------------------ fabfile/util.py | 32 +++++++-- 4 files changed, 155 insertions(+), 77 deletions(-) diff --git a/Cokefile b/Cokefile index 4cb0a92..efbf256 100644 --- a/Cokefile +++ b/Cokefile @@ -83,7 +83,10 @@ task \clean 'Clean up environment and artifacts' -> task \source_list 'Print a list of the source file paths.' -> invoke \setup {sources} = require 'kraken/server/view-helpers' - say sources("www/modules.yaml", 'dev').join '\n' + say do + sources("www/modules.yaml", 'dev') + .map -> it.slice 1 + .join '\n' # task \dist 'Assemble a distribution package for deploy' -> # invoke \cleanup_tests diff --git a/deploy.sh b/deploy.sh index 72276bc..4620bd2 100755 --- a/deploy.sh +++ b/deploy.sh @@ -120,7 +120,7 @@ log "Building Vendor Bundle ($target) ..." for js in $(coke source_list | grep vendor); do log " file: $js" for d in static var tmp/dist; do - f="$d$js" + f="$d/$js" log " checking '$f' ..." [ -f $f ] && break done @@ -140,7 +140,7 @@ target=tmp/dist/js/kraken/app-bundle uglifyjs=$(which uglifyjs || echo "node_modules/uglify-js/bin/uglifyjs") log "Building App JS Bundle (${target}.js) ..." log "cat \n$(coke source_list | grep -v vendor | sed 's/^/ var/')" -cat $(coke source_list | grep -v vendor | sed 's/^/var/') > ${target}.js +cat $(coke source_list | grep -v vendor | sed 's/^/var\//') > ${target}.js $uglifyjs < ${target}.js > ${target}.min.js log "ok!" diff --git a/fabfile/__init__.py b/fabfile/__init__.py index 0c24425..25debfb 100644 --- a/fabfile/__init__.py +++ b/fabfile/__init__.py @@ -1,84 +1,106 @@ #!/usr/bin/env fab # -*- coding: utf-8 -*- +"GraphKit Deployer" -""" GraphKit Fabric File -""" # Deal with the fact that we aren't *really* a Python project, # so we haven't declared python dependencies. import sys try: import fabric - from path import path + from path import path as p # fabric.api.path conflicts except ImportError: - print "ERROR: You're missing a dependency! " - "To build this project using fabric, you'll need to install fabric (and some other stuff):" - "\n\n" - "pip install -U fabric path.py" + print """ + ERROR: You're missing a dependency! + To build this project using fabric, you'll need to install fabric (and some other stuff): + + pip install -U fabric path.py + """ sys.exit(1) - +from itertools import dropwhile from fabric.api import * from fabric.colors import white, blue, cyan, green, yellow, red, magenta from fabric.contrib.project import rsync_project +from util import * + + +### Fabric Config + +env.project_name = 'kraken-ui' +env.use_ssh_config = True # this is not working for me! +env.colors = True + + +### Project Config -env.project_name = 'kraken-ui' -env.use_ssh_config = True # this is not working for me! -env.colors = True +env.dev_server = 'localhost:8081' +env.minify_cmd = 'uglifyjs' +env.dist = p('dist') +env.local_tmp = p('tmp') +env.work_dir = env.local_tmp/env.dist +env.browserify_js = 'vendor/browserify.js' +env.work_browserify_js = env.work_dir/env.browserify_js +env.vendor_search_dirs = map(p, ['static', 'var', env.work_dir]) +env.vendor_bundle = env.work_dir/'vendor/vendor-bundle.min.js' +env.app_bundle = env.work_dir/'js/kraken/app-bundle.js' +env.app_bundle_min = p(env.app_bundle.replace('.js', '.min.js')) -# defaults -# Hm. -env.local_dist_directory = path('tmp/dist') -env.dist_directory = path('dist') -env.dev_server = 'localhost:8081' + +### Deploy Environments # There should be a way to do this using stages. # See: http://tav.espians.com/fabric-python-with-cleaner-api-and-parallel-deployment-support.html # env.config_file = False -# env.stages = ['production', 'staging'] +# env.stages = ['prod', 'staging'] -# environments -@task(alias='prod') -def production(): - """Set production environment variables""" - env.hosts = ['reportcard2.pmtpa.wmflabs'] - env.directory = '/srv/reportcard/kraken-ui' - env.owner = 'www-data' - env.group = 'www' +@task +def prod(): + """ Set deploy environment to production. + """ + env.deploy_env = 'prod' + env.hosts = ['reportcard2.pmtpa.wmflabs'] + env.target_dir = '/srv/reportcard/kraken-ui' + env.owner = 'www-data' + env.group = 'www' @task def staging(): - """Set staging environment variables""" - env.hosts = ['less.ly'] - env.directory = '/home/wmf/projects/kraken-ui' - env.user = 'wmf' - env.owner = 'wmf' - env.group = 'www' + """ Set deploy environment to staging. + """ + env.deploy_env = 'staging' + env.hosts = ['less.ly'] + env.target_dir = '/home/wmf/projects/kraken-ui' + env.user = 'wmf' + env.owner = 'wmf' + env.group = 'www' + ### Build Deploy Bundle @task -def update_version(): - """ Ensure `lib/version.js` has up to date git revision. +def bundle(): + """ Bundles both vendor and application files. """ - local('coke update_version') + update_version() + collapse_trees() + bundle_vendor() + bundle_app() +@msg('Collapsing Serve Trees') @task def collapse_trees(): """ Collapse the serve trees into one directory. """ - dist = env.local_dist_directory - - # Update version (derf) update_version() # Ensure clean dist directory - dist.rmtree(ignore_errors=True) - dist.makedirs() + env.work_dir.rmtree(ignore_errors=True) + env.work_dir.makedirs() # XXX: Unfortunately, we can't use rsync_project() for local-to-local copies, as it insists on # inserting a : before the remote path, which indicates a local-to-remote copy. :( @@ -86,60 +108,97 @@ def collapse_trees(): # Copy the static files, derived files, and the whole data directory (bc lack of trailing /) # into dist. Note that you will need to load all the site pages in your browser to populate var # with the derived files. - local('rsync -Ca static/ var/ data %s/' % dist) + local('rsync -Ca static/ var/ data %(work_dir)s/' % env) # We copy lib (which contains .co source files) to src to make it easy to link source content # to each other. Finding it in gitweb is a pain. Finding it in gerrit is almost impossible. # But this could go away when we move to github. - local('rsync -Ca lib/ %s/src/' % dist) + local('rsync -Ca lib/ %(work_dir)s/src/' % env) # For some reason, the shell tool does not generate a file identical to the middleware. So whatever. # We curl here because we know that version works. - # local('browserify -o %s/vendor/browserify.js -r events -r seq' % dist) - with open('%s/vendor/browserify.js' % dist, 'w') as f: - f.write( local('curl --silent --fail --url http://%s/vendor/browserify.js', capture=True) ) + # local('browserify -o %(work_dir)s/%(browserify_js)s -r events -r seq' % env) + with env.work_browserify_js.open('w') as f: + f.write( local('curl --silent --fail --url http://%(dev_server)s/%(browserify_js)s' % env, capture=True) ) +@msg('Building Vendor Bundle') @task -def bundle(): - """Builds and bundles static files in tmp/dist for deployment.""" +def bundle_vendor(): + """ Bundles vendor files. + """ + update_version() + with env.vendor_bundle.open('w') as vendor_bundle: + + for js in local('coke source_list | grep vendor', capture=True).split('\n'): + try: + # Search for matching vendor file as it might be derived (.mod.js) + vendor_file = ( d/js for d in env.vendor_search_dirs if (d/js).exists() ).next() + except StopIteration: + abort("Unable to locate vendor file '%s'!" % js) + + vendor_bundle.write("\n;\n") + with vendor_file.open() as f: + vendor_bundle.write(f.read()) + +@msg('Building App Bundle') +@task +def bundle_app(): + """ Bundles and minifies app files. + """ + update_version() + # XXX: Meh. Maybe this should become python code. + local('cat $(coke source_list | grep -v vendor | sed "s/^/var\//") > %(app_bundle)s' % env) + local('%(minify_cmd)s %(app_bundle)s > %(app_bundle_min)s' % env) + + + +### Deploy Tasks + +@task +def full_deploy(): + """ Full deploy. + """ pass @task def fix_permissions(): - """Recursively fixes permissions on the deployment host.""" - sudo('chmod -R g+w %s') - sudo('chown -R %(owner)s:%(group)s %(directory)s' % env) - pass + """ Recursively fixes permissions on the deployment host. + """ + sudo('chmod -R g+w %(target_dir)s' % env) + sudo('chown -R %(owner)s:%(group)s %(target_dir)s' % env) @task def update(): - """Runs git pull on the deployment host.""" - with cd(env.directory): + """ Runs git pull on the deployment host. + """ + with cd(env.target_dir): run('git pull') - @task def distribute(): - """Copies tmp/dist to deployment host.""" - local("rsync -Caz -v %(local_dist_directory)s %(user)s@%(host)s:%(directory)s/%(dist_directory)s" % env) + """ Copies `dist` package to deployment host. + """ # TODO: make sure the following works. - # rsync_project(local_dir=env.local_dist_directory, remote_dir="%(user)s@%(host)s:%(directory)s/%(dist_directory)s" % env) - pass + # rsync_project(local_dir=env.work_dir, remote_dir="%(user)s@%(host)s:%(target_dir)s/%(dist)s" % env) + local("rsync -Caz -v %(work_dir)s %(user)s@%(host)s:%(target_dir)s/%(dist)s" % env) @task def restart_server(): - """Restarts node.js server on the deployment host.""" + """ Restarts node.js server on the deployment host. + """ # need to work on this for less.ly sudo("supervisor restart reportcard") - pass -@task -def deploy(): - """Full deployment of latest project""" - pass +### Misc + @task -def test(): - puts("user: %(user)s" % env) - run('whoami') +@runs_once +def update_version(): + """ Ensure `lib/version.js` has up to date git revision. + """ + local('coke update_version') + print + + diff --git a/fabfile/util.py b/fabfile/util.py index bcffa5f..7673775 100644 --- a/fabfile/util.py +++ b/fabfile/util.py @@ -1,19 +1,35 @@ #!/usr/bin/env fab # -*- coding: utf-8 -*- -# Liberated from fabric source: -# https://github.com/fabric/fabric/blob/master/fabfile/utils.py - from __future__ import with_statement from contextlib import contextmanager from fabric.api import hide, puts +from fabric.colors import white, blue, cyan, green, yellow, red, magenta -__all__ = ('msg',) +__all__ = ('quietly', 'msg',) @contextmanager -def msg(txt): +def quietly(txt): + "Wrap a block in a message, suppressing other output." puts(txt + "...", end='', flush=True) - with hide('everything'): - yield - puts("done.", show_prefix=False, flush=True) + with hide('everything'): yield + puts("woo.", show_prefix=False, flush=True) + + +def msg(txt, quiet=False): + "Decorator to wrap a task in a message, optionally suppressing all output." + def outer(fn): + def inner(*args, **kwargs): + if quiet: + puts(green(txt + '...', bold=True), end='', flush=True) + with hide('everything'): + result = fn(*args, **kwargs) + puts(white('Woo.\n'), show_prefix=False, flush=True) + else: + puts(green(txt + '...', bold=True)) + result = fn(*args, **kwargs) + puts(white('Woo.\n')) + return result + return inner + return outer -- 1.7.0.4