From 37f5287282e49f5c475e565b2f635e2fa770aaa9 Mon Sep 17 00:00:00 2001 From: dsc Date: Tue, 8 May 2012 23:01:41 -0700 Subject: [PATCH] Moves bundle and deploy tasks into submodules. --- fabfile/__init__.py | 169 ++++----------------------------------------------- fabfile/bundle.py | 80 ++++++++++++++++++++++++ fabfile/deploy.py | 77 +++++++++++++++++++++++ fabfile/util.py | 24 +++++++- 4 files changed, 191 insertions(+), 159 deletions(-) create mode 100644 fabfile/bundle.py create mode 100644 fabfile/deploy.py diff --git a/fabfile/__init__.py b/fabfile/__init__.py index 25debfb..d720782 100644 --- a/fabfile/__init__.py +++ b/fabfile/__init__.py @@ -3,11 +3,13 @@ "GraphKit Deployer" +import sys +from functools import wraps + # Deal with the fact that we aren't *really* a Python project, # so we haven't declared python dependencies. -import sys try: - import fabric + from fabric.api import * from path import path as p # fabric.api.path conflicts except ImportError: print """ @@ -19,12 +21,6 @@ except ImportError: sys.exit(1) -from itertools import dropwhile -from fabric.api import * -from fabric.colors import white, blue, cyan, green, yellow, red, magenta -from fabric.contrib.project import rsync_project -from util import * - ### Fabric Config @@ -49,156 +45,15 @@ env.app_bundle = env.work_dir/'js/kraken/app-bundle.js' env.app_bundle_min = p(env.app_bundle.replace('.js', '.min.js')) +# Rename to avoid conflicts when we make top-level wrapper tasks +import bundle +import deploy -### Deploy Environments - -# There should be a way to do this using stages. -# See: http://tav.espians.com/fabric-python-with-cleaner-api-and-parallel-deployment-support.html -# env.config_file = False -# env.stages = ['prod', 'staging'] -@task -def prod(): - """ Set deploy environment to production. +@task(default=True) +def gogogo(): + """ Bundles and deploys the project. [Default] """ - env.deploy_env = 'prod' - env.hosts = ['reportcard2.pmtpa.wmflabs'] - env.target_dir = '/srv/reportcard/kraken-ui' - env.owner = 'www-data' - env.group = 'www' - -@task -def staging(): - """ Set deploy environment to staging. - """ - env.deploy_env = 'staging' - env.hosts = ['less.ly'] - env.target_dir = '/home/wmf/projects/kraken-ui' - env.user = 'wmf' - env.owner = 'wmf' - env.group = 'www' - - - -### Build Deploy Bundle - -@task -def bundle(): - """ Bundles both vendor and application files. - """ - update_version() - collapse_trees() - bundle_vendor() - bundle_app() - -@msg('Collapsing Serve Trees') -@task -def collapse_trees(): - """ Collapse the serve trees into one directory. - """ - update_version() - - # Ensure clean dist directory - env.work_dir.rmtree(ignore_errors=True) - env.work_dir.makedirs() - - # XXX: Unfortunately, we can't use rsync_project() for local-to-local copies, as it insists on - # inserting a : before the remote path, which indicates a local-to-remote copy. :( - - # Copy the static files, derived files, and the whole data directory (bc lack of trailing /) - # into dist. Note that you will need to load all the site pages in your browser to populate var - # with the derived files. - local('rsync -Ca static/ var/ data %(work_dir)s/' % env) - - # We copy lib (which contains .co source files) to src to make it easy to link source content - # to each other. Finding it in gitweb is a pain. Finding it in gerrit is almost impossible. - # But this could go away when we move to github. - local('rsync -Ca lib/ %(work_dir)s/src/' % env) - - # For some reason, the shell tool does not generate a file identical to the middleware. So whatever. - # We curl here because we know that version works. - # local('browserify -o %(work_dir)s/%(browserify_js)s -r events -r seq' % env) - with env.work_browserify_js.open('w') as f: - f.write( local('curl --silent --fail --url http://%(dev_server)s/%(browserify_js)s' % env, capture=True) ) - -@msg('Building Vendor Bundle') -@task -def bundle_vendor(): - """ Bundles vendor files. - """ - update_version() - with env.vendor_bundle.open('w') as vendor_bundle: - - for js in local('coke source_list | grep vendor', capture=True).split('\n'): - try: - # Search for matching vendor file as it might be derived (.mod.js) - vendor_file = ( d/js for d in env.vendor_search_dirs if (d/js).exists() ).next() - except StopIteration: - abort("Unable to locate vendor file '%s'!" % js) - - vendor_bundle.write("\n;\n") - with vendor_file.open() as f: - vendor_bundle.write(f.read()) - -@msg('Building App Bundle') -@task -def bundle_app(): - """ Bundles and minifies app files. - """ - update_version() - # XXX: Meh. Maybe this should become python code. - local('cat $(coke source_list | grep -v vendor | sed "s/^/var\//") > %(app_bundle)s' % env) - local('%(minify_cmd)s %(app_bundle)s > %(app_bundle_min)s' % env) - - - -### Deploy Tasks - -@task -def full_deploy(): - """ Full deploy. - """ - pass - -@task -def fix_permissions(): - """ Recursively fixes permissions on the deployment host. - """ - sudo('chmod -R g+w %(target_dir)s' % env) - sudo('chown -R %(owner)s:%(group)s %(target_dir)s' % env) - -@task -def update(): - """ Runs git pull on the deployment host. - """ - with cd(env.target_dir): - run('git pull') - -@task -def distribute(): - """ Copies `dist` package to deployment host. - """ - # TODO: make sure the following works. - # rsync_project(local_dir=env.work_dir, remote_dir="%(user)s@%(host)s:%(target_dir)s/%(dist)s" % env) - local("rsync -Caz -v %(work_dir)s %(user)s@%(host)s:%(target_dir)s/%(dist)s" % env) - -@task -def restart_server(): - """ Restarts node.js server on the deployment host. - """ - # need to work on this for less.ly - sudo("supervisor restart reportcard") - - - -### Misc - -@task -@runs_once -def update_version(): - """ Ensure `lib/version.js` has up to date git revision. - """ - local('coke update_version') - print - + bundle.bundle_all() + deploy.full_deploy() diff --git a/fabfile/bundle.py b/fabfile/bundle.py new file mode 100644 index 0000000..81553c8 --- /dev/null +++ b/fabfile/bundle.py @@ -0,0 +1,80 @@ +#!/usr/bin/env fab +# -*- coding: utf-8 -*- + +from fabric.api import * +from fabric.colors import white, blue, cyan, green, yellow, red, magenta +from util import * + + +### Build Deploy Bundle + +@task(default=True) +def bundle_all(): + """ Bundles vendor and application files. + """ + update_version() + collapse_trees() + bundle_vendor() + bundle_app() + +@task +@msg('Collapsing Serve Trees') +def collapse_trees(): + """ Collapse the serve trees into one directory. + """ + update_version() + + # Ensure clean dist directory + env.work_dir.rmtree(ignore_errors=True) + env.work_dir.makedirs() + + # XXX: Unfortunately, we can't use rsync_project() for local-to-local copies, as it insists on + # inserting a : before the remote path, which indicates a local-to-remote copy. :( + + # Copy the static files, derived files, and the whole data directory (bc lack of trailing /) + # into dist. Note that you will need to load all the site pages in your browser to populate var + # with the derived files. + local('rsync -Ca static/ var/ data %(work_dir)s/' % env) + + # We copy lib (which contains .co source files) to src to make it easy to link source content + # to each other. Finding it in gitweb is a pain. Finding it in gerrit is almost impossible. + # But this could go away when we move to github. + local('rsync -Ca lib/ %(work_dir)s/src/' % env) + + # For some reason, the shell tool does not generate a file identical to the middleware. So whatever. + # We curl here because we know that version works. + # local('browserify -o %(work_dir)s/%(browserify_js)s -r events -r seq' % env) + with env.work_browserify_js.open('w') as f: + f.write( local('curl --silent --fail --url http://%(dev_server)s/%(browserify_js)s' % env, capture=True) ) + +@task +@msg('Building Vendor Bundle') +def bundle_vendor(): + """ Bundles vendor files. + """ + update_version() + with env.vendor_bundle.open('w') as vendor_bundle: + + for js in local('coke source_list | grep vendor', capture=True).split('\n'): + try: + # Search for matching vendor file as it might be derived (.mod.js) + vendor_file = ( d/js for d in env.vendor_search_dirs if (d/js).exists() ).next() + except StopIteration: + abort("Unable to locate vendor file '%s'!" % js) + + vendor_bundle.write("\n;\n") + with vendor_file.open() as f: + vendor_bundle.write(f.read()) + +@task +@msg('Building App Bundle') +def bundle_app(): + """ Bundles and minifies app files. + """ + update_version() + # XXX: Meh. Maybe this should become python code. + local('cat $(coke source_list | grep -v vendor | sed "s/^/var\//") > %(app_bundle)s' % env) + local('%(minify_cmd)s %(app_bundle)s > %(app_bundle_min)s' % env) + + + diff --git a/fabfile/deploy.py b/fabfile/deploy.py new file mode 100644 index 0000000..d3e71bc --- /dev/null +++ b/fabfile/deploy.py @@ -0,0 +1,77 @@ +#!/usr/bin/env fab +# -*- coding: utf-8 -*- + +from fabric.api import * +from fabric.colors import white, blue, cyan, green, yellow, red, magenta +from fabric.contrib.project import rsync_project +from util import * + + +### Deploy Environments + +# There should be a way to do this using stages. +# See: http://tav.espians.com/fabric-python-with-cleaner-api-and-parallel-deployment-support.html +# env.config_file = False +# env.stages = ['prod', 'staging'] + +@task +def prod(): + """ Set deploy environment to production. + """ + env.deploy_env = 'prod' + env.hosts = ['reportcard2.pmtpa.wmflabs'] + env.target_dir = '/srv/reportcard/kraken-ui' + env.owner = 'www-data' + env.group = 'www' + +@task +def staging(): + """ Set deploy environment to staging. + """ + env.deploy_env = 'staging' + env.hosts = ['less.ly'] + env.target_dir = '/home/wmf/projects/kraken-ui' + env.user = 'wmf' + env.owner = 'wmf' + env.group = 'www' + + + +### Deploy Tasks + +@task(default=True) +def full_deploy(): + """ Deploy the project. + """ + pass + +@task +def fix_permissions(): + """ Recursively fixes permissions on the deployment host. + """ + sudo('chmod -R g+w %(target_dir)s' % env) + sudo('chown -R %(owner)s:%(group)s %(target_dir)s' % env) + +@task +def update(): + """ Runs git pull on the deployment host. + """ + with cd(env.target_dir): + run('git pull') + +@task +def distribute(): + """ Copies `dist` package to deployment host. + """ + # TODO: make sure the following works. + # rsync_project(local_dir=env.work_dir, remote_dir="%(user)s@%(host)s:%(target_dir)s/%(dist)s" % env) + local("rsync -Caz -v %(work_dir)s %(user)s@%(host)s:%(target_dir)s/%(dist)s" % env) + +@task +def restart_server(): + """ Restarts node.js server on the deployment host. + """ + # need to work on this for less.ly + sudo("supervisor restart reportcard") + + diff --git a/fabfile/util.py b/fabfile/util.py index 7673775..b61d664 100644 --- a/fabfile/util.py +++ b/fabfile/util.py @@ -3,10 +3,12 @@ from __future__ import with_statement from contextlib import contextmanager -from fabric.api import hide, puts +from functools import wraps + +from fabric.api import * from fabric.colors import white, blue, cyan, green, yellow, red, magenta -__all__ = ('quietly', 'msg',) +__all__ = ('quietly', 'msg', 'coke', 'update_version',) @contextmanager @@ -20,6 +22,7 @@ def quietly(txt): def msg(txt, quiet=False): "Decorator to wrap a task in a message, optionally suppressing all output." def outer(fn): + @wraps(fn) def inner(*args, **kwargs): if quiet: puts(green(txt + '...', bold=True), end='', flush=True) @@ -33,3 +36,20 @@ def msg(txt, quiet=False): return result return inner return outer + + +### Misc + +def coke(args, capture=False): + """ Invokes project Cokefile. + """ + local('coke %s' % args, capture=capture) + +@runs_once +def update_version(): + """ Ensure `lib/version.js` has up to date git revision. + """ + coke('update_version') + print '' + + -- 1.7.0.4