From: dsc <dsc@wikimedia.org>
Date: Wed, 16 May 2012 06:27:08 +0000 (-0700)
Subject: Adds branch support to deployer.
X-Git-Url: http://git.less.ly:3516/?a=commitdiff_plain;h=b56d5c30c257260618e14bb748aaa4c30d56d233;p=kraken-ui.git

Adds branch support to deployer.
---

diff --git a/fabfile/__init__.py b/fabfile/__init__.py
index e13d825..60acf02 100644
--- a/fabfile/__init__.py
+++ b/fabfile/__init__.py
@@ -34,40 +34,51 @@ from util import *
 
 # TODO: env.rcfile = 'fabfile/fabricrc.conf'
 
-# We `update()` here so --set on the commandline will override.
-for k, v in dict(
-    project_name       = 'kraken-ui',
+# We use `defaults()` here so --set on the commandline will override.
+defaults(env, dict(
+    project_name       = 'GraphKit',
     colors             = True,
     use_ssh_config     = True,
     
+    git_origin         = 'git@less.ly:kraken-ui.git',
     dev_server         = 'localhost:8081',
     minify_cmd         = 'uglifyjs',
     
+    supervisor_job     = 'reportcard',
+    
+    ### Paths
     dist               = 'dist',
     local_tmp          = 'tmp',
+    work_dir           = '%(local_tmp)s/%(dist)s',
+    
     browserify_js      = 'vendor/browserify.js',
-    supervisor_job     = 'reportcard',
-).iteritems():
-    env.setdefault(k, v)
-
-for k in ('dist', 'local_tmp',):
+    work_browserify_js = '%(work_dir)s/%(browserify_js)s',
+    
+    vendor_search_dirs = ['static', 'var', '%(work_dir)s']
+    vendor_bundle      = '%(work_dir)s/vendor/vendor-bundle.min.js',
+    app_bundle         = '%(work_dir)s/js/kraken/app-bundle.js',
+))
+
+env_paths = (
+    'dist', 'local_tmp', 'work_dir', 
+    'browserify_js', 'work_browserify_js',
+    'vendor_bundle', 'app_bundle',
+)
+for k in env_paths:
     env[k] = p(env[k])
 
-env.work_dir           = env.local_tmp/env.dist
-env.work_browserify_js = env.work_dir/env.browserify_js
-env.vendor_search_dirs = map(p, ['static', 'var', env.work_dir])
-env.vendor_bundle      = env.work_dir/'vendor/vendor-bundle.min.js'
-env.app_bundle         = env.work_dir/'js/kraken/app-bundle.js'
+env.vendor_search_dirs = [ expand(p(vd)) for vd in env.vendor_search_dirs ]
 env.app_bundle_min     = p(env.app_bundle.replace('.js', '.min.js'))
 
 
+
 ### Setup Staging Environments
 
 # Envs aren't declared with @task in the stages module so that we can
 # decorate them here and have them show up at the top level.
 import stages
 
-for name in stages.NAMES:
+for name in stages.STAGE_NAMES:
     globals()[name] = task(getattr(stages, name))
 
 
diff --git a/fabfile/bundle.py b/fabfile/bundle.py
index 0884c7f..33956a3 100644
--- a/fabfile/bundle.py
+++ b/fabfile/bundle.py
@@ -9,6 +9,7 @@ from util import *
 
 
 @task(default=True)
+@expand_env
 def bundle_all():
     """ Bundles vendor and application files.
     """
@@ -18,6 +19,7 @@ def bundle_all():
     bundle_app()
 
 @task
+@expand_env
 @msg('Collapsing Serve Trees')
 def collapse_trees():
     """ Collapse the serve trees into one directory.
@@ -37,7 +39,7 @@ def collapse_trees():
     local('rsync -Ca static/ var/ data %(work_dir)s/' % env)
     
     # We copy lib (which contains .co source files) to src to make it easy to link source content
-    # to each other. Finding it in gitweb is a pain. Finding it in gerrit is almost impossible. 
+    # to each other. Finding it in gitweb is a pain. Finding it in gerrit is almost impossible.
     # But this could go away when we move to github.
     local('rsync -Ca lib/ %(work_dir)s/src/' % env)
     
@@ -48,6 +50,7 @@ def collapse_trees():
         f.write( local('curl --silent --fail --url http://%(dev_server)s/%(browserify_js)s' % env, capture=True) )
 
 @task
+@expand_env
 @msg('Building Vendor Bundle')
 def bundle_vendor():
     """ Bundles vendor files.
@@ -68,6 +71,7 @@ def bundle_vendor():
         vendor_bundle.write('\n')
 
 @task
+@expand_env
 @msg('Building App Bundle')
 def bundle_app():
     """ Bundles and minifies app files.
diff --git a/fabfile/deploy.py b/fabfile/deploy.py
index 4111088..6f7e6cb 100644
--- a/fabfile/deploy.py
+++ b/fabfile/deploy.py
@@ -4,6 +4,7 @@
 
 from fabric.api import *
 from fabric.colors import white, blue, cyan, green, yellow, red, magenta
+from fabric.contrib.files import exists
 from fabric.contrib.project import rsync_project
 
 from stages import ensure_stage
@@ -12,18 +13,21 @@ from util import *
 
 
 @task(default=True)
+@expand_env
 @ensure_stage
 def deploy_and_update():
     """ Deploy the project.
     """
     fix_permissions()
-    pull()
+    update_branch()
     sync_files()
     fix_permissions()
     restart_node()
 
 @task
+@expand_env
 @ensure_stage
+@msg('Fixing Permissions')
 def fix_permissions(user=None, group=None):
     """ Recursively fixes permissions on the deployment host.
     """
@@ -33,14 +37,42 @@ def fix_permissions(user=None, group=None):
     sudo('chown -R %s:%s %s' % (user, group, env.target_dir))
 
 @task
+@expand_env
 @ensure_stage
-def pull():
+@msg('Cloning Origin')
+def clone():
+    """ Clones source on deployment host if not present.
+    """
+    if exists(env.target_dir): return
+    with cd(env.target_dir.dirname()):
+        run('git clone %(git_origin)s' % env)
+
+@task
+@expand_env
+@ensure_stage
+def checkout():
+    """ Checks out proper branch on deployment host.
+    """
+    # TODO: Locally saved data files will cause yelling?
+    with cd(env.target_dir):
+        run('git fetch --all')
+        run('git checkout --track origin/%(git_branch)s' % env)
+
+@task
+@expand_env
+@ensure_stage
+@msg('Updating Branch')
+def update_branch():
     """ Runs git pull on the deployment host.
     """
-    with cd(env.target_dir): run('git pull')
+    with cd(env.target_dir):
+        execute(checkout)
+        run('git pull origin %(git_branch)s' % env)
 
 @task
+@expand_env
 @ensure_stage
+@msg('Syncing Files')
 def sync_files():
     """ Copies `dist` package to deployment host.
     """
@@ -49,7 +81,9 @@ def sync_files():
     # rsync_project(local_dir=env.work_dir, remote_dir="%(user)s@%(host)s:%(target_dir)s/" % env)
 
 @task
+@expand_env
 @ensure_stage
+@msg('Restarting Node.js')
 def restart_node():
     """ Restarts node.js server on the deployment host.
     """
diff --git a/fabfile/stages.py b/fabfile/stages.py
index 2b58c72..21ba7d2 100644
--- a/fabfile/stages.py
+++ b/fabfile/stages.py
@@ -6,23 +6,31 @@ import sys
 from functools import wraps
 from fabric.api import env, abort, prompt, execute
 from fabric.colors import white, blue, cyan, green, yellow, red, magenta
+from fabric.contrib.console import confirm
 
-__all__ = ['NAMES', 'validate_stage', 'prompt_for_stage', 'ensure_stage', 'stage',]
 
-# (otto) There should be a way to do this using stages.
-# See: http://tav.espians.com/fabric-python-with-cleaner-api-and-parallel-deployment-support.html
-# (dsc) ...except that most of those changes are not actually in Fabric :(
+__all__ = [
+    'STAGE_NAMES', 'prompt_for_stage', 'ensure_stage', 
+    'working_branch', 'check_branch',
+]
 
-# env.config_file = False
-# env.stages = ['prod', 'staging']
 
+STAGE_NAMES = []
 
-NAMES = []
+def stage(fn):
+    """ Decorator indicating this function sets a stage environment.
+    """
+    STAGE_NAMES.append(fn.__name__)
+    __all__.append(fn.__name__)
+    return fn
 
 def validate_stage(name):
-    "Tests whether given name is a valid staging environment."
+    """ Tests whether given name is a valid staging environment.
+        
+            name = fabric.api.prompt(msg, validate=validate_stage)
+    """
     name = name.strip()
-    if name not in NAMES:
+    if name not in STAGE_NAMES:
         raise Exception("%r is not a valid staging environment!" % name)
     return name
 
@@ -33,14 +41,13 @@ def prompt_for_stage(fn):
     @wraps(fn)
     def wrapper(*args, **kwargs):
         if 'deploy_env' not in env:
-            name = prompt(white('Please select a staging target %s:' % NAMES, bold=True), validate=validate_stage)
+            name = prompt(white('Please select a staging target %s:' % STAGE_NAMES, bold=True), validate=validate_stage)
             execute(name)
         # Must call `execute` on the supplied task, otherwise the host-lists won't be updated.
         execute(fn, *args, **kwargs)
     
     return wrapper
 
-
 def ensure_stage(fn):
     "Decorator that ensures a stage is set."
     
@@ -52,18 +59,33 @@ def ensure_stage(fn):
     
     return wrapper
 
-def stage(fn):
-    """ Decorator indicating this function sets a stage environment:
-        
-            @stage
-            def mystage():
-                env.deploy_env = 'mystage'
-                ...
-        
-    """
-    NAMES.append(fn.__name__)
-    __all__.append(fn.__name__)
-    return fn
+
+def working_branch():
+    "Determines the working branch."
+    return [ branch.split()[1] for branch in local('git branch', capture=True).split('\n') if '*' in branch ][0]
+
+def check_branch(fn):
+    "Decorator that ensures deploy branch matches current branch."
+    
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        env.working_branch = working_branch()
+        if env.working_branch != env.git_branch:
+            question = 'Your working branch %(working_branch)r does not match the expected branch %(git_branch)r for %(deploy_env)s. Proceed anyway?' % env
+            confirm(yellow(question, bold=True))
+        return fn(*args, **kwargs)
+    
+    return wrapper
+
+
+### Stages
+
+# env.stages = ['prod', 'staging']
+
+# (otto) There should be a way to do this using stages.
+# See: http://tav.espians.com/fabric-python-with-cleaner-api-and-parallel-deployment-support.html
+# (dsc) ...except that most of those changes are not actually in Fabric :(
+
 
 
 ###
@@ -81,9 +103,11 @@ def prod():
     env.hosts      = ['reportcard2.pmtpa.wmflabs']
     env.gateway    = 'bastion.wmflabs.org'
     env.target_dir = '/srv/reportcard/kraken-ui'
+    env.git_branch = 'master'
     env.owner      = 'www-data'
     env.group      = 'www'
 
+
 @stage
 def test():
     """ Set deploy environment to test.
@@ -92,10 +116,12 @@ def test():
     env.hosts      = ['kripke.pmtpa.wmflabs']
     env.gateway    = 'bastion.wmflabs.org'
     env.target_dir = '/srv/test-reportcard.wmflabs.org/kraken-ui'
+    env.git_branch = 'rc'
     env.owner      = 'www-data'
     env.group      = 'www'
     env.supervisor_job = 'test-reportcard'
 
+
 @stage
 def dev():
     """ Set deploy environment to dev.
@@ -104,10 +130,12 @@ def dev():
     env.hosts      = ['kripke.pmtpa.wmflabs']
     env.gateway    = 'bastion.wmflabs.org'
     env.target_dir = '/srv/dev-reportcard.wmflabs.org/kraken-ui'
+    env.git_branch = 'develop'
     env.owner      = 'www-data'
     env.group      = 'www'
     env.supervisor_job = 'dev-reportcard'
 
+
 @stage
 def lessly():
     """ Set deploy environment to lessly.
@@ -115,6 +143,7 @@ def lessly():
     env.deploy_env = 'lessly'
     env.hosts      = ['less.ly']
     env.target_dir = '/home/wmf/projects/kraken-ui'
+    env.git_branch = 'develop'
     env.owner      = 'wmf'
     env.group      = 'www'
     del env['gateway']
diff --git a/fabfile/util.py b/fabfile/util.py
index 736cbb4..c5e7ef5 100644
--- a/fabfile/util.py
+++ b/fabfile/util.py
@@ -4,13 +4,19 @@
 from __future__ import with_statement
 from contextlib import contextmanager
 from functools import wraps
+from path import path as p # renamed to avoid conflict w/ fabric.api.path
 
 from fabric.api import *
 from fabric.colors import white, blue, cyan, green, yellow, red, magenta
 
-__all__ = ('quietly', 'msg', 'coke', 'update_version',)
+__all__ = (
+    'quietly', 'msg', 'coke', 'update_version',
+    'defaults', 'expand', 'expand_env', 'format', 'expand_env',
+)
 
 
+### Context Managers
+
 @contextmanager
 def quietly(txt):
     "Wrap a block in a message, suppressing other output."
@@ -19,6 +25,8 @@ def quietly(txt):
     puts("woo.", show_prefix=False, flush=True)
 
 
+### Decorators
+
 def msg(txt, quiet=False):
     "Decorator to wrap a task in a message, optionally suppressing all output."
     def outer(fn):
@@ -38,13 +46,12 @@ def msg(txt, quiet=False):
     return outer
 
 
-
-### Misc
+### Coke Integration
 
 def coke(args, capture=False):
     """ Invokes project Cokefile.
     """
-    local('coke %s' % args, capture=capture)
+    return local('coke %s' % args, capture=capture)
 
 @runs_once
 def update_version():
@@ -54,3 +61,48 @@ def update_version():
     print ''
 
 
+
+### Misc
+
+def defaults(target, *sources):
+    "Update target dict using `setdefault()` for each key in each source."
+    for source in sources:
+        for k, v in source.iteritems():
+            target.setdefault(k, v)
+    return target
+
+def expand(s):
+    "Recursively expands given string using the `env` dict."
+    is_path = isinstance(s, p)
+    prev = None
+    while prev != s:
+        prev = s
+        s = s % env
+    return p(s) if is_path else s
+
+def format(s):
+    "Recursively formats string using the `env` dict."
+    is_path = isinstance(s, p)
+    prev = None
+    while prev != s:
+        prev = s
+        s = s.format(**env)
+    return p(s) if is_path else s
+
+
+@runs_once
+def _expand_env():
+    for k, v in env.iteritems():
+        if not isinstance(v, basestring): continue
+        env[k] = expand(env[k])
+
+def expand_env(fn):
+    "Decorator expands all strings in `env`."
+    
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        _expand_env()
+        return fn(*args, **kwargs)
+    
+    return wrapper
+