From: David Schoonover Date: Thu, 5 Jul 2012 21:16:13 +0000 (-0700) Subject: Refactors server into middleware; changes 'dev' to 'development' when testing environ... X-Git-Url: http://git.less.ly:3516/?a=commitdiff_plain;h=12916d14b6c7fa95717f3ef7bc37d1972cb39e21;p=limn.git Refactors server into middleware; changes 'dev' to 'development' when testing environment, as per node convention. --- diff --git a/Cokefile b/Cokefile index efbf256..b457b03 100644 --- a/Cokefile +++ b/Cokefile @@ -4,8 +4,7 @@ require 'buildtools/tasks' {removeSync:remove} = require 'remove' -MODULE_LINK = 'node_modules/kraken' -EXPRESS_DEP_MIME = 'node_modules/express/node_modules/mime' +MODULE_LINK = 'node_modules/kraken' @@ -20,10 +19,6 @@ task \link 'Link package source to node_modules so the name resolves correctly' task \install 'Install project dependencies.' -> <- sh 'npm install', {-verbose, +errors, +die} - - # Remove express's copy of node-mime so we can add our own extensions - # and have it pick them up by default. - remove EXPRESS_DEP_MIME, {+ignoreMissing} task \setup 'Ensure project is set up for development.' -> invoke \install @@ -76,7 +71,7 @@ task \cleanup_tests 'Removes compiled tests' -> task \clean 'Clean up environment and artifacts' -> invoke \cleanup_tests - remove [MODULE_LINK, EXPRESS_DEP_MIME, 'var', 'tmp/dist'], {+ignoreMissing, +verbose} + remove [MODULE_LINK, 'var', 'tmp/dist'], {+ignoreMissing, +verbose} @@ -84,7 +79,7 @@ task \source_list 'Print a list of the source file paths.' -> invoke \setup {sources} = require 'kraken/server/view-helpers' say do - sources("www/modules.yaml", 'dev') + sources("www/modules.yaml", 'development') .map -> it.slice 1 .join '\n' diff --git a/lib/server/middleware.co b/lib/server/middleware.co new file mode 100755 index 0000000..6cc9e73 --- /dev/null +++ b/lib/server/middleware.co @@ -0,0 +1,256 @@ +#!/usr/bin/env coco + + +fs = {existsSync:exists} = require 'fs' +path = {dirname} = require 'path' +subproc = {exec, spawn} = require 'child_process' + +glob = require 'glob' +yaml = require 'js-yaml' +Seq = require 'seq' +express = require 'express' + +{_, op} = require '../util' +{mkdirp, mkdirpAsync} = require './mkdirp' +{readFilesAsync} = require './files' +Controller = require './controller' + + + +/** + * Limn project-internals + */ +BASE = dirname dirname __dirname +WWW = "#BASE/www" +VAR = "#BASE/var" +STATIC = "#BASE/static" +DIST = "#BASE/dist" +DATA = "#BASE/data" + +NODE_ENV = process.env.NODE_ENV or 'development' +IS_DEV = NODE_ENV is 'development' +IS_PROD = NODE_ENV is 'production' + +LOG_LEVEL = process.env.KRAKEN_LOG_LEVEL +LOG_LEVEL or= if IS_DEV then 'INFO' else 'WARN' +LOG_LEVEL .= toUpperCase() + +REV = process.env.KRAKEN_REV or 'HEAD' +try REV = require '../version' catch e + + + +DEFAULT_OPTIONS = + dataDir : './data' # Path to directory where data files should be written. + # datasetDir : null # Path to a directory to statically serve as `/data` if present. + proxy : + enabled : false # Enables remote dataset proxy. If omitted, the proxy will be enabled if either `proxy.whitelist` or `proxy.blacklist` are set. + whitelist : null # Array of domain patterns to whitelist for proxy. Strings are matched via glob-like syntax, but regular expressions may also be passed. If `proxy.enabled` is true but no whitelist is provided, it defaults to `['*']`. + blacklist : null # Array of domain patterns to blacklist from proxying. Strings are matched via glob-like syntax, but regular expressions may also be passed. + + + + +# expose API +exports = module.exports = limn + + +/** + * Create a new instance of the Limn middleware. + * @param {Object} [options={}] Options: + */ +function limn(options) + app = express.createServer() + app = _.extend app, application + app.init() + app + + + +application = limn.application = + + /** + * @constructor + */ + init : (opts={}) -> + @REV = REV + @BASE = BASE + + ### setup options with defaults + opts = _.merge {}, DEFAULT_OPTIONS, opts + opx = opts.proxy + opx.enabled = true if opx.enabled is false and (opx.whitelist or opx.blacklist) + if opx.enabled + opx.whitelist or= [ /.*/ ] + opx.blacklist or= [] + @set 'limn options', opts + + + ### apply configuration + + # config shared by all environments + @configure -> + @set 'views', WWW + @set 'view engine', 'jade' + @set 'view options', { + layout : false + version : REV + } import require './view-helpers' + + # @helpers require './view-helpers' + + # Parse URL, fiddle + @use require('./reqinfo')({}) + + # Parse form submits into req.params + @use express.bodyParser() + + # Allow "spoofing" HTTP methods that IE doesn't support + @use express.methodOverride() + + # Route to the web services + @use @router + + # Browserify some deps + @use require('browserify') do + mount : '/vendor/browserify.js' + require : <[ seq d3 events ]> + cache : "#BASE/.cache/browserify/cache.json" + + # production config + @configure 'production', -> + # log all requests + @use express.logger() + + # set a one year max-age for static files + @set 'static max age', 31_557_600_000_ms + + # show simple errors + @use express.errorHandler() + + + # development config + @configure 'development', -> + # show exceptions, pretty stack traces + @use express.errorHandler { +dumpExceptions, +showStack } + + # turn on pretty-printing of views + @set('view options').pretty = true + + # import dev-only deps here, so prod users don't need them + compiler = require 'connect-compiler-extras' + + # transparently recompile modules that have changed + @use compiler do + enabled : <[ coco jade-browser stylus yaml ]> + src : WWW + dest : VAR + options : stylus : { nib:true, include:"#WWW/css" } + log_level : LOG_LEVEL + @use compiler do + enabled : 'yaml' + src : DATA + dest : "#VAR/data" + log_level : LOG_LEVEL + + # wrap modules in commonjs closure for browser + @use compiler do + enabled : 'commonjs_define' + src : [ STATIC ] + dest : VAR + options : + commonjs : { drop_path_parts:1, drop_full_ext:false } + commonjs_define : { drop_path_parts:1, drop_full_ext:false } + log_level : LOG_LEVEL + @use compiler do + enabled : 'commonjs_define' + src : [ VAR, WWW ] + dest : VAR + options : + commonjs : { drop_path_parts:1, drop_full_ext:true } + commonjs_define : { drop_path_parts:1, drop_full_ext:true } + log_level : LOG_LEVEL + + # serve static files + @configure -> + opts = @set('static file options') or {} + @use express.static DIST, ^opts if exists DIST + @use express.static WWW, ^opts + @use express.static VAR, ^opts + @use express.static STATIC, ^opts + + + + # data controllers + @controller require './controllers/graph' + @controller require './controllers/dashboard' + + YAML_EXT_PAT = /\.ya?ml$/i + @get '/datasources/all', (req, res, next) -> + data = {} + Seq() + .seq glob, 'data/datasources/**/*.@(yaml|json)', {+nocase, +nosort}, Seq + .seq (paths) -> readFilesAsync paths, this + .seq (txts) -> @ok _.items txts + .flatten false + .parMap ([f, text]) -> + # console.log "parsing file[#i]: '#f' -> text[#{text.length}]..." + k = f.replace YAML_EXT_PAT, '.json' + v = data[k] = {} + try + if YAML_EXT_PAT.test f + v = data[k] = yaml.load text + else + v = data[k] = JSON.parse text + # console.log "#f ok!", data + @ok v + catch err + console.error "[/data/all] catch! #err" + console.error err + console.error that if err.stack + res.send { error:String(err), partial_data:data } + .seq -> res.send data + .catch (err) -> + console.error '[/data/all] catch!' + console.error err + console.error that if err.stack + res.send { error:String(err), partial_data:data } + + @controller require './controllers/datasource' + + + # proxy endpoint + if opts.proxy.enabled + proxy = require('./proxy')({ + blacklist: opts.proxy.blacklist + whitelist: opts.proxy.whitelist + }) + @get '/x', proxy + @get '/x/*', proxy + + + # non-controller endpoints + @get '/', (req, res) -> + res.render 'dashboard' + + @get '/geo', (req, res) -> + res.render 'geo' + + @get '/:type/:action/?', (req, res, next) -> + {type, action} = req.params + if exists "#WWW/#type/#action.jade" + res.render "#type/#action" + else + next() + + @get '/:type/?', (req, res, next) -> + {type} = req.params + if exists "#WWW/#type.jade" + res.render "#type" + else + next() + + this + + + diff --git a/lib/server/server.co b/lib/server/server.co index 1f8788a..7c7459b 100755 --- a/lib/server/server.co +++ b/lib/server/server.co @@ -1,227 +1,18 @@ #!/usr/bin/env coco -fs = require 'fs' -path = require 'path' -glob = require 'glob' -{existsSync:exists} = path -{exec, spawn} = require 'child_process' -{mkdirp, mkdirpAsync} = require './mkdirp' -{ readFilesAsync, -} = require './files' +fs = require 'fs' +path = require 'path' +{exec, spawn, +} = require 'child_process' +exists = fs.existsSync or path.existsSync -Seq = require 'seq' -yaml = require 'js-yaml' -mime = require 'mime' -express = require 'express' -# Resource = require 'express-resource' -compiler = require 'connect-compiler-extras' -_ = require '../util/underscore' -op = require '../util/op' -files = require './files' +express = require 'express' +LimnMiddleware = require './middleware' - -### Config - -CWD = process.cwd() -WWW = "#CWD/www" -VAR = "#CWD/var" -STATIC = "#CWD/static" -DIST = "#CWD/dist" -DATA = "#CWD/data" - -PORT = 8081 -PORT = parseInt(that, 10) if process.env.KRAKEN_PORT - -NODE_ENV = process.env.NODE_ENV or 'dev' - -LOG_LEVEL = process.env.KRAKEN_LOG_LEVEL -LOG_LEVEL or= if _.startsWith NODE_ENV, 'dev' then 'INFO' else 'WARN' -LOG_LEVEL .= toUpperCase() - -USE_LOGGER = op.parseBool(process.env.KRAKEN_USE_LOGGER) or LOG_LEVEL is 'DEBUG' or NODE_ENV is 'prod' - -VERSION = process.env.KRAKEN_VERSION or 'dev' -err, stdout, stderr <- exec 'git rev-parse --short HEAD', {cwd:CWD, env:process.env} -throw err if err -VERSION = stdout.trim! - - -### Setup - -app = express.createServer() - -app.start = -> - console.log s = "Kraken Server (port=#PORT, env=#NODE_ENV, version=#VERSION)" - console.log '=' * s.length - app.listen PORT - app - -app.configure -> - mime.define 'text/plain' : <[ jade co styl stylus yaml md ]> - - # Set up - app.set 'views', WWW - app.set 'view engine', 'jade' - app.set 'view options', { - layout : false - pretty : true - version : VERSION - } import require './view-helpers' - - app.use express.logger() if USE_LOGGER - - # Parse URL, fiddle - app.use require('./reqinfo')({}) - - # Parse form submits into req.params - app.use express.bodyParser() - - # Allow "spoofing" HTTP methods that IE doesn't support - app.use express.methodOverride() - - # Route to the web services - app.use app.router - - # Transparently recompile modules that have changed - app.use compiler do - enabled : <[ coco jade-browser stylus yaml ]> - src : WWW - dest : VAR - options : stylus : { nib:true, include:"#WWW/css" } - log_level : LOG_LEVEL - - app.use compiler do - enabled : 'yaml' - src : DATA - dest : "#VAR/data" - log_level : LOG_LEVEL - - # wrap modules in commonjs closure for browser - app.use compiler do - enabled : 'commonjs_define' - src : [ STATIC ] - dest : VAR - options : - commonjs : { drop_path_parts:1, drop_full_ext:false } - commonjs_define : { drop_path_parts:1, drop_full_ext:false } - log_level : LOG_LEVEL - app.use compiler do - enabled : 'commonjs_define' - src : [ VAR, WWW ] - dest : VAR - options : - commonjs : { drop_path_parts:1, drop_full_ext:true } - commonjs_define : { drop_path_parts:1, drop_full_ext:true } - log_level : LOG_LEVEL - app.use require('browserify') do - mount : '/vendor/browserify.js' - require : <[ seq d3 events ]> - cache : false - # cache : "#CWD/.cache/browserify/cache.json" - - # Serve static files - app.use express.static WWW - app.use express.static VAR - app.use express.static STATIC - # app.use express.static DIST if exists DIST - app.use express.static DIST if NODE_ENV is 'prod' - - # Serve directory listings - # app.use express.directory WWW - # app.use express.directory VAR - # app.use express.directory STATIC - # app.use express.directory DIST - - app.use express.errorHandler do - dumpExceptions : true - showStack : true - - - -/* * * * Routes and Controllers * * * {{{ */ - -Controller = require './controller' -app.controller require './controllers/graph' -app.controller require './controllers/dashboard' - -YAML_EXT_PAT = /\.ya?ml$/i -app.get '/datasources/all', (req, res, next) -> - data = {} - Seq() - .seq glob, 'data/datasources/**/*.@(yaml|json)', {+nocase, +nosort}, Seq - .seq (paths) -> readFilesAsync paths, this - .seq (txts) -> @ok _.items txts - .flatten false - .parMap ([f, text]) -> - # console.log "parsing file[#i]: '#f' -> text[#{text.length}]..." - k = f.replace YAML_EXT_PAT, '.json' - v = data[k] = {} - try - if YAML_EXT_PAT.test f - v = data[k] = yaml.load text - else - v = data[k] = JSON.parse text - # console.log "#f ok!", data - @ok v - catch err - console.error "[/data/all] catch! #err" - console.error err - console.error that if err.stack - res.send { error:String(err), partial_data:data } - .seq -> res.send data - .catch (err) -> - console.error '[/data/all] catch!' - console.error err - console.error that if err.stack - res.send { error:String(err), partial_data:data } - -app.controller require './controllers/datasource' - - -# Data Proxy -PROXY_DOMAIN_BLACKLIST = <[ - *.ru -]> -PROXY_DOMAIN_WHITELIST = <[ - wmflabs.org mediawiki.org wikimedia.org wikipedia.org - *.wmflabs.org *.mediawiki.org *.wikimedia.org *.wikipedia.org -]> - -if NODE_ENV is 'dev' - PROXY_DOMAIN_WHITELIST.push 'localhost', 'github.com', '*.github.com' - - -proxy = require('./proxy')({ - blacklist: PROXY_DOMAIN_BLACKLIST - whitelist: PROXY_DOMAIN_WHITELIST -}) -app.get '/x', proxy -app.get '/x/*', proxy - - - -app.get '/', (req, res) -> - res.render 'dashboard' - -app.get '/geo', (req, res) -> - res.render 'geo' - -app.get '/:type/:action/?', (req, res, next) -> - {type, action} = req.params - if exists "#WWW/#type/#action.jade" - res.render "#type/#action" - else - next() - -app.get '/:type/?', (req, res, next) -> - {type} = req.params - if exists "#WWW/#type.jade" - res.render "#type" - else - next() +app = exports = module.exports = express.createServer() /** * Handle webhook notification to pull from origin. @@ -241,15 +32,27 @@ app.all '/webhook/post-update', (req, res) -> else res.send "$ #cmd\n\n#stdout\n\n#stderr", 200 -# }}} +/** + * Load Limn middleware + */ +app.use limn = app.limn = LimnMiddleware() + -exports import { - CWD, WWW, VAR, STATIC, - PORT, LOG_LEVEL, NODE_ENV, VERSION, - app, module, require, -} mainfile = path.basename require.main?.filename if require.main is module or 'Cokefile' is mainfile - app.start() + PORT = 8081 + PORT = parseInt(that, 10) if process.env.KRAKEN_PORT + + NODE_ENV = process.env.NODE_ENV or 'development' + + REV = process.env.KRAKEN_REV or 'HEAD' + try REV = require '../version' catch e + err, stdout, stderr <- exec 'git rev-parse --short HEAD', {cwd:process.cwd(), env:process.env} + throw err if err + REV = stdout.trim! unless REV + + console.log s = "Kraken Server (port=#PORT, env=#NODE_ENV, rev=#REV, base_dir=#{limn.BASE})" + console.log '=' * s.length + app.listen PORT diff --git a/lib/server/view-helpers.co b/lib/server/view-helpers.co index 70a7c66..c96ccc7 100644 --- a/lib/server/view-helpers.co +++ b/lib/server/view-helpers.co @@ -5,15 +5,12 @@ WWW = exports.WWW = "#CWD/www" VAR = exports.VAR = "#CWD/var" STATIC = exports.STATIC = "#CWD/static" -VERSION = 'dev' +VERSION = 'HEAD' try VERSION = require '../version' catch e -exports.VERSION = VERSION +exports.VERSION = exports.version = VERSION -# maybe! -exports.require = require - -fs = exports.fs = require 'fs' -path = exports.path = require 'path' +fs = exports.fs = require 'fs' +path = exports.path = require 'path' _ = exports._ = require 'underscore' _.str = require 'underscore.string' @@ -24,22 +21,23 @@ yaml = exports.yaml = require 'js-yaml' jade = exports.jade = require 'jade' exports.env = process.env -NODE_ENV = exports.NODE_ENV = (process.env.NODE_ENV or 'dev').toLowerCase() -# IS_PROD = exports.IS_PROD = _ NODE_ENV .startsWith 'prod' -# IS_TEST = exports.IS_TEST = _ NODE_ENV .startsWith 'test' -# IS_DEV = exports.IS_DEV = not (IS_PROD or IS_TEST) or _.startsWith NODE_ENV, 'dev' -# # Canonicalize -# NODE_ENV = exports.NODE_ENV = if IS_PROD then 'prod' else if IS_TEST then 'test' else 'dev' +NODE_ENV = exports.NODE_ENV = (process.env.NODE_ENV or 'development').toLowerCase() +IS_PROD = exports.IS_PROD = NODE_ENV is 'production' +IS_TEST = exports.IS_TEST = NODE_ENV is 'test' +IS_DEV = exports.IS_DEV = not (IS_PROD or IS_TEST) + + +SOURCES_ENV = if process.env.KRAKEN_FORCE_BUNDLES then 'production' else NODE_ENV -SOURCES_ENV = if process.env.KRAKEN_FORCE_BUNDLES then 'prod' else NODE_ENV +/** + * Reify a modules.yaml file + */ sources = exports.sources = (modulesFile, node_env=SOURCES_ENV) -> - # node_env = 'dev' if _.startsWith node_env, 'dev' mods = yaml.load fs.readFileSync modulesFile, 'utf8' modlist = (mods.all or []).concat (mods[node_env] or []) _.flatten modlist.map ({suffix or '', paths}) -> joinTree('', paths).map -> it+suffix - joinTree = exports.joinTree = function joinTree(root, tree) return ["#root/#tree"] if typeof tree is 'string' _ tree .reduce do @@ -52,6 +50,3 @@ joinTree = exports.joinTree = function joinTree(root, tree) acc [] - -# util = exports.util = require '../util' - diff --git a/lib/template/dashboard/dashboard.jade b/lib/template/dashboard/dashboard.jade index 5081dbd..2445010 100644 --- a/lib/template/dashboard/dashboard.jade +++ b/lib/template/dashboard/dashboard.jade @@ -10,13 +10,13 @@ section#dashboard.centered li: h3 Graphs li.active: a(href="#core-graphs", data-toggle="tab") Core li: a(href="#other-graphs", data-toggle="tab") Others - if ENV === 'dev' + if IS_DEV li: a(href="#dev-graphs", data-toggle="tab") Dev .tab-content .core-graphs-pane.tab-pane.active(id="core-graphs") .other-graphs-pane.tab-pane(id="other-graphs") - if ENV === 'dev' + if IS_DEV .dev-graphs-pane.tab-pane(id="dev-graphs") diff --git a/lib/template/graph/graph-display.jade b/lib/template/graph/graph-display.jade index 24de1a0..9097a84 100644 --- a/lib/template/graph/graph-display.jade +++ b/lib/template/graph/graph-display.jade @@ -25,7 +25,7 @@ section.graph-display.graph(id=graph_id) .graph-links-row.row .span6.offset3.ug.graph-permalink input.span6(value="#{model.toPermalink()}", readonly="readonly") - if ENV == 'dev' + if IS_DEV a.span1.export-button.btn(href="#") i.icon-file | Export diff --git a/package.co b/package.co index fd932fd..13371fc 100644 --- a/package.co +++ b/package.co @@ -26,13 +26,12 @@ dependencies : 'glob' : '== 3.1.9' 'findit' : '== 0.1.2' 'remove' : '>= 0.1.1' - 'mime' : '== 1.2.5' - 'express' : '== 2.5.9' + 'express' : '== 2.5.11' 'express-resource' : '== 0.2.4' 'connect-compiler' : 'https://github.com/dsc/connect-compiler/tarball/master' 'connect-compiler-extras' : 'https://github.com/dsc/connect-compiler-extras/tarball/master' 'jade' : '== 0.22.1' - 'stylus' : '== 0.25.0' + 'stylus' : '== 0.27.2' 'nib' : '== 0.4.0' 'browserify' : '== 1.9.4' 'backbone' : '== 0.9.1' @@ -46,5 +45,5 @@ devDependencies : # scripts : test:'expresso' repository : type:'git', url:'git://less.ly/kraken-ui.git' -engine : node:'>=0.6.2' +engine : node:'>=0.8.1' license : 'MIT' diff --git a/package.json b/package.json index 624f0aa..ce86a74 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,12 @@ "glob": "== 3.1.9", "findit": "== 0.1.2", "remove": ">= 0.1.1", - "mime": "== 1.2.5", - "express": "== 2.5.9", + "express": "== 2.5.11", "express-resource": "== 0.2.4", "connect-compiler": "https://github.com/dsc/connect-compiler/tarball/master", "connect-compiler-extras": "https://github.com/dsc/connect-compiler-extras/tarball/master", "jade": "== 0.22.1", - "stylus": "== 0.25.0", + "stylus": "== 0.27.2", "nib": "== 0.4.0", "browserify": "== 1.9.4", "backbone": "== 0.9.1", @@ -50,7 +49,7 @@ "url": "git://less.ly/kraken-ui.git" }, "engine": { - "node": ">=0.6.2" + "node": ">=0.8.1" }, "license": "MIT" } diff --git a/www/layout.jade b/www/layout.jade index 391f61f..b7d9e7a 100644 --- a/www/layout.jade +++ b/www/layout.jade @@ -41,7 +41,7 @@ html(lang="en", dir="ltr") i.icon-home.icon-white a(href="/") Home //- FIXME: put link to /graphs only if dev env, for now. - if NODE_ENV == 'dev' + if IS_DEV li i.icon-th.icon-white a(href='/graphs') Browse diff --git a/www/mixins/helpers.jade b/www/mixins/helpers.jade index 9ca8302..a9ccc24 100644 --- a/www/mixins/helpers.jade +++ b/www/mixins/helpers.jade @@ -1,6 +1,6 @@ - root = (function(){ return this; })(); -- if (typeof version == 'undefined') root.version = 'dev'; -- if (version == null || !version || version === 'dev') { version = String(new Date().getTime()); } +- if (typeof version == 'undefined') root.version = 'HEAD'; +- if (version == null || !version || version === 'HEAD') { version = String(new Date().getTime()); } mixin css(href, media, path_root) - media = media || 'screen' diff --git a/www/modules.yaml b/www/modules.yaml index 0f884a3..b538834 100644 --- a/www/modules.yaml +++ b/www/modules.yaml @@ -1,12 +1,12 @@ all: [] -prod: +production: - suffix: .js paths: - vendor/vendor-bundle.min - js/kraken/app-bundle -dev: +development: - suffix: .js paths: - vendor: @@ -80,7 +80,7 @@ dev: - timeseries - csv - index - - op + - op - backbone - parser - cascade