From 157ab4b27e195e4df177d14f0e87e605aafa3b97 Mon Sep 17 00:00:00 2001 From: dsc Date: Sat, 27 Nov 2010 03:56:43 -0800 Subject: [PATCH] Adds module builder. --- .gitignore | 1 + bin/cjs.py | 293 ++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/cjs/module.js | 17 +++ lib/cjs/require.js | 50 +++++++++ tanks.php | 17 +++- test/jsc/bar.js | 3 - test/jsc/baz.js | 2 - test/jsc/foo.js | 2 - test/jsc/jsc.py | 198 ----------------------------------- test/jsc/lol.js | 1 - 10 files changed, 377 insertions(+), 207 deletions(-) create mode 100755 bin/cjs.py create mode 100644 lib/cjs/module.js create mode 100644 lib/cjs/require.js delete mode 100644 test/jsc/bar.js delete mode 100644 test/jsc/baz.js delete mode 100644 test/jsc/foo.js delete mode 100755 test/jsc/jsc.py delete mode 100644 test/jsc/lol.js diff --git a/.gitignore b/.gitignore index d98effc..ec39ced 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.md.html tmp/ +build/ diff --git a/bin/cjs.py b/bin/cjs.py new file mode 100755 index 0000000..95706e3 --- /dev/null +++ b/bin/cjs.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python +# encoding: utf-8 +__author__ = 'David Schoonover ' +__date__ = '2010-11-24' +__version__ = (0, 0, 1) + +__all__ = ('ResolutionError', 'Module', 'JSResolver',) + + +import sys, re +from itertools import chain, repeat +from collections import defaultdict +from subprocess import Popen, check_output, STDOUT, PIPE +from glob import glob + +from bunch import * +from path import path +import pystache + + +REQUIRE_PAT = re.compile(r'require\(\s*([\'"])(.*?)\1\s*\)') +ROOT = path(__file__).abspath().dirname().dirname() +LIB = ROOT/'lib/cjs' +CWD = path('.') +BLANK = path('') +MODULE_TEMPLATE = '' + +try: + with (LIB/'module.js').open('rU') as f : + MODULE_TEMPLATE = f.read() +except Exception as ex: + print + print 'ROOT={ROOT}, LIB={LIB}, module.js={}'.format(LIB/'module.js', **locals()) + raise + + +class ResolutionError(Exception): pass + + +def trim(s='', *chars): + for chs in chars: + if chs and s.endswith(chs): + s = s[:-len(chs)] + return s + +def partition(it, sep): + """ Partitions an iterable at the first occurrence of sep, and return a 3-tuple + containing the list of items before the separator, the separator itself, + and after the seperator. If the separator is not found, return a 3-tuple of + (the exhausted iterable as a list, None, []). + """ + before = [] + it = iter(it) + for val in it: + if val == sep: + break + else: + before.append(val) + else: + return (before, None, []) + return (before, sep, list(it)) + +def canonicalise(query, base=None): + """ query: A module ID (relative or absolute) or filepath + base: Optional. The referring module ID. + """ + if isinstance(base, Module): + base = base.id + + query = trim(query, '/index.cjs', '.cjs', '/index.js', '.js') + basedir = path(base or '').dirname() + print "canonicalise(query={query}, basedir={basedir})".format(**locals()) + + if query.startswith('.'): + id = ( basedir / query ).normpath() + else: + id = query + + if id.startswith('..'): + raise ResolutionError('Impossible to resolve {} from {}!'.format(query, base)) + + print " -->", str(id) + return str(id) + + + +class Module(Bunch): + DEFAULTS = { + 'id' : '', # Canonical Module ID + 'file' : None, # path + 'name' : '', # Module Name + 'uri' : '', # Module URI (TODO) + 'text' : None, # Unmodified module text + 'contents' : None, # Compiled module text + '_requires' : None, + 'outpath' : None, + } + + + def __init__(self, id, file, out=None, compile=True): + self.update(Module.DEFAULTS) + + self.id = id + self.file = path(file) + self.name = self.id.split('/').pop().capitalize() + + if out: + out = (path(out) / self.id) + '.js' + else: + out = self.file.replace('.cjs', '.js') + self.outpath = path(out) + + if compile: self.compile() + print "new!", repr(self) + + def compile(self, uri=''): + if uri: self.uri = uri.format(**self) # TODO: calc uri + + if not self.file.endswith('.cjs'): + return self + + outdir = self.outpath.dirname() + if not outdir.exists(): + outdir.makedirs() + + with self.outpath.open('w') as out: + out.write(self.contents) + + return self + + def read(self, attr='contents'): + if self['text'] is None: + with self.file.open('rU') as f: + txt = f.read() + self['text'] = txt + self['contents'] = pystache.render(MODULE_TEMPLATE, self) + return self[attr] + + @property + def text(self): + return self.read('text') + + @property + def contents(self): + return self.read('contents') + + @property + def requires(self): + if self._requires is None: + self._requires = [ canonicalise(m.group(2), self) for m in REQUIRE_PAT.finditer(self.text) ] + return self._requires + + def __hash__(self): + return hash(self.id) + + def __cmp__(self, other): + return cmp(self.id, other.id) + + def __eq__(self, other): + return self.id == other.id + + def __str__(self): + return str(self.id) + + def __repr__(self): + return '{self.__class__.__name__}(id={self.id}, file={self.file!s})'.format(**locals()) + + + + +class CommonJS(object): + """ Compiles JS modules into browser-safe JS files. """ + + @staticmethod + def discover(files, repos, out=None): + "Discover listed modules and their dependencies." + cjs = CommonJS(repos, out) + queue = [ cjs.lookup(f) for f in files ] + seen = set() + while queue: + mod = queue.pop(0) + seen.add(mod) + print mod, "requirements:", mod.requires + for query in mod.requires: + print mod, "requires", query + req = cjs.lookup(query, mod) + if req not in seen: + queue.append(req) + return cjs + + + repos = [] + modules = {} # id -> Module + _graph = None # id -> set(dependants) + + + def __init__(self, repos, out=None): + self.modules = {} + self.repos = set( path(path(p).abspath()+'/') for p in repos) + self.out = out + + def register(self, id, file): + mod = self.modules[id] = Module(id=id, file=file, out=self.out) + return mod + + def lookup(self, query, base=None): + absquery = path(query).abspath() + id = canonicalise(query, base) + print "lookup(query={query}, base={base}) -> id={id}".format(**locals()) + + if id in self.modules: + return self.modules[id] + + for repo in self.repos: + if absquery.startswith(repo) and absquery.isfile(): + id = canonicalise(absquery[len(repo):]) + return self.register(id, absquery) + + p = (repo / id).abspath() + for f in (p+'.cjs', p/'index.cjs', p+'.js', p/'index.js'): + if f.isfile(): + return self.register(id, f) + + raise ResolutionError('Unable to find file for (query={query}, id={id}, base={base}) in repos={}!'.format(map(str, self.repos), **locals())) + + @property + def graph(self): + if self._graph is None: + graph = self._graph = defaultdict(set) + for mod in self.modules.values(): + for req in mod.requires: + graph[req].add(mod) + return self._graph + + @property + def deplist(self): + return '\n'.join( '%s %s' % (id, dep.id) for (id, deps) in self.graph.iteritems() for dep in deps ) + + @property + def dependencies(self): + p = Popen(['tsort'], stderr=STDOUT, stdin=PIPE, stdout=PIPE) + p.stdin.write( self.deplist ) + p.stdin.close() + if p.wait() != 0: + raise ResolutionError('Cannot resolve dependencies! Requirements must be acyclic!') + return p.stdout.read().strip().split('\n') + + def cat(self): + for dep in self.dependencies: + print self.modules[dep].contents + + + + + + +def main(): + from optparse import OptionParser + + parser = OptionParser( + usage = 'usage: %prog [options] file[...] [-- [repo_path[...]]]', + description = 'Resolves imports in JS files', + version = '%prog'+" %i.%i.%i" % __version__) + parser.add_option("-p", "--repo-paths", dest='repos', default='', + help="Comma-seperated paths to search for unqualified modules. [default: .]") + parser.add_option("-o", "--out", default=None, + help="Root directory to write compiled JS files. [default: Module's directory]") + + (options, args) = parser.parse_args() + + if not args: + parser.error("You must specify at least one JS file to resolve!") + + (files, sep, repos) = partition(sys.argv, '--') + files = filter(lambda f: f in args, files) + repos = filter(lambda f: f in args, repos) + repos.extend( filter(None, options.repos.split(',')) ) + + print 'files:', files, 'repos:', (repos or ['.']) + js = CommonJS.discover(files=files, repos=repos or ['.'], out=options.out) + + print 'deplist:' + print js.deplist + print + print 'dependencies:' + print '\n'.join(js.dependencies) + print + + return 0 + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/lib/cjs/module.js b/lib/cjs/module.js new file mode 100644 index 0000000..7f225e1 --- /dev/null +++ b/lib/cjs/module.js @@ -0,0 +1,17 @@ +require.install( +// Module ID +'{{id}}', + +// Module metadata +{ + 'name' : '{{name}}', + 'id' : '{{id}}', + 'uri' : '{{uri}}' +}, + +// Module Setup function +function setup{{name}}(require, exports, module){ + +{{text}} + +}); \ No newline at end of file diff --git a/lib/cjs/require.js b/lib/cjs/require.js new file mode 100644 index 0000000..98d77d7 --- /dev/null +++ b/lib/cjs/require.js @@ -0,0 +1,50 @@ +;require = (function(){ + +var modules = require.modules = {} +, cache = require.cache = {}; + +function require(id){ + if ( cache.hasOwnProperty(id) ) + return cache[id]; + + if ( !modules.hasOwnProperty(id) ) + throw new Error('No such module "'+id+'"!'); + + var module = modules[ids] + , exports = module.exports = {}; + + // TODO: load the module + // if (module.hasOwnProperty('load')) { + // /* load needs to load the file synchronously using XHR + // we should probably supply a lib function, like require.load(). + // */ + // module.load.call(exports, require, exports, module); + // } + + // setup the module + module.setup.call(exports, require, exports, module); + + return exports; +}; + +require.install = function(id, module, setup){ + if ( !id ) + throw new Error('No module ID specified: id='+id+'!'); + + if ( modules.hasOwnProperty(id) ) + throw new Error('Module "'+id+'" already exists!'); + + if ( module instanceof Function ) { + setup = module; + module = { 'id':id }; + } + + module.id = id; + module.setup = setup; + modules[id] = module; + + return module; +}; + +return require; +})(); diff --git a/tanks.php b/tanks.php index 6488545..38ae17b 100644 --- a/tanks.php +++ b/tanks.php @@ -10,6 +10,10 @@ class Tanks { const ALL_SCRIPTS = 7; // MAIN_SCRIPTS | LIB_SCRIPTS | SRC_SCRIPTS; + static $mainCJS = array( + "src/tanks/ui/main.cjs" + ); + static $mainScripts = array( "src/tanks/ui/main.js" ); @@ -55,6 +59,7 @@ class Tanks { // "lib/uki/uki-theme/aristo.js", "src/lessly/future.js", + "lib/cjs/require.js", "src/Y/y.js.php", "src/Y/modules/y.event.js", @@ -85,13 +90,23 @@ class Tanks { "src/easel/loop/cooldown.js", ); - static function writeTags($scripts=null, $prefix="") { + static function writeTags($scripts=null, $prefix="", $recompile=true) { $scripts = $scripts ? $scripts : Tanks::SRC_AND_LIB; + + if ($recompile) Tanks::compile(); + if ($scripts & Tanks::LIB_SCRIPTS) foreach (self::$libScripts as $s) js($s, $prefix); if ($scripts & Tanks::SRC_SCRIPTS) foreach (self::$srcScripts as $s) js($s, $prefix); if ($scripts & Tanks::MAIN_SCRIPTS) foreach (self::$mainScripts as $s) js($s, $prefix); } + static function compile($main=null) { + $main = ($main ? $main : self::$mainCJS); + $scripts = join($main, " "); + shell_exec("cjs.py $scripts"); + } + + } function js($src, $prefix="") { diff --git a/test/jsc/bar.js b/test/jsc/bar.js deleted file mode 100644 index 38eaaa4..0000000 --- a/test/jsc/bar.js +++ /dev/null @@ -1,3 +0,0 @@ -require('./lol.js'); -require('./baz.js'); -var bar = true; \ No newline at end of file diff --git a/test/jsc/baz.js b/test/jsc/baz.js deleted file mode 100644 index d12b446..0000000 --- a/test/jsc/baz.js +++ /dev/null @@ -1,2 +0,0 @@ -require('./lol.js'); -var baz = true; \ No newline at end of file diff --git a/test/jsc/foo.js b/test/jsc/foo.js deleted file mode 100644 index 514e202..0000000 --- a/test/jsc/foo.js +++ /dev/null @@ -1,2 +0,0 @@ -require('./bar.js'); -var foo = true; \ No newline at end of file diff --git a/test/jsc/jsc.py b/test/jsc/jsc.py deleted file mode 100755 index 0775b8d..0000000 --- a/test/jsc/jsc.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 -__author__ = 'David Schoonover ' -__date__ = '2010-11-24' -__version__ = (0, 0, 1) - -__all__ = ('ResolutionError', 'Module', 'JSResolver',) - - -import sys, re -from itertools import chain, repeat -from collections import defaultdict -from subprocess import Popen, check_output, STDOUT, PIPE -from path import path - - -class ResolutionError(Exception): pass - -REQUIRE_PAT = re.compile(r'require\(\s*([\'"])(.*?)\1\s*\)') -CWD = path('.') -BLANK = path('') - -class Module(object): - module_paths = [] - - @staticmethod - def canonicalise(module, basefile=BLANK): - if isinstance(basefile, Module): - basefile = path(basefile.key) - - if module.endswith('/index.js'): - module = module[:-9] - elif module.endswith('.js'): - module = module[:-3] - - if module.startswith('./') and basefile.dirname(): - key = ( basefile.dirname() / module ).normpath() - if basefile.startswith('./'): - key = CWD / key - else: - key = module - - return str(key) - - - key = '' - query = '' - basefile = BLANK - filepath = BLANK - _contents = None - _requires = None - - - def __init__(self, query, basefile=BLANK): - if isinstance(basefile, Module): - basefile = basefile.key - - self.query = query - self.basefile = path(basefile) - self.key = Module.canonicalise(query, self.basefile) - - self.filepath = self.lookup() - - def lookup(self): - if self.key.startswith('./'): - search = [ CWD ] - else: - search = self.module_paths - for f in chain(*( (p+'.js', p/'index.js') for p in ((base / self.key).abspath() for base in search) )): - if f.isfile(): - return f - raise ResolutionError('Unable to find %s' % self.key) - - @property - def contents(self): - if self._contents is None: - with self.filepath.open('rU') as f: - self._contents = f.read() - return self._contents - - @property - def requires(self): - if self._requires is None: - self._requires = [ Module.canonicalise(m.group(2), self) for m in REQUIRE_PAT.finditer(self.contents) ] - return self._requires - - def __hash__(self): - return hash(self.key) - - def __cmp__(self, other): - return cmp(self.key, other.key) - - def __eq__(self, other): - return self.key == other.key - - # def __str__(self): - # return str(self.key) - - def __repr__(self): - return '{self.__class__.__name__}(key={self.key!r}, query={self.query!r}, basefile={self.basefile!r})'.format(**locals()) - - - - -class JSResolver(object): - """ Resolves JS imports. """ - - def __init__(self, files, module_paths=None): - Module.module_paths = set( path(p).abspath() for p in (module_paths or []) ) - - self.graph = defaultdict(set) - self.files = [ Module(CWD / path(f).normpath()) for f in files ] - self.modules = dict( (mod.key,mod) for mod in self.files ) - - def run(self): - queue = self.files[:] - - while queue: - mod = queue.pop(0) - - for req in mod.requires: - - if req in self.modules: - m = self.modules[req] - else: - m = Module(req, mod) - self.modules[req] = m - queue.append(m) - - # self.graph[mod.key].add(m) - self.graph[m.key].add(mod) - - return self - - @property - def deplist(self): - return '\n'.join( '%s %s' % (key, dep.key) for (key, deps) in self.graph.iteritems() for dep in deps ) - - @property - def dependencies(self): - # p = Process(['tsort', '-'], stdin=PIPE) - # p._process.stdin.write( self.deplist() ) - # p._process.stdin.flush() - # return p() - p = Popen(['tsort'], stderr=STDOUT, stdin=PIPE, stdout=PIPE) - p.stdin.write( self.deplist ) - p.stdin.close() - if p.wait() != 0: - raise ResolutionError('Cannot resolve dependencies! Requirements must be acyclic!') - return p.stdout.read().strip().split('\n') - - # TODO wrap in closure; collect the `module.id` info; assign `exports` to the module definition - # TODO create a modules repo for handling require() calls at runtime - - def cat(self): - for dep in self.dependencies: - print '// %s' % dep - print self.modules[dep].contents - - - - - - -def main(): - from optparse import OptionParser - - parser = OptionParser( - usage = 'usage: %prog [options] file[...] [-- [module_path[...]]]', - description = 'Resolves imports in JS files', - version = '%prog'+" %i.%i.%i" % __version__) - parser.add_option("-p", "--module-paths", default='', - help="Comma-seperated paths to search for unqualified modules.") - - (options, args) = parser.parse_args() - - if not args: - parser.error("You must specify at least one JS file to resolve!") - - js = JSResolver(args, module_paths=options.module_paths.split(',')) - js.run() - print 'deplist:' - print js.deplist - print - print 'dependencies:' - print '\n'.join(js.dependencies) - # print js.dependencies - print - print 'cat:' - js.cat() - print '---' - print - - return 0 - -if __name__ == '__main__': - sys.exit(main()) - diff --git a/test/jsc/lol.js b/test/jsc/lol.js deleted file mode 100644 index 832976f..0000000 --- a/test/jsc/lol.js +++ /dev/null @@ -1 +0,0 @@ -var lol = true; -- 1.7.0.4