Merge pull request #11 from oncletom/feature-11

rules as regexp

.travis.yml

@@ -5,4 +5,7 @@ node_js:
 notifications:
   email:
     on_success: change
-    on_failure: change
+    on_failure: change
+before_script:
+  - npm install -g grunt
+  - npm run-script build

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 1.1
+
+Nothing new: only optimization and cleanup.
+
+There is a **Backward Compatibility Break** if you used directly `rules.json`. Format has changed.
+Use `rules-legacy.json` to benefit from the formatted data from version `1.0.x`.
+
+* builder now relies on [grunt](http://gruntjs.com/)
+* reduced the number of dependencies, all of them are `devDependencies` specific only
+* reduced the filesize of `rules.json` (from 600K to less than 60K, it's about 21K gzipped)
+
 ## 1.0.3
 
 * updated `rules.json` with latest public rules change

README.md

@@ -16,8 +16,8 @@ Thanks Mozilla!
 
 *tld.js* is available under [NPM](http://npmjs.org/) registry.
 
-```javascript
-npm install tldjs --save
+```bash
+npm install tldjs --save --production
 ```
 
 And to include it in any relevant script:
@@ -46,25 +46,26 @@ tld.isValid('t.go'); // returns `false`
 
 ```
 
-## Browser-side
+## Updating TLDs List
+
+Many libraries offer a list of TLDs. But, are they up-to-date? And how to update them?
 
-**Notice**: this part has not been developed yet.
+Hopefully for you, even if I'm flying over the world, if I've lost my Internet connection or even if
+you do manage your own list, you can update it by yourself, painlessly.
 
-The library is designed to be useable on the browser-side, in an framework agnostic fashion. No `jQuery.tld()`.
+You may have installed the package with its `devDependencies` with:
 
-```javascript
-<script src="/path/to/tld.js"></script>
-<script>
-tld.getDomain(window.location.host); //returns the current domain
-</script>
+```bash
+npm install tldjs --save
 ```
 
-## Rebuilding TLDs List
+If you did not install with the `--production` flag, then it's okay.
 
-Many libraries offer a list of TLDs. But, are they up-to-date? And how to update them?
+[grunt](http://gruntjs.com/) is mandatory to build and update rules. If it's not installed globally, then do so:
 
-Hopefully for you, even if I'm flying over the world, if I've lost my Internet connection or even if
-you do manage your own list, you can update it by yourself, painlessly.
+```bash
+npm install -g grunt
+```
 
 How? By typing this in your console
 

lib/exports/index.js

@@ -0,0 +1,13 @@
+"use strict";
+
+/*jshint node:true strict: true */
+
+/*
+ * Default list of exporters
+ */
+var exporters = [
+  require(__dirname + '/standard-json.js'),
+  require(__dirname + '/legacy-json.js')
+];
+
+module.exports = exporters;

lib/exports/legacy-json.js

@@ -0,0 +1,18 @@
+"use strict";
+
+/*jshint node:true strict: true */
+
+var Rule = require(__dirname + '/../rule.js');
+
+function legacyJSONExport(xlds){
+  var data = [];
+
+  xlds.forEach(function(xld){
+    data.push( new Rule(xld) );
+  });
+
+  return ['rules-legacy.json', data];
+}
+
+
+module.exports = legacyJSONExport;

lib/exports/standard-json.js

@@ -0,0 +1,39 @@
+"use strict";
+
+/*jshint node:true strict: true */
+
+var Rule = require(__dirname + '/../rule.js');
+
+function standardJSONExport(xlds){
+  var data = {};
+
+  xlds.forEach(function(xld){
+    var rule = new Rule(xld);
+
+    if (!data[rule.firstLevel]){
+      data[rule.firstLevel] = [];
+    }
+
+    if (rule.wildcard){
+      data[rule.firstLevel].push('*' + (rule.secondLevel || ''));
+    }
+
+    if (rule.exception){
+      data[rule.firstLevel].push('!' + (rule.secondLevel || ''));
+    }
+
+    if (!rule.exception && !rule.wildcard && rule.secondLevel){
+      data[rule.firstLevel].push(rule.secondLevel);
+    }
+  });
+
+  // Compressing data
+  for (var tld in data){
+    data[tld] = data[tld].join('|');
+  }
+
+  return ['rules.json', data];
+}
+
+
+module.exports = standardJSONExport;

lib/grunt/update.js

@@ -3,7 +3,7 @@
 /*jshint node:true strict: true */
 
 var request = require('request');
-var parser = require(__dirname + '/../rule.js');
+var parser = require(__dirname + '/../parsers/publicsuffix-org.js');
 
 module.exports = function(grunt){
   var _ = grunt.utils._;
@@ -36,19 +36,34 @@ module.exports = function(grunt){
    * @param {string|undefined} provider
    */
   return function tldUpdate(provider){
-    var done = this.async();
-    var config = computeConfiguration(this.name);
+    var done, config;
+
+    done = this.async();
+    config = computeConfiguration(this.name);
     provider = computeProvider(provider, config);
 
     request.get(config.providers[provider], function (err, response, body) {
+      var queue, tlds;
+
       if (err) {
         throw new Error(err);
       }
 
-      var tlds = parser.parse(body);
-      grunt.file.write(__dirname + '/../../src/rules.json', JSON.stringify(tlds));
+      tlds = parser.parse(body);
+      queue = grunt.utils.async.queue(function(exportTask, callback){
+        var task_result = exportTask(tlds);
+        var filename = task_result[0];
+        var data = task_result[1];
+
+        data = JSON.stringify(data);
+
+        grunt.file.write(__dirname + '/../../src/'+filename, data);
+
+        callback();
+      }, 5);
 
-      done();
+      queue.drain = done;
+      queue.push(require(__dirname + '/../exports/index.js'));
     });
   };
 };

lib/parsers/publicsuffix-org.js

@@ -0,0 +1,71 @@
+"use strict";
+
+/*jshint node:true strict: true */
+
+function PublicSuffixOrgParser(){
+
+}
+
+/**
+ * Parse a one-domain-per-line file
+ *
+ * @param body {String}
+ * @return {Array}
+ */
+PublicSuffixOrgParser.parse = function (body){
+  return (body+'')
+    .split(/\n/m)
+    .filter(PublicSuffixOrgParser.filterRow)
+    .map(PublicSuffixOrgParser.domainBuilder);
+};
+
+/**
+ * Returns a rule based on string analysis
+ *
+ * @param rule {PublicSuffixRule}
+ */
+PublicSuffixOrgParser.domainBuilder = function (row){
+  var rule = {};
+  row = row.trim();
+
+  //setting initial rule
+  rule.source = row;
+
+  //exception
+  row = row.replace(/^!(.+)$/, function(m, tld){
+    rule.exception = true;
+
+    return tld;
+  });
+
+  //wildcard
+  row = row.replace(/^(\*\.)(.+)$/, function(m, dummy, tld){
+    rule.wildcard = true;
+
+    return tld;
+  });
+
+  //splitting domains
+  row.replace(/^((.+)\.)?([^\.]+)$/, function(m, dummy, secondLevel, firstLevel){
+    rule.firstLevel = firstLevel;
+
+    if (secondLevel){
+      rule.secondLevel = secondLevel;
+    }
+
+  });
+
+  return rule;
+};
+
+/**
+ * Filters a commented or empty line
+ *
+ * @param row {String}
+ * @return {String|null}
+ */
+PublicSuffixOrgParser.filterRow = function (row) {
+  return (/^\/\//).test(row) ? null : row;
+};
+
+module.exports = PublicSuffixOrgParser;