From 44c3a826b671b6e2fbc660aba778f2301a0cdbd2 Mon Sep 17 00:00:00 2001 From: The8472 Date: Thu, 11 Feb 2016 17:19:34 +0100 Subject: [PATCH 1/6] support for fs: URIs without redirecting to http: fixes #48 --- lib/gateways.js | 6 +++ lib/main.js | 1 + lib/protocols.js | 85 ++++++++++++++++++++++++++-------- lib/request-fix.js | 49 ++++++++++++++++++++ package.json | 6 +++ test/test-canonical-fs-uris.js | 79 +++++++++++++++++++++++++++++++ test/test-linkify.js | 4 +- 7 files changed, 209 insertions(+), 21 deletions(-) create mode 100644 lib/request-fix.js create mode 100644 test/test-canonical-fs-uris.js diff --git a/lib/gateways.js b/lib/gateways.js index a7d86156c..b96954c52 100644 --- a/lib/gateways.js +++ b/lib/gateways.js @@ -99,3 +99,9 @@ Object.defineProperty(exports, 'linkify', { return prefs.linkify } }) + +Object.defineProperty(exports, 'fsUris', { + get: function () { + return prefs.fsUris + } +}) diff --git a/lib/main.js b/lib/main.js index ef3a5fed2..e0cbea8bd 100644 --- a/lib/main.js +++ b/lib/main.js @@ -7,6 +7,7 @@ const parent = require('sdk/remote/parent') exports.main = function (options, callbacks) { // eslint-disable-line no-unused-vars require('./redirects.js') + require('./request-fix.js') require('./peer-watch.js') protocols.register() parent.remoteRequire('./child-main.js', module) diff --git a/lib/protocols.js b/lib/protocols.js index 750f1f607..27a744c8b 100644 --- a/lib/protocols.js +++ b/lib/protocols.js @@ -35,14 +35,39 @@ const HANDLERS = { 'web+ipns': WebIpnsProtocolHandler } +/* + nsIURI does not not resolve relative paths properly + nsIStandardURL does, but always uses /// for authority-less URLs + thus we convert back and forth +*/ +function resolveRelative (base, relative) { + let newBase = Cc['@mozilla.org/network/standard-url;1'].createInstance(Ci.nsIStandardURL) + newBase.QueryInterface(Ci.nsIURI) + newBase.init(Ci.nsIStandardURL.URLTYPE_NO_AUTHORITY, -1, base.spec, null, null) + let resolved = Cc['@mozilla.org/network/standard-url;1'].createInstance(Ci.nsIStandardURL) + resolved.init(Ci.nsIStandardURL.URLTYPE_NO_AUTHORITY, -1, relative, null, newBase) + resolved.QueryInterface(Ci.nsIURI) + + // console.log("rel", base, relative, newBase, resolved) + + return resolved.spec +} + CommonProtocolHandler.prototype = Object.freeze({ defaultPort: -1, allowPort: function (port, scheme) { // eslint-disable-line no-unused-vars return false }, + /* + since ipfs is a public network and pages could just embed their own js-ipfs code anyway to fetch the data + we allow any page to embed ipfs uris and waive CORS restrictions + + TODO: consider adding URI_SAFE_TO_LOAD_IN_SECURE_CONTEXT (discussion needed) + */ protocolFlags: Ci.nsIProtocolHandler.URI_NOAUTH | - Ci.nsIProtocolHandler.URI_LOADABLE_BY_ANYONE, + Ci.nsIProtocolHandler.URI_LOADABLE_BY_ANYONE | + Ci.nsIProtocolHandler.URI_FETCHABLE_BY_ANYONE, normalizedIpfsPath: function (uriSpec) { let schemeExp = this.scheme.replace(/\+/, '\\+') // fix for web+fs etc @@ -56,41 +81,61 @@ CommonProtocolHandler.prototype = Object.freeze({ newURI: function (aSpec, aOriginCharset, aBaseURI) { // console.info('Detected newURI with IPFS protocol: ' + aSpec) + // console.log(aSpec, aOriginCharset, aBaseURI && aBaseURI.spec) if (aBaseURI && aBaseURI.scheme === this.scheme) { // presence of aBaseURI means a dependent resource or a relative link - // and we need to return correct http URI - let http = gw.publicUri.spec + this.pathPrefix + aBaseURI.path - let base = ioservice.newURI(http, null, null) - let uri = ioservice.newURI(aSpec, aOriginCharset, base) - return uri + // resolve relative within the current protocol + // leave the decision to convert to http to the next steps + aSpec = resolveRelative(aBaseURI, aSpec) } - /* Left for future use (if we enable channel.originalURI in newChannel method) - let uri = Cc['@mozilla.org/network/simple-uri;1'].createInstance(Ci.nsIURI) - uri.spec = aSpec - return uri - */ + let normalized = this.normalizedIpfsPath(aSpec) + + // console.log("norm", normalized) + + if (gw.fsUris && this.scheme === FS_SCHEME && new RegExp(`^${this.scheme}:`).test(aSpec)) { + let uri = Cc['@mozilla.org/network/simple-uri;1'].createInstance(Ci.nsIURI) + uri.spec = this.scheme + ':/' + normalized + return uri + } - let http = gw.publicUri.spec + this.normalizedIpfsPath(aSpec) - let uri = ioservice.newURI(http, aOriginCharset, null) + let uri = ioservice.newURI(normalized, aOriginCharset, gw.publicUri) // console.info('newURI routed to HTTP gateway: ' + uri.spec) return uri }, - newChannel: function (aURI) { + newChannel2: function (aURI, loadInfo) { // console.info('Detected newChannel for IPFS protocol: ' + aURI.spec) - let http = gw.publicUri.spec + this.pathPrefix + aURI.path - let channel = ioservice.newChannel(http, aURI.originCharset, null) + let normalized = this.normalizedIpfsPath(this.pathPrefix + aURI.path) + let gwUri = gw.publicUri + let httpUri = ioservice.newURI(normalized, aURI.originCharset, gwUri) + let channel = null + + if (loadInfo !== null) { + channel = ioservice.newChannelFromURIWithLoadInfo(httpUri, loadInfo) + } else { + channel = ioservice.newChannelFromURI(httpUri) + } - // line below would keep nice protocol URL in GUI - // but is disabled for now due to issues like - // https://github.com/lidel/ipfs-firefox-addon/issues/3 - // channel.originalURI = aURI + if (gw.fsUris && this.scheme === FS_SCHEME) { + channel.originalURI = aURI + channel.loadFlags &= ~Ci.nsIChannel.LOAD_REPLACE + // channel.loadFlags |= Ci.nsIChannel.LOAD_ANONYMOUS + } + + channel.loadInfo = loadInfo + + channel.QueryInterface(Ci.nsIWritablePropertyBag) + channel.QueryInterface(Ci.nsIWritablePropertyBag2) + channel.setPropertyAsAString('ipfs-uri', aURI.spec) // console.info('newChannel routed to HTTP gateway: ' + channel.URI.spec) return channel + }, + newChannel: function (aUri) { + return this.newChannel2(aUri, null) } }) diff --git a/lib/request-fix.js b/lib/request-fix.js new file mode 100644 index 000000000..21ea7f4be --- /dev/null +++ b/lib/request-fix.js @@ -0,0 +1,49 @@ +'use strict' + +const {Ci} = require('chrome') +const events = require('sdk/system/events') +const gw = require('./gateways.js') + +/* +const catman = Cc['@mozilla.org/categorymanager;1'].getService(CI.nsICategoryManager) +// category ("net-channel-event-sinks") +catman.addCategoryEntry(in string aCategory, in string aEntry, in string aValue, in boolean aPersist, in boolean aReplace) +*/ + +function fixupRequest (e) { + if (e.type !== 'http-on-modify-request') { + return + } + if (!gw.fsUris) { + return + } + const channel = e.subject.QueryInterface(Ci.nsIHttpChannel) + + channel.QueryInterface(Ci.nsIHttpChannelInternal) + channel.QueryInterface(Ci.nsIPropertyBag2) + channel.QueryInterface(Ci.nsIPropertyBag) + + let isIpfsReq = null + try { + isIpfsReq = channel.hasKey('ipfs-uri') + } catch (e) { + // console.log(e) + } + + if (isIpfsReq && channel.originalURI.scheme === 'fs') { + /* + // TODO: investigate effects of the following flags + // cookies make no sense in the ipfs context, we don't want to carry different gateway cookies into the page + channel.loadFlags |= Ci.nsIRequest.LOAD_ANONYMOUS + // should only do that for /ipfs/ paths since those are stable + channel.loadFlags |= Ci.nsIRequest.VALIDATE_NEVER + */ + + // prevent redirects from replacing the effective URI + channel.loadFlags &= ~Ci.nsIChannel.LOAD_REPLACE + + } + +} + +events.on('http-on-modify-request', fixupRequest) diff --git a/package.json b/package.json index 510eccf47..64a43090f 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,12 @@ "title": "Extended IPFS Link Support", "type": "bool", "value": false + }, + { + "name": "fsUris", + "title": "[Experimental] display fs:/ URIs as-is instead of rewriting to http://", + "type": "bool", + "value": false } ], "permissions": { diff --git a/test/test-canonical-fs-uris.js b/test/test-canonical-fs-uris.js new file mode 100644 index 000000000..3ca6a4f67 --- /dev/null +++ b/test/test-canonical-fs-uris.js @@ -0,0 +1,79 @@ +'use strict' + +const tabs = require('sdk/tabs') +const protocols = require('../lib/protocols.js') +const fsFactory = protocols.fs + +protocols.register() + +const fs = fsFactory.createInstance() +const gw = require('../lib/gateways.js') +const self = require('sdk/self') +const testpage = self.data.url('linkify-demo.html') +const sripage = 'fs:/ipfs/QmSrCRJmzE4zE1nAfWPbzVfanKQNBhp7ZWmMnEdbiLvYNh/mdown#sample.md' +const parent = require('sdk/remote/parent') + +const childMain = require.resolve('../lib/child-main.js') + +parent.remoteRequire(childMain) + +const {Cc, Ci} = require('chrome') +const ioservice = Cc['@mozilla.org/network/io-service;1'].getService(Ci.nsIIOService) + +ioservice.newURI('fs:/ipns/foo', null, null) + +exports['test newURI'] = function (assert) { + require('sdk/simple-prefs').prefs.fsUris = true + + assert.equal(fs.newURI('fs:/ipns/foo', null, null).spec, 'fs:/ipns/foo', 'keeps fs:/ uris as-is') +} + +exports['test newChannel'] = function (assert) { + require('sdk/simple-prefs').prefs.fsUris = true + + let uri = fs.newURI('fs:///ipns/foo', null, null) + let chan = fs.newChannel(uri) + + assert.equal(chan.originalURI.spec, 'fs:/ipns/foo', "keeps fs: URI as channel's originalURI") + + // double and triple slashes lead to gateway redirects, which cause CORS troubles -> check normalization + assert.equal(chan.URI.spec, 'https://ipfs.io/ipns/foo', 'channel has normalized http urls') +} + +// https://github.com/lidel/ipfs-firefox-addon/issues/3 +exports['test subresource loading'] = function (assert, done) { + require('sdk/simple-prefs').prefs.fsUris = true + gw.redirectEnabled = false + + tabs.open({ + url: testpage, + onReady: (tab) => { + + // first load somehow doesn't have protocol handlers registered. so load resource:// first, then redirect to fs:/ page + if (tab.url !== sripage) { + tab.url = sripage + tab.reload() + return + } + + let worker = tab.attach({ + contentScript: ` + let obs = new MutationObserver(function(mutations) { + let result = (document.querySelector("#ipfs-markdown-reader") instanceof HTMLHeadingElement) + self.port.emit("test result", {result: result}) + }) + obs.observe(document.body,{childList: true}) + ` + }) + worker.port.on('test result', (msg) => { + assert.equal(msg.result, true, 'subresource loaded successfully') + + require('sdk/simple-prefs').prefs.fsUris = false + tab.close(done) + }) + } + }) +} + +require('./prefs-util.js').isolateTestCases(exports) +require('sdk/test').run(exports) diff --git a/test/test-linkify.js b/test/test-linkify.js index 5f9af176f..361f038e2 100644 --- a/test/test-linkify.js +++ b/test/test-linkify.js @@ -4,12 +4,14 @@ const tabs = require('sdk/tabs') const parent = require('sdk/remote/parent') const self = require('sdk/self') const testpage = self.data.url('linkify-demo.html') +const prefs = require('sdk/simple-prefs').prefs require('../lib/rewrite-pages.js') parent.remoteRequire('resource://ipfs-firefox-addon-at-lidel-dot-org/lib/rewrite-pages.js') exports['test link processing, plain text conversion'] = function (assert, done) { - require('sdk/simple-prefs').prefs.linkify = true + prefs.linkify = true + prefs.fsUris = true tabs.open({ url: testpage, From 68041061a9f5a2386c929286cba8ae64f195adbc Mon Sep 17 00:00:00 2001 From: The8472 Date: Thu, 11 Feb 2016 21:15:10 +0100 Subject: [PATCH 2/6] tests pass locally, not on travis. let's see if this helps --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a3e072fe9..4318f84ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: node_js node_js: - - "0.12" + - "4.1" sudo: false addons: apt: From 18cb54304bb2b33bb216b2447882046a5b0cd923 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 12 Feb 2016 22:31:10 +0100 Subject: [PATCH 3/6] backup prefs once, restore from backup before tests --- test/prefs-util.js | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/test/prefs-util.js b/test/prefs-util.js index 363b6a73b..0ddb7dc6a 100644 --- a/test/prefs-util.js +++ b/test/prefs-util.js @@ -1,32 +1,27 @@ 'use strict' -const { before, after } = require('sdk/test/utils') +const { before } = require('sdk/test/utils') const { prefs } = require('sdk/simple-prefs') -exports.storePrefs = (backup) => { - // console.log('Backing up simple-prefs') - if (!backup) { - backup = new Map() - } - for (let key in prefs) { - backup.set(key, prefs[key]) - } - return backup + +const backup = new Map() + +for (let key in prefs) { + backup.set(key, prefs[key]) } -exports.restorePrefs = (backup) => { + +function restorePrefs() { // console.log('Restoring simple-prefs') for (let [key, data] of backup) { prefs[key] = data } } +exports.restorePrefs = restorePrefs + exports.isolateTestCases = (testCases) => { - let backup = null before(testCases, (name, assert) => { - backup = exports.storePrefs() - }) - after(testCases, (name, assert) => { - exports.restorePrefs(backup) + restorePrefs() }) } From fb7fb15de0fdce0d49a0be33dd91aa88e1a76b71 Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 12 Feb 2016 22:31:57 +0100 Subject: [PATCH 4/6] protocol handlers emit http urls by default --- test/test-protocols.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test-protocols.js b/test/test-protocols.js index 7c3636a56..d4dcb0ee7 100644 --- a/test/test-protocols.js +++ b/test/test-protocols.js @@ -1,5 +1,6 @@ var proto = require('../lib/protocols.js') var gw = require('../lib/gateways.js') +const prefs = require('sdk/simple-prefs').prefs const pubGwUri = gw.publicUri const ipfsPath = 'QmTkzDwWqPbnAh5YiV5VwcTLnGdwSNsNTn2aDxdXBFca7D/example#/ipfs/QmSsNVuALPa1TW1GDahup8fFDqo95iFyPE7E6HpqDivw3p/readme.md' const ipnsPath = 'ipfs.git.sexy' @@ -164,10 +165,12 @@ exports['test newURI(web+fs:///ipns/)'] = function (assert) { } exports['test protocol rewrite'] = function (assert) { + prefs.fsUris = true + assert.equal(proto.rewrite('ipfs:QmYHNYAaYK5hm3ZhZFx5W9H6xydKDGimjdgJMrMSdnctEm'), - 'ipfs:QmYHNYAaYK5hm3ZhZFx5W9H6xydKDGimjdgJMrMSdnctEm', '#1') + 'https://ipfs.io/ipfs/QmYHNYAaYK5hm3ZhZFx5W9H6xydKDGimjdgJMrMSdnctEm', '#1') - assert.equal(proto.rewrite('ipns:ipfs.io'), 'ipns:ipfs.io', '#2') + assert.equal(proto.rewrite('ipns:ipfs.io'), 'https://ipfs.io/ipns/ipfs.io', '#2') assert.equal(proto.rewrite('fs:/ipns/ipfs.io'), 'fs:/ipns/ipfs.io', '#3') From 20a53e2bd51b414168ec7715382a5bcdfbe96eba Mon Sep 17 00:00:00 2001 From: The8472 Date: Fri, 12 Feb 2016 22:39:56 +0100 Subject: [PATCH 5/6] pacify eslint --- test/prefs-util.js | 6 ++---- test/test-protocols.js | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/prefs-util.js b/test/prefs-util.js index 0ddb7dc6a..73d6746d1 100644 --- a/test/prefs-util.js +++ b/test/prefs-util.js @@ -3,15 +3,13 @@ const { before } = require('sdk/test/utils') const { prefs } = require('sdk/simple-prefs') - const backup = new Map() for (let key in prefs) { - backup.set(key, prefs[key]) + backup.set(key, prefs[key]) } - -function restorePrefs() { +function restorePrefs () { // console.log('Restoring simple-prefs') for (let [key, data] of backup) { prefs[key] = data diff --git a/test/test-protocols.js b/test/test-protocols.js index d4dcb0ee7..9b41188b6 100644 --- a/test/test-protocols.js +++ b/test/test-protocols.js @@ -165,7 +165,7 @@ exports['test newURI(web+fs:///ipns/)'] = function (assert) { } exports['test protocol rewrite'] = function (assert) { - prefs.fsUris = true + prefs.fsUris = true assert.equal(proto.rewrite('ipfs:QmYHNYAaYK5hm3ZhZFx5W9H6xydKDGimjdgJMrMSdnctEm'), 'https://ipfs.io/ipfs/QmYHNYAaYK5hm3ZhZFx5W9H6xydKDGimjdgJMrMSdnctEm', '#1') From 0b085707442b45333e8adef73aa1c3728d0622ba Mon Sep 17 00:00:00 2001 From: The8472 Date: Sun, 14 Feb 2016 12:59:35 +0100 Subject: [PATCH 6/6] canonical uris with private gateway enabled --- lib/gateways.js | 6 ++++++ lib/protocols.js | 3 +-- test/test-canonical-fs-uris.js | 9 ++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/gateways.js b/lib/gateways.js index b96954c52..5dcb778ae 100644 --- a/lib/gateways.js +++ b/lib/gateways.js @@ -79,6 +79,12 @@ Object.defineProperty(exports, 'publicUri', { } }) +Object.defineProperty(exports, 'gwUri', { + get: function () { + return this.redirectEnabled ? CUSTOM_GATEWAY_URI : PUBLIC_GATEWAY_URI + } +}) + Object.defineProperty(exports, 'publicHosts', { get: function () { return PUBLIC_GATEWAY_HOSTS diff --git a/lib/protocols.js b/lib/protocols.js index 27a744c8b..cc273864a 100644 --- a/lib/protocols.js +++ b/lib/protocols.js @@ -109,8 +109,7 @@ CommonProtocolHandler.prototype = Object.freeze({ newChannel2: function (aURI, loadInfo) { // console.info('Detected newChannel for IPFS protocol: ' + aURI.spec) let normalized = this.normalizedIpfsPath(this.pathPrefix + aURI.path) - let gwUri = gw.publicUri - let httpUri = ioservice.newURI(normalized, aURI.originCharset, gwUri) + let httpUri = ioservice.newURI(normalized, aURI.originCharset, gw.gwUri) let channel = null if (loadInfo !== null) { diff --git a/test/test-canonical-fs-uris.js b/test/test-canonical-fs-uris.js index 3ca6a4f67..d2132daa2 100644 --- a/test/test-canonical-fs-uris.js +++ b/test/test-canonical-fs-uris.js @@ -30,6 +30,7 @@ exports['test newURI'] = function (assert) { exports['test newChannel'] = function (assert) { require('sdk/simple-prefs').prefs.fsUris = true + gw.redirectEnabled = false let uri = fs.newURI('fs:///ipns/foo', null, null) let chan = fs.newChannel(uri) @@ -37,7 +38,13 @@ exports['test newChannel'] = function (assert) { assert.equal(chan.originalURI.spec, 'fs:/ipns/foo', "keeps fs: URI as channel's originalURI") // double and triple slashes lead to gateway redirects, which cause CORS troubles -> check normalization - assert.equal(chan.URI.spec, 'https://ipfs.io/ipns/foo', 'channel has normalized http urls') + assert.equal(chan.URI.spec, 'https://ipfs.io/ipns/foo', 'redirect off, channel has normalized http urls') + + gw.redirectEnabled = true + + chan = fs.newChannel(uri) + + assert.equal(chan.URI.spec, 'http://127.0.0.1:8080/ipns/foo', 'redirect on, channel has normalized http urls') } // https://github.com/lidel/ipfs-firefox-addon/issues/3