enable hooks for fetching code behind solidity import statements

[serapath]

@todo

• enable hooks for fetching and caching code behind solidity import statements

---

I think we should make another module which imports/requires solc-js and many resolvers, and exports a compile function, which takes sourcecode and uses the correct resolvers, so users dont have to bother

maybe here is some resolver code

• https://github.com/Crypto-Punkers/resolver-engine
• https://github.com/ethereum/remix/issues/1059
• https://github.com/dominictarr/npmd-git-resolve/blob/master/index.js
• https://github.com/ipfs/js-ipfs#use-in-the-browser

const compile = require('solc-js')

const resolve_http = async x => await fetch(x).then(r => r.text())
const resolve_swarm = require('resolve-swarm')

const output = await compile(sourceode, async (importpath) => await resolve_http (importpath))

async function compile (sourcecode, getContent) {
  const allimports = await getAllImports(sourcecode)
  const localSources = {}
  // e.g. runs 10x when 10 imports in sourcecode
  for (path of allimports) {
    if (isSwarm(path)) localSources[path] = await resolve_swarm (path)
    if (isHttp(path) localSources[path] = await resolve_http (path)
    // ...
  }
  const output = solcjs(sourcecode, localSources)
  // let localSources = [{
    // path: 'lib.sol',
    // content: 'library L { function f() internal returns (uint) { return 7; } }'
  // }];
  return output
}

const myDB = { } // maybe it looks like:

const sourcecode = `
  import 'https://github.com/OpenZeppelin/openzeppelin-solidity/contracts/math/a.sol';
  import 'https://github.com/OpenZeppelin/openzeppelin-solidity/contracts/math/b.sol';
  import 'https://github.com/OpenZeppelin/openzeppelin-solidity/contracts/math/c.sol';
  import 'https://github.com/OpenZeppelin/openzeppelin-solidity/contracts/math/d.sol';
  import 'https://github.com/OpenZeppelin/openzeppelin-solidity/contracts/math/e.sol';
  import 'lib1.sol'
  import 'lib2.sol'
  import 'lib3.sol'
  import 'lib4.sol'
  import 'lib5.sol'

  library OldLibrary {
      function someFunction(uint8 a) public returns(bool);
  }

  contract NewContract {
      function f(uint8 a) public returns (bool) {
          return OldLibrary.someFunction(a);
      }
  }`

test('sourcecode with 10 imports calls callback 10 times', function (t) {
  t.plan(10)
  const output = await compile(sourcecode, async (path) => {
    t.assert(true)
    console.log(path)
    var output
    if (myDB.contains(path)) output = await myDB.get(path)
    else output = new Promise(resolve => {
      resolver.import(path, null, ({ content }) => resolve(content)
    })
    return output
  })
})

/*
  'https://github.com/OpenZeppelin/openzeppelin-solidity/contracts/math/a.sol';
  'https://github.com/OpenZeppelin/openzeppelin-solidity/contracts/math/b.sol';
  'https://github.com/OpenZeppelin/openzeppelin-solidity/contracts/math/c.sol';
  'https://github.com/OpenZeppelin/openzeppelin-solidity/contracts/math/d.sol';
  'https://github.com/OpenZeppelin/openzeppelin-solidity/contracts/math/e.sol';
  'lib1.sol'
  'lib2.sol'
  'lib3.sol'
  'lib4.sol'
  'lib5.sol'
*/

const output = await compile(sourcecode, async (path) => 
    myDB.contains(path) ? await myDB.get(path) : await getImportContent(path)
)

// so "getImportContent(...)" is a replacement for
new Promise(resolve => resolver.import(path, null, ({ content }) => resolve(content))

---

readCallback function can't be a async function, because emscripten doesn't support it very well.

• Support async read callbacks. argotorg/solc-js#140
• https://github.com/ethereum/solidity/blob/develop/libsolidity/interface/StandardCompiler.cpp#L410
• https://github.com/ethereum/solidity/blob/develop/libsolidity/interface/ReadFile.h#L30

but I think another workaround is provided "getReadCallback" API.

example 1:

let readCallback = await getReadCallback(sourceCode);
let output = await compiler(sourceCode, readCallback);

example 2:

let localSources = [{
       path: 'lib.sol',
       content: 'library L { function f() internal returns (uint) { return 7; } }'
     }];
 
let readCallback = await getReadCallback(sourceCode, localSources);
let output = await compiler(sourceCode, readCallback);

https://github.com/ethereum-play/solc-js/pull/1/commits/1db7eb24755acf1446487e7693073e2ca1be7301#diff-3b8192acb0d491039ab68dea0b430bfcR62

https://github.com/ethereum-play/solc-js/pull/1/commits/1db7eb24755acf1446487e7693073e2ca1be7301#diff-3b8192acb0d491039ab68dea0b430bfcR39

about imports

• https://gitter.im/ethereum/remix-dev?at=5c077fac80986419d5406c0a
• Don't fully compiler/check imports. argotorg/solidity#2862
• https://github.com/ethereum/remix/issues/1059#issuecomment-444394390
• check https://github.com/ethereum/remix/tree/remixResolve/remix-url-resolver

---

• https://docs.google.com/document/d/1PZBSCBWBwd6AqWCgXqLnw8FNQ4HRurP5usrXuKuU0a0/edit#heading=h.qp8fp07k2avk

---

• https://blog.zeppelin.solutions/solidity-compiler-audit-8cfc0316a420

---

keywords

• combineSource
• resolver
• import parser

---

• ethereum solution:
  • https://github.com/ethereum/remix/tree/remix-resolve/remix-resolve
  • relation issue
    • https://github.com/ethereum/remix/issues/1063
    • Don't fully compiler/check imports. argotorg/solidity#2862
    • https://github.com/ethereum/remix/issues/1059
    • https://github.com/ethereum/remix/issues/917
  • pull request / code
    • minor fixes remix-project-org/remix#1085
    • https://github.com/0mkara/etheratom/blob/master/lib/helpers/compiler-imports.js
    • https://github.com/0mkara/etheratom/blob/master/lib/web3/methods.js#L39
• truffle solution: compile once, then get the error message to find import path.
  • https://github.com/trufflesuite/truffle/blob/develop/packages/truffle-compile/parser.js
  • https://github.com/trufflesuite/truffle/tree/develop/packages/truffle-resolver
• etheratom solution:
  • code
    • https://github.com/0mkara/etheratom/blob/7479f2f9a003616bf5220b70ca8503d88f560834/lib/helpers/compiler-imports.js
    • https://github.com/0mkara/etheratom/blob/c2523b28d2587da9f9f7ef02dca98e383a00f6e0/lib/web3/web3.js#L242
  • relation issue
    • combineSources for large library imports does not work 0mkara/etheratom#201
• resolver-engine solution
  • https://github.com/resolver-engine/resolver-engine
  • relation issue
    • FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory pubkey/solidity-cli#2
• features
  • Support async read callbacks. argotorg/solc-js#140