(WIP) Return headers as JSON array instead of concatenated string

docs/protocol-methods.rst

@@ -76,7 +76,7 @@ With *cp_height* 8::
 blockchain.block.headers
 ========================
 
-Return a concatenated chunk of block headers from the main chain.
+Return a chunk of block headers from the main chain.
 
 **Signature**
 
@@ -85,6 +85,7 @@ Return a concatenated chunk of block headers from the main chain.
   .. versionchanged:: 1.4
      *cp_height* parameter added
   .. versionchanged:: 1.4.1
+  .. versionchanged:: 1.5
 
   *start_height*
 
@@ -112,11 +113,19 @@ Return a concatenated chunk of block headers from the main chain.
     the available headers will be returned.  If more headers than
     *max* were requested at most *max* will be returned.
 
+  * *headers*
+
+    An array containing the binary block headers in-order; each header is a
+    hexadecimal string.  AuxPoW data (if present in the original header) is
+    truncated if *cp_height* is nonzero.  Only present in version 1.5 and
+    higher.
+
   * *hex*
 
     The binary block headers concatenated together in-order as a
     hexadecimal string.  Starting with version 1.4.1, AuxPoW data (if present
-    in the original header) is truncated if *cp_height* is nonzero.
+    in the original header) is truncated if *cp_height* is nonzero.  Only
+    present in versions lower than 1.5.
 
   * *max*
 
@@ -149,7 +158,10 @@ See :ref:`here <cp_height example>` for an example of *root* and
 
   {
     "count": 2,
-    "hex": "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e36299"
+    "headers":
+    [
+      "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c", "010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e36299"
+    ],
     "max": 2016
   }
 

electrumx/server/session.py

@@ -936,7 +936,7 @@ class ElectrumX(SessionBase):
     '''A TCP server that handles incoming Electrum connections.'''
 
     PROTOCOL_MIN = (1, 4)
-    PROTOCOL_MAX = (1, 4, 2)
+    PROTOCOL_MAX = (1, 5, 0)
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -1214,6 +1214,8 @@ async def block_headers(self, start_height, count, cp_height=0):
         start_height and count must be non-negative integers.  At most
         MAX_CHUNK_SIZE headers will be returned.
         '''
+        if self.protocol_tuple >= (1, 5, 0):
+            return await self.block_headers_array(start_height, count, cp_height)
         start_height = non_negative_integer(start_height)
         count = non_negative_integer(count)
         cp_height = non_negative_integer(cp_height)
@@ -1230,6 +1232,38 @@ async def block_headers(self, start_height, count, cp_height=0):
         self.bump_cost(cost)
         return result
 
+    async def block_headers_array(self, start_height, count, cp_height=0):
+        '''Return block headers in an array for the main chain;
+        starting at start_height.
+        start_height and count must be non-negative integers.  At most
+        MAX_CHUNK_SIZE headers will be returned.
+        '''
+        start_height = non_negative_integer(start_height)
+        count = non_negative_integer(count)
+        cp_height = non_negative_integer(cp_height)
+        cost = count / 50
+
+        max_size = self.MAX_CHUNK_SIZE
+        count = min(count, max_size)
+        headers, count = await self.db.read_headers(start_height, count)
+        result = {'count': count, 'max': max_size, 'headers': []}
+        if count and cp_height:
+            cost += 1.0
+            last_height = start_height + count - 1
+            result.update(await self._merkle_proof(cp_height, last_height))
+
+        cursor = 0
+        height = 0
+        while cursor < len(headers):
+            next_cursor = self.db.header_offset(height + 1)
+            header = headers[cursor:next_cursor]
+            result['headers'].append(header.hex())
+            cursor = next_cursor
+            height += 1
+
+        self.bump_cost(cost)
+        return result
+
     def is_tor(self):
         '''Try to detect if the connection is to a tor hidden service we are
         running.'''
@@ -1747,33 +1781,38 @@ async def block_header(self, height, cp_height=0):
             return result
 
         # Covered by a checkpoint; truncate AuxPoW data
-        result['header'] = self.truncate_auxpow(result['header'], height)
+        result['header'] = self.truncate_auxpow_single(result['header'])
         return result
 
     async def block_headers(self, start_height, count, cp_height=0):
-        result = await super().block_headers(start_height, count, cp_height)
-
         # Older protocol versions don't truncate AuxPoW
         if self.protocol_tuple < (1, 4, 1):
-            return result
+            return await super().block_headers(start_height, count, cp_height)
 
         # Not covered by a checkpoint; return full AuxPoW data
         if cp_height == 0:
-            return result
+            return await super().block_headers(start_height, count, cp_height)
+
+        result = await super().block_headers_array(start_height, count, cp_height)
 
         # Covered by a checkpoint; truncate AuxPoW data
-        result['hex'] = self.truncate_auxpow(result['hex'], start_height)
-        return result
+        result['headers'] = self.truncate_auxpow_headers(result['headers'])
 
-    def truncate_auxpow(self, headers_full_hex, start_height):
-        height = start_height
-        headers_full = util.hex_to_bytes(headers_full_hex)
-        cursor = 0
-        headers = bytearray()
+        # Return headers in array form
+        if self.protocol_tuple >= (1, 5, 0):
+            return result
 
-        while cursor < len(headers_full):
-            headers += headers_full[cursor:cursor+self.coin.TRUNCATED_HEADER_SIZE]
-            cursor += self.db.dynamic_header_len(height)
-            height += 1
+        # Return headers in concatenated form
+        result['hex'] = ''.join(result['headers'])
+        del result['headers']
+        return result
+
+    def truncate_auxpow_headers(self, headers):
+        result = []
+        for header in headers:
+            result.append(self.truncate_auxpow_single(header))
+        return result
 
-        return headers.hex()
+    def truncate_auxpow_single(self, header: str):
+        # 2 hex chars per byte
+        return header[:2*self.coin.TRUNCATED_HEADER_SIZE]