Dapps: How to get elements of array in a contract.
When I develop decentralized applications, I have often encountered the situation (e.g. get listings of hotel) that I want to get all or some elements of array.
But in the current solidity specification, a function cannot return a dynamic array, so we need to get an element by calling function to get element each by each like this.
// This is a mock code to explain what I've done. all web3 code should be assumed run synchronously.
// Instantiate a contract in Javascript.
const registry = SomeRegistry(web3).deployed();
try {
const numOfElements = registry.numOfElements();
const results = [];
for (let i = 0; i < numOfElements; i++) {
const elem = registry.getElementOfArray(i);
results.push(elem);
}
// Do something with the results.
} catch (e) {
// handling error.
}By this code, you have to call an JSON-RPC methods of web3, which means there are a lot of communication overheads between your Dapps and an Ethereum node.
So, I’d like to introduce a way to serialize an array in solidity as bytes and deserialize bytes in an external code such as Javascript.
Array of fixed size types.
First of all, I started to work on implementing serialization function for array of fixed size type such as integer, address or so.
It’s not so hard to serialize/deserialize that kind of array, because we could have an assumption for size of elements of array.
The solidity codes are below:
// Integer array storage.
contract IntArrayStorage {
uint256[] arr;
function getArrLength()
returns (uint256)
{
return arr.length;
}
function setIntArr(uint256 _index, uint256 _value)
{
arr[_index] = _value;
}
// Convenient function for out of evm world.
function getAsBytes(uint256 _from, uint256 _to)
public
constant
returns (bytes)
{
require(_from >= 0 && _to >= _from && arr.length >= _to);
// Size of bytes
uint256 size = 32 * (_to - _from + 1);
uint256 counter = 0;
bytes memory b = new bytes(size);
for (uint256 x = _from; x < _to + 1; x++) {
uint256 elem = arr[x];
for (uint y = 0; y < 32; y++) {
b[counter] = byte(uint8(elem / (2 ** (8 * (31 - y)))));
counter++;
}
}
return b;
}
}// address array storage.
contract AddressArrayStorage {
address[] arr;
function getArrLength()
returns (uint256)
{
return arr.length;
}
function setIntArr(uint256 _index, address _value)
{
arr[_index] = _value;
}
// Convenient function for out of evm world.
function getAsBytes(uint256 _from, uint256 _to)
public
constant
returns (bytes)
{
require(_from >= 0 && _to >= _from && arr.length >= _to);
// Size of bytes
uint256 size = 20 * (_to - _from + 1);
uint256 counter = 0;
bytes memory b = new bytes(size);
for (uint256 x = _from; x < _to + 1; x++) {
bytes memory elem = toBytes(arr[x]);
for (uint y = 0; y < 20; y++) {
b[counter] = elem[y];
counter++;
}
}
return b;
}
// from: https://ethereum.stackexchange.com/questions/884/how-to-convert-an-address-to-bytes-in-solidity
function toBytes(address a) constant returns (bytes b){
assembly {
let m := mload(0x40)
mstore(add(m, 20), xor(0x140000000000000000000000000000000000000000, a))
mstore(0x40, add(m, 52))
b := m
}
}
}
The most important part of these codes is getAsBytes function. these function convert all elements of array into bytes and concatenate these bytes to a single bytes.
From your dapps code, you can call the getAsBytes code to get all or some part of elements in array simultaneously like this.
// web3 js returns bytes as hex in string, so treat returned value as hex.
function stripHexPrefix(str) {
return str.startsWith('0x') ? str.slice(2) : str;
}
// Convert byte string to int array.
function bytesToIntArray(byteString) {
let stripped = stripHexPrefix(byteString);
return stripped.match(/.{1,64}/g).map(s => parseInt("0x" + s));
}
// Convert byte string to address array
function bytesToAddressArray(byteString) {
let stripped = stripHexPrefix(byteString);
return stripped.match(/.{1,40}/g).map(s => "0x" + s);
}Array of dynamic size types
There are dynamic size types in Solidity, so in some case, we could not have an assumption of size of elements in array.
To handle these cases, I adopted some kind of serialization format consist of three part like:
<meta_length><content_length><content>
- <meta_length>: bytes of uint8 which tells size of content_length part*.
- <content_length>: bytes of uint8-uint256 which tells length of subsequent content.
- <content>: bytes of content.
The reason why meta_length is added is strike balance between memory efficiency and flexibility (of content length).
If we take uint8 for content_length part, we can't handle any string longer than 255. On contrary, if we take uint256 for content_length part, we have to use 32 bytes for tiny string. so for now, I introduce meta_length part.
To understand this format, let’s say we have a array which have Hello world in Solidity.
And the length of bytes of Hello World is 11, so we just only need uint8 for content_length. Therefore meta_length and content_length would be 01 and 0b in hex.
so the complete format we would get in external code would be 0x010b48656c6c6f20576f726c64 (utf8 string of 'Hello World' in hex is '48656c6c6f20576f726c64')
the code in solidity and js is below:
contract StringArrayStorage {
uint8 constant max8 = 2**8 - 1;
uint16 constant max16 = 2**16 - 1;
uint32 constant max32 = 2**32 - 1;
uint64 constant max64 = 2**64 - 1;
uint128 constant max128 = 2**128 - 1;
uint256 constant max256 = 2**256 - 1;
string[] arr;
function getArrLength()
returns (uint256)
{
return arr.length;
}
function setStrArr(uint256 _index, string _value)
{
arr[_index] = _value;
}
// Convenient function for out of evm world.
function getAsBytes(uint256 _from, uint256 _to)
public
constant
returns (bytes)
{
require(_from >= 0 && _to >= _from && arr.length >= _to);
uint256 bytesSize = 0;
bytes memory elem;
uint8 len;
// calculate required length of bytes.
for (uint256 a = _from; a < _to + 1; a++) {
elem = bytes(arr[a]);
len = lengthBytes(elem.length);
require(len != 255);
bytesSize += 1 + len + elem.length;
}
uint256 counter = 0;
bytes memory b = new bytes(bytesSize);
for (uint256 x = _from; x < _to + 1; x++) {
elem = bytes(arr[x]);
len = lengthBytes(elem.length);
// length of next integer specifying string content size.
b[counter] = byte(len);
counter++;
// bytes of integer to specify string bytes size
// string content it self.
for (uint y = 0; y < len; y++) {
b[counter] = byte(uint8(elem.length / (2 ** (8 * (len - 1 - y)))));
counter++;
}
// string content it self.
for (uint z = 0; z < elem.length ; z++) {
b[counter] = elem[z];
counter++;
}
}
return b;
}
function lengthBytes(uint256 length)
internal
returns
(uint8)
{
// can't handle too long string in this way.
require(length <= max256);
// 8bit
if (length >=0 && length <= max8) {
return 1;
}
// 16bit
if (length > max8 && length <= max16) {
return 2;
}
//32 bit
if (length >= max16 && length < max32) {
return 4;
}
//64bit
if (length >= max32 && length < max64) {
return 8;
}
//128bit
if (length >= max64 && length < max128) {
return 16;
}
//256 bit
return 32;
}
}The javascript code to deserialize a returned value would be like below:
function hexToStr(hex) {
var str = '';
for (var i = 0; i < hex.length; i += 2) {
var v = parseInt(hex.substr(i, 2), 16);
if (v) str += String.fromCharCode(v);
}
return str;
}
class StringReader {
constructor(str) {
this.str = str;
this.cursor = 0;
console.log(this.cursor);
}
read(count) {
let right = this.str.length > (this.cursor + count) ? (this.cursor + count) : this.str.length;
let res = this.str.substring(this.cursor, right);
this.cursor = right;
return res;
}
isEnd() {
return this.cursor == this.str.length;
}
}
function readOneString(strReader) {
let metaSize = strReader.read(2);
let strSize = strReader.read(parseInt("0x" + metaSize) * 2);
let strBytes = strReader.read(parseInt("0x" + strSize) * 2);
return hexToStr(strBytes);
}
export function bytesToStringArray(byteString) {
let stripped = stripHexPrefix(byteString);
let stringReader = new StringReader(stripped);
let result = [];
while(!stringReader.isEnd()) {
let res = readOneString(stringReader);
result.push(res);
}
console.log(result);
return result;
}Further improvements.
These codes could be extended for other values such like bytes array.
In addition, we may need an standard way to serialize/deserialize types/enums/structs for the sake of performance. I think we could also extend these way to that kind of user-defined types.
But, I think (hope) these code would be just a workaround until EVM will be able to handle dynamic arrays as returned value.