node_utils.cpp

#include <cstring>
#include <vector>
#include <cstdlib>
#include <ctime>
#include <algorithm>
#include <chrono>
#include <memory>
#include <stdexcept>
#include <cinttypes>

#include "defines.h"
#include "structs.h"
#include "connection.h"
#include "node_utils.h"
#include "logger.h"
#include "k12_and_key_utils.h"
#include "key_utils.h"
#include "wallet_utils.h"

static CurrentTickInfo getTickInfoFromNode(QCPtr qc)
{
    CurrentTickInfo result;
    struct {
        RequestResponseHeader header;
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(REQUEST_CURRENT_TICK_INFO);
    qc->sendData((uint8_t *) &packet, packet.header.size());

    try
    {
        result = qc->receivePacketWithHeaderAs<CurrentTickInfo>();
    } 
    catch (std::exception)
    {
        memset(&result, 0, sizeof(CurrentTickInfo));
    }

    return result;
}

uint32_t getTickNumberFromNode(QCPtr qc)
{
    auto curTickInfo = getTickInfoFromNode(qc);
    int retryCounter = 0;
    while (curTickInfo.tick == 0 && retryCounter++ < 3)
    {
        qc->resolveConnection();
        curTickInfo = getTickInfoFromNode(qc);
    }
    return curTickInfo.tick;
}

void printTickInfoFromNode(const char* nodeIp, int nodePort)
{
    auto qc = make_qc(nodeIp, nodePort);
    auto curTickInfo = getTickInfoFromNode(qc);
    if (curTickInfo.epoch != 0)
    {
        LOG("Tick: %u\n", curTickInfo.tick);
        LOG("Epoch: %u\n", curTickInfo.epoch);
        LOG("Number Of Aligned Votes: %u\n", curTickInfo.numberOfAlignedVotes);
        LOG("Number Of Misaligned Votes: %u\n", curTickInfo.numberOfMisalignedVotes);
        LOG("Initial tick: %u\n", curTickInfo.initialTick);
    }
    else
    {
        LOG("Error while getting tick info from %s:%d\n", nodeIp, nodePort);
    }
}

CurrentSystemInfo getSystemInfoFromNode(QCPtr qc)
{
    CurrentSystemInfo result;
    struct {
        RequestResponseHeader header;
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(REQUEST_SYSTEM_INFO);
    qc->sendData((uint8_t *) &packet, packet.header.size());

    try 
    {
        result = qc->receivePacketWithHeaderAs<CurrentSystemInfo>();
    }
    catch (std::logic_error)
    {
        memset(&result, 0, sizeof(CurrentSystemInfo));
    }

    return result;
}

void printSystemInfoFromNode(const char* nodeIp, int nodePort)
{
    auto qc = make_qc(nodeIp, nodePort);
    CurrentSystemInfo curSystemInfo = getSystemInfoFromNode(qc);
    if (curSystemInfo.epoch != 0)
    {
        LOG("Version: %u\n", curSystemInfo.version);
        LOG("Epoch: %u\n", curSystemInfo.epoch);
        LOG("Tick: %u\n", curSystemInfo.tick);
        LOG("InitialTick: %u\n", curSystemInfo.initialTick);
        LOG("LatestCreatedTick: %u\n", curSystemInfo.latestCreatedTick);
        LOG("NumberOfEntities: %u\n", curSystemInfo.numberOfEntities);
        LOG("TotalSpectrumAmount: %llu\n", curSystemInfo.totalSpectrumAmount);
        LOG("EntityBalanceDustThreshold: %llu\n", curSystemInfo.currentEntityBalanceDustThreshold);
        LOG("NumberOfTransactions: %u\n", curSystemInfo.numberOfTransactions);
        char hex[65];
        byteToHex(curSystemInfo.randomMiningSeed, hex, 32);
        LOG("RandomMiningSeed: %s\n", hex);
        LOG("SolutionThreshold: %u\n", curSystemInfo.solutionThreshold);
        LOG("AdditionSolutionThreshold: %llu\n", curSystemInfo.solutionAdditionalThreshold);

        // todo: add initial time

        LOG("\nAbout EntityBalanceDustThreshold: Entity balances less or euqal this value will be burned if number of entites rises to 75%% of spectrum capacity. Starts to be meaningful if >50%% of spectrum is filled but may still change after that.\n");
    }
    else
    {
        LOG("Error while getting system info from %s:%d\n", nodeIp, nodePort);
    }
}

// Mirrors core computeMultiDimRevenue in src/revenue.h
// Constants match the core: NUMBER_OF_COMPUTORS = 676 -> QUORUM = 451, REVENUE_SCALE = 1024.
static const unsigned long long REVENUE_SCALE = 1024;
static const unsigned int REVENUE_QUORUM = 451;

// QUORUM-th largest score (== core getQuorumScore), min 1.
static unsigned long long revQuorumThreshold(const unsigned long long* scores)
{
    std::vector<unsigned long long> sorted(scores, scores + NUMBER_OF_COMPUTORS);
    std::sort(sorted.begin(), sorted.end()); // ascending; index N-QUORUM is the QUORUM-th largest
    unsigned long long th = sorted[NUMBER_OF_COMPUTORS - REVENUE_QUORUM];
    return th ? th : 1ULL;
}

// Per-computor factor in [0, SCALE]. noActivityValve mirrors the oracle/doge valve: if the whole
// network is idle in this dimension, everyone gets the full factor (tx has no valve in core).
static void revComputeFactor(const unsigned long long* scores, bool noActivityValve, unsigned long long* out)
{
    if (noActivityValve)
    {
        bool any = false;
        for (unsigned int i = 0; i < NUMBER_OF_COMPUTORS; i++)
        {
            if (scores[i])
            {
                any = true;
                break;
            }
        }
        if (!any)
        {
            for (unsigned int i = 0; i < NUMBER_OF_COMPUTORS; i++)
            {
                out[i] = REVENUE_SCALE;
            }
            return;
        }
    }
    unsigned long long th = revQuorumThreshold(scores);
    for (unsigned int i = 0; i < NUMBER_OF_COMPUTORS; i++)
    {
        if (scores[i] == 0)
        {
            out[i] = 0;
        }
        else if (scores[i] >= th)
        {
            out[i] = REVENUE_SCALE;
        }
        else
        {
            out[i] = REVENUE_SCALE * scores[i] / th;
        }
    }
}

static unsigned long long revIpow(unsigned long long v, unsigned int e)
{
    unsigned long long r = 1;
    for (unsigned int i = 0; i < e; i++)
    {
        r *= v;
    }
    return r;
}

// true if x^k <= n, computed without overflow
static bool revPowLE(unsigned long long x, unsigned int k, unsigned long long n)
{
    unsigned long long r = 1;
    for (unsigned int i = 0; i < k; i++)
    {
        if (x != 0 && r > n / x)
        {
            return false;
        }
        r *= x;
    }
    return r <= n;
}

// floor(n^(1/k)), matches core math_lib::irootK64<k>
static unsigned long long revIroot(unsigned long long n, unsigned int k)
{
    if (k <= 1)
    {
        return n;
    }
    if (n == 0)
    {
        return 0;
    }
    unsigned long long lo = 0, hi = n;
    while (lo < hi)
    {
        unsigned long long mid = lo + (hi - lo + 1) / 2;
        if (revPowLE(mid, k, n))
        {
            lo = mid;
        }
        else
        {
            hi = mid - 1;
        }
    }
    return lo;
}

void dumpRevenueDataFromNode(const char* nodeIp, int nodePort, const char* outputFile)
{
    auto qc = make_qc(nodeIp, nodePort);

    struct {
        RequestResponseHeader header;
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(REQUEST_REVENUE_DATA);
    qc->sendData((uint8_t *) &packet, packet.header.size());

    // RevenueData is ~16 KB; receive into a heap buffer to avoid a large stack frame.
    auto result = std::make_unique<RevenueData>();
    try
    {
        qc->receivePacketWithHeaderAs<RevenueData>(*result);
    }
    catch (std::logic_error)
    {
        LOG("Error while getting revenue data from %s:%d\n", nodeIp, nodePort);
        return;
    }

    // Compute the factors and revenue exactly as the node does at end of epoch (approximate mid-epoch).
    auto txF = std::make_unique<unsigned long long[]>(NUMBER_OF_COMPUTORS);
    auto oracleF = std::make_unique<unsigned long long[]>(NUMBER_OF_COMPUTORS);
    auto dogeF = std::make_unique<unsigned long long[]>(NUMBER_OF_COMPUTORS);
    revComputeFactor(result->txScore, /*noActivityValve=*/false, txF.get());
    revComputeFactor(result->oracleScore, /*noActivityValve=*/true, oracleF.get());
    revComputeFactor(result->dogeScore, /*noActivityValve=*/true, dogeF.get());

    const unsigned int K = result->dogeK ? result->dogeK : 1;
    const unsigned long long sPowKm1 = revIpow(REVENUE_SCALE, K - 1);
    const unsigned long long divisor = REVENUE_SCALE * REVENUE_SCALE * REVENUE_SCALE;

    FILE* f = fopen(outputFile, "w");
    if (!f)
    {
        LOG("Failed to open %s for writing\n", outputFile);
        return;
    }
    fprintf(f, "Tick,DogeK,IPC,ComputorIndex,txScore,oracleScore,dogeScore,txFactor,oracleFactor,dogeFactor,revenue,revenuePercent\n");

    for (unsigned int i = 0; i < NUMBER_OF_COMPUTORS; i++)
    {
        const unsigned long long dogeRoot = revIroot(dogeF[i] * sPowKm1, K);
        const unsigned long long revenue =
            (unsigned long long)result->ipc * txF[i] * oracleF[i] * dogeRoot / divisor;
        const double pct = result->ipc ? (100.0 * (double)revenue / (double)result->ipc) : 0.0;
        fprintf(f, "%u,%u,%lld,%u,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%.4f\n",
                result->tick, result->dogeK, result->ipc,
                i, result->txScore[i], result->oracleScore[i], result->dogeScore[i],
                txF[i], oracleF[i], dogeF[i], revenue, pct);
    }

    fclose(f);
}

static void getTickTransactions(QCPtr qc, const uint32_t requestedTick, int nTx,
                                std::vector<Transaction>& txs, // out
                                std::vector<TxhashStruct>* hashes, // out
                                std::vector<ExtraDataStruct>* extraData, // out
                                std::vector<SignatureStruct>* sigs // out
)
{
    txs.resize(0);
    if (hashes != nullptr)
    {
        hashes->resize(0);
    }
    if (extraData != nullptr)
    {
        extraData->resize(0);
    }
    if (sigs != nullptr)
    {
        sigs->resize(0);
    }

    struct RequestPacket 
    {
        RequestResponseHeader header;
        RequestedTickTransactions txs;
    };
    auto packet = std::make_unique<RequestPacket>();
    packet->header.setSize(sizeof(RequestPacket));
    packet->header.randomizeDejavu();
    packet->header.setType(REQUEST_TICK_TRANSACTIONS);
    packet->txs.tick = requestedTick;
    for (int i = 0; i < (nTx+7)/8; i++) packet->txs.transactionFlags[i] = 0;
    for (int i = (nTx+7)/8; i < NUMBER_OF_TRANSACTIONS_PER_TICK/8; i++) packet->txs.transactionFlags[i] = 0xff;
    qc->sendData((uint8_t *) packet.get(), packet->header.size());

    constexpr unsigned long long bufferSize = sizeof(RequestResponseHeader) + MAX_TRANSACTION_SIZE;
    uint8_t buffer[bufferSize];
    int recvByte = qc->receiveData(buffer, sizeof(RequestResponseHeader));
    int recvTx = 0;
    while (recvByte == sizeof(RequestResponseHeader))
    {
        auto header = (RequestResponseHeader*)buffer;
        if (header->type() == BROADCAST_TRANSACTION)
        {
            recvByte = qc->receiveAllDataOrThrowException(buffer + sizeof(RequestResponseHeader), sizeof(Transaction));
            auto tx = (Transaction*)(buffer + sizeof(RequestResponseHeader));
            txs.push_back(*tx);
            if (tx->inputSize > MAX_INPUT_SIZE)
            {
                LOG("Received tx with invalid inputSize!\n");
                exit(1);
            }
            ++recvTx;
            recvByte = qc->receiveAllDataOrThrowException(buffer + sizeof(RequestResponseHeader) + sizeof(Transaction), tx->inputSize + SIGNATURE_SIZE);
            if (hashes != nullptr)
            {
                uint8_t digest[32] = { 0 };
                KangarooTwelve(reinterpret_cast<const uint8_t*>(tx),
                    sizeof(Transaction) + tx->inputSize + SIGNATURE_SIZE,
                    digest,
                    32);
                TxhashStruct hash;
                char txHash[128] = { 0 };
                getTxHashFromDigest(digest, txHash);
                memcpy(hash.hash, txHash, 60);
                hashes->push_back(hash);
            }
            if (extraData != nullptr)
            {
                ExtraDataStruct ed;
                ed.vecU8.resize(tx->inputSize);
                if (tx->inputSize != 0)
                {
                    memcpy(ed.vecU8.data(), reinterpret_cast<const uint8_t*>(tx) + sizeof(Transaction), tx->inputSize);
                }
                extraData->push_back(ed);
            }
            if (sigs != nullptr)
            {
                SignatureStruct sig;
                memcpy(sig.sig, reinterpret_cast<const uint8_t*>(tx) + sizeof(Transaction) + tx->inputSize, SIGNATURE_SIZE);
                sigs->push_back(sig);
            }
        }
        recvByte = qc->receiveData(buffer, sizeof(RequestResponseHeader));
        // check only after receive because response has an EndResponse header at the end
        if (recvTx == nTx)
            break;
    }

    LOG("Received %d tick transactions\n", recvTx);
}

static bool getTickData(const char* nodeIp, const int nodePort, const uint32_t tick, TickData& result)
{
    auto qc = make_qc(nodeIp, nodePort);
    return getTickData(qc, tick, result);
}

bool getTickData(QCPtr qc, const uint32_t tick, TickData& result)
{
    struct
    {
        RequestResponseHeader header;
        RequestTickData requestTickData;
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(REQUEST_TICK_DATA);
    packet.requestTickData.requestedTickData.tick = tick;
    qc->sendData((uint8_t*)&packet, packet.header.size());

    try
    {
        qc->receivePacketWithHeaderAs<TickData>(result);
    }
    catch (const std::logic_error& e)
    {
        LOG("%s\n", e.what());
        memset(&result, 0, sizeof(TickData));
        return false;
    }
    catch (const EndResponseReceived)
    {
        // EndResponse is sent if tick is empty or not in tick storage
        memset(&result, 0, sizeof(TickData));
    }
    return true;
}

int getMoneyFlewStatus(QCPtr qc, const char* txHash, const uint32_t requestedTick)
{
    struct {
        RequestResponseHeader header;
        RequestTxStatus rts;
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(REQUEST_TX_STATUS);
    packet.rts.tick = requestedTick;
    qc->sendData((uint8_t *) &packet, packet.header.size());
    RespondTxStatus result;
    try
    {
        result = qc->receivePacketWithHeaderAs<RespondTxStatus>();
        // notice: the node not always return full size of RESPOND_TX_STATUS
        // it only returns enough digests
        // -> set remainder in array memory which may contain junk to 0
        memset(result.txDigests[result.txCount], 0, (NUMBER_OF_TRANSACTIONS_PER_TICK - result.txCount) * 32);
    }
    catch (std::logic_error)
    {
        memset(&result, 0, sizeof(RespondTxStatus));
        // it's expected to catch this error on some node that not turn on tx status
        return -1;
    }

    int tx_id = -1;
    for (uint32_t i = 0; i < result.txCount; i++)
    {
        char tx_hash[60];
        memset(tx_hash, 0, 60);
        getIdentityFromPublicKey(result.txDigests[i], tx_hash, true);
        if (memcmp(tx_hash, txHash, 60) == 0)
        {
            tx_id = i;
            break;
        }
    }
    if (tx_id == -1)
    {
        return -1; // not found !?
    }
    return (result.moneyFlew[tx_id >> 3] & (1<<(tx_id & 7))) ? 1 : 0;
}

bool checkTxOnTick(QCPtr qc, const char* txHash, uint32_t requestedTick, bool printTxReceipt)
{
    // conditions:
    // - current Tick is higher than requested tick
    // - has tick data
    // - has txHash in tick transactions
    uint32_t currentTick = getTickNumberFromNode(qc);
    if (currentTick <= requestedTick)
    {
        LOG("Please wait a bit more. Requested tick %u, current tick %u\n", requestedTick, currentTick);
        return false;
    }
    auto td = std::make_unique<TickData>();
    if (!getTickData(qc, requestedTick, *td))
    {
        return false;
    }
    else if (td->epoch == 0)
    {
        LOG("Tick %u is empty, not in current epoch or in the future\n", requestedTick);
        return false;
    }
    int numTx = 0;
    uint8_t all_zero[32] = {0};
    for (int i = 0; i < NUMBER_OF_TRANSACTIONS_PER_TICK; i++)
    {
        if (memcmp(all_zero, td->transactionDigests[i], 32) != 0) numTx++;
    }

    auto txs = std::make_unique<std::vector<Transaction>>();
    txs->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto txHashesFromTick = std::make_unique<std::vector<TxhashStruct>>();
    txHashesFromTick->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto extraData = std::make_unique<std::vector<ExtraDataStruct>>();
    extraData->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);

    getTickTransactions(qc, requestedTick, numTx, *txs, txHashesFromTick.get(), extraData.get(), /*sigs=*/nullptr);

    for (int i = 0; i < txHashesFromTick->size(); i++)
    {
        if (memcmp(txHashesFromTick->at(i).hash, txHash, 60) == 0)
        {
            LOG("Found tx %s on tick %u\n", txHash, requestedTick);
            if (printTxReceipt)
            {
                // check for moneyflew status
                qc->resolveConnection();
                int moneyFlew = getMoneyFlewStatus(qc, txHash, requestedTick);
                printReceipt(txs->at(i), txHash, extraData->at(i).vecU8.data(), moneyFlew);
            }
            return true;
        }
    }
    LOG("Can NOT find tx %s on tick %u\n", txHash, requestedTick);
    return false;
}

bool checkTxOnTick(const char* nodeIp, const int nodePort, const char* txHash, uint32_t requestedTick, bool printTxReceipt)
{
    auto qc = std::make_shared<QubicConnection>(nodeIp, nodePort);
    return checkTxOnTick(qc, txHash, requestedTick, printTxReceipt);
}

// @return:
// - 0: ok
// - 1: hash doesn't exist
// - 2: failed connection
int _GetInputDataFromTxHash(QCPtr& qc, const char* txHash, uint8_t* outData, int& dataSize)
{
    char txUpperHash[61] = {0};
    for (int i = 0; txHash[i] != '\0'; ++i)
    {
        txUpperHash[i] = std::toupper(txHash[i]);
    }
    struct {
        RequestResponseHeader header;
        RequestedTransactionInfo txs;
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(REQUEST_TRANSACTION_INFO);
    getPublicKeyFromIdentity(txUpperHash, packet.txs.transactionDigest);

    qc->sendData((uint8_t *) &packet, packet.header.size());

    // Received the respond and print the receipt
    bool receivedTx = false;
    constexpr unsigned long long bufferSize = sizeof(RequestResponseHeader) + MAX_TRANSACTION_SIZE;
    uint8_t buffer[bufferSize];
    int recvByte = qc->receiveData(buffer, bufferSize);
    if (recvByte > 0)
    {
        auto header = (RequestResponseHeader*)buffer;
        if (header->type() == BROADCAST_TRANSACTION)
        {
            auto tx = (Transaction*)(buffer + sizeof(RequestResponseHeader));
            uint8_t digest[32] = {0};
            char respondTxHash[61] = {0};
            KangarooTwelve(
                    reinterpret_cast<const uint8_t*>(tx),
                    sizeof(Transaction) + tx->inputSize + SIGNATURE_SIZE,
                    digest,
                    32);
            getTxHashFromDigest(digest, respondTxHash);
            // Check the digest of respond transaction
            if (memcmp(txHash, respondTxHash, 60) == 0)
            {
                ExtraDataStruct ed;
                ed.vecU8.resize(tx->inputSize);
                if (tx->inputSize != 0)
                {
                    memcpy(
                            ed.vecU8.data(),
                            reinterpret_cast<const uint8_t*>(tx) + sizeof(Transaction),
                            tx->inputSize);
                }

                receivedTx = true;
                memcpy(outData, ed.vecU8.data(), tx->inputSize);
                dataSize = tx->inputSize;
            }
        }
    }
    if (!receivedTx)
    {
        LOG("Failed to get tx info. Please check the tx hash and try again later.\n");
        return 1;
    }
    return 0;
}

// @return:
// - 0: ok
// - 1: hash doesn't exist
// - 2: failed connection
int _GetTxInfo(QCPtr& qc, const char* txHash)
{
    char txUpperHash[61] = {0};
    for (int i = 0; txHash[i] != '\0'; ++i)
    {
        txUpperHash[i] = std::toupper(txHash[i]);
    }
    struct {
        RequestResponseHeader header;
        RequestedTransactionInfo txs;
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(REQUEST_TRANSACTION_INFO);
    getPublicKeyFromIdentity(txUpperHash, packet.txs.transactionDigest);

    qc->sendData((uint8_t *) &packet, packet.header.size());

    // Received the respond and print the receipt
    bool receivedTx = false;
    constexpr unsigned long long bufferSize = sizeof(RequestResponseHeader) + MAX_TRANSACTION_SIZE;
    uint8_t buffer[bufferSize];
    int recvByte = qc->receiveData(buffer, bufferSize);
    if (recvByte > 0)
    {
        auto header = (RequestResponseHeader*)(buffer);
        if (header->type() == BROADCAST_TRANSACTION)
        {
            auto tx = (Transaction*)(buffer + sizeof(RequestResponseHeader));
            uint8_t digest[32] = {0};
            char respondTxHash[61] = {0};
            KangarooTwelve(
                    reinterpret_cast<const uint8_t*>(tx),
                    sizeof(Transaction) + tx->inputSize + SIGNATURE_SIZE,
                    digest,
                    32);
            getTxHashFromDigest(digest, respondTxHash);
            // Check the digest of respond transaction
            if (memcmp(txHash, respondTxHash, 60) == 0)
            {
                ExtraDataStruct ed;
                ed.vecU8.resize(tx->inputSize);
                if (tx->inputSize != 0)
                {
                    memcpy(
                            ed.vecU8.data(),
                            reinterpret_cast<const uint8_t*>(tx) + sizeof(Transaction),
                            tx->inputSize);
                }

                receivedTx = true;
                printReceipt(*tx, respondTxHash, ed.vecU8.data(), -1);
            }
        }
    }
    if (!receivedTx)
    {
        LOG("Failed to get tx info. Please check the tx hash and try again later.\n");
        return 1;
    }
    return 0;
}

int getTxInfo(const char* nodeIp, const int nodePort, const char* txHash)
{
    auto qc = std::make_shared<QubicConnection>(nodeIp, nodePort);
    return _GetTxInfo(qc, txHash);
}

static void dumpQuorumTick(const Tick& A, bool dumpComputorIndex = true)
{
    char digest[64] = {0};
    if (dumpComputorIndex) LOG("Computor index: %d\n", A.computorIndex);
    LOG("Epoch: %d\n", A.epoch);
    LOG("Tick: %d\n", A.tick);
    LOG("Time: 20%02u-%02u-%02u %02u:%02u:%02u.%04u\n", A.year, A.month, A.day, A.hour, A.minute, A.second, A.millisecond);
    LOG("prevResourceTestingDigest: %u\n", A.prevResourceTestingDigest);
    getIdentityFromPublicKey(A.prevSpectrumDigest, digest, true);
    LOG("prevSpectrumDigest: %s\n", digest);
    getIdentityFromPublicKey(A.prevUniverseDigest, digest, true);
    LOG("prevUniverseDigest: %s\n", digest);
    getIdentityFromPublicKey(A.prevComputerDigest, digest, true);
    LOG("prevComputerDigest: %s\n", digest);
    getIdentityFromPublicKey(A.transactionDigest, digest, true);
    LOG("prevTransactionBodyDigest: %u\n", A.prevTransactionBodyDigest);
    LOG("transactionDigest: %s\n", digest);
    getIdentityFromPublicKey(A.expectedNextTickTransactionDigest, digest, true);
    LOG("expectedNextTickTransactionDigest: %s\n", digest);
}

bool compareVote(const Tick&A, const Tick&B, bool compareExpectedNextTickDigest)
{
    bool sameExpectedNextTickDigest = true;
    if (compareExpectedNextTickDigest)
    {
        sameExpectedNextTickDigest = memcmp(A.expectedNextTickTransactionDigest, B.expectedNextTickTransactionDigest, 32) == 0;
    }
    return (A.epoch == B.epoch) && (A.tick == B.tick) &&
           (A.year == B.year) && (A.month == B.month) && (A.day == B.day) && (A.hour == B.hour) && (A.minute == B.minute) && (A.second == B.second) &&
           (A.millisecond == B.millisecond) &&
           (A.prevResourceTestingDigest == B.prevResourceTestingDigest) &&
           (memcmp(A.prevSpectrumDigest, B.prevSpectrumDigest, 32) == 0) &&
           (memcmp(A.prevUniverseDigest, B.prevUniverseDigest, 32) == 0) &&
           (memcmp(A.prevComputerDigest, B.prevComputerDigest, 32) == 0) &&
           (memcmp(A.transactionDigest, B.transactionDigest, 32) == 0) &&
            sameExpectedNextTickDigest;
}

bool verifyVoteWithSalt(const Tick&A,
                        const BroadcastComputors& bc,
                        const unsigned int prevResourceDigest,
                        const uint8_t* prevSpectrumDigest,
                        const uint8_t* prevUniverseDigest,
                        const uint8_t* prevComputerDigest,
                        const unsigned int prevTransactionBodyDigest,
                        const uint8_t* nextTickTransactionDigest)
{
    int cid = A.computorIndex;
    uint8_t saltedData[64];
    uint8_t saltedDigest[32];
    memset(saltedData, 0, 64);
    memcpy(saltedData, bc.computors.publicKeys[cid], 32);
    memcpy(saltedData+32, &prevResourceDigest, 4);
    KangarooTwelve(saltedData, 36, saltedDigest, 4);
    if (A.saltedResourceTestingDigest != *((unsigned int*)(saltedDigest)))
    {
        LOG("Mismatched saltedResourceTestingDigest. Computor index: %d\n", cid);
        return false;
    }
    memcpy(saltedData+32, prevSpectrumDigest, 32);
    KangarooTwelve(saltedData, 64, saltedDigest, 32);
    if (memcmp(saltedDigest, A.saltedSpectrumDigest, 32) != 0)
    {
        LOG("Mismatched saltedSpectrumDigest. Computor index: %d\n", cid);
        return false;
    }

    memcpy(saltedData+32, prevUniverseDigest, 32);
    KangarooTwelve(saltedData, 64, saltedDigest, 32);
    if (memcmp(saltedDigest, A.saltedUniverseDigest, 32) != 0)
    {
        LOG("Mismatched saltedUniverseDigest. Computor index: %d\n", cid);
        return false;
    }

    memcpy(saltedData+32, prevComputerDigest, 32);
    KangarooTwelve(saltedData, 64, saltedDigest, 32);
    if (memcmp(saltedDigest, A.saltedComputerDigest, 32) != 0)
    {
        LOG("Mismatched saltedComputerDigest. Computor index: %d\n", cid);
        return false;
    }
    bool should_check_txBodyDigest = isArrayZero(A.expectedNextTickTransactionDigest, 32) == isArrayZero(nextTickTransactionDigest, 32);
    if (!isArrayZero(A.expectedNextTickTransactionDigest, 32) && !isArrayZero(nextTickTransactionDigest, 32))
    {
        if (A.expectedNextTickTransactionDigest != nextTickTransactionDigest)
        {
            should_check_txBodyDigest = false;
        }
    }
    if(should_check_txBodyDigest){
        memset(saltedData+32, 0, 32);
        memcpy(saltedData+32, &prevTransactionBodyDigest, 4);
        KangarooTwelve(saltedData, 36, saltedDigest, 4);
        if (A.saltedTransactionBodyDigest != *((unsigned int*)(saltedDigest)))
        {
            LOG("Mismatched saltedTransactionBodyDigest. Computor index: %d\n%u\n%u\n", cid, A.saltedTransactionBodyDigest, *((unsigned int*)(saltedDigest)));
            return false;
        }
    }
    return true;
}

std::string indexToAlphabet(int index){
    std::string result = "";
    result += char('A' + (index/26));
    result += char('A' + (index%26));
    return result;
}

void getUniqueVotes(std::vector<Tick>& votes, std::vector<Tick>& uniqueVote, std::vector<std::vector<int>>& voteIndices, int N,
                    bool verifySalt = false,
                    bool compareExpectedNextTickDigest = false,
                    BroadcastComputors* pBC = nullptr,
                    const unsigned int prevResourceDigest = 0,
                    const uint8_t* prevSpectrumDigest = nullptr,
                    const uint8_t* prevUniverseDigest = nullptr,
                    const uint8_t* prevComputerDigest = nullptr,
                    const unsigned int prevTransactionBodyDigest = 0,
                    const uint8_t* nextTickTransactionDigest = 0)
{
    if (votes.size() == 0) return;
    if (verifySalt)
    {
        std::vector<Tick> new_votes;
        LOG("Performing salt check...\n");
        bool all_passed = true;
        for (int i = 0; i < N; i++)
        {
            if (!verifyVoteWithSalt(votes[i], *pBC, prevResourceDigest, prevSpectrumDigest, prevUniverseDigest, prevComputerDigest, prevTransactionBodyDigest, nextTickTransactionDigest))
            {
                LOG("Vote %d failed to pass salt check\n", i);
                dumpQuorumTick(votes[i]);
                all_passed = false;
            }
            else
            {
                new_votes.push_back(votes[i]);
            }
        }
        if (all_passed)
        {
            LOG("ALL votes PASSED salts check\n");
        }
        else
        {
            votes = new_votes;
        }
    }
    uniqueVote.resize(0);
    uniqueVote.push_back(votes[0]);
    voteIndices.resize(1);
    voteIndices[0].push_back(votes[0].computorIndex);
    for (int i = 1; i < votes.size(); i++)
    {
        int vote_indice = -1;
        for (int j = 0; j < uniqueVote.size(); j++)
        {
            if (compareVote(votes[i], uniqueVote[j], compareExpectedNextTickDigest))
            {
                vote_indice = j;
                break;
            }
        }
        if (vote_indice != -1)
        {
            voteIndices[vote_indice].push_back(votes[i].computorIndex);
        }
        else
        {
            uniqueVote.push_back(votes[i]);
            voteIndices.resize(voteIndices.size() + 1);
            int M = int(voteIndices.size() -1);
            voteIndices[M].resize(0);
            voteIndices[M].push_back(votes[i].computorIndex);
        }
    }
}

void getQuorumTick(const char* nodeIp, const int nodePort, uint32_t requestedTick, const char* compFileName)
{
    auto qc = std::make_shared<QubicConnection>(nodeIp, nodePort);
    BroadcastComputors bc;
    {
        FILE* f = fopen(compFileName, "rb");
        if (fread(&bc, 1, sizeof(BroadcastComputors), f) != sizeof(BroadcastComputors))
        {
            LOG("Failed to read comp list\n");
            fclose(f);
            return;
        }
        fclose(f);
    }

    static struct
    {
        RequestResponseHeader header;
        RequestedQuorumTick rqt;
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(RequestedQuorumTick::type); // REQUEST_TICK_DATA
    packet.rqt.tick = requestedTick;
    memset(packet.rqt.voteFlags, 0, (676 + 7) / 8);
    qc->sendData(reinterpret_cast<uint8_t *>(&packet), sizeof(packet));
    auto votes = qc->getLatestVectorPacketAs<Tick>();
    LOG("Received %d quorum tick #%u (votes)\n", votes.size(), requestedTick);

    packet.rqt.tick = requestedTick+1;
    memset(packet.rqt.voteFlags, 0, (676 + 7) / 8);
    qc->sendData(reinterpret_cast<uint8_t *>(&packet), sizeof(packet));
    auto votes_next = qc->getLatestVectorPacketAs<Tick>();
    LOG("Received %d quorum tick #%u (votes)\n", votes_next.size(), requestedTick+1);

    int N = int(votes.size());
    if (N == 0)
    {
        return;
    }

    for (int i = 0; i < N; i++)
    {
        uint8_t digest[64] = {0};
        votes[i].computorIndex ^= Tick::type();
        KangarooTwelve((uint8_t*)&votes[i], sizeof(Tick) - SIGNATURE_SIZE, digest, 32);
        votes[i].computorIndex ^= Tick::type();
        int comp_index = votes[i].computorIndex;
        if (!verify(bc.computors.publicKeys[comp_index], digest, votes[i].signature))
        {
            LOG("Signature of vote %d is not correct\n", i);
            dumpQuorumTick(votes[i]);
            return;
        }
    }
    std::vector<Tick> uniqueVote, uniqueVoteNext;
    std::vector<std::vector<int>> voteIndices, voteIndicesNext;
    getUniqueVotes(votes_next, uniqueVoteNext, voteIndicesNext, N, false /*verifySalt*/, false /*Compare expected next digest*/);
    if (votes_next.size() < 451)
    {
        printf("Failed to get votes for tick %d, this will not perform salt check\n", requestedTick+1);
        getUniqueVotes(votes, uniqueVote, voteIndices, N, false /*verifySalt*/, true /*Compare expected next digest*/);
    }
    else
    {
        int max_id = 0;
        for (int i = 1; i < uniqueVoteNext.size(); i++)
        {
            if (voteIndicesNext[max_id].size() < voteIndicesNext[i].size())
            {
                max_id = i;
            }
        }
        if (voteIndicesNext[max_id].size() >= 451)
        {
            auto vote_next = uniqueVoteNext[max_id];
            getUniqueVotes(votes, uniqueVote, voteIndices, N, true, true, &bc,
                vote_next.prevResourceTestingDigest,
                vote_next.prevSpectrumDigest,
                vote_next.prevUniverseDigest,
                vote_next.prevComputerDigest,
                vote_next.prevTransactionBodyDigest,
                vote_next.transactionDigest);
        }
        else
        {
            LOG("WARNING: No quorum on tick %u (maximum aligned vote: %d). Skip salt check...\n", requestedTick + 1, int(voteIndicesNext[max_id].size()));
            getUniqueVotes(votes, uniqueVote, voteIndices, N, false /*verifySalt*/, true /*Compare expected next digest*/);
        }
    }

    LOG("Number of unique votes: %d\n", uniqueVote.size());
    bool flag[676] = {false};
    for (int i = 0; i < uniqueVote.size(); i++)
    {
        LOG("Vote #%d (voted by %d computors ID) ", i, voteIndices[i].size());
        const bool dumpComputorIndex = false;
        dumpQuorumTick(uniqueVote[i], dumpComputorIndex);
        LOG("Voted by: ");
        std::sort(voteIndices[i].begin(), voteIndices[i].end());
        for (int j = 0; j < voteIndices[i].size(); j++)
        {
            int index = voteIndices[i][j];
            flag[index] = true;
            auto alphabet = indexToAlphabet(index);
            if (j < voteIndices[i].size() - 1)
            {
                LOG("%d(%s), ", index, alphabet.c_str());
            }
            else
            {
                LOG("%d(%s)\n", index, alphabet.c_str());
            }
        }
    }

    LOG("Missing comps: \n");
    for (int i = 0; i < 676; i++)
    {
        if (!flag[i])
        {
            auto alphabet = indexToAlphabet(i);
            char iden[64] = {0};
            getIdentityFromPublicKey(bc.computors.publicKeys[i], iden, false);
            LOG("%d\t%s\t%s\n", i, alphabet.c_str(), iden);
        }
    }
}

void getTickDataToFile(const char* nodeIp, const int nodePort, uint32_t requestedTick, const char* fileName)
{
    auto qc = std::make_shared<QubicConnection>(nodeIp, nodePort);
    auto td = std::make_unique<TickData>();
    if (!getTickData(nodeIp, nodePort, requestedTick, *td))
    {
        return;
    }
    else if (td->epoch == 0)
    {
        LOG("Tick %u not in current epoch or in the future\n", requestedTick);
        return;
    }
    int numTx = 0;
    uint8_t all_zero[32] = {0};
    for (numTx = NUMBER_OF_TRANSACTIONS_PER_TICK; numTx > 0; numTx--)
    {
        if (memcmp(all_zero, td->transactionDigests[numTx-1], 32) != 0) break;
    }
    LOG("Found %d transactions in tick %u\n", numTx, requestedTick);

    auto txs = std::make_unique<std::vector<Transaction>>();
    txs->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto extraData = std::make_unique<std::vector<ExtraDataStruct>>();
    extraData->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto signatures = std::make_unique<std::vector<SignatureStruct>>();
    signatures->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);

    getTickTransactions(qc, requestedTick, numTx, *txs, /*hashes=*/nullptr, extraData.get(), signatures.get());

    FILE* f = fopen(fileName, "wb");
    fwrite(td.get(), 1, sizeof(TickData), f);
    for (int i = 0; i < txs->size(); i++)
    {
        fwrite(&txs->at(i), 1, sizeof(Transaction), f);
        int extraDataSize = txs->at(i).inputSize;
        if (extraDataSize != 0)
        {
            fwrite(extraData->at(i).vecU8.data(), 1, extraDataSize, f);
        }
        fwrite(signatures->at(i).sig, 1, SIGNATURE_SIZE, f);
    }
    fclose(f);
    LOG("Tick data and tick transactions have been written to %s\n", fileName);
}

void readTickDataFromFile(const char* fileName, TickData& td,
                          std::vector<Transaction>& txs,
                          std::vector<ExtraDataStruct>* extraData,
                          std::vector<SignatureStruct>* signatures,
                          std::vector<TxhashStruct>* txHashes)
{
    uint8_t extraDataBuffer[1024] = {0};
    uint8_t signatureBuffer[128] = {0};
    char txHashBuffer[128] = {0};
    uint8_t digest[32] = {0};

    FILE* f = fopen(fileName, "rb");
    if (fread(&td, 1, sizeof(TickData), f) != sizeof(TickData))
    {
        LOG("Failed to read TickData\n");
        fclose(f);
        return;
    }

    int numTx = 0;
    uint8_t all_zero[32] = {0};
    LOG("List of transactions on tickData (correct order):\n");
    for (int i = 0; i < NUMBER_OF_TRANSACTIONS_PER_TICK; i++)
    {
        if (memcmp(all_zero, td.transactionDigests[i], 32) != 0)
        {
            numTx++;
            char digestHex[65] = {0};
            getIdentityFromPublicKey(td.transactionDigests[i], digestHex, true);
            LOG("%s\n", digestHex);
        }
    }
    LOG("Total number of transaction digests: %d\n", numTx);

    auto vDigests = std::make_unique<std::vector<TxDigestStruct>>();
    vDigests->resize(numTx);
    for (int i = 0; i < numTx; i++)
    {
        Transaction tx;
        if (fread(&tx, 1, sizeof(Transaction), f) != sizeof(Transaction))
        {
            LOG("Failed to read Transaction %d\n", i);
            break;
        }
        int extraDataSize = tx.inputSize;
        if (extraData != nullptr)
        {
            ExtraDataStruct eds;
            if (extraDataSize != 0)
            {
                if (fread(extraDataBuffer, 1, extraDataSize, f) != extraDataSize)
                {
                    LOG("Failed to read Transaction payload for tx %d\n", i);
                    fclose(f);
                    return;
                }
                eds.vecU8.resize(extraDataSize);
                memcpy(eds.vecU8.data(), extraDataBuffer, extraDataSize);
            }
            extraData->push_back(eds);
        }

        if (fread(signatureBuffer, 1, SIGNATURE_SIZE, f) != SIGNATURE_SIZE)
        {
            LOG("Failed to read signature for tx %d\n", i);
            fclose(f);
            return;
        }
        if (signatures != nullptr)
        {
            SignatureStruct sig;
            memcpy(sig.sig, signatureBuffer, SIGNATURE_SIZE);
            signatures->push_back(sig);
        }
        {
            std::vector<uint8_t> raw_data;
            raw_data.resize(sizeof(Transaction) + tx.inputSize + SIGNATURE_SIZE);
            auto ptr = raw_data.data();
            memcpy(ptr, &tx, sizeof(Transaction));
            memcpy(ptr + sizeof(Transaction), extraDataBuffer, tx.inputSize);
            memcpy(ptr + sizeof(Transaction) + tx.inputSize, signatureBuffer, SIGNATURE_SIZE);
            KangarooTwelve(ptr,
                           uint32_t(raw_data.size()),
                           digest,
                           32);
            memcpy(vDigests->at(i).digest, digest, 32);
        }
        if (txHashes != nullptr)
        {
            TxhashStruct tx_hash;
            getTxHashFromDigest(digest, txHashBuffer);
            memcpy(tx_hash.hash, txHashBuffer, 60);
            txHashes->push_back(tx_hash);
        }
        txs.push_back(tx);
    }

    // if fread failed for a transaction, fill remaining txs with all zero
    for (int i = int(txs.size()); i < numTx; i++)
    {
        Transaction tx;
        memset(&tx, 0, sizeof(Transaction));
        txs.push_back(tx);
        if (extraData != nullptr)
        {
            ExtraDataStruct eds;
            eds.vecU8.resize(0);
            extraData->push_back(eds);
        }
        if (signatures != nullptr)
        {
            SignatureStruct ss;
            memset(ss.sig, 0, 64);
            signatures->push_back(ss);
        }
        if (txHashes != nullptr)
        {
            TxhashStruct tx_hash;
            memset(tx_hash.hash, 0, 60);
            txHashes->push_back(tx_hash);
        }
    }

    // put in correct order by tickdata
    auto _txs = std::make_unique<std::vector<Transaction>>();
    auto _extraData = std::make_unique<std::vector<ExtraDataStruct>>();
    auto _signatures = std::make_unique<std::vector<SignatureStruct>>();
    auto _txHashes = std::make_unique<std::vector<TxhashStruct>>();

    _txs->resize(numTx);
    if (extraData != nullptr) _extraData->resize(numTx);
    if (signatures != nullptr) _signatures->resize(numTx);
    if (txHashes != nullptr) _txHashes->resize(numTx);

    unsigned int numNotMatched = 0;
    for (int i = 0; i < numTx; i++)
    {
        bool foundMatch = false;
        for (int j = 0; j < numTx; j++)
        {
            // guaranteed by the protocol, if any duplicated here it's a bug on core side
            if (memcmp(vDigests->at(j).digest, td.transactionDigests[i], 32) == 0)
            {
                foundMatch = true;
                _txs->at(i) = txs[j];
                if (extraData != nullptr) _extraData->at(i) =  extraData->at(j);
                if (signatures != nullptr) _signatures->at(i) = signatures->at(j);
                if (txHashes != nullptr) _txHashes->at(i) = txHashes->at(j);
                break;
            }
        }
        if (!foundMatch)
        {
            numNotMatched++;
            LOG("Did not find a matching transaction for digest %d in TickData: ", i);
            char digestHex[65] = { 0 };
            getIdentityFromPublicKey(td.transactionDigests[i], digestHex, true);
            LOG("%s\n", digestHex);

            memset(&_txs->at(i), 0, sizeof(Transaction));
            if (extraData != nullptr) _extraData->at(i).vecU8.resize(0);
            if (signatures != nullptr) memset(_signatures->at(i).sig, 0, 64);
            if (txHashes != nullptr) memset(_txHashes->at(i).hash, 0, 60);
        }
    }
    if (numNotMatched > 0)
    {
        LOG("In total: did not find a matching transaction for %u digests\n", numNotMatched);
    }

    txs = *_txs;
    if (extraData != nullptr) (*extraData) = *_extraData;
    if (signatures != nullptr) (*signatures) = *_signatures;
    if (txHashes != nullptr) (*txHashes) = *_txHashes;
    fclose(f);
}

BroadcastComputors readComputorListFromFile(const char* fileName);

void printTickDataFromFile(const char* fileName, const char* compFile)
{
    auto td = std::make_unique<TickData>();
    auto txs = std::make_unique< std::vector<Transaction>>();
    txs->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto extraData = std::make_unique< std::vector<ExtraDataStruct>>();
    extraData->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto signatures = std::make_unique<std::vector<SignatureStruct>>();
    signatures->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto txHashes = std::make_unique< std::vector<TxhashStruct>>();
    txHashes->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);

    readTickDataFromFile(fileName, *td, *txs, extraData.get(), signatures.get(), txHashes.get());
    
    // verify everything
    uint8_t digest[32];
    BroadcastComputors bc;
    bc = readComputorListFromFile(compFile);
    if (bc.computors.epoch != td->epoch)
    {
        LOG("Computor list epoch (%u) and tick data epoch (%u) are not matched\n", bc.computors.epoch, td->epoch);
    }
    KangarooTwelve((uint8_t*)td.get(), sizeof(TickData), digest, 32);
    {
        char tddigest[61]={0};
        getIdentityFromPublicKey(digest, tddigest, true);
        LOG("Tickdata: %s\n", tddigest);
    }
    int computorIndex = td->computorIndex;
    td->computorIndex ^= BROADCAST_FUTURE_TICK_DATA;
    KangarooTwelve(reinterpret_cast<const uint8_t *>(td.get()),
                   sizeof(TickData) - SIGNATURE_SIZE,
                   digest,
                   32);
    uint8_t* computorOfThisTick = bc.computors.publicKeys[computorIndex];
    if (verify(computorOfThisTick, digest, td->signature))
    {
        char computorID[61] = {0};
        getIdentityFromPublicKey(computorOfThisTick, computorID, false);
        LOG("Tick is VERIFIED (signed by correct computor).\n");
        LOG("Comp ID: %s\n", computorID);
    }
    else
    {
        LOG("Tick is NOT verified (not signed by correct computor).\n");
    }
    LOG("Epoch: %u\n", td->epoch);
    LOG("Tick: %u\n", td->tick);
    LOG("Computor index: %u\n", computorIndex);
    LOG("Datetime: %u-%u-%u %u:%u:%u.%u\n", td->day, td->month, td->year, td->hour, td->minute, td->second, td->millisecond);

    unsigned int numZero = 0;
    for (int i = 0; i < txs->size(); i++)
    {
        if (isArrayZero((uint8_t*)&txs->at(i), sizeof(Transaction)))
        {
            LOG("Detect a zero transaction - Ignoring\n");
            numZero++;
            continue;
        }
        uint8_t* extraDataPtr = extraData->at(i).vecU8.empty() ? nullptr : extraData->at(i).vecU8.data();
        printReceipt(txs->at(i), txHashes->at(i).hash, extraDataPtr);
        if (verifyTx(txs->at(i), extraData->at(i).vecU8.data(), signatures->at(i).sig))
        {
            LOG("Transaction is VERIFIED\n");
        }
        else
        {
            LOG("Transaction is NOT VERIFIED. Incorrect signature\n");
        }
    }
    if (numZero > 0)
    {
        LOG("Total number of zero transactions %u\n", numZero);
    }
}

bool checkTxOnFile(const char* txHash, const char* fileName)
{
    auto td = std::make_unique<TickData>();
    auto txs = std::make_unique< std::vector<Transaction>>();
    txs->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto extraData = std::make_unique< std::vector<ExtraDataStruct>>();
    extraData->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto signatures = std::make_unique<std::vector<SignatureStruct>>();
    signatures->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto txHashes = std::make_unique< std::vector<TxhashStruct>>();
    txHashes->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);

    readTickDataFromFile(fileName, *td, *txs, extraData.get(), signatures.get(), txHashes.get());

    for (int i = 0; i < txs->size(); i++)
    {
        if (memcmp(txHashes->at(i).hash, txHash, 60) == 0)
        {
            LOG("Found tx %s on file %s\n", txHash, fileName);
            printReceipt(txs->at(i), txHash, extraData->at(i).vecU8.data());
            return true;
        }
    }
    LOG("Can NOT find tx %s on file %s\n", txHash, fileName);
    return false;
}

void sendRawPacket(const char* nodeIp, const int nodePort, int rawPacketSize, uint8_t* rawPacket)
{
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData(rawPacket, rawPacketSize);
    LOG("Sent %d bytes\n", rawPacketSize);
    RequestResponseHeader header;
    uint8_t* headerPtr = (uint8_t*)&header;
    qc->receiveData(headerPtr, sizeof(RequestResponseHeader));
    std::vector<uint8_t> buffer;
    if (header.size() > sizeof(RequestResponseHeader))
    {
        unsigned long long remainingSize = header.size() - sizeof(RequestResponseHeader);
        buffer.resize(remainingSize);
        qc->receiveData(buffer.data(), int(remainingSize));
    }
    LOG("Received %d bytes\n", header.size());
    for (int i = 0; i < sizeof(RequestResponseHeader); ++i)
    {
        LOG("%02x", headerPtr[i]);
    }
    for (int i = 0; i < buffer.size(); ++i)
    {
        LOG("%02x", buffer[i]);
    }
    LOG("\n");
}

void sendSpecialCommand(const char* nodeIp, const int nodePort, const char* seed, int command)
{
    uint8_t privateKey[32] = {0};
    uint8_t sourcePublicKey[32] = {0};
    uint8_t subseed[32] = {0};
    uint8_t digest[32] = {0};
    uint8_t signature[64] = {0};

    struct {
        RequestResponseHeader header;
        SpecialCommand cmd;
        uint8_t signature[64];
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(PROCESS_SPECIAL_COMMAND);
    uint64_t curTime = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
    uint64_t commandByte = (uint64_t)(command) << 56;
    packet.cmd.everIncreasingNonceAndCommandType = commandByte | curTime;

    getSubseedFromSeed((uint8_t*)seed, subseed);
    getPrivateKeyFromSubSeed(subseed, privateKey);
    getPublicKeyFromPrivateKey(privateKey, sourcePublicKey);
    KangarooTwelve((unsigned char*)&packet.cmd,
                   sizeof(packet.cmd),
                   digest,
                   32);
    sign(subseed, sourcePublicKey, digest, signature);
    memcpy(packet.signature, signature, 64);
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData((uint8_t *) &packet, packet.header.size());

    SpecialCommand response;
    try
    {
        response = qc->receivePacketWithHeaderAs<SpecialCommand>();
    }
    catch (std::logic_error& e)
    {
        memset(&response, 0, sizeof(SpecialCommand));
        if (command != SPECIAL_COMMAND_REFRESH_PEER_LIST)
        {
            LOG("%s\n", e.what());
            return;
        }
    }

    if (response.everIncreasingNonceAndCommandType == packet.cmd.everIncreasingNonceAndCommandType)
    {
        LOG("Node received special command\n");
    } 
    else
    {
        if (command != SPECIAL_COMMAND_REFRESH_PEER_LIST)
        {
            LOG("Failed to send special command\n");
        } 
        else 
        {
            LOG("Sent special command\n"); // the connection is refreshed right after this command, no way to verify remotely
        }
    }
}

void toggleMainAux(const char* nodeIp, const int nodePort, const char* seed, std::string mode0, std::string mode1)
{
    uint8_t privateKey[32] = {0};
    uint8_t sourcePublicKey[32] = {0};
    uint8_t subseed[32] = {0};
    uint8_t digest[32] = {0};
    uint8_t signature[64] = {0};

    struct {
        RequestResponseHeader header;
        SpecialCommandToggleMainModeResquestAndResponse cmd;
        uint8_t signature[64];
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(PROCESS_SPECIAL_COMMAND);
    uint64_t curTime = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
    uint64_t commandByte = (uint64_t)(SPECIAL_COMMAND_TOGGLE_MAIN_MODE_REQUEST) << 56;
    packet.cmd.everIncreasingNonceAndCommandType = commandByte | curTime;
    uint8_t flag = 0;
    if (mode0 == "MAIN") flag |= 1;
    if (mode1 == "MAIN") flag |= 2;
    packet.cmd.mainModeFlag = flag;
    memset(packet.cmd.padding, 0, 7);

    getSubseedFromSeed((uint8_t*)seed, subseed);
    getPrivateKeyFromSubSeed(subseed, privateKey);
    getPublicKeyFromPrivateKey(privateKey, sourcePublicKey);
    KangarooTwelve((unsigned char*)&packet.cmd,
                   sizeof(packet.cmd),
                   digest,
                   32);
    sign(subseed, sourcePublicKey, digest, signature);
    memcpy(packet.signature, signature, 64);
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData((uint8_t *) &packet, packet.header.size());

    SpecialCommandToggleMainModeResquestAndResponse response;
    try
    {
        response = qc->receivePacketWithHeaderAs<SpecialCommandToggleMainModeResquestAndResponse>();
    }
    catch (std::logic_error)
    {
        memset(&response, 0, sizeof(SpecialCommandToggleMainModeResquestAndResponse));
    }

    if (response.everIncreasingNonceAndCommandType == packet.cmd.everIncreasingNonceAndCommandType)
    {
        if (response.mainModeFlag == packet.cmd.mainModeFlag)
        {
            LOG("Successfully set MAINAUX flag\n");
        } 
        else 
        {
            LOG("The packet is successfully sent but failed set MAINAUX flag\n");
        }
    }
    else
    {
        LOG("Failed set MAINAUX flag\n");
    }
}

void setSolutionThreshold(const char* nodeIp, const int nodePort, const char* seed, int epoch, int threshold, int algo)
{
    uint8_t privateKey[32] = {0};
    uint8_t sourcePublicKey[32] = {0};
    uint8_t subseed[32] = {0};
    uint8_t digest[32] = {0};
    uint8_t signature[64] = {0};

    struct {
        RequestResponseHeader header;
        SpecialCommandSetSolutionThresholdResquestAndResponse cmd;
        uint8_t signature[64];
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(PROCESS_SPECIAL_COMMAND);
    uint64_t curTime = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
    uint64_t commandByte = (uint64_t)(SPECIAL_COMMAND_SET_SOLUTION_THRESHOLD_REQUEST) << 56;
    packet.cmd.everIncreasingNonceAndCommandType = commandByte | curTime;
    packet.cmd.epoch = epoch;
    packet.cmd.threshold = threshold;
    packet.cmd.algoType = algo;

    getSubseedFromSeed((uint8_t*)seed, subseed);
    getPrivateKeyFromSubSeed(subseed, privateKey);
    getPublicKeyFromPrivateKey(privateKey, sourcePublicKey);
    KangarooTwelve((unsigned char*)&packet.cmd,
                   sizeof(packet.cmd),
                   digest,
                   32);
    sign(subseed, sourcePublicKey, digest, signature);
    memcpy(packet.signature, signature, 64);
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData((uint8_t *) &packet, packet.header.size());

    SpecialCommandSetSolutionThresholdResquestAndResponse response;
    try
    {
        response = qc->receivePacketWithHeaderAs<SpecialCommandSetSolutionThresholdResquestAndResponse>();
    }
    catch (std::logic_error) 
    {
        memset(&response, 0, sizeof(SpecialCommandSetSolutionThresholdResquestAndResponse));
    }

    if (response.everIncreasingNonceAndCommandType == packet.cmd.everIncreasingNonceAndCommandType)
    {
        if (response.epoch == packet.cmd.epoch && response.threshold == packet.cmd.threshold && response.algoType == packet.cmd.algoType)
        {
            LOG("Successfully set solution threshold\n");
        } 
        else 
        {
            LOG("The packet is successfully sent but failed set solution threshold\n");
        }
    } 
    else
    {
        LOG("Failed set solution threshold\n");
    }
}

void logTime(const UtcTime& time)
{
    LOG("%u-%02u-%02u %02u:%02u:%02u.%09u", time.year, time.month, time.day, time.hour, time.minute, time.second, time.nanosecond);
}

UtcTime convertTime(std::chrono::system_clock::time_point time)
{
    using namespace std;
    using namespace std::chrono;

    time_t tt = system_clock::to_time_t(time);
    tm utc_tm = *gmtime(&tt);
    UtcTime utcTime;
    utcTime.year = utc_tm.tm_year + 1900;
    utcTime.month = utc_tm.tm_mon + 1;
    utcTime.day = utc_tm.tm_mday;
    utcTime.hour = utc_tm.tm_hour;
    utcTime.minute = utc_tm.tm_min;
    utcTime.second = utc_tm.tm_sec;

    // get nanoseconds
    typedef duration<int, ratio_multiply<hours::period, ratio<24> >::type> days;
    system_clock::duration tp = time.time_since_epoch();
    days d = duration_cast<days>(tp);
    tp -= d;
    hours h = duration_cast<hours>(tp);
    tp -= h;
    minutes m = duration_cast<minutes>(tp);
    tp -= m;
    seconds s = duration_cast<seconds>(tp);
    tp -= s;
    utcTime.nanosecond = uint32_t(duration_cast<nanoseconds>(tp).count());

    return utcTime;
}

void syncTime(const char* nodeIp, const int nodePort, const char* seed)
{
    uint8_t privateKey[32] = { 0 };
    uint8_t sourcePublicKey[32] = { 0 };
    uint8_t subseed[32] = { 0 };
    uint8_t digest[32] = { 0 };
    uint8_t signature[64] = { 0 };
    getSubseedFromSeed((uint8_t*)seed, subseed);
    getPrivateKeyFromSubSeed(subseed, privateKey);
    getPublicKeyFromPrivateKey(privateKey, sourcePublicKey);

    LOG("---------------------------------------------------------------------------------\n");
    LOG("This sets the node clock to roughly be in sync with the local clock.\n");
    LOG("CAUTION: MAKE SURE THAT YOUR LOCAL CLOCK IS SET CORRECTLY, FOR EXAMPLE USING NTP.\n");
    LOG("---------------------------------------------------------------------------------\n\n");

    auto qc = make_qc(nodeIp, nodePort);

    using namespace std::chrono;
    // get time from node and measure round trip time
    unsigned long long roundTripTimeNanosec = 0;
    unsigned long long finalizePacketTimeNanosec = 0;
    {
        LOG("Querying node time ...\n\n");

        struct {
            RequestResponseHeader header;
            SpecialCommand cmd;
            uint8_t signature[64];
        } queryTimeMsg;
        queryTimeMsg.header.setSize(sizeof(queryTimeMsg));
        queryTimeMsg.header.randomizeDejavu();
        queryTimeMsg.header.setType(PROCESS_SPECIAL_COMMAND);
        uint64_t curTime = duration_cast<microseconds>(system_clock::now().time_since_epoch()).count();
        uint64_t commandByte = (uint64_t)(SPECIAL_COMMAND_QUERY_TIME) << 56;
        queryTimeMsg.cmd.everIncreasingNonceAndCommandType = commandByte | curTime;
        LOG("Current unix time (in us) used for nonce (query time): %" PRIu64 "\n\n", curTime);

        // we need to measure the time it takes to finalize the packet because these steps are also executed after the sendTime packet is created later
        auto finalizePacketStartTime = steady_clock::now();

        KangarooTwelve((unsigned char*)&queryTimeMsg.cmd,
                       sizeof(queryTimeMsg.cmd),
                       digest,
                       32);
        sign(subseed, sourcePublicKey, digest, signature);
        memcpy(queryTimeMsg.signature, signature, 64);

        auto startTime = steady_clock::now();

        qc->sendData((uint8_t*)&queryTimeMsg, queryTimeMsg.header.size());
        
        SpecialCommandSendTime response;
        try
        {
            response = qc->receivePacketWithHeaderAs<SpecialCommandSendTime>();
        }
        catch (std::logic_error)
        {
            memset(&response, 0, sizeof(SpecialCommandSendTime));
        }        
        
        auto endTime = steady_clock::now();
        auto nowLocal = system_clock::now();

        if ((response.everIncreasingNonceAndCommandType & 0xFFFFFFFFFFFFFF) != (queryTimeMsg.cmd.everIncreasingNonceAndCommandType & 0xFFFFFFFFFFFFFF)) 
        {
            LOG("Failed to query node time!\n");
            return;
        }

        finalizePacketTimeNanosec = duration<unsigned long long, std::nano>(startTime - finalizePacketStartTime).count();
        roundTripTimeNanosec = duration<unsigned long long, std::nano>(endTime - startTime).count();
        LOG("Clock status before sync:\n");
        LOG("\tNode time (UTC):  "); logTime(response.utcTime); LOG("  -  round trip time %llu ms\n", roundTripTimeNanosec / 1000000);
        LOG("\tLocal time (UTC): "); logTime(convertTime(nowLocal)); LOG("\n\n");
    }

    if (roundTripTimeNanosec > 3000000000LLU)
    {
        LOG("Round trip time is too large. Sync skipped, because it would be very inaccurate!");
        return;
    }

    // set node clock to local UTC time + finalize packet time + half round trip time (not very accurate but simple and sufficient for requirements of 5 seconds)
    {
        struct {
            RequestResponseHeader header;
            SpecialCommandSendTime cmd;
            uint8_t signature[64];
        } sendTimeMsg;
        sendTimeMsg.header.setSize(sizeof(sendTimeMsg));
        sendTimeMsg.header.randomizeDejavu();
        sendTimeMsg.header.setType(PROCESS_SPECIAL_COMMAND);
        uint64_t curTime = duration_cast<microseconds>(system_clock::now().time_since_epoch()).count();
        uint64_t commandByte = (uint64_t)(SPECIAL_COMMAND_SEND_TIME) << 56;
        sendTimeMsg.cmd.everIncreasingNonceAndCommandType = commandByte | curTime;
        LOG("Current unix time (in us) used for nonce (send time): %" PRIu64 "\n\n", curTime);

        auto finalizePacketTime = duration_cast<system_clock::duration>(nanoseconds(finalizePacketTimeNanosec));
        auto halfRoudTripTime = duration_cast<system_clock::duration>(nanoseconds(roundTripTimeNanosec / 2));
        auto now = system_clock::now();
        UtcTime timeToSet = convertTime(now + finalizePacketTime + halfRoudTripTime);
        sendTimeMsg.cmd.utcTime = timeToSet;
        LOG("Setting node time to "); logTime(timeToSet); LOG(" ...\n\n");

        auto finalizePacketStartTime = steady_clock::now();

        KangarooTwelve((unsigned char*)&sendTimeMsg.cmd,
                       sizeof(sendTimeMsg.cmd),
                       digest,
                       32);
        sign(subseed, sourcePublicKey, digest, signature);
        memcpy(sendTimeMsg.signature, signature, 64);

        auto startTime = steady_clock::now();
        
        qc->sendData((uint8_t*)&sendTimeMsg, sendTimeMsg.header.size());

        SpecialCommandSendTime response;
        try
        {
            response = qc->receivePacketWithHeaderAs<SpecialCommandSendTime>();
        }
        catch (std::logic_error) 
        {
            memset(&response, 0, sizeof(SpecialCommandSendTime));
        }

        auto endTime = steady_clock::now();
        auto nowLocal = system_clock::now();

        if (response.everIncreasingNonceAndCommandType != sendTimeMsg.cmd.everIncreasingNonceAndCommandType) 
        {
            LOG("Failed to set node time!\n");
            return;
        }

        finalizePacketTimeNanosec = duration<unsigned long long, std::nano>(startTime - finalizePacketStartTime).count();
        roundTripTimeNanosec = duration<unsigned long long, std::nano>(endTime - startTime).count();
        LOG("Clock status after sync:\n");
        LOG("\tNode time (UTC):  "); logTime(response.utcTime); LOG("  -  round trip time %llu ms\n", roundTripTimeNanosec / 1000000);
        LOG("\tLocal time (UTC): "); logTime(convertTime(nowLocal)); LOG("\n\n");
    }
}

void setLoggingMode(const char* nodeIp, const int nodePort, const char* seed, char mode)
{
    uint8_t privateKey[32] = { 0 };
    uint8_t sourcePublicKey[32] = { 0 };
    uint8_t subseed[32] = { 0 };
    uint8_t digest[32] = { 0 };
    uint8_t signature[64] = { 0 };

    struct {
        RequestResponseHeader header;
        SpecialCommandSetConsoleLoggingModeRequestAndResponse cmd;
        uint8_t signature[64];
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(PROCESS_SPECIAL_COMMAND);
    uint64_t curTime = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
    uint64_t commandByte = (uint64_t)(SPECIAL_COMMAND_SET_CONSOLE_LOGGING_MODE) << 56;
    packet.cmd.everIncreasingNonceAndCommandType = commandByte | curTime;
    packet.cmd.loggingMode = mode;

    getSubseedFromSeed((uint8_t*)seed, subseed);
    getPrivateKeyFromSubSeed(subseed, privateKey);
    getPublicKeyFromPrivateKey(privateKey, sourcePublicKey);
    KangarooTwelve((unsigned char*)&packet.cmd,
        sizeof(packet.cmd),
        digest,
        32);
    sign(subseed, sourcePublicKey, digest, signature);
    memcpy(packet.signature, signature, 64);
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData((uint8_t*)&packet, packet.header.size());

    SpecialCommandSetConsoleLoggingModeRequestAndResponse response;
    try
    {
        response = qc->receivePacketWithHeaderAs<SpecialCommandSetConsoleLoggingModeRequestAndResponse>();
    }
    catch (std::logic_error)
    {
        memset(&response, 0, sizeof(SpecialCommandSetConsoleLoggingModeRequestAndResponse));
    }

    if (response.everIncreasingNonceAndCommandType == packet.cmd.everIncreasingNonceAndCommandType)
    {
        LOG("Successfully set logging mode\n");
    }
    else
    {
        LOG("Failed set logging mode\n");
    }
}

void broadcastCompChat(const char* nodeIp, const int nodePort, const char* seed, char* compChatMsg)
{
    uint8_t privateKey[32] = { 0 };
    uint8_t sourcePublicKey[32] = { 0 };
    uint8_t subseed[32] = { 0 };
    uint8_t digest[32] = { 0 };
    getSubseedFromSeed((uint8_t*)seed, subseed);
    getPrivateKeyFromSubSeed(subseed, privateKey);
    getPublicKeyFromPrivateKey(privateKey, sourcePublicKey);
    std::string compChatStr(compChatMsg);
    char rand_str[5] = {0};
    rand_str[0] = 'a' + (getRand32() % 26);
    rand_str[1] = 'a' + (getRand32() % 26);
    rand_str[2] = 'a' + (getRand32() % 26);
    rand_str[3] = 'a' + (getRand32() % 26);
    compChatStr = "DISCORD" + std::string(rand_str) + compChatStr;
    std::vector<uint8_t> vData;
    vData.resize(sizeof(RequestResponseHeader) + 32*3+ SIGNATURE_SIZE + compChatStr.size());
    RequestResponseHeader* header = (RequestResponseHeader*)vData.data();
    uint8_t* signature_ptr = (vData.data() + vData.size() - 64);
    header->setSize(uint32_t(vData.size()));
    header->zeroDejavu();
    header->setType(BROADCAST_MESSAGE);
    uint8_t* ptr = vData.data() + sizeof(RequestResponseHeader);
    memcpy(ptr, sourcePublicKey, 32);
    ptr += 32;
    memset(ptr, 0, 32);
    ptr += 32;
    memset(ptr, 0, 32);
    ptr += 32;
    memcpy(ptr, compChatStr.data(), compChatStr.size());

    KangarooTwelve(vData.data() + sizeof(RequestResponseHeader),
                   uint32_t(vData.size() - sizeof(RequestResponseHeader) - SIGNATURE_SIZE),
                   digest,
                   32);
    sign(subseed, sourcePublicKey, digest, signature_ptr);
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData(vData.data(), int(vData.size()));
    LOG("Broadcasted message to network\n");
}

BroadcastComputors readComputorListFromFile(const char* fileName)
{
    BroadcastComputors result;
    FILE* f = fopen(fileName, "rb");
    if (fread(&result, 1, sizeof(BroadcastComputors), f) != sizeof(BroadcastComputors))
    {
        LOG("Failed to read comp list\n");
        fclose(f);
        return result;
    }
    fclose(f);
    uint8_t digest[32] = {0};
    uint8_t arbPubkey[32] = {0};
    // verify with arb
    getPublicKeyFromIdentity(ARBITRATOR, arbPubkey);
    KangarooTwelve(reinterpret_cast<const uint8_t *>(&result),
                   sizeof(BroadcastComputors) - SIGNATURE_SIZE,
                   digest,
                   32);
    if (verify(arbPubkey, digest, result.computors.signature))
    {
        LOG("Computor list is VERIFIED (signed by ARBITRATOR)\n");
    } 
    else 
    {
        LOG("Computor list is NOT verified\n");
    }
    return result;
}

bool getComputorFromNode(const char* nodeIp, const int nodePort, BroadcastComputors& result)
{
    static struct
    {
        RequestResponseHeader header;
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(REQUEST_COMPUTORS);
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData((uint8_t *) &packet, packet.header.size());

    try
    {
        result = qc->receivePacketWithHeaderAs<BroadcastComputors>();
        return true;
    }
    catch (const std::logic_error& e)
    {
        LOG("%s\n", e.what());
        memset(&result, 0, sizeof(BroadcastComputors));
        return false;
    }
    catch (const EndResponseReceived)
    {
        LOG("Node does not have a verified computor list yet\n");
        memset(&result, 0, sizeof(BroadcastComputors));
        return false;
    }
}

void getComputorListToFile(const char* nodeIp, const int nodePort, const char* fileName)
{
    BroadcastComputors bc;
    if (!getComputorFromNode(nodeIp, nodePort, bc))
    {
        LOG("Failed to get valid computor list!\n");
        return;
    }
    uint8_t digest[32] = {0};
    uint8_t arbPubkey[32] = {0};
    // verify with arb, dump data
    getPublicKeyFromIdentity(ARBITRATOR, arbPubkey);
    for (int i = 0; i < NUMBER_OF_COMPUTORS; i++)
    {
        char identity[128] = {0};
        const bool isLowerCase = false;
        getIdentityFromPublicKey(bc.computors.publicKeys[i], identity, isLowerCase);
        LOG("%d %s\n", i, identity);
    }
    LOG("Epoch: %u\n", bc.computors.epoch);
    KangarooTwelve(reinterpret_cast<const uint8_t *>(&bc),
                   sizeof(BroadcastComputors) - SIGNATURE_SIZE,
                   digest,
                   32);
    if (verify(arbPubkey, digest, bc.computors.signature))
    {
        LOG("Computor list is VERIFIED (signed by ARBITRATOR)\n");
    } 
    else 
    {
        LOG("Computor list is NOT verified\n");
    }
    FILE* f = fopen(fileName, "wb");
    fwrite(&bc, 1, sizeof(BroadcastComputors), f);
    fclose(f);
}

std::vector<std::string> _getNodeIpList(const char* nodeIp, const int nodePort)
{
    std::vector<std::string> result;
    QCPtr qc;
    try
    {
        qc = make_qc(nodeIp, nodePort);
    } 
    catch (std::logic_error)
    {
        return result;
    }

    struct {
        RequestResponseHeader header;
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    std::vector<uint8_t> buffer;
    qc->getHandshakeData(buffer);
    uint8_t* data = buffer.data();
    int recvByte = int(buffer.size());
    if (recvByte == 0)
    {
        return result;
    }
    auto epp = (ExchangePublicPeers*)(data);
    for (int i = 0; i < 4; i++)
    {
        if (epp->peers[i][0] == 0 && epp->peers[i][1] == 0 && epp->peers[i][2] == 0 && epp->peers[i][3] == 0) continue;
        std::string new_ip = std::to_string(epp->peers[i][0]) + "." + std::to_string(epp->peers[i][1]) + "." + std::to_string(epp->peers[i][2]) + "." + std::to_string(epp->peers[i][3]);
        result.push_back(new_ip);
    }
    return result;
}

void getNodeIpList(const char* nodeIp, const int nodePort)
{
    LOG("Fetching node ip list from %s\n", nodeIp);
    std::vector<std::string> result = _getNodeIpList(nodeIp, nodePort);
    int count = 0;
    for (int i = 0; i < result.size() && count++ < 4; i++)
    {
        std::vector<std::string> new_result = _getNodeIpList(result[i].c_str(), nodePort);
        result.insert(result.end(), new_result.begin(), new_result.end());
    }
    std::sort(result.begin(), result.end());
    auto last = std::unique(result.begin(), result.end());
    result.erase(last, result.end());
    for (auto s : result)
    {
        LOG("%s\n", s.c_str());
    }
}

static bool isEmptyEntity(const Entity& e){
    bool is_pubkey_zero = true;
    for (int i = 0; i < 32; i++)
    {
        if (e.publicKey[i] != 0)
        {
            is_pubkey_zero = false;
            break;
        }
    }
    if (is_pubkey_zero) return true;
    if (e.outgoingAmount == 0 && e.incomingAmount == 0) return true;
    if (e.latestIncomingTransferTick == 0 && e.latestOutgoingTransferTick == 0) return true;
    return false;
}

void dumpSpectrumToCSV(const char* input, const char* output){
    const size_t SPECTRUM_CAPACITY = 0x1000000ULL; // may be changed in the future
    Entity* spectrum = (Entity*)malloc(SPECTRUM_CAPACITY*sizeof(Entity));
    FILE* f = fopen(input, "rb");
    if (fread(spectrum, 1, SPECTRUM_CAPACITY*sizeof(Entity), f) != SPECTRUM_CAPACITY*sizeof(Entity))
    {
        LOG("Failed to read spectrum\n");
        fclose(f);
        return;
    }
    fclose(f);
    f = fopen(output, "w");
    {
        std::string header ="ID,LastInTick,LastOutTick,AmountIn,AmountOut,Balance\n";
        fwrite(header.c_str(), 1, header.size(), f);
    }
    char buffer[128] = {0};
    for (int i = 0; i < SPECTRUM_CAPACITY; i++)
    {
        if (!isEmptyEntity(spectrum[i]))
        {
            memset(buffer, 0, 128);
            getIdentityFromPublicKey(spectrum[i].publicKey, buffer, false);
            std::string id = buffer;
            std::string line = id + "," + std::to_string(spectrum[i].latestIncomingTransferTick)
                               + "," + std::to_string(spectrum[i].latestOutgoingTransferTick)
                               + "," + std::to_string(spectrum[i].incomingAmount)
                               + "," + std::to_string(spectrum[i].outgoingAmount)
                               + "," + std::to_string(spectrum[i].incomingAmount-spectrum[i].outgoingAmount) + "\n";
            fwrite(line.c_str(), 1, line.size(), f);
        }
    }
    free(spectrum);
    fclose(f);
}

// only print ownership
void dumpUniverseToCSV(const char* input, const char* output){
    const size_t ASSETS_CAPACITY = 0x1000000ULL; // may be changed in the future
    AssetRecord* asset = (AssetRecord*)malloc(ASSETS_CAPACITY*sizeof(Entity));
    FILE* f = fopen(input, "rb");
    if (fread(asset, 1, ASSETS_CAPACITY*sizeof(AssetRecord), f) != ASSETS_CAPACITY * sizeof(AssetRecord))
    {
        LOG("Failed to read assets\n");
        fclose(f);
        return;
    }
    fclose(f);
    f = fopen(output, "w");
    {
        std::string header ="Index,Type,ID,OwnerIndex,ContractIndex,AssetName,AssetIssuer,Amount\n";
        fwrite(header.c_str(), 1, header.size(), f);
    }
    char buffer[128] = {0};
    for (int i = 0; i < ASSETS_CAPACITY; i++)
    {
        if (asset[i].varStruct.ownership.type == OWNERSHIP)
        {
            memset(buffer, 0, 128);
            getIdentityFromPublicKey(asset[i].varStruct.ownership.publicKey, buffer, false);
            std::string id = buffer;
            std::string asset_name = "null";
            std::string issuerID = "null";
            size_t issue_index = asset[i].varStruct.ownership.issuanceIndex;
            {
                // get asset name
                memset(buffer, 0, 128);
                memcpy(buffer, asset[issue_index].varStruct.issuance.name, 7);
                asset_name = buffer;
            }
            {
                // get issuer
                memset(buffer, 0, 128);
                getIdentityFromPublicKey(asset[issue_index].varStruct.issuance.publicKey, buffer, false);
                issuerID = buffer;
            }
            std::string line = std::to_string(i) + ",OWNERSHIP,"+ id
                               + "," + std::to_string(i) + ","
                               + std::to_string(asset[i].varStruct.ownership.managingContractIndex) + "," + asset_name
                               + "," + issuerID
                               + "," + std::to_string(asset[i].varStruct.ownership.numberOfShares) + "\n";
            fwrite(line.c_str(), 1, line.size(), f);
        }
        if (asset[i].varStruct.ownership.type == POSSESSION)
        {
            memset(buffer, 0, 128);
            getIdentityFromPublicKey(asset[i].varStruct.possession.publicKey, buffer, false);
            std::string id = buffer;
            std::string asset_name = "null";
            std::string issuerID = "null";
            std::string str_index = std::to_string(i);
            int owner_index = asset[i].varStruct.possession.ownershipIndex;
            int contract_index = asset[i].varStruct.possession.managingContractIndex;
            std::string str_owner_index = std::to_string(owner_index);
            std::string str_contract_index = std::to_string(contract_index);
            std::string str_amount = std::to_string(asset[i].varStruct.possession.numberOfShares);
            {
                // get asset name
                int issuance_index = asset[owner_index].varStruct.ownership.issuanceIndex;
                memset(buffer, 0, 128);
                memcpy(buffer, asset[issuance_index].varStruct.issuance.name, 7);
                asset_name = buffer;
                memset(buffer, 0, 128);
                getIdentityFromPublicKey(asset[issuance_index].varStruct.issuance.publicKey, buffer, false);
                issuerID = buffer;
            }
            std::string line = str_index + ",POSSESSION," + id + "," + str_owner_index + "," +
                               str_contract_index + "," + asset_name + "," + issuerID + "," + str_amount + "\n";
            fwrite(line.c_str(), 1, line.size(), f);
        }
        if (asset[i].varStruct.ownership.type == ISSUANCE)
        {
            memset(buffer, 0, 128);
            getIdentityFromPublicKey(asset[i].varStruct.issuance.publicKey, buffer, false);
            std::string id = buffer;
            std::string asset_name = "null";
            std::string issuerID = "null";
            std::string str_index = std::to_string(i);
            std::string str_owner_index = std::to_string(0);
            std::string str_contract_index = std::to_string(1); // don't know how to get this yet
            std::string str_amount = std::to_string(asset[i].varStruct.possession.numberOfShares);
            {
                // get asset name
                memset(buffer, 0, 128);
                memcpy(buffer, asset[i].varStruct.issuance.name, 7);
                asset_name = buffer;
                memset(buffer, 0, 128);
                getIdentityFromPublicKey(asset[i].varStruct.issuance.publicKey, buffer, false);
                issuerID = buffer;
            }
            // std::string header ="Index,Type,ID,OwnerIndex,ContractIndex,AssetName,AssetIssuer,Amount\n";
            std::string line = str_index + ",ISSUANCE," + id + "," + str_owner_index + "," +
                               str_contract_index + "," + asset_name + "," + issuerID + "," + str_amount + "\n";
            fwrite(line.c_str(), 1, line.size(), f);
        }
    }
    free(asset);
    fclose(f);
}

void getMiningScoreRanking(const char* nodeIp, const int nodePort, const char* seed)
{
    uint8_t privateKey[32] = {0};
    uint8_t sourcePublicKey[32] = {0};
    uint8_t subseed[32] = {0};
    uint8_t digest[32] = {0};
    uint8_t signature[64] = {0};

    struct {
        RequestResponseHeader header;
        SpecialCommand cmd;
        uint8_t signature[64];
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(PROCESS_SPECIAL_COMMAND);
    uint64_t curTime = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
    uint64_t commandByte = (uint64_t)(SPECIAL_COMMAND_GET_MINING_SCORE_RANKING) << 56;
    packet.cmd.everIncreasingNonceAndCommandType = commandByte | curTime;
    getSubseedFromSeed((uint8_t*)seed, subseed);
    getPrivateKeyFromSubSeed(subseed, privateKey);
    getPublicKeyFromPrivateKey(privateKey, sourcePublicKey);
    KangarooTwelve((unsigned char*)&packet.cmd,
                   sizeof(packet.cmd),
                   digest,
                   32);
    sign(subseed, sourcePublicKey, digest, signature);
    memcpy(packet.signature, signature, 64);
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData((uint8_t *) &packet, packet.header.size());

    SpecialCommandGetMiningScoreRanking response;

    int headerSize = sizeof(RequestResponseHeader) + 8 + 4; // header and everIncreasingNonceAndCommandType and numberOfRankings
    std::vector<uint8_t> buffer;
    buffer.resize(headerSize);
    qc->receiveData(buffer.data(), headerSize);
    int contentSize = 0;
    {
        uint8_t* data = buffer.data() + sizeof(RequestResponseHeader);
        response.everIncreasingNonceAndCommandType = *((unsigned long long*)data);
        response.numRankings = *((unsigned int*)(data+8));
        contentSize = response.numRankings * 36; // 32 pubkey + 4 bytes score
    }
    if (response.everIncreasingNonceAndCommandType != packet.cmd.everIncreasingNonceAndCommandType)
    {
        LOG("Failed to get mining score ranking!\n");
        return;
    }
    if (contentSize < 676 * 36)
    {
        LOG("Malformed content size\n");
        return;
    }
    buffer.resize(contentSize);
    qc->receiveAllDataOrThrowException(buffer.data(), contentSize);
    uint8_t* data = buffer.data();
    // Get data out
    unsigned char* ptr = data;

    // Get the number of miners
    LOG("Total miner: %u\n", response.numRankings);
    response.rankings.resize(response.numRankings);

    // Get detail ranking score of miners
    unsigned long long total_score = 0;
    if (response.numRankings > 0)
    {
        memcpy(response.rankings.data(), ptr, response.numRankings * sizeof(SpecialCommandGetMiningScoreRanking::ScoreEntry));
        LOG("%-8s%-64s%s\n", "ArrayIndex", "Identity", "Score");
        for (unsigned int i = 0; i < response.numRankings ; i++)
        {
            SpecialCommandGetMiningScoreRanking::ScoreEntry miner = response.rankings[i];
            char publicIdentity[128] = {0};
            getIdentityFromPublicKey(miner.minerPublicKey, publicIdentity, false);
            LOG("%-8u%-64s%u\n", i + 1, publicIdentity, miner.minerScore);
            total_score += miner.minerScore;
        }
        LOG("WARNING: This is raw miner score data from the node and is NOT the final ranking list.\n");
        LOG("Check processTick function in core node to have the latest ranking logic\n");
    }
    else
    {
        LOG("No updated yet.\n");
    }
    LOG("Total score: %llu\n", total_score);
}

unsigned int extract10Bit(const unsigned char* data, unsigned int idx)
{
    //TODO: simplify this
    unsigned int byte0 = data[idx + (idx >> 2)];
    unsigned int byte1 = data[idx + (idx >> 2) + 1];
    unsigned int last_bit0 = 8 - (idx & 3) * 2;
    unsigned int first_bit1 = 10 - last_bit0;
    unsigned int res = (byte0 & ((1 << last_bit0) - 1)) << first_bit1;
    res |= (byte1 >> (8 - first_bit1));
    return res;
}

void getVoteCounterTransaction(const char* nodeIp, const int nodePort, unsigned int requestedTick, const char* compFileName)
{
    BroadcastComputors bc;
    {
        FILE* f = fopen(compFileName, "rb");
        if (fread(&bc, 1, sizeof(BroadcastComputors), f) != sizeof(BroadcastComputors))
        {
            LOG("Failed to read comp list\n");
            fclose(f);
            return;
        }
        fclose(f);
    }
    auto qc = make_qc(nodeIp, nodePort);

    auto txs = std::make_unique<std::vector<Transaction>>();
    txs->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);
    auto extraData = std::make_unique<std::vector<ExtraDataStruct>>();
    extraData->reserve(NUMBER_OF_TRANSACTIONS_PER_TICK);

    getTickTransactions(qc, requestedTick, NUMBER_OF_TRANSACTIONS_PER_TICK, *txs, /*hashes=*/nullptr, extraData.get(), /*sigs=*/nullptr);
    
    auto td = std::make_unique<TickData>();
    getTickData(qc, requestedTick, *td);

    unsigned int votes[676];
    int nTx = int(txs->size());
    LOG("Finding in %d transactions\n", nTx);
    for (int i = 0; i < nTx; i++)
    {
        if (extraData->at(i).vecU8.size() == 848 + 32)
        {
            int comp_idx = requestedTick % 676;
            if (memcmp(txs->at(i).sourcePublicKey, bc.computors.publicKeys[comp_idx], 32) == 0)
            {
                uint8_t* data = extraData->at(i).vecU8.data();
                uint32_t sum = 0;
                if (txs->at(i).inputType == 1)
                {
                    LOG("---Type: vote counter---\n");
                }
                if (txs->at(i).inputType == 8)
                {
                    LOG("---Type: custom mining counter---\n");
                }
                if (memcmp(td->timelock, data + 848, 32) == 0)
                {
                    LOG("Matched data lock\n");
                }
                else
                {
                    LOG("Mismatched data lock\n");
                    char hex[128] = { 0 };
                    byteToHex(data + 848, hex, 32);
                    LOG("have: %s\n", hex);
                    byteToHex(td->timelock, hex, 32);
                    LOG("want: %s\n", hex);
                    continue;
                }
                if (txs->at(i).inputType == 1)
                {
                    for (int j = 0; j < 676; j++)
                    {
                        votes[j] = extract10Bit(data, j);
                        sum += votes[j];
                        auto alphabet = indexToAlphabet(j);
                        LOG("%s: %u | ", alphabet.c_str(), votes[j]);
                        if ((j + 1) % 26 == 0) LOG("\n");
                    }
                    if (sum < 676 * 451)
                    {
                        LOG("Invalid sum votes: %u\n", sum);
                    }
                    if (votes[comp_idx] != 0)
                    {
                        LOG("Invalid comp votes\n");
                    }
                }
                if (txs->at(i).inputType == 8)
                {
                    for (int j = 0; j < 676; j++)
                    {
                        votes[j] = extract10Bit(data, j);
                        sum += votes[j];
                        auto alphabet = indexToAlphabet(j);
                        LOG("%s: %u | ", alphabet.c_str(), votes[j]);
                        if ((j + 1) % 26 == 0) LOG("\n");
                    }
                }
            }
        }
    }
}

void saveSnapshot(const char* nodeIp, const int nodePort, const char* seed)
{
    uint8_t privateKey[32] = {0};
    uint8_t sourcePublicKey[32] = {0};
    uint8_t subseed[32] = {0};
    uint8_t digest[32] = {0};
    uint8_t signature[64] = {0};

    struct {
        RequestResponseHeader header;
        SpecialCommandSaveSnapshotRequestAndResponse cmd;
        uint8_t signature[64];
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(PROCESS_SPECIAL_COMMAND);
    uint64_t curTime = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
    uint64_t commandByte = (uint64_t)(SPECIAL_COMMAND_SAVE_SNAPSHOT) << 56;
    packet.cmd.everIncreasingNonceAndCommandType = commandByte | curTime;

    getSubseedFromSeed((uint8_t*)seed, subseed);
    getPrivateKeyFromSubSeed(subseed, privateKey);
    getPublicKeyFromPrivateKey(privateKey, sourcePublicKey);
    KangarooTwelve((unsigned char*)&packet.cmd,
                   sizeof(packet.cmd),
                   digest,
                   32);
    sign(subseed, sourcePublicKey, digest, signature);
    memcpy(packet.signature, signature, 64);
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData((uint8_t *) &packet, packet.header.size());

    SpecialCommandSaveSnapshotRequestAndResponse response;
    try
    {
        response = qc->receivePacketWithHeaderAs<SpecialCommandSaveSnapshotRequestAndResponse>();
    }
    catch (std::logic_error) 
    {
        memset(&response, 0, sizeof(SpecialCommandSaveSnapshotRequestAndResponse));
    }

    if (response.everIncreasingNonceAndCommandType == packet.cmd.everIncreasingNonceAndCommandType)
    {
        if (response.currentTick > 0 && response.status == SpecialCommandSaveSnapshotRequestAndResponse::SAVING_TRIGGERED)
        {
            LOG("Successfully triggered saving snapshot at tick %u. It will be saved at the next tick.\n", response.currentTick);
            LOG("Please wait until the node connection is re-enabled.\n");
        } 
        else 
        {
            LOG("The packet was successfully sent, but no save was triggered.\n");
            LOG("- Tick: %u\n", response.currentTick);
            LOG("- Status: ");
            switch (response.status)
            {
            case SpecialCommandSaveSnapshotRequestAndResponse::REMOTE_SAVE_MODE_DISABLED :
                LOG("REMOTE_SAVE_MODE_DISABLED.\n");
                break;
            case SpecialCommandSaveSnapshotRequestAndResponse::SAVING_IN_PROGRESS :
                LOG("SAVING_IN_PROGRESS.\n");
                break;
            case SpecialCommandSaveSnapshotRequestAndResponse::UNKNOWN_FAILURE :
            default:
                LOG("UNKNOWN_FAILURE.\n");
                break;
            }
        }
    } 
    else
    {
        LOG("Failed to trigger saving snapshot\n");
    }
}

void setExecutionFeeMultiplier(const char* nodeIp, const int nodePort, const char* seed, unsigned long long multiplierNumerator, unsigned long long multiplierDenominator)
{
    uint8_t privateKey[32] = { 0 };
    uint8_t sourcePublicKey[32] = { 0 };
    uint8_t subseed[32] = { 0 };
    uint8_t digest[32] = { 0 };
    uint8_t signature[64] = { 0 };

    struct {
        RequestResponseHeader header;
        SpecialCommandExecutionFeeMultiplierRequestAndResponse cmd;
        uint8_t signature[64];
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(PROCESS_SPECIAL_COMMAND);
    uint64_t curTime = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
    uint64_t commandByte = (uint64_t)(SPECIAL_COMMAND_SET_EXECUTION_FEE_MULTIPLIER) << 56;
    packet.cmd.everIncreasingNonceAndCommandType = commandByte | curTime;
    packet.cmd.multiplierNumerator = multiplierNumerator;
    packet.cmd.multiplierDenominator = multiplierDenominator;

    getSubseedFromSeed((uint8_t*)seed, subseed);
    getPrivateKeyFromSubSeed(subseed, privateKey);
    getPublicKeyFromPrivateKey(privateKey, sourcePublicKey);
    KangarooTwelve((unsigned char*)&packet.cmd,
        sizeof(packet.cmd),
        digest,
        32);
    sign(subseed, sourcePublicKey, digest, signature);
    memcpy(packet.signature, signature, 64);
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData((uint8_t*)&packet, packet.header.size());

    SpecialCommandExecutionFeeMultiplierRequestAndResponse response;
    try
    {
        response = qc->receivePacketWithHeaderAs<SpecialCommandExecutionFeeMultiplierRequestAndResponse>();
    }
    catch (std::logic_error)
    {
        memset(&response, 0, sizeof(SpecialCommandExecutionFeeMultiplierRequestAndResponse));
    }

    if (response.everIncreasingNonceAndCommandType == packet.cmd.everIncreasingNonceAndCommandType)
    {
        LOG("Successfully set execution fee multiplier\n");
    }
    else
    {
        LOG("Failed to set execution fee multiplier\n");
    }
}

void getExecutionFeeMultiplier(const char* nodeIp, const int nodePort, const char* seed)
{
    uint8_t privateKey[32] = { 0 };
    uint8_t sourcePublicKey[32] = { 0 };
    uint8_t subseed[32] = { 0 };
    uint8_t digest[32] = { 0 };
    uint8_t signature[64] = { 0 };

    struct {
        RequestResponseHeader header;
        SpecialCommand cmd;
        uint8_t signature[64];
    } packet;
    packet.header.setSize(sizeof(packet));
    packet.header.randomizeDejavu();
    packet.header.setType(PROCESS_SPECIAL_COMMAND);
    uint64_t curTime = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
    uint64_t commandByte = (uint64_t)(SPECIAL_COMMAND_GET_EXECUTION_FEE_MULTIPLIER) << 56;
    packet.cmd.everIncreasingNonceAndCommandType = commandByte | curTime;

    getSubseedFromSeed((uint8_t*)seed, subseed);
    getPrivateKeyFromSubSeed(subseed, privateKey);
    getPublicKeyFromPrivateKey(privateKey, sourcePublicKey);
    KangarooTwelve((unsigned char*)&packet.cmd,
        sizeof(packet.cmd),
        digest,
        32);
    sign(subseed, sourcePublicKey, digest, signature);
    memcpy(packet.signature, signature, 64);
    auto qc = make_qc(nodeIp, nodePort);
    qc->sendData((uint8_t*)&packet, packet.header.size());

    SpecialCommandExecutionFeeMultiplierRequestAndResponse response;
    try
    {
        response = qc->receivePacketWithHeaderAs<SpecialCommandExecutionFeeMultiplierRequestAndResponse>();
    }
    catch (std::logic_error)
    {
        memset(&response, 0, sizeof(SpecialCommandExecutionFeeMultiplierRequestAndResponse));
    }

    if (response.everIncreasingNonceAndCommandType == packet.cmd.everIncreasingNonceAndCommandType)
    {
        LOG("Execution fee multiplier is currently set to:\n");
        LOG("- Numerator: %" PRIu64 "\n", response.multiplierNumerator);
        LOG("- Denominator: %" PRIu64 "\n", response.multiplierDenominator);
    }
    else
    {
        LOG("Failed to get execution fee multiplier\n");
    }
}