commit 779f12fe5a69166a65135ddf9563e1d83ee348bf
parent 9c2c9642f95c9c6334f25ed94c6a5d5062ce43c7
Author: nolash <dev@holbrook.no>
Date: Tue, 9 Feb 2021 09:42:46 +0100
Add gas transfer script
Diffstat:
11 files changed, 515 insertions(+), 12 deletions(-)
diff --git a/CHANGELOG b/CHANGELOG
@@ -0,0 +1,5 @@
+- 0.0.1
+ * Add eth tx decode
+ * Add eth balance query
+ * Add eth checksum address
+ * Add eth subscribe monitor
diff --git a/cic_tools/cic_eth/runnable/users.py b/cic_tools/cic_eth/runnable/users.py
@@ -0,0 +1,252 @@
+#!/usr/bin/python
+
+# standard imports
+import json
+import time
+import datetime
+import random
+import logging
+import os
+import base64
+import hashlib
+import sys
+import uuid
+import argparse
+
+# third-party imports
+import redis
+import vobject
+import celery
+from faker import Faker
+import cic_registry
+import confini
+from cic_eth.api import Api
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+fake = Faker(['sl', 'en_US', 'no', 'de', 'ro'])
+
+#f = open('cic.conf', 'r')
+#config = json.load(f)
+#f.close()
+#
+
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
+argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+argparser.add_argument('--timeout', default=1.0, type=int, help='timeout to wait for account create callback')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+argparser.add_argument('count', help='Number of users to generate', type=int)
+args = argparser.parse_args()
+
+if args.v == True:
+ logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+ logging.getLogger().setLevel(logging.DEBUG)
+
+config_dir = os.path.join(args.c)
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+args_override = {
+ 'ETH_PROVIDER': getattr(args, 'p'),
+ 'CIC_CHAIN_SPEC': getattr(args, 'i'),
+}
+config.dict_override(args_override, 'cli flag')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+
+
+dt_now = datetime.datetime.utcnow()
+dt_then = dt_now - datetime.timedelta(weeks=150)
+ts_now = int(dt_now.timestamp())
+ts_then = int(dt_then.timestamp())
+
+queue = args.q
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+redis_host = config.get('REDIS_HOST')
+redis_port = config.get('REDIS_PORT')
+redis_db = config.get('REDIS_DB')
+redis_channel = str(uuid.uuid4())
+r = redis.Redis(redis_host, redis_port, redis_db)
+ps = r.pubsub()
+ps.subscribe(redis_channel)
+ps.get_message()
+
+api = Api(
+ config.get('CIC_CHAIN_SPEC'),
+ queue=args.q,
+ callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
+ callback_task='cic_eth.callbacks.redis.redis',
+ callback_queue=queue,
+ )
+
+gift_max = 10000
+gift_factor = (10**9)
+
+user_count = args.count
+
+categories = [
+ "food/water",
+ "fuel/energy",
+ "education",
+ "health",
+ "shop",
+ "environment",
+ "transport",
+ "farming/labor",
+ "savingsgroup",
+ ]
+
+phone_idx = []
+
+
+def genPhoneIndex(phone):
+ h = hashlib.new('sha256')
+ h.update(phone.encode('utf-8'))
+ h.update(b'cic.msisdn')
+ return h.digest().hex()
+
+
+def genId(addr, typ):
+ h = hashlib.new('sha256')
+ h.update(bytes.fromhex(addr[2:]))
+ h.update(typ.encode('utf-8'))
+ return h.digest().hex()
+
+
+def genDate():
+
+ logg.info(ts_then)
+ ts = random.randint(ts_then, ts_now)
+ return datetime.datetime.fromtimestamp(ts).timestamp()
+
+
+def genPhone():
+ return fake.msisdn()
+
+
+def genPersonal(phone):
+ fn = fake.first_name()
+ ln = fake.last_name()
+ e = fake.email()
+
+ v = vobject.vCard()
+ first_name = fake.first_name()
+ last_name = fake.last_name()
+ v.add('n')
+ v.n.value = vobject.vcard.Name(family=last_name, given=first_name)
+ v.add('fn')
+ v.fn.value = '{} {}'.format(first_name, last_name)
+ v.add('tel')
+ v.tel.typ_param = 'CELL'
+ v.tel.value = phone
+ v.add('email')
+ v.email.value = fake.email()
+
+ vcard_serialized = v.serialize()
+ vcard_base64 = base64.b64encode(vcard_serialized.encode('utf-8'))
+
+ return vcard_base64.decode('utf-8')
+
+
+def genCats():
+ i = random.randint(0, 3)
+ return random.choices(categories, k=i)
+
+
+def genAmount():
+ return random.randint(0, gift_max) * gift_factor
+
+
+def gen():
+ old_blockchain_address = '0x' + os.urandom(20).hex()
+ t = api.create_account(register=True)
+
+ ps.get_message()
+ m = ps.get_message(timeout=args.timeout)
+ new_blockchain_address = json.loads(m['data'])
+
+ #new_blockchain_address = t.get()
+ gender = random.choice(['female', 'male', 'other'])
+ phone = genPhone()
+ v = genPersonal(phone)
+ o = {
+ 'date_registered': genDate(),
+ 'vcard': v,
+ 'gender': gender,
+ 'key': {
+ 'ethereum': [
+ old_blockchain_address,
+ new_blockchain_address,
+ ],
+ },
+ 'location': {
+ 'latitude': str(fake.latitude()),
+ 'longitude': str(fake.longitude()),
+ 'external': { # add osm lookup
+ }
+ },
+ 'selling': genCats(),
+ }
+ uid = genId(new_blockchain_address, 'cic.person')
+
+ return (uid, phone, o)
+
+
+def prepareLocalFilePath(datadir, address):
+ parts = [
+ address[:2],
+ address[2:4],
+ ]
+ dirs = '{}/{}/{}'.format(
+ datadir,
+ parts[0],
+ parts[1],
+ )
+ os.makedirs(dirs, exist_ok=True)
+ return dirs
+
+
+if __name__ == '__main__':
+
+ os.makedirs('data/person', exist_ok=True)
+ os.makedirs('data/phone', exist_ok=True)
+
+ fa = open('./data/amounts', 'w')
+ fb = open('./data/addresses', 'w')
+
+ #for i in range(10):
+ for i in range(int(user_count)):
+
+ (uid, phone, o) = gen()
+ eth = o['key']['ethereum'][1]
+
+ print(o)
+
+ d = prepareLocalFilePath('./data/person', uid)
+ f = open('{}/{}'.format(d, uid), 'w')
+ json.dump(o, f)
+ f.close()
+
+ pidx = genPhoneIndex(phone)
+ d = prepareLocalFilePath('./data/phone', uid)
+ f = open('{}/{}'.format(d, pidx), 'w')
+ f.write(eth)
+ f.close()
+
+ amount = genAmount()
+ fa.write('{},{}\n'.format(eth,amount))
+ fb.write('{}\n'.format(eth))
+ logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
+
+ fb.close()
+ fa.close()
diff --git a/cic_tools/eth/constant.py b/cic_tools/eth/constant.py
@@ -1,2 +1,3 @@
-zero_address = '0x{:040x}'.format(0)
-zero_content = '0x{:064x}'.format(0)
+ZERO_ADDRESS = '0x{:040x}'.format(0)
+ZERO_CONTENT = '0x{:064x}'.format(0)
+MINIMUM_FEE_UNITS = 21000
diff --git a/cic_tools/eth/gas.py b/cic_tools/eth/gas.py
@@ -0,0 +1,56 @@
+# third-party imports
+from hexathon import (
+ add_0x,
+ strip_0x,
+ )
+from crypto_dev_signer.eth.transaction import EIP155Transaction
+
+# local imports
+from cic_tools.eth.method import (
+ jsonrpc_template,
+ )
+from cic_tools.eth.tx import TxFactory
+from cic_tools.eth.hash import keccak256_hex_to_hex
+
+
+def price():
+ o = jsonrpc_template()
+ o['method'] = 'eth_gasPrice'
+ return o
+
+
+def balance(address):
+ o = jsonrpc_template()
+ o['method'] = 'eth_getBalance'
+ o['params'].append(address)
+ return o
+
+
+class GasTxFactory(TxFactory):
+
+ def create(self, sender, recipient, value):
+ tx = self.template(sender, recipient)
+ txe = EIP155Transaction(tx, tx['nonce'], tx['chainId'])
+ self.signer.signTransaction(txe)
+ tx_raw = txe.rlp_serialize()
+ tx_raw_hex = add_0x(tx_raw.hex())
+ tx_hash_hex = add_0x(keccak256_hex_to_hex(tx_raw_hex))
+
+ o = jsonrpc_template()
+ o['method'] = 'eth_sendRawTransaction'
+ o['params'].append(tx_raw_hex)
+
+ return (tx_hash_hex, o)
+
+
+class DefaultGasOracle:
+
+ def __init__(self, conn):
+ self.conn = conn
+
+
+ def get(self):
+ o = price()
+ r = self.conn.do(o)
+ n = strip_0x(r)
+ return int(n, 16)
diff --git a/cic_tools/eth/hash.py b/cic_tools/eth/hash.py
@@ -1,7 +1,29 @@
+# third-party imports
import sha3
+from hexathon import (
+ add_0x,
+ strip_0x,
+ )
def keccak256_hex(s):
h = sha3.keccak_256()
h.update(s.encode('utf-8'))
return h.digest().hex()
+
+
+def keccak256_string_to_hex(s):
+ return keccak256_hex(s)
+
+
+def keecak256_bytes_to_hex(b):
+ h = sha3.keccak_256()
+ h.update(b)
+ return h.digest().hex()
+
+
+def keccak256_hex_to_hex(hx):
+ h = sha3.keccak_256()
+ b = bytes.fromhex(strip_0x(hx))
+ h.update(b)
+ return h.digest().hex()
diff --git a/cic_tools/eth/method.py b/cic_tools/eth/method.py
@@ -4,13 +4,13 @@ import uuid
from hexathon import add_0x
from eth_abi import encode_single
-from .hash import keccak256_hex
-from .constant import zero_address
+from .hash import keccak256_string_to_hex
+from .constant import ZERO_ADDRESS
# TODO: move to cic-contracts
-erc20_balance_signature = keccak256_hex('balanceOf(address)')[:8]
-erc20_decimals_signature = keccak256_hex('decimals()')[:8]
+erc20_balance_signature = keccak256_string_to_hex('balanceOf(address)')[:8]
+erc20_decimals_signature = keccak256_string_to_hex('decimals()')[:8]
def jsonrpc_template():
@@ -22,7 +22,7 @@ def jsonrpc_template():
}
-def erc20_balance(contract_address, address, sender_address=zero_address):
+def erc20_balance(contract_address, address, sender_address=ZERO_ADDRESS):
o = jsonrpc_template()
o['method'] = 'eth_call'
data = erc20_balance_signature
@@ -34,7 +34,7 @@ def erc20_balance(contract_address, address, sender_address=zero_address):
return o
-def erc20_decimals(contract_address, sender_address=zero_address):
+def erc20_decimals(contract_address, sender_address=ZERO_ADDRESS):
o = jsonrpc_template()
o['method'] = 'eth_call'
arg = add_0x(erc20_decimals_signature)
@@ -45,7 +45,7 @@ def erc20_decimals(contract_address, sender_address=zero_address):
return o
-def call(contract_address, data, sender_address=zero_address):
+def call(contract_address, data, sender_address=ZERO_ADDRESS):
return {
'from': sender_address,
'to': contract_address,
diff --git a/cic_tools/eth/nonce.py b/cic_tools/eth/nonce.py
@@ -0,0 +1,32 @@
+# third-party imports
+from hexathon import (
+ add_0x,
+ strip_0x,
+ )
+
+# local imports
+from cic_tools.eth.method import (
+ jsonrpc_template,
+ )
+
+
+def nonce(address):
+ o = jsonrpc_template()
+ o['method'] = 'eth_getTransactionCount'
+ o['params'].append(address)
+ o['params'].append('pending')
+ return o
+
+
+class DefaultNonceOracle:
+
+ def __init__(self, address, conn):
+ self.address = address
+ self.conn = conn
+
+
+ def next(self):
+ o = nonce(self.address)
+ r = self.conn.do(o)
+ n = strip_0x(r)
+ return int(n, 16)
diff --git a/cic_tools/eth/runnable/balance.py b/cic_tools/eth/runnable/balance.py
@@ -57,9 +57,6 @@ if args.v:
conn = HTTPConnection(args.p)
def main():
-# w3 = web3.Web3(web3.Web3.HTTPProvider(args.p))
-# REPLACE WITH URLLIB
-
account = to_checksum(args.account)
if not args.u and account != add_0x(args.account):
raise ValueError('invalid checksum address')
diff --git a/cic_tools/eth/runnable/gas.py b/cic_tools/eth/runnable/gas.py
@@ -0,0 +1,112 @@
+#!python3
+
+"""Gas transfer script
+
+.. moduleauthor:: Louis Holbrook <dev@holbrook.no>
+.. pgp:: 0826EDA1702D1E87C6E2875121D2E7BB88C2A746
+
+"""
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# standard imports
+import sys
+import os
+import json
+import argparse
+import logging
+
+# third-party imports
+from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
+from crypto_dev_signer.keystore import DictKeystore
+from crypto_dev_signer.eth.helper import EthTxExecutor
+from hexathon import (
+ add_0x,
+ strip_0x,
+ )
+
+# local imports
+from cic_tools.eth.address import to_checksum
+from cic_tools.eth.connection import HTTPConnection
+from cic_tools.eth.method import (
+ jsonrpc_template,
+ )
+from cic_tools.eth.nonce import DefaultNonceOracle
+from cic_tools.eth.gas import (
+ DefaultGasOracle,
+ GasTxFactory,
+ )
+from cic_tools.eth.gas import balance as gas_balance
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+
+default_abi_dir = '/usr/share/local/cic/solidity/abi'
+default_eth_provider = os.environ.get('ETH_PROVIDER', 'http://localhost:8545')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+argparser.add_argument('-w', action='store_true', help='Wait for the last transaction to be confirmed')
+argparser.add_argument('-ww', action='store_true', help='Wait for every transaction to be confirmed')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string')
+argparser.add_argument('-a', '--signer-address', dest='a', type=str, help='Signing address')
+argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-u', '--unsafe', dest='u', action='store_true', help='Auto-convert address to checksum adddress')
+argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+argparser.add_argument('recipient', type=str, help='Ethereum address of recipient')
+argparser.add_argument('amount', type=int, help='Amount of tokens to mint and gift')
+args = argparser.parse_args()
+
+
+if args.vv:
+ logg.setLevel(logging.DEBUG)
+elif args.v:
+ logg.setLevel(logging.INFO)
+
+block_last = args.w
+block_all = args.ww
+
+signer_address = None
+keystore = DictKeystore()
+if args.y != None:
+ logg.debug('loading keystore file {}'.format(args.y))
+ signer_address = keystore.import_keystore_file(args.y)
+ logg.debug('now have key for signer address {}'.format(signer_address))
+signer = EIP155Signer(keystore)
+
+conn = HTTPConnection(args.p)
+nonce_oracle = DefaultNonceOracle(signer_address, conn)
+gas_oracle = DefaultGasOracle(conn)
+
+chain_pair = args.i.split(':')
+chain_id = int(chain_pair[1])
+
+
+def balance(address):
+ o = gas_balance(address)
+ r = conn.do(o)
+ hx = strip_0x(r)
+ return int(hx, 16)
+
+
+def main():
+ recipient = to_checksum(args.recipient)
+ if not args.u and recipient != add_0x(args.recipient):
+ raise ValueError('invalid checksum address')
+
+ value = args.amount
+
+ logg.debug('sender {} balance before: {}'.format(signer_address, balance(signer_address)))
+ logg.debug('recipient {} balance before: {}'.format(recipient, balance(recipient)))
+
+ g = GasTxFactory(signer, gas_oracle, nonce_oracle, chain_id=chain_id)
+ (tx_hash_hex, o) = g.create(signer_address, recipient, value)
+ conn.do(o)
+
+ print(tx_hash_hex)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/cic_tools/eth/tx.py b/cic_tools/eth/tx.py
@@ -10,6 +10,7 @@ from rlp import encode as rlp_encode
# local imports
from .address import to_checksum
+from .constant import MINIMUM_FEE_UNITS
logg = logging.getLogger(__name__)
@@ -81,3 +82,27 @@ def unpack_signed(tx_raw_bytes, chain_id=1):
}
+class TxFactory:
+
+ def __init__(self, signer, gas_oracle, nonce_oracle, chain_id=1):
+ self.gas_oracle = gas_oracle
+ self.nonce_oracle = nonce_oracle
+ self.chain_id = chain_id
+ self.signer = signer
+
+
+ def template(self, sender, recipient):
+ gas_price = self.gas_oracle.get()
+ logg.debug('using gas price {}'.format(gas_price))
+ nonce = self.nonce_oracle.next()
+ logg.debug('using nonce {} for address {}'.format(nonce, sender))
+ return {
+ 'from': sender,
+ 'to': recipient,
+ 'value': 0,
+ 'data': '0x',
+ 'nonce': nonce,
+ 'gasPrice': gas_price,
+ 'gas': MINIMUM_FEE_UNITS,
+ 'chainId': self.chain_id,
+ }
diff --git a/requirements.txt b/requirements.txt
@@ -13,3 +13,4 @@ hexathon==0.0.1a2
eth-abi==2.1.1
eth-keys==0.3.3
websocket-client==0.57.0
+redis==3.5.3
