dash/test/functional/test_framework/authproxy.py

210 lines
8.9 KiB
Python
Raw Normal View History

Backports 0.15 pr2 (#2597) * Merge #9815: Trivial: use EXIT_ codes instead of magic numbers a87d02a use EXIT_ codes instead of magic numbers (Marko Bencun) * Merge #9801: Removed redundant parameter from mempool.PrioritiseTransaction eaea2bb Removed redundant parameter from mempool.PrioritiseTransaction (gubatron) * remove extra parameter (see 3a3745bb) in dash specific code * Merge #9819: Remove harmless read of unusued priority estimates bc8fd12 Remove harmless read of unusued priority estimates (Alex Morcos) * Merge #9766: Add --exclude option to rpc-tests.py c578408 Add exclude option to rpc-tests.py (John Newbery) * Merge #9577: Fix docstrings in qa tests 3f95a80 Fix docstrings in qa tests (John Newbery) * Merge #9823: qa: Set correct path for binaries in rpc tests 3333ad0 qa: Set correct path for binaries in rpc tests (MarcoFalke) * Merge #9833: Trivial: fix comments referencing AppInit2 ef9f495 Trivial: fix comments referencing AppInit2 (Marko Bencun) * Merge #9612: [trivial] Rephrase the definition of difficulty. dc222f8 Trivial: Rephrase the definition of difficulty in the code. (Karl-Johan Alm) * Merge #9847: Extra test vector for BIP32 30aedcb BIP32 extra test vector (Pieter Wuille) * Merge #9839: [qa] Make import-rescan.py watchonly check reliable 864890a [qa] Make import-rescan.py watchonly check reliable (Russell Yanofsky) Tree-SHA512: ea0e2b1d4fc8f35174c3d575fb751b428daf6ad3aa944fad4e3ddcc9195e4f17051473acabc54203b1d27cca64cf911b737ab92e986c40ef384410652e2dbea1 * Change back file params
2019-01-07 10:55:35 +01:00
# Copyright (c) 2011 Jeff Garzik
#
# Previous copyright, from python-jsonrpc/jsonrpc/proxy.py:
#
# Copyright (c) 2007 Jan-Klaas Kollhof
#
# This file is part of jsonrpc.
#
# jsonrpc is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this software; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""HTTP proxy for opening RPC connection to dashd.
Backports 0.15 pr2 (#2597) * Merge #9815: Trivial: use EXIT_ codes instead of magic numbers a87d02a use EXIT_ codes instead of magic numbers (Marko Bencun) * Merge #9801: Removed redundant parameter from mempool.PrioritiseTransaction eaea2bb Removed redundant parameter from mempool.PrioritiseTransaction (gubatron) * remove extra parameter (see 3a3745bb) in dash specific code * Merge #9819: Remove harmless read of unusued priority estimates bc8fd12 Remove harmless read of unusued priority estimates (Alex Morcos) * Merge #9766: Add --exclude option to rpc-tests.py c578408 Add exclude option to rpc-tests.py (John Newbery) * Merge #9577: Fix docstrings in qa tests 3f95a80 Fix docstrings in qa tests (John Newbery) * Merge #9823: qa: Set correct path for binaries in rpc tests 3333ad0 qa: Set correct path for binaries in rpc tests (MarcoFalke) * Merge #9833: Trivial: fix comments referencing AppInit2 ef9f495 Trivial: fix comments referencing AppInit2 (Marko Bencun) * Merge #9612: [trivial] Rephrase the definition of difficulty. dc222f8 Trivial: Rephrase the definition of difficulty in the code. (Karl-Johan Alm) * Merge #9847: Extra test vector for BIP32 30aedcb BIP32 extra test vector (Pieter Wuille) * Merge #9839: [qa] Make import-rescan.py watchonly check reliable 864890a [qa] Make import-rescan.py watchonly check reliable (Russell Yanofsky) Tree-SHA512: ea0e2b1d4fc8f35174c3d575fb751b428daf6ad3aa944fad4e3ddcc9195e4f17051473acabc54203b1d27cca64cf911b737ab92e986c40ef384410652e2dbea1 * Change back file params
2019-01-07 10:55:35 +01:00
AuthServiceProxy has the following improvements over python-jsonrpc's
ServiceProxy class:
- HTTP connections persist for the life of the AuthServiceProxy object
(if server supports HTTP/1.1)
- sends protocol 'version', per JSON-RPC 1.1
- sends proper, incrementing 'id'
- sends Basic HTTP authentication headers
- parses all JSON numbers that look like floats as Decimal
- uses standard Python json lib
"""
import base64
import decimal
from http import HTTPStatus
import http.client
2014-07-08 20:31:13 +02:00
import json
import logging
import os
import socket
import time
import urllib.parse
HTTP_TIMEOUT = 30
USER_AGENT = "AuthServiceProxy/0.1"
2014-07-08 20:31:13 +02:00
log = logging.getLogger("BitcoinRPC")
class JSONRPCException(Exception):
def __init__(self, rpc_error, http_status=None):
try:
errmsg = '%(message)s (%(code)i)' % rpc_error
except (KeyError, TypeError):
errmsg = ''
super().__init__(errmsg)
self.error = rpc_error
self.http_status = http_status
2014-07-08 20:31:13 +02:00
def EncodeDecimal(o):
if isinstance(o, decimal.Decimal):
return str(o)
2014-07-08 20:31:13 +02:00
raise TypeError(repr(o) + " is not JSON serializable")
class AuthServiceProxy():
2014-07-08 20:31:13 +02:00
__id_count = 0
# ensure_ascii: escape unicode as \uXXXX, passed to json.dumps
def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None, ensure_ascii=True):
self.__service_url = service_url
self._service_name = service_name
self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests
self.__url = urllib.parse.urlparse(service_url)
user = None if self.__url.username is None else self.__url.username.encode('utf8')
passwd = None if self.__url.password is None else self.__url.password.encode('utf8')
authpair = user + b':' + passwd
self.__auth_header = b'Basic ' + base64.b64encode(authpair)
Merge bitcoin/bitcoin#25950: test: fix test abort for high timeout values (and `--timeout-factor 0`) 14302a4802e2dbb41f5189de88f99ddd5dda7736 test: fix test abort for high timeout values (and `--timeout-factor 0`) (Sebastian Falbesoner) Pull request description: On master, the functional tests's option `--timeout-factor 0` (which according to the test docs and parameter description should disable the RPC timeouts) currently fails, same as high values like `--timeout-factor 999999`: ``` $ ./test/functional/wallet_basic.py --timeout-factor 0 2022-08-29T01:26:39.561000Z TestFramework (INFO): Initializing test directory /tmp/bitcoin_func_test_f24yxzp5 2022-08-29T01:26:40.262000Z TestFramework (ERROR): Assertion failed Traceback (most recent call last): File "/home/honey/bitcoin/test/functional/test_framework/test_framework.py", line 549, in start_nodes node.wait_for_rpc_connection() File "/home/honey/bitcoin/test/functional/test_framework/test_node.py", line 234, in wait_for_rpc_connection rpc.getblockcount() File "/home/honey/bitcoin/test/functional/test_framework/coverage.py", line 49, in __call__ return_val = self.auth_service_proxy_instance.__call__(*args, **kwargs) File "/home/honey/bitcoin/test/functional/test_framework/authproxy.py", line 142, in __call__ response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) File "/home/honey/bitcoin/test/functional/test_framework/authproxy.py", line 107, in _request self.__conn.request(method, path, postdata, headers) File "/usr/local/lib/python3.9/http/client.py", line 1285, in request self._send_request(method, url, body, headers, encode_chunked) File "/usr/local/lib/python3.9/http/client.py", line 1331, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/usr/local/lib/python3.9/http/client.py", line 1280, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/usr/local/lib/python3.9/http/client.py", line 1040, in _send_output self.send(msg) File "/usr/local/lib/python3.9/http/client.py", line 980, in send self.connect() File "/usr/local/lib/python3.9/http/client.py", line 946, in connect self.sock = self._create_connection( File "/usr/local/lib/python3.9/socket.py", line 844, in create_connection raise err File "/usr/local/lib/python3.9/socket.py", line 832, in create_connection sock.connect(sa) OSError: [Errno 22] Invalid argument ``` This is caused by a high timeout value that Python's HTTP(S) client library can't cope with. Fix this by clamping down the connection's set timeout value in AuthProxy. The change can easily be tested by running an arbitrary test with `--timeout-factor 0` on master (should fail), on this PR (should pass) and on this PR with the clamping value increased by 1 (should fail). // EDIT: The behaviour was observed on OpenBSD 7.1 and Python 3.9.12. ACKs for top commit: MarcoFalke: lgtm ACK 14302a4802e2dbb41f5189de88f99ddd5dda7736 Tree-SHA512: 6469e8ac699f1bb7dea11d5fb8b3ae54d895bb908570587c5631144cd41fe980ca0b1e6d0b7bfa07983307cba15fb26ae92e6766375672bf5be838d8e5422dbc
2023-02-20 17:15:27 +01:00
# clamp the socket timeout, since larger values can cause an
# "Invalid argument" exception in Python's HTTP(S) client
# library on some operating systems (e.g. OpenBSD, FreeBSD)
self.timeout = min(timeout, 2147483)
self._set_conn(connection)
def __getattr__(self, name):
if name.startswith('__') and name.endswith('__'):
# Python internal stuff
raise AttributeError
if self._service_name is not None:
name = "%s.%s" % (self._service_name, name)
return AuthServiceProxy(self.__service_url, name, connection=self.__conn)
def _request(self, method, path, postdata):
'''
Do a HTTP request, with retry if we get disconnected (e.g. due to a timeout).
This is a workaround for https://bugs.python.org/issue3566 which is fixed in Python 3.5.
'''
headers = {'Host': self.__url.hostname,
'User-Agent': USER_AGENT,
'Authorization': self.__auth_header,
'Content-type': 'application/json'}
if os.name == 'nt':
# Windows somehow does not like to re-use connections
# TODO: Find out why the connection would disconnect occasionally and make it reusable on Windows
# Avoid "ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine"
self._set_conn()
try:
self.__conn.request(method, path, postdata, headers)
return self._get_response()
except (BrokenPipeError, ConnectionResetError):
# Python 3.5+ raises BrokenPipeError when the connection was reset
# ConnectionResetError happens on FreeBSD
self.__conn.close()
self.__conn.request(method, path, postdata, headers)
return self._get_response()
except OSError as e:
Merge #19368: test: improve functional tests compatibility with BSD/macOS 3a7e79478ab41af7c53ce14d9fca9815bffe1f73 test: retry when write to a socket fails on macOS (Ivan Metlushko) 8cf9d15b823d91d2a74fc83832fccca2219342c9 test: use pgrep for better compatibility (Ivan Metlushko) Pull request description: Rationale: a few minor changes to make experience of running tests on macOS a bit better 1.`pidof` is not available on BSD/macOS, while `pgrep` is present on BSD, Linux and macOS 2. Add retry as a workaround for a weird behavior when writing to a socket (https://bugs.python.org/issue33450). Stacktrace attached Man pages: https://www.freebsd.org/cgi/man.cgi?query=pgrep&apropos=0&sektion=1&manpath=FreeBSD+6.0-RELEASE&arch=default&format=html https://man7.org/linux/man-pages/man1/pgrep.1.html Related to #19281 Stacktrace example: ``` ... 33/161 - feature_abortnode.py failed, Duration: 63 s stdout: 2020-06-11T10:46:43.947000Z TestFramework (INFO): Initializing test directory /var/folders/2q/d5w9zh614r7g5c8r74ln3g400000gq/T/test_runner_₿_🏃_20200611_174102/feature_abortnode_128 2020-06-11T10:46:45.199000Z TestFramework (INFO): Waiting for crash 2020-06-11T10:47:15.921000Z TestFramework (INFO): Node crashed - now verifying restart fails 2020-06-11T10:47:47.068000Z TestFramework (INFO): Stopping nodes [node 1] Cleaning up leftover process stderr: Traceback (most recent call last): File "/Users/xxx/Projects/bitcoin/test/functional/feature_abortnode.py", line 50, in <module> AbortNodeTest().main() File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_framework.py", line 142, in main exit_code = self.shutdown() File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_framework.py", line 266, in shutdown self.stop_nodes() File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_framework.py", line 515, in stop_nodes node.stop_node(wait=wait) File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_node.py", line 318, in stop_node self.stop(wait=wait) File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/coverage.py", line 47, in __call__ return_val = self.auth_service_proxy_instance.__call__(*args, **kwargs) File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/authproxy.py", line 142, in __call__ response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/authproxy.py", line 107, in _request self.__conn.request(method, path, postdata, headers) File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 1107, in request self._send_request(method, url, body, headers) File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 1152, in _send_request self.endheaders(body) File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 1103, in endheaders self._send_output(message_body) File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 936, in _send_output self.send(message_body) File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 908, in send self.sock.sendall(data) OSError: [Errno 41] Protocol wrong type for socket ``` ACKs for top commit: laanwj: ACK 3a7e79478ab41af7c53ce14d9fca9815bffe1f73 Tree-SHA512: fefbe40ce94ab29f18bbbed2a434194b1384ffa5279b1d04db7a3708e3dd422bd9e450f1db3f95a1a851fac5a626ab533c6ebcfd7ede96f8ccae9e6f3e9fff92
2020-07-01 15:06:29 +02:00
# Workaround for a bug on macOS. See https://bugs.python.org/issue33450
retry = '[Errno 41] Protocol wrong type for socket' in str(e)
if retry:
self.__conn.close()
self.__conn.request(method, path, postdata, headers)
return self._get_response()
else:
raise
def get_request(self, *args, **argsn):
2014-07-08 20:31:13 +02:00
AuthServiceProxy.__id_count += 1
log.debug("-{}-> {} {}".format(
AuthServiceProxy.__id_count,
self._service_name,
json.dumps(args or argsn, default=EncodeDecimal, ensure_ascii=self.ensure_ascii),
))
if args and argsn:
params = dict(args=args, **argsn)
else:
params = args or argsn
return {'version': '1.1',
'method': self._service_name,
'params': params,
'id': AuthServiceProxy.__id_count}
def __call__(self, *args, **argsn):
postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
response, status = self._request('POST', self.__url.path, postdata.encode('utf-8'))
if response['error'] is not None:
raise JSONRPCException(response['error'], status)
elif 'result' not in response:
raise JSONRPCException({
'code': -343, 'message': 'missing JSON-RPC result'}, status)
elif status != HTTPStatus.OK:
raise JSONRPCException({
'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status)
else:
return response['result']
def batch(self, rpc_call_list):
postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
log.debug("--> " + postdata)
response, status = self._request('POST', self.__url.path, postdata.encode('utf-8'))
if status != HTTPStatus.OK:
raise JSONRPCException({
'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status)
return response
def _get_response(self):
req_start_time = time.time()
try:
http_response = self.__conn.getresponse()
except socket.timeout:
raise JSONRPCException({
'code': -344,
'message': '%r RPC took longer than %f seconds. Consider '
'using larger timeout for calls that take '
'longer to return.' % (self._service_name,
self.__conn.timeout)})
if http_response is None:
raise JSONRPCException({
'code': -342, 'message': 'missing HTTP response from server'})
content_type = http_response.getheader('Content-Type')
if content_type != 'application/json':
raise JSONRPCException(
{'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)},
http_response.status)
2014-07-08 20:31:13 +02:00
responsedata = http_response.read().decode('utf8')
response = json.loads(responsedata, parse_float=decimal.Decimal)
elapsed = time.time() - req_start_time
2014-07-08 20:31:13 +02:00
if "error" in response and response["error"] is None:
log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii)))
2014-07-08 20:31:13 +02:00
else:
log.debug("<-- [%.6f] %s" % (elapsed, responsedata))
return response, http_response.status
def __truediv__(self, relative_uri):
return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn)
def _set_conn(self, connection=None):
port = 80 if self.__url.port is None else self.__url.port
if connection:
self.__conn = connection
self.timeout = connection.timeout
elif self.__url.scheme == 'https':
self.__conn = http.client.HTTPSConnection(self.__url.hostname, port, timeout=self.timeout)
else:
self.__conn = http.client.HTTPConnection(self.__url.hostname, port, timeout=self.timeout)