X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=src%2Fmain.rs;h=288cdd24d229168e09ee270e9fdb988b430cf23a;hb=852e939be31574ea13b9f0da3056393d23d55753;hp=19dcb308f6a12144df21afcebd4afe69e7bc75d5;hpb=c1b7bcd7a464506a551f618a5afbbaf6ecbc0f1c;p=dnsseed-rust diff --git a/src/main.rs b/src/main.rs index 19dcb30..288cdd2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,21 +5,20 @@ mod bgp_client; mod timeout_stream; mod datastore; -use std::{cmp, env}; +use std::env; use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::sync::atomic::{Ordering, AtomicBool}; use std::time::{Duration, Instant}; use std::net::{SocketAddr, ToSocketAddrs}; -use bitcoin_hashes::sha256d; - use bitcoin::blockdata::block::Block; use bitcoin::blockdata::constants::genesis_block; -use bitcoin::network::constants::Network; +use bitcoin::hash_types::{BlockHash}; +use bitcoin::network::constants::{Network, ServiceFlags}; use bitcoin::network::message::NetworkMessage; -use bitcoin::network::message_blockdata::{GetHeadersMessage, Inventory, InvType}; -use bitcoin::util::hash::BitcoinHash; +use bitcoin::network::message_blockdata::{GetHeadersMessage, Inventory}; +//use bitcoin::util::hash::BitcoinHash; use printer::{Printer, Stat}; use peer::Peer; @@ -31,18 +30,61 @@ use bgp_client::BGPClient; use tokio::prelude::*; use tokio::timer::Delay; -static mut REQUEST_BLOCK: Option>>> = None; -static mut HIGHEST_HEADER: Option>> = None; -static mut HEADER_MAP: Option>>> = None; -static mut HEIGHT_MAP: Option>>> = None; +static mut REQUEST_BLOCK: Option>>> = None; +static mut HIGHEST_HEADER: Option>> = None; +static mut HEADER_MAP: Option>>> = None; +static mut HEIGHT_MAP: Option>>> = None; static mut DATA_STORE: Option> = None; static mut PRINTER: Option> = None; static mut TOR_PROXY: Option = None; pub static START_SHUTDOWN: AtomicBool = AtomicBool::new(false); static SCANNING: AtomicBool = AtomicBool::new(false); + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::ptr; +use std::sync::atomic::AtomicUsize; + +// We keep track of all memory allocated by Rust code, refusing new allocations if it exceeds +// 1.75GB. +// +// Note that while Rust's std, in general, should panic in response to a null allocation, it +// is totally conceivable that some code will instead dereference this null pointer, which +// would violate our guarantees that Rust modules should never crash the entire application. +// +// In the future, as upstream Rust explores a safer allocation API (eg the Alloc API which +// returns Results instead of raw pointers, or redefining the GlobalAlloc API to allow +// panic!()s inside of alloc calls), we should switch to those, however these APIs are +// currently unstable. +const TOTAL_MEM_LIMIT_BYTES: usize = (1024 + 756) * 1024 * 1024; +static TOTAL_MEM_ALLOCD: AtomicUsize = AtomicUsize::new(0); +struct MemoryLimitingAllocator; +unsafe impl GlobalAlloc for MemoryLimitingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let len = layout.size(); + if len > TOTAL_MEM_LIMIT_BYTES { + return ptr::null_mut(); + } + if TOTAL_MEM_ALLOCD.fetch_add(len, Ordering::AcqRel) + len > TOTAL_MEM_LIMIT_BYTES { + TOTAL_MEM_ALLOCD.fetch_sub(len, Ordering::AcqRel); + return ptr::null_mut(); + } + System.alloc(layout) + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + System.dealloc(ptr, layout); + TOTAL_MEM_ALLOCD.fetch_sub(layout.size(), Ordering::AcqRel); + } +} + +#[global_allocator] +static ALLOC: MemoryLimitingAllocator = MemoryLimitingAllocator; + + struct PeerState { - request: Arc<(u64, sha256d::Hash, Block)>, + request: Arc<(u64, BlockHash, Block)>, + pong_nonce: u64, node_services: u64, msg: (String, bool), fail_reason: AddressState, @@ -51,7 +93,6 @@ struct PeerState { recvd_pong: bool, recvd_addrs: bool, recvd_block: bool, - pong_nonce: u64, } pub fn scan_node(scan_time: Instant, node: SocketAddr, manual: bool) { @@ -81,19 +122,8 @@ pub fn scan_node(scan_time: Instant, node: SocketAddr, manual: bool) { Peer::new(node.clone(), unsafe { TOR_PROXY.as_ref().unwrap() }, Duration::from_secs(timeout), printer) }); tokio::spawn(peer.and_then(move |(mut write, read)| { - TimeoutStream::new_timeout(read, scan_time + Duration::from_secs(store.get_u64(U64Setting::RunTimeout))).map_err(move |err| { - match err { - bitcoin::consensus::encode::Error::UnrecognizedNetworkCommand(ref msg) => { - // If we got here, we hit one of the explicitly disallowed messages indicating - // a bogus "node". - let mut state_lock = err_peer_state.lock().unwrap(); - state_lock.msg = (format!("(bad msg type {})", msg), true); - state_lock.fail_reason = AddressState::EvilNode; - }, - _ => {}, - } - () - }).for_each(move |msg| { + TimeoutStream::new_timeout(read, scan_time + Duration::from_secs(store.get_u64(U64Setting::RunTimeout))) + .map_err(|_| ()).for_each(move |msg| { let mut state_lock = peer_state.lock().unwrap(); macro_rules! check_set_flag { ($recvd_flag: ident, $msg: expr) => { { @@ -125,7 +155,7 @@ pub fn scan_node(scan_time: Instant, node: SocketAddr, manual: bool) { state_lock.fail_reason = AddressState::LowVersion; return future::err(()); } - if ver.services & (1 | (1 << 10)) == 0 { + if !ver.services.has(ServiceFlags::NETWORK) && !ver.services.has(ServiceFlags::NETWORK_LIMITED) { state_lock.msg = (format!("({}: services {:x})", safe_ua, ver.services), true); state_lock.fail_reason = AddressState::NotFullNode; return future::err(()); @@ -136,8 +166,11 @@ pub fn scan_node(scan_time: Instant, node: SocketAddr, manual: bool) { return future::err(()); } check_set_flag!(recvd_version, "version"); - state_lock.node_services = ver.services; + state_lock.node_services = ver.services.as_u64(); state_lock.msg = (format!("(subver: {})", safe_ua), false); + if let Err(_) = write.try_send(NetworkMessage::SendAddrV2) { + return future::err(()); + } if let Err(_) = write.try_send(NetworkMessage::Verack) { return future::err(()); } @@ -173,10 +206,7 @@ pub fn scan_node(scan_time: Instant, node: SocketAddr, manual: bool) { } if addrs.len() > 10 { if !state_lock.recvd_addrs { - if let Err(_) = write.try_send(NetworkMessage::GetData(vec![Inventory { - inv_type: InvType::WitnessBlock, - hash: state_lock.request.1, - }])) { + if let Err(_) = write.try_send(NetworkMessage::GetData(vec![Inventory::WitnessBlock(state_lock.request.1)])) { return future::err(()); } } @@ -184,6 +214,23 @@ pub fn scan_node(scan_time: Instant, node: SocketAddr, manual: bool) { } unsafe { DATA_STORE.as_ref().unwrap() }.add_fresh_nodes(&addrs); }, + Some(NetworkMessage::AddrV2(addrs)) => { + if addrs.len() > 1000 { + state_lock.fail_reason = AddressState::ProtocolViolation; + state_lock.msg = (format!("due to oversized addr: {}", addrs.len()), true); + state_lock.recvd_addrs = false; + return future::err(()); + } + if addrs.len() > 10 { + if !state_lock.recvd_addrs { + if let Err(_) = write.try_send(NetworkMessage::GetData(vec![Inventory::WitnessBlock(state_lock.request.1)])) { + return future::err(()); + } + } + state_lock.recvd_addrs = true; + } + unsafe { DATA_STORE.as_ref().unwrap() }.add_fresh_nodes_v2(&addrs); + }, Some(NetworkMessage::Block(block)) => { if block != state_lock.request.2 { state_lock.fail_reason = AddressState::ProtocolViolation; @@ -195,10 +242,13 @@ pub fn scan_node(scan_time: Instant, node: SocketAddr, manual: bool) { }, Some(NetworkMessage::Inv(invs)) => { for inv in invs { - if inv.inv_type == InvType::Transaction { - state_lock.fail_reason = AddressState::EvilNode; - state_lock.msg = ("due to unrequested inv tx".to_string(), true); - return future::err(()); + match inv { + Inventory::Transaction(_) | Inventory::WitnessTransaction(_) => { + state_lock.fail_reason = AddressState::EvilNode; + state_lock.msg = ("due to unrequested inv tx".to_string(), true); + return future::err(()); + } + _ => {}, } } }, @@ -207,6 +257,14 @@ pub fn scan_node(scan_time: Instant, node: SocketAddr, manual: bool) { state_lock.msg = ("due to unrequested transaction".to_string(), true); return future::err(()); }, + Some(NetworkMessage::Unknown { command, .. }) => { + if command.as_ref() == "gnop" { + let mut state_lock = err_peer_state.lock().unwrap(); + state_lock.msg = (format!("(bad msg type {})", command), true); + state_lock.fail_reason = AddressState::EvilNode; + return future::err(()); + } + }, _ => {}, } future::ok(()) @@ -279,17 +337,19 @@ fn scan_net() { let printer = unsafe { PRINTER.as_ref().unwrap() }; let store = unsafe { DATA_STORE.as_ref().unwrap() }; + let start_time = Instant::now(); let mut scan_nodes = store.get_next_scan_nodes(); printer.add_line(format!("Got {} addresses to scan", scan_nodes.len()), false); - let per_iter_time = Duration::from_millis(1000 / store.get_u64(U64Setting::ConnsPerSec)); - let start_time = Instant::now(); - let mut iter_time = start_time; + if !scan_nodes.is_empty() { + let per_iter_time = Duration::from_millis(datastore::SECS_PER_SCAN_RESULTS * 1000 / scan_nodes.len() as u64); + let mut iter_time = start_time; - for node in scan_nodes.drain(..) { - scan_node(iter_time, node, false); - iter_time += per_iter_time; + for node in scan_nodes.drain(..) { + scan_node(iter_time, node, false); + iter_time += per_iter_time; + } } - Delay::new(cmp::max(iter_time, start_time + Duration::from_secs(1))).then(move |_| { + Delay::new(start_time + Duration::from_secs(datastore::SECS_PER_SCAN_RESULTS)).then(move |_| { if !START_SHUTDOWN.load(Ordering::Relaxed) { scan_net(); } @@ -343,24 +403,23 @@ fn make_trusted_conn(trusted_sockaddr: SocketAddr, bgp_client: Arc) { if let Some(height) = header_map.get(&headers[0].prev_blockhash).cloned() { for i in 0..headers.len() { - let hash = headers[i].bitcoin_hash(); + let hash = headers[i].block_hash(); if i < headers.len() - 1 && headers[i + 1].prev_blockhash != hash { return future::err(()); } - header_map.insert(headers[i].bitcoin_hash(), height + 1 + (i as u64)); - height_map.insert(height + 1 + (i as u64), headers[i].bitcoin_hash()); + header_map.insert(headers[i].block_hash(), height + 1 + (i as u64)); + height_map.insert(height + 1 + (i as u64), headers[i].block_hash()); } let top_height = height + headers.len() as u64; *unsafe { HIGHEST_HEADER.as_ref().unwrap() }.lock().unwrap() - = (headers.last().unwrap().bitcoin_hash(), top_height); + = (headers.last().unwrap().block_hash(), top_height); printer.set_stat(printer::Stat::HeaderCount(top_height)); if top_height >= starting_height as u64 { - if let Err(_) = trusted_write.try_send(NetworkMessage::GetData(vec![Inventory { - inv_type: InvType::WitnessBlock, - hash: height_map.get(&(top_height - 216)).unwrap().clone(), - }])) { + if let Err(_) = trusted_write.try_send(NetworkMessage::GetData(vec![ + Inventory::WitnessBlock(height_map.get(&(top_height - 216)).unwrap().clone()) + ])) { return future::err(()); } } @@ -377,7 +436,7 @@ fn make_trusted_conn(trusted_sockaddr: SocketAddr, bgp_client: Arc) { } }, Some(NetworkMessage::Block(block)) => { - let hash = block.header.bitcoin_hash(); + let hash = block.block_hash(); let header_map = unsafe { HEADER_MAP.as_ref().unwrap() }.lock().unwrap(); let height = *header_map.get(&hash).expect("Got loose block from trusted peer we coulnd't have requested"); if height == unsafe { HIGHEST_HEADER.as_ref().unwrap() }.lock().unwrap().1 - 216 { @@ -416,13 +475,13 @@ fn main() { unsafe { HEADER_MAP = Some(Box::new(Mutex::new(HashMap::with_capacity(600000)))) }; unsafe { HEIGHT_MAP = Some(Box::new(Mutex::new(HashMap::with_capacity(600000)))) }; - unsafe { HEADER_MAP.as_ref().unwrap() }.lock().unwrap().insert(genesis_block(Network::Bitcoin).bitcoin_hash(), 0); - unsafe { HEIGHT_MAP.as_ref().unwrap() }.lock().unwrap().insert(0, genesis_block(Network::Bitcoin).bitcoin_hash()); - unsafe { HIGHEST_HEADER = Some(Box::new(Mutex::new((genesis_block(Network::Bitcoin).bitcoin_hash(), 0)))) }; - unsafe { REQUEST_BLOCK = Some(Box::new(Mutex::new(Arc::new((0, genesis_block(Network::Bitcoin).bitcoin_hash(), genesis_block(Network::Bitcoin)))))) }; + unsafe { HEADER_MAP.as_ref().unwrap() }.lock().unwrap().insert(genesis_block(Network::Bitcoin).block_hash(), 0); + unsafe { HEIGHT_MAP.as_ref().unwrap() }.lock().unwrap().insert(0, genesis_block(Network::Bitcoin).block_hash()); + unsafe { HIGHEST_HEADER = Some(Box::new(Mutex::new((genesis_block(Network::Bitcoin).block_hash(), 0)))) }; + unsafe { REQUEST_BLOCK = Some(Box::new(Mutex::new(Arc::new((0, genesis_block(Network::Bitcoin).block_hash(), genesis_block(Network::Bitcoin)))))) }; let trt = tokio::runtime::Builder::new() - .blocking_threads(2).core_threads(num_cpus::get().max(1) * 3) + .blocking_threads(2).core_threads(num_cpus::get().max(1) + 1) .build().unwrap(); let _ = trt.block_on_all(future::lazy(|| {